Added libbe/encoding.py to wrap input/output/file access appropriately.

I borrowed most of the code for this.

get_encoding() is from Trac
  http://trac.edgewall.org/browser/trunk/trac/util/datefmt.py
  format_datetime()
Trac has a BSD license
  http://trac.edgewall.org/wiki/TracLicense
I don't know if such a small snippet requires us to "reproduce the
above copyright" or where we need to reproduce it if it is needed.

The stdout/stdin replacement code follows
  http://wiki.python.org/moin/ShellRedirectionFails

Because of the stdout replacement, the doctests executes now need an
optional 'test' argument to turn off replacement during the doctests,
otherwise doctest flips out (since it had set up stdout to catch
output, and then we clobbered it's setup).

References:
  http://wiki.python.org/moin/Unicode
  http://www.amk.ca/python/howto/unicode
  http://www.python.org/dev/peps/pep-0100/


I also split libbe/editor.py off from libbe.utility.py and started
explaining the motivation for the BugDir init flags in it's docstring.

14 files changed, 363 insertions, 159 deletions

diff --git a/libbe/arch.py b/libbe/arch.py
index fd953a4..1173535 100644
--- a/libbe/arch.py
+++ b/libbe/arch.py
@@ -14,10 +14,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+import codecs
 import os
+import re
 import shutil
 import time
-import re
 import unittest
 import doctest
 
@@ -133,13 +134,16 @@ class Arch(RCS):
         """
         tagpath = os.path.join(path, "{arch}", "=tagging-method")
         lines_out = []
-        for line in file(tagpath, "rb"):
-            line.decode("utf-8")
+        f = codecs.open(tagpath, "r", self.encoding)
+        for line in f:
             if line.startswith("source "):
                 lines_out.append("source ^[._=a-zA-X0-9].*$\n")
             else:
                 lines_out.append(line)
-        file(tagpath, "wb").write("".join(lines_out).encode("utf-8"))
+        f.close()
+        f = codecs.open(tagpath, "w", self.encoding)
+        f.write("".join(lines_out))
+        f.close()
 
     def _add_project_code(self, path):
         # http://mwolson.org/projects/GettingStartedWithArch.html
@@ -215,7 +219,9 @@ class Arch(RCS):
         return [os.path.join(root, p) for p in inv_str.split('\n')]
     def _add_dir_rule(self, rule, dirname, root):
         inv_path = os.path.join(dirname, '.arch-inventory')
-        file(inv_path, "ab").write(rule)
+        f = codecs.open(inv_path, "a", self.encoding)
+        f.write(rule)
+        f.close()
         if os.path.realpath(inv_path) not in self._list_added(root):
             paranoid = self.paranoid
             self.paranoid = False
@@ -233,12 +239,16 @@ class Arch(RCS):
         pass
     def _rcs_get_file_contents(self, path, revision=None):
         if revision == None:
-            return file(self._u_abspath(path), "rb").read()
+            return RCS._rcs_get_file_contents(self, path, revision)
         else:
             status,output,error = \
                 self._invoke_client("file-find", path, revision)
-            path = output.rstrip('\n')
-            return file(self._u_abspath(path), "rb").read()
+            relpath = output.rstrip('\n')
+            abspath = os.path.join(self.rootdir, relpath)
+            f = codecs.open(abspath, "r", self.encoding)
+            contents = f.read()
+            f.close()
+            return contents
     def _rcs_duplicate_repo(self, directory, revision=None):
         if revision == None:
             RCS._rcs_duplicate_repo(self, directory, revision)
diff --git a/libbe/bug.py b/libbe/bug.py
index 68cff7b..67051f2 100644
--- a/libbe/bug.py
+++ b/libbe/bug.py
@@ -182,7 +182,10 @@ class Bug(object):
         if show_comments == True:
             if self._comments_loaded == False:
                 self.load_comments()
-            comout = self.comment_root.string_thread(auto_name_map=True,
+            # take advantage of the string_thread(auto_name_map=True)
+            # SIDE-EFFECT of sorting by bug time.
+            comout = self.comment_root.string_thread(flatten=False,
+                                                     auto_name_map=True,
                                                      bug_shortname=shortname)
             output = bugout + '\n' + comout.rstrip('\n')
         else :
diff --git a/libbe/bugdir.py b/libbe/bugdir.py
index 596922f..6bb6a43 100644
--- a/libbe/bugdir.py
+++ b/libbe/bugdir.py
@@ -24,8 +24,9 @@ import doctest
 
 import mapfile
 import bug
-import utility
 import rcs
+import encoding
+import utility
 
 
 class NoBugDir(Exception):
@@ -64,7 +65,9 @@ class MultipleBugMatches(ValueError):
 TREE_VERSION_STRING = "Bugs Everywhere Tree 1 0\n"
 
 
-def setting_property(name, valid=None, doc=None):
+def setting_property(name, valid=None, default=None, doc=None):
+    if default != None:
+        raise NotImplementedError
     def getter(self):
         value = self.settings.get(name) 
         if valid is not None:
@@ -88,7 +91,27 @@ def setting_property(name, valid=None, doc=None):
 
 class BugDir (list):
     """
-    File-system access:
+    Sink to existing root
+    ======================
+    
+    Consider the following usage case:
+    You have a bug directory rooted in
+      /path/to/source
+    by which I mean the '.be' directory is at
+      /path/to/source/.be
+    However, you're of in some subdirectory like
+      /path/to/source/GUI/testing
+    and you want to comment on a bug.  Setting sink_to_root=True wen
+    you initialize your BugDir will cause it to search for the '.be'
+    file in the ancestors of the path you passed in as 'root'.
+      /path/to/source/GUI/testing/.be     miss
+      /path/to/source/GUI/.be             miss
+      /path/to/source/.be                 hit!
+    So it still roots itself appropriately without much work for you.
+        
+    File-system access
+    ==================
+    
     When rooted in non-bugdir directory, BugDirs live completely in
     memory until the first call to .save().  This creates a '.be'
     sub-directory containing configurations options, bugs, comments,
@@ -98,12 +121,35 @@ class BugDir (list):
     will only load information from the file system when it loads new
     bugs/comments that it doesn't already have in memory, or when it
     explicitly asked to do so (e.g. .load() or __init__(from_disk=True)).
+    
+    Allow RCS initialization
+    ========================
+    
+    This one is for testing purposes.  Setting it to True allows the
+    BugDir to search for an installed RCS backend and initialize it in
+    the root directory.  This is a convenience option for supporting
+    tests of versioning functionality (e.g. .duplicate_bugdir).
+
+    Disable encoding manipulation
+    =============================
+    
+    This one is for testing purposed.  You might have non-ASCII
+    Unicode in your bugs, comments, files, etc.  BugDir instances try
+    and support your preferred encoding scheme (e.g. "utf-8") when
+    dealing with stream and file input/output.  For stream output,
+    this involves replacing sys.stdout and sys.stderr
+    (libbe.encode.set_IO_stream_encodings).  However this messes up
+    doctest's output catching.  In order to support doctest tests
+    using BugDirs, set manipulate_encodings=False, and stick to ASCII
+    in your tests.
     """
     def __init__(self, root=None, sink_to_existing_root=True,
                  assert_new_BugDir=False, allow_rcs_init=False,
+                 manipulate_encodings=True,
                  from_disk=False, rcs=None):
         list.__init__(self)
         self._save_user_id = False
+        self._manipulate_encodings = manipulate_encodings
         self.settings = {}
         if root == None:
             root = os.getcwd()
@@ -131,7 +177,8 @@ class BugDir (list):
         """
         if not os.path.exists(path):
             raise NoRootEntry(path)
-        versionfile = utility.search_parent_directories(path, os.path.join(".be", "version"))
+        versionfile=utility.search_parent_directories(path,
+                                                      os.path.join(".be", "version"))
         if versionfile != None:
             beroot = os.path.dirname(versionfile)
             root = os.path.dirname(beroot)
@@ -142,11 +189,11 @@ class BugDir (list):
                 raise NoBugDir(path)
             return beroot
         
-    def get_version(self, path=None):
-        if self.rcs_name == None:
-            # Use a temporary RCS to check the version for the first time
+    def get_version(self, path=None, use_none_rcs=False):
+        if use_none_rcs == True:
             RCS = rcs.rcs_by_name("None")
             RCS.root(self.root)
+            RCS.encoding = encoding.get_encoding()
         else:
             RCS = self.rcs
 
@@ -159,47 +206,65 @@ class BugDir (list):
         self.rcs.set_file_contents(self.get_path("version"),
                                    TREE_VERSION_STRING)
 
-    rcs_name = setting_property("rcs_name",
-                                ("None", "bzr", "git", "Arch", "hg"),
-                                doc=
-"""The name of the current RCS.  Kept seperate to make saving/loading
-settings easy.  Don't set this attribute.  Set .rcs instead, and
-.rcs_name will be automatically adjusted.""")
-
-    _rcs = None
+    def _get_encoding(self):
+        if self._encoding == None:
+            return encoding.get_encoding()
+        else:
+            return self._encoding
+    def _set_encoding(self, new_encoding):
+        if new_encoding != None:
+            if encoding.known_encoding(new_encoding) == False:
+                raise InvalidValue("encoding", new_encoding)
+        self._encoding = new_encoding
+        if self._manipulate_encodings == True:
+            encoding.set_IO_stream_encodings(self.encoding)
+        if hasattr(self, "rcs"):
+            if self.rcs != None:
+                self.rcs.encoding = self.encoding
+    _encoding = setting_property("encoding",
+                                 doc=
+"""The default input/output encoding to use (e.g. "utf-8").
+Dont' set this attribute, set .encoding instead.""")
+    encoding = property(_get_encoding, _set_encoding, doc=
+"""The default input/output encoding to use (e.g. "utf-8").""")
 
     def _get_rcs(self):
         return self._rcs
-
     def _set_rcs(self, new_rcs):
         if new_rcs == None:
             new_rcs = rcs.rcs_by_name("None")
+        new_rcs.encoding = self.encoding
         self._rcs = new_rcs
         new_rcs.root(self.root)
         self.rcs_name = new_rcs.name
-
+    _rcs = None
     rcs = property(_get_rcs, _set_rcs,
                    doc="A revision control system (RCS) instance")
+    rcs_name = setting_property("rcs_name",
+                                ("None", "bzr", "git", "Arch", "hg"),
+                                doc=
+"""The name of the current RCS.  Kept seperate to make saving/loading
+settings easy.  Don't set this attribute.  Set .rcs instead, and
+.rcs_name will be automatically adjusted.""")
 
-    _user_id = setting_property("user_id", doc=
-"""The user's prefered name.  Kept seperate to make saving/loading
-settings easy.  Don't set this attribute.  Set .user_id instead,
-and ._user_id will be automatically adjusted.  This setting is
-only saved if ._save_user_id == True""")
 
     def _get_user_id(self):
         if self._user_id == None and self.rcs != None:
             self._user_id = self.rcs.get_user_id()
         return self._user_id
-
     def _set_user_id(self, user_id):
         if self.rcs != None:
             self.rcs.user_id = user_id
         self._user_id = user_id
-
     user_id = property(_get_user_id, _set_user_id, doc=
 """The user's prefered name, e.g 'John Doe <jdoe@example.com>'.  Note
 that the Arch RCS backend *enforces* ids with this format.""")
+    _user_id = setting_property("user_id", doc=
+"""The user's prefered name.  Kept seperate to make saving/loading
+settings easy.  Don't set this attribute.  Set .user_id instead,
+and ._user_id will be automatically adjusted.  This setting is
+only saved if ._save_user_id == True""")
+
 
     target = setting_property("target",
                               doc="The current project development target")
@@ -231,7 +296,7 @@ that the Arch RCS backend *enforces* ids with this format.""")
         return new_rcs
 
     def load(self):
-        version = self.get_version()
+        version = self.get_version(use_none_rcs=True)
         if version != TREE_VERSION_STRING:
             raise NotImplementedError, \
                 "BugDir cannot handle version '%s' yet." % version
@@ -241,6 +306,7 @@ that the Arch RCS backend *enforces* ids with this format.""")
             self.settings = self._get_settings(self.get_path("settings"))
             
             self.rcs = rcs.rcs_by_name(self.rcs_name)
+            self.encoding = self.encoding # setup encoding, IO_stream_encoding...
             if self.settings.get("user_id") != None:
                 self.save_user_id()  # was a user name in the settings file
 
@@ -292,6 +358,8 @@ that the Arch RCS backend *enforces* ids with this format.""")
                 if "user_id" in settings:
                     settings = copy.copy(settings)
                     del settings["user_id"]
+            if settings.get("encoding") == encoding.get_encoding():
+                del settings["encoding"] # don't duplicate system default
         allow_no_rcs = not self.rcs.path_in_root(settings_path)
         # allow_no_rcs=True should only be for the special case of
         # configuring duplicate bugdir settings
@@ -310,7 +378,7 @@ that the Arch RCS backend *enforces* ids with this format.""")
             duplicate_settings["user_id"] = self.user_id
             self._save_settings(duplicate_settings_path, duplicate_settings)
 
-        return BugDir(duplicate_path, from_disk=True)
+        return BugDir(duplicate_path, from_disk=True, manipulate_encodings=self._manipulate_encodings)
 
     def remove_duplicate_bugdir(self):
         self.rcs.remove_duplicate_repo()
@@ -423,7 +491,8 @@ def simple_bug_dir():
     """
     dir = utility.Dir()
     assert os.path.exists(dir.path)
-    bugdir = BugDir(dir.path, sink_to_existing_root=False, allow_rcs_init=True)
+    bugdir = BugDir(dir.path, sink_to_existing_root=False, allow_rcs_init=True,
+                    manipulate_encodings=False)
     bugdir._dir_ref = dir # postpone cleanup since dir.__del__() removes dir.
     bug_a = bugdir.new_bug("a", summary="Bug A")
     bug_a.creator = "John Doe <jdoe@example.com>"
diff --git a/libbe/bzr.py b/libbe/bzr.py
index a0ae715..38af6bb 100644
--- a/libbe/bzr.py
+++ b/libbe/bzr.py
@@ -55,7 +55,7 @@ class Bzr(RCS):
         pass
     def _rcs_get_file_contents(self, path, revision=None):
         if revision == None:
-            return file(os.path.join(self.rootdir, path), "rb").read()
+            return RCS._rcs_get_file_contents(self, path, revision)
         else:
             status,output,error = \
                 self._u_invoke_client("cat","-r",revision,path)
diff --git a/libbe/cmdutil.py b/libbe/cmdutil.py
index 6d7ab01..1a321e9 100644
--- a/libbe/cmdutil.py
+++ b/libbe/cmdutil.py
@@ -16,14 +16,14 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 import optparse
 import os
-import locale
 from textwrap import TextWrapper
 from StringIO import StringIO
 import doctest
 
 import bugdir
 import plugin
-import utility
+import encoding
+
 
 class UserError(Exception):
     def __init__(self, msg):
@@ -34,6 +34,13 @@ class UserErrorWrap(UserError):
         UserError.__init__(self, str(exception))
         self.exception = exception
 
+class GetHelp(Exception):
+    pass
+
+class UsageError(Exception):
+    pass
+
+
 def iter_commands():
     for name, module in plugin.iter_plugins("becommands"):
         yield name.replace("_", "-"), module
@@ -52,9 +59,11 @@ def get_command(command_name):
         raise UserError("Unknown command %s" % command_name)
     return cmd
 
+
 def execute(cmd, args):
-    encoding = locale.getpreferredencoding() or 'ascii'
-    return get_command(cmd).execute([a.decode(encoding) for a in args])
+    enc = encoding.get_encoding()
+    get_command(cmd).execute([a.decode(enc) for a in args])
+    return 0
 
 def help(cmd=None):
     if cmd != None:
@@ -71,17 +80,8 @@ def help(cmd=None):
             ret.append("be %s%*s    %s" % (name, numExtraSpaces, "", desc))
         return "\n".join(ret)
 
-class GetHelp(Exception):
-    pass
-
-
-class UsageError(Exception):
-    pass
-
-
 def raise_get_help(option, opt, value, parser):
     raise GetHelp
-
         
 class CmdOptionParser(optparse.OptionParser):
     def __init__(self, usage):
diff --git a/libbe/comment.py b/libbe/comment.py
index 579e294..bd085fa 100644
--- a/libbe/comment.py
+++ b/libbe/comment.py
@@ -243,11 +243,19 @@ class Comment(Tree):
         self.add_reply(reply)
         return reply
 
-    def string_thread(self, name_map={}, indent=0,
+    def string_thread(self, name_map={}, indent=0, flatten=True,
                       auto_name_map=False, bug_shortname=None):
         """
         Return a sting displaying a thread of comments.
         bug_shortname is only used if auto_name_map == True.
+        
+        SIDE-EFFECT: if auto_name_map==True, calls comment_shornames()
+        which will sort the tree by comment.time.  Avoid by calling
+          name_map = {}
+          for shortname,comment in comm.comment_shortnames(bug_shortname):
+              name_map[comment.uuid] = shortname
+          comm.sort(key=lambda c : c.From) # your sort
+          comm.string_thread(name_map=name_map)
 
         >>> a = Comment(bug=None, uuid="a", body="Insightful remarks")
         >>> a.time = utility.str_to_time("Thu, 20 Nov 2008 01:00:00 +0000")
@@ -261,7 +269,7 @@ class Comment(Tree):
         >>> d.uuid = "d"
         >>> d.time = utility.str_to_time("Thu, 20 Nov 2008 04:00:00 +0000")
         >>> a.sort(key=lambda comm : comm.time)
-        >>> print a.string_thread()
+        >>> print a.string_thread(flatten=True)
         --------- Comment ---------
         Name: a
         From: 
@@ -317,7 +325,7 @@ class Comment(Tree):
             for shortname,comment in self.comment_shortnames(bug_shortname):
                 name_map[comment.uuid] = shortname
         stringlist = []
-        for depth,comment in self.thread(flatten=True):
+        for depth,comment in self.thread(flatten=flatten):
             ind = 2*depth+indent
             if comment.uuid in name_map:
                 sname = name_map[comment.uuid]
diff --git a/libbe/config.py b/libbe/config.py
index 79c0d6f..94c700e 100644
--- a/libbe/config.py
+++ b/libbe/config.py
@@ -15,30 +15,40 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 import ConfigParser
+import codecs
+import locale
 import os.path
+import sys
 import doctest
 
+default_encoding = sys.getfilesystemencoding() or locale.getpreferredencoding()
+
 def path():
     """Return the path to the per-user config file"""
     return os.path.expanduser("~/.bugs_everywhere")
 
-def set_val(name, value, section="DEFAULT"):
+def set_val(name, value, section="DEFAULT", encoding=None):
     """Set a value in the per-user config file
 
     :param name: The name of the value to set
     :param value: The new value to set (or None to delete the value)
     :param section: The section to store the name/value in
     """
+    if encoding == None:
+        encoding = default_encoding
     config = ConfigParser.ConfigParser()
-    config.read(path())
+    f = codecs.open(path(), "r", encoding)
+    config.readfp(f, path())
+    f.close()
     if value is not None:
         config.set(section, name, value)
     else:
         config.remove_option(section, name)
-    config.write(file(path(), "wb"))
-    pass
+    f = codecs.open(path(), "w", encoding)
+    config.write(f)
+    f.close()
 
-def get_val(name, section="DEFAULT"):
+def get_val(name, section="DEFAULT", encoding=None):
     """
     Get a value from the per-user config file
 
@@ -49,13 +59,17 @@ def get_val(name, section="DEFAULT"):
     True
     >>> set_val("junk", "random")
     >>> get_val("junk")
-    'random'
+    u'random'
     >>> set_val("junk", None)
     >>> get_val("junk") is None
     True
     """
+    if encoding == None:
+        encoding = default_encoding
     config = ConfigParser.ConfigParser()
-    config.read(path())
+    f = codecs.open(path(), "r", encoding)
+    config.readfp(f, path())
+    f.close()
     try:
         return config.get(section, name)
     except ConfigParser.NoOptionError:
diff --git a/libbe/diff.py b/libbe/diff.py
index 86a91ca..5fc0166 100644
--- a/libbe/diff.py
+++ b/libbe/diff.py
@@ -46,12 +46,13 @@ def diff_report(diff_data, bug_dir):
     added.sort(cmp_severity)
     removed.sort(cmp_severity)
     modified.sort(modified_cmp)
-
+    lines = []
+    
     if len(added) > 0:
-        print "New bug reports:"
+        lines.append("New bug reports:")
         for bug in added:
-            print bug.string(shortlist=True)
-        print ""
+            lines.extend(bug.string(shortlist=True).splitlines())
+        lines.append("")
 
     if len(modified) > 0:
         printed = False
@@ -61,15 +62,18 @@ def diff_report(diff_data, bug_dir):
                 continue
             if not printed:
                 printed = True
-                print "Modified bug reports:"
-            print change_str
-        print ""
+                lines.append("Modified bug reports:")
+            lines.extend(change_str.splitlines())
+        if printed == True:
+            lines.append("")
 
-    if len(removed) > 0: 
-        print "Removed bug reports:"
+    if len(removed) > 0:
+        lines.append("Removed bug reports:")
         for bug in removed:
-            print bug.string(shortlist=True)
-        print ""
+            lines.extend(bug.string(shortlist=True).splitlines())
+        lines.append("")
+    
+    return '\n'.join(lines)
 
 def change_lines(old, new, attributes):
     change_list = []    
diff --git a/libbe/editor.py b/libbe/editor.py
new file mode 100644
index 0000000..4a63e5c
--- /dev/null
+++ b/libbe/editor.py
@@ -0,0 +1,103 @@
+# Bugs Everywhere, a distributed bugtracker
+# Copyright (C) 2005 Aaron Bentley and Panometrics, Inc.
+# <abentley@panoramicfeedback.com>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#    MA 02110-1301, USA
+
+import codecs
+import locale
+import os
+import sys
+import tempfile
+import doctest
+
+default_encoding = sys.getfilesystemencoding() or locale.getpreferredencoding()
+
+comment_marker = u"== Anything below this line will be ignored\n"
+
+class CantFindEditor(Exception):
+    def __init__(self):
+        Exception.__init__(self, "Can't find editor to get string from")
+
+def editor_string(comment=None, encoding=None):
+    """Invokes the editor, and returns the user_produced text as a string
+
+    >>> if "EDITOR" in os.environ:
+    ...     del os.environ["EDITOR"]
+    >>> if "VISUAL" in os.environ:
+    ...     del os.environ["VISUAL"]
+    >>> editor_string()
+    Traceback (most recent call last):
+    CantFindEditor: Can't find editor to get string from
+    >>> os.environ["EDITOR"] = "echo bar > "
+    >>> editor_string()
+    u'bar\\n'
+    >>> os.environ["VISUAL"] = "echo baz > "
+    >>> editor_string()
+    u'baz\\n'
+    >>> del os.environ["EDITOR"]
+    >>> del os.environ["VISUAL"]
+    """
+    if encoding == None:
+        encoding = default_encoding
+    for name in ('VISUAL', 'EDITOR'):
+        try:
+            editor = os.environ[name]
+            break
+        except KeyError:
+            pass
+    else:
+        raise CantFindEditor()
+    fhandle, fname = tempfile.mkstemp()
+    try:
+        if comment is not None:
+            os.write(fhandle, '\n'+comment_string(comment))
+        os.close(fhandle)
+        oldmtime = os.path.getmtime(fname)
+        os.system("%s %s" % (editor, fname))
+        f = codecs.open(fname, "r", encoding)
+        output = trimmed_string(f.read())
+        f.close()
+        if output.rstrip('\n') == "":
+            output = None
+    finally:
+        os.unlink(fname)
+    return output
+
+
+def comment_string(comment):
+    """
+    >>> comment_string('hello') == comment_marker+"hello"
+    True
+    """
+    return comment_marker + comment
+
+
+def trimmed_string(instring):
+    """
+    >>> trimmed_string("hello\\n"+comment_marker)
+    u'hello\\n'
+    >>> trimmed_string("hi!\\n" + comment_string('Booga'))
+    u'hi!\\n'
+    """
+    out = []
+    for line in instring.splitlines(True):
+        if line.startswith(comment_marker):
+            break
+        out.append(line)
+    return ''.join(out)
+
+suite = doctest.DocTestSuite()
diff --git a/libbe/encoding.py b/libbe/encoding.py
new file mode 100644
index 0000000..8b0ef73
--- /dev/null
+++ b/libbe/encoding.py
@@ -0,0 +1,53 @@
+# Bugs Everywhere, a distributed bugtracker
+# Copyright (C) 2005 Aaron Bentley and Panometrics, Inc.
+# <abentley@panoramicfeedback.com>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#    MA 02110-1301, USA
+import codecs
+import locale
+import sys
+import doctest
+
+def get_encoding():
+    """
+    Guess a useful input/output/filesystem encoding...  Maybe we nees
+    seperate encodings for input/output and filessytem?  Hmm...
+    """
+    encoding = locale.getpreferredencoding() or sys.getdefaultencoding()
+    if sys.platform != 'win32' or sys.version_info[:2] > (2, 3):
+        encoding = locale.getlocale(locale.LC_TIME)[1] or encoding
+        # Python 2.3 on windows doesn't know about 'XYZ' alias for 'cpXYZ'
+    return encoding
+
+def known_encoding(encoding):
+    """
+    >>> known_encoding("highly-unlikely-encoding")
+    False
+    >>> known_encoding(get_encoding())
+    True
+    """
+    try:
+        codecs.lookup(encoding)
+        return True
+    except LookupError:
+        return False
+
+def set_IO_stream_encodings(encoding):
+    sys.stdin = codecs.getreader(encoding)(sys.__stdin__)
+    sys.stdout = codecs.getwriter(encoding)(sys.__stdout__)
+    sys.stderr = codecs.getwriter(encoding)(sys.__stderr__)
+
+suite = doctest.DocTestSuite()
diff --git a/libbe/git.py b/libbe/git.py
index 046e72e..98fda2b 100644
--- a/libbe/git.py
+++ b/libbe/git.py
@@ -69,7 +69,7 @@ class Git(RCS):
         self._rcs_add(path)
     def _rcs_get_file_contents(self, path, revision=None):
         if revision == None:
-            return file(self._u_abspath(path), "rb").read()
+            return RCS._rcs_get_file_contents(self, path, revision)
         else:
             arg = "%s:%s" % (revision,path)
             status,output,error = self._u_invoke_client("show", arg)
diff --git a/libbe/hg.py b/libbe/hg.py
index 27cbb79..c00d7e2 100644
--- a/libbe/hg.py
+++ b/libbe/hg.py
@@ -58,14 +58,14 @@ class Hg(RCS):
         pass
     def _rcs_get_file_contents(self, path, revision=None):
         if revision == None:
-            return file(os.path.join(self.rootdir, path), "rb").read()
+            return RCS._rcs_get_file_contents(self, path, revision)
         else:
             status,output,error = \
                 self._u_invoke_client("cat","-r",revision,path)
             return output
     def _rcs_duplicate_repo(self, directory, revision=None):
         if revision == None:
-            RCS._rcs_duplicate_repo(self, directory, revision)
+            return RCS._rcs_duplicate_repo(self, directory, revision)
         else:
             self._u_invoke_client("archive", "--rev", revision, directory)
     def _rcs_commit(self, commitfile):
diff --git a/libbe/rcs.py b/libbe/rcs.py
index 3519c3d..786f9dd 100644
--- a/libbe/rcs.py
+++ b/libbe/rcs.py
@@ -15,13 +15,14 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 from subprocess import Popen, PIPE
+import codecs
 import os
 import os.path
-from socket import gethostname
 import re
+from socket import gethostname
+import shutil
 import sys
 import tempfile
-import shutil
 import unittest
 import doctest
 
@@ -77,8 +78,9 @@ class PathNotInRoot(Exception):
         self.root = root
 
 class NoSuchFile(Exception):
-    def __init__(self, pathname):
-        Exception.__init__(self, "No such file: %s" % pathname)
+    def __init__(self, pathname, root="."):
+        path = os.path.abspath(os.path.join(root, pathname))
+        Exception.__init__(self, "No such file: %s" % path)
 
 
 def new():
@@ -97,12 +99,13 @@ class RCS(object):
     name = "None"
     client = "" # command-line tool for _u_invoke_client
     versioned = False
-    def __init__(self, paranoid=False):
+    def __init__(self, paranoid=False, encoding=sys.getdefaultencoding()):
         self.paranoid = paranoid
         self.verboseInvoke = False
         self.rootdir = None
         self._duplicateBasedir = None
         self._duplicateDirname = None
+        self.encoding = encoding
     def __del__(self):
         self.cleanup()
 
@@ -171,15 +174,15 @@ class RCS(object):
         pass
     def _rcs_get_file_contents(self, path, revision=None):
         """
-        Get the file contents as they were in a given revision.  Don't
-        worry about decoding the contents, the RCS.get_file_contents()
-        method will handle that.
-        
+        Get the file contents as they were in a given revision.
         Revision==None specifies the current revision.
         """
         assert revision == None, \
             "The %s RCS does not support revision specifiers" % self.name
-        return file(os.path.join(self.rootdir, path), "rb").read()
+        f = codecs.open(os.path.join(self.rootdir, path), "r", self.encoding)
+        contents = f.read()
+        f.close()
+        return contents
     def _rcs_duplicate_repo(self, directory, revision=None):
         """
         Get the repository as it was in a given revision.
@@ -297,14 +300,18 @@ class RCS(object):
             relpath = self._u_rel_path(path)
             contents = self._rcs_get_file_contents(relpath,revision)
         else:
-            contents = file(path, "rb").read()
-        return contents.decode("utf-8")
+            f = codecs.open(path, "r", self.encoding)
+            contents = f.read()
+            f.close()
+        return contents
     def set_file_contents(self, path, contents, allow_no_rcs=False):
         """
         Set the file contents under version control.
         """
         add = not os.path.exists(path)
-        file(path, "wb").write(contents.encode("utf-8"))
+        f = codecs.open(path, "w", self.encoding)
+        f.write(contents)
+        f.close()
         
         if self._use_rcs(path, allow_no_rcs):
             if add:
@@ -537,13 +544,13 @@ class RCS(object):
         Split the commitfile created in self.commit() back into
         summary and header lines.
         """
-        f = file(commitfile, "rb")
+        f = codecs.open(commitfile, "r", self.encoding)
         summary = f.readline()
         body = f.read()
         body.lstrip('\n')
         if len(body) == 0:
             body = None
-        f.close
+        f.close()
         return (summary, body)
         
 
diff --git a/libbe/utility.py b/libbe/utility.py
index 2c77fcf..30240a9 100644
--- a/libbe/utility.py
+++ b/libbe/utility.py
@@ -15,10 +15,11 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 import calendar
-import time
+import codecs
 import os
-import tempfile
 import shutil
+import tempfile
+import time
 import doctest
 
 
@@ -87,73 +88,5 @@ def str_to_time(str_time):
 def handy_time(time_val):
     return time.strftime("%a, %d %b %Y %H:%M", time.localtime(time_val))
 
-class CantFindEditor(Exception):
-    def __init__(self):
-        Exception.__init__(self, "Can't find editor to get string from")
-
-def editor_string(comment=None):
-
-    """Invokes the editor, and returns the user_produced text as a string
-
-    >>> if "EDITOR" in os.environ:
-    ...     del os.environ["EDITOR"]
-    >>> if "VISUAL" in os.environ:
-    ...     del os.environ["VISUAL"]
-    >>> editor_string()
-    Traceback (most recent call last):
-    CantFindEditor: Can't find editor to get string from
-    >>> os.environ["EDITOR"] = "echo bar > "
-    >>> editor_string()
-    u'bar\\n'
-    >>> os.environ["VISUAL"] = "echo baz > "
-    >>> editor_string()
-    u'baz\\n'
-    >>> del os.environ["EDITOR"]
-    >>> del os.environ["VISUAL"]
-    """
-    for name in ('VISUAL', 'EDITOR'):
-        try:
-            editor = os.environ[name]
-            break
-        except KeyError:
-            pass
-    else:
-        raise CantFindEditor()
-    fhandle, fname = tempfile.mkstemp()
-    try:
-        if comment is not None:
-            os.write(fhandle, '\n'+comment_string(comment))
-        os.close(fhandle)
-        oldmtime = os.path.getmtime(fname)
-        os.system("%s %s" % (editor, fname))
-        output = trimmed_string(file(fname, "rb").read().decode("utf-8"))
-        if output.rstrip('\n') == "":
-            output = None
-    finally:
-        os.unlink(fname)
-    return output
-
-
-def comment_string(comment):
-    """
-    >>> comment_string('hello')
-    '== Anything below this line will be ignored ==\\nhello'
-    """
-    return '== Anything below this line will be ignored ==\n' + comment
-
-
-def trimmed_string(instring):
-    """
-    >>> trimmed_string("hello\\n== Anything below this line will be ignored")
-    'hello\\n'
-    >>> trimmed_string("hi!\\n" + comment_string('Booga'))
-    'hi!\\n'
-    """
-    out = []
-    for line in instring.splitlines(True):
-        if line.startswith('== Anything below this line will be ignored'):
-            break
-        out.append(line)
-    return ''.join(out)
 
 suite = doctest.DocTestSuite()