Merged mostly completed `be email-bugs'.

Highlights:
 * new be commands 'email-bugs' and 'import-xml'
 * standardized <be-xml> format for XML files.
 * new be-handle-mail interface '[be-bug:xml]'
 * restrict_file_access security patch
 * new subprocess handling submodule libbe.subproc
 * test.py adjusted to use an installed VCS for most tests.
 * assorted bugfixes

Altered interfaces to the following be commands:
 * comment --xml     tag gone, use import-xml.
 * show --xml        xml format updated to <be-xml> format.

Also adjusted be-mbox-to-xml and be-xml-to-mbox to handle new <be-xml>
format and provide better handling of *.extra_strings.

5 files changed, 120 insertions, 43 deletions

diff --git a/interfaces/email/interactive/README b/interfaces/email/interactive/README
index 79ef9a9..2070973 100644
--- a/interfaces/email/interactive/README
+++ b/interfaces/email/interactive/README
@@ -23,13 +23,15 @@ execution.
 
 Once be-handle-mail receives the email, the parsing method is selected
 according to the subject tag that procmail used grab the email in the
-first place.  There are three parsing styles:
+first place.  There are four parsing styles:
     Style                 Subject
     creating bugs         [be-bug:submit] new bug summary
     commenting on bugs    [be-bug:<bug-id>] commit message
     control               [be-bug] commit message
+    xml                   [be-bug:xml] commit message
 These are analogous to submit@bugs.debian.org, nnn@bugs.debian.org,
-and control@bugs.debian.org respectively.
+and control@bugs.debian.org respectively.  The xml style has no Debian
+analog.
 
 Creating bugs
 =============
@@ -106,6 +108,15 @@ shlex.split().
     --
     Goofy tagline ignored.
 
+XML
+===
+
+This interface allows users without access to the versioned source of
+the program to conveniently submit bugs and comments using be.  You
+should not attempt to compose emails for this interface by hand.  See
+the documentation for the `email-bugs' and `import-xml' be commands
+for details.
+
 Example emails
 ==============
 
diff --git a/interfaces/email/interactive/be-handle-mail b/interfaces/email/interactive/be-handle-mail
index fa80698..e0e3490 100755
--- a/interfaces/email/interactive/be-handle-mail
+++ b/interfaces/email/interactive/be-handle-mail
@@ -81,6 +81,7 @@ SUBJECT_TAG_START = None
 SUBJECT_TAG_NEW = None
 SUBJECT_TAG_COMMENT = None
 SUBJECT_TAG_CONTROL = None
+SUBJECT_TAG_XML = None
 
 BREAK = u"--"
 NEW_REQUIRED_PSEUDOHEADERS = [u"Version"]
@@ -90,7 +91,7 @@ NEW_OPTIONAL_PSEUDOHEADERS = [u"Reporter", u"Assign", u"Depend", u"Severity",
 CONTROL_COMMENT = u"#"
 ALLOWED_COMMANDS = [u"assign", u"comment", u"commit", u"depend", u"help",
                     u"list", u"merge", u"new", u"open", u"severity", u"show",
-                    u"status", u"subscribe", u"tag", u"target"]
+                    u"status", u"subscribe", u"tag", u"target", u"import-xml"]
 
 AUTOCOMMIT = True
 
@@ -241,7 +242,8 @@ class Command (object):
         os.chdir(BE_DIR)
         try:
             self.ret = libbe.cmdutil.execute(self.command, self.args,
-                                             manipulate_encodings=False)
+                                             manipulate_encodings=False,
+                                             restrict_file_access=True)
         except libbe.cmdutil.GetHelp:
             print libbe.cmdutil.help(command)
         except libbe.cmdutil.GetCompletions:
@@ -402,8 +404,8 @@ class Message (object):
     def _subject_tag_type(self):
         """
         Parse subject tag, return (type, value), where type is one of
-        None, "new", "comment", or "control"; and value is None except
-        in the case of "comment", in which case it's the bug
+        None, "new", "comment", "control", or "xml"; and value is None
+        except in the case of "comment", in which case it's the bug
         ID/shortname.
         """
         tag,subject = self._split_subject()
@@ -413,6 +415,8 @@ class Message (object):
             type = u"new"
         elif tag == SUBJECT_TAG_CONTROL:
             type = u"control"
+        elif tag == SUBJECT_TAG_XML:
+            type = u"xml"
         else:
             match = SUBJECT_TAG_COMMENT.match(tag)
             if len(match.groups()) == 1:
@@ -506,6 +510,8 @@ class Message (object):
             commands = self.parse_comment(value)
         elif tag_type == u"control":
             commands = self.parse_control()
+        elif tag_type == u"xml":
+            commands = self.parse_xml()
         else:
             raise Exception, u"Unrecognized tag type '%s'" % tag_type
         return commands
@@ -564,8 +570,10 @@ class Message (object):
         if mime_type == "text/plain":
             body = self._strip_footer(body)
         content_type = mime_type
-        args = [u"--author", author, u"--alt-id", alt_id,
-                u"--content-type", content_type, bug_id, u"-"]
+        args = [u"--author", author]
+        if alt_id != None:
+            args.extend([u"--alt-id", alt_id])
+        args.extend([u"--content-type", content_type, bug_id, u"-"])
         commands = [Command(self, command, args, stdin=body)]
         return commands
     def parse_control(self):
@@ -583,6 +591,16 @@ class Message (object):
         if len(commands) == 0:
             raise InvalidEmail(self, u"No commands in control email.")
         return commands
+    def parse_xml(self):
+        command = u"import-xml"
+        body,mime_type = list(self._get_bodies_and_mime_types())[0]
+        if mime_type != "text/xml":
+            raise InvalidEmail(self,
+                u"Emails to %s must have MIME type 'text/xml', not '%s'."
+                % (SUBJECT_TAG_XML, mime_type))
+        args = [u"-"]
+        commands = [Command(self, command, args, stdin=body)]
+        return commands
     def run(self):
         self._begin_response()
         commands = self.parse()
@@ -736,13 +754,15 @@ def generate_global_tags(tag_base=u"be-bug"):
     Generate a series of tags from a base tag string.
     """
     global SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \
-        SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL
+        SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL, \
+        SUBJECT_TAG_XML
     SUBJECT_TAG_BASE = tag_base
     SUBJECT_TAG_START = u"[%s" % tag_base
     SUBJECT_TAG_RESPONSE = u"[%s]" % tag_base
     SUBJECT_TAG_NEW = u"[%s:submit]" % tag_base
     SUBJECT_TAG_COMMENT = re.compile(u"\[%s:([\-0-9a-z]*)]" % tag_base)
     SUBJECT_TAG_CONTROL = SUBJECT_TAG_RESPONSE
+    SUBJECT_TAG_XML = u"[%s:xml]" % tag_base
 
 def open_logfile(logpath=None):
     """
@@ -945,6 +965,10 @@ class GenerateGlobalTagsTestCase (unittest.TestCase):
         m = SUBJECT_TAG_COMMENT.match("[projectX-bug:xyz-123]")
         self.failUnlessEqual(len(m.groups()), 1)
         self.failUnlessEqual(m.group(1), u"xyz-123")
+    def test_subject_tag_xml(self):
+        "Should set SUBJECT_TAG_XML global correctly"
+        generate_global_tags(u"projectX-bug")
+        self.failUnlessEqual(SUBJECT_TAG_XML, u"[projectX-bug:xml]")
 
 if __name__ == "__main__":
     main(sys.argv)
diff --git a/interfaces/email/interactive/examples/email_bugs b/interfaces/email/interactive/examples/email_bugs
new file mode 100644
index 0000000..949e1c1
--- /dev/null
+++ b/interfaces/email/interactive/examples/email_bugs
@@ -0,0 +1,37 @@
+From jdoe@example.com Fri Apr 18 12:00:00 2008
+Content-Type: text/xml; charset="utf-8"
+MIME-Version: 1.0
+Content-Transfer-Encoding: quoted-printable
+From: jdoe@example.com
+To: a@b.com
+Date: Fri, 18 Apr 2008 12:00:00 +0000
+Subject: [be-bug:xml] Updates to a, b
+
+<?xml version="1.0" encoding="utf-8" ?>
+<be-xml>
+ <version>
+   <tag>1.0.0</tag>
+   <branch-nick>be</branch-nick>
+   <revno>446</revno>
+   <revision-id>wking@drexel.edu-20091119214553-iqyw2cpqluww3zna</revision-id>
+ </version>
+ <bug>
+   <uuid>a</uuid>
+   <short-name>a</short-name>
+   <severity>minor</severity>
+   <status>open</status>
+   <creator>John Doe &lt;jdoe@example.com&gt;</creator>
+   <created>Thu, 01 Jan 1970 00:00:00 +0000</created>
+   <summary>Bug A</summary>
+ </bug>
+ <bug>
+   <uuid>b</uuid>
+   <short-name>b</short-name>
+   <severity>minor</severity>
+   <status>closed</status>
+   <creator>Jane Doe &lt;jdoe@example.com&gt;</creator>
+   <created>Thu, 01 Jan 1970 00:00:00 +0000</created>
+   <summary>Bug B</summary>
+ </bug>
+</be-xml>
+
diff --git a/interfaces/xml/be-mbox-to-xml b/interfaces/xml/be-mbox-to-xml
index a740117..3af2978 100755
--- a/interfaces/xml/be-mbox-to-xml
+++ b/interfaces/xml/be-mbox-to-xml
@@ -15,8 +15,8 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 """
-Convert an mbox into xml suitable for imput into be.
-  $ cat mbox | be-mbox-to-xml | be comment --xml <ID> -
+Convert an mbox into xml suitable for input into be.
+  $ be-mbox-to-xml file.mbox | be import-xml -c <ID> -
 mbox is a flat-file format, consisting of a series of messages.
 Messages begin with a a From_ line, followed by RFC 822 email,
 followed by a blank line.
@@ -140,10 +140,10 @@ def comment_message_to_xml(message, fields=None):
 def main(mbox_filename):
     mb = mbox(mbox_filename)
     print u'<?xml version="1.0" encoding="%s" ?>' % DEFAULT_ENCODING
-    print u"<comment-list>"
+    print u"<be-xml>"
     for message in mb:
         print comment_message_to_xml(message)
-    print u"</comment-list>"
+    print u"</be-xml>"
 
 
 if __name__ == "__main__":
diff --git a/interfaces/xml/be-xml-to-mbox b/interfaces/xml/be-xml-to-mbox
index 7960d56..dc4524e 100755
--- a/interfaces/xml/be-xml-to-mbox
+++ b/interfaces/xml/be-xml-to-mbox
@@ -86,21 +86,23 @@ class Bug (LimitedAttrDict):
               u"comments",
               u"extra-strings"]
     def print_to_mbox(self):
-        name,addr = email.utils.parseaddr(self["creator"])
-        print "From %s %s" % (addr, rfc2822_to_asctime(self["created"]))
-        print "Message-ID: <%s@%s>" % (self["uuid"], DEFAULT_DOMAIN)
-        print "Date: %s" % self["created"]
-        print "From: %s" % self["creator"]
-        print "Content-Type: %s; charset=%s" % ("text/plain", DEFAULT_ENCODING)
-        print "Content-Transfer-Encoding: 8bit"
-        print "Subject: %s: %s" % (self["short-name"], self["summary"])
-        print ""
-        print self["summary"]
-        print ""
-        if "extra-strings" in self:
-            print "extra strings:\n  ",
-            print '\n  '.join(self["extra_strings"])
-        print ""
+        if "creator" in self:
+            # otherwise, probably a `be show` uuid-only bug to avoid
+            # root comments.
+            name,addr = email.utils.parseaddr(self["creator"])
+            print "From %s %s" % (addr, rfc2822_to_asctime(self["created"]))
+            print "Message-id: <%s@%s>" % (self["uuid"], DEFAULT_DOMAIN)
+            print "Date: %s" % self["created"]
+            print "From: %s" % self["creator"]
+            print "Content-Type: %s; charset=%s" % ("text/plain", DEFAULT_ENCODING)
+            print "Content-Transfer-Encoding: 8bit"
+            print "Subject: %s: %s" % (self["short-name"], self["summary"])
+            if "extra-strings" in self:
+                for estr in self["extra_strings"]:
+                    print "X-Extra-String: %s" % estr
+            print ""
+            print self["summary"]
+            print ""
         if "comments" in self:
             for comment in self["comments"]:
                 comment.print_to_mbox(self)            
@@ -131,7 +133,8 @@ class Comment (LimitedAttrDict):
               u"author",
               u"date",
               u"content-type",
-              u"body"]
+              u"body",
+              u"extra-strings"]
     def print_to_mbox(self, bug=None):
         if bug == None:
             bug = Bug()
@@ -142,7 +145,7 @@ class Comment (LimitedAttrDict):
         elif "alt-id" in self: id = self["alt-id"]
         else:                  id = None
         if id != None:
-            print "Message-ID: <%s@%s>" % (id, DEFAULT_DOMAIN)
+            print "Message-id: <%s@%s>" % (id, DEFAULT_DOMAIN)
         print "Date: %s" % self["date"]
         print "From: %s" % self["author"]
         subject = ""
@@ -156,6 +159,9 @@ class Comment (LimitedAttrDict):
         if "in-reply-to" not in self.keys():
             self["in-reply-to"] = bug["uuid"]
         print "In-Reply-To: <%s@%s>" % (self["in-reply-to"], DEFAULT_DOMAIN)
+        if "extra-strings" in self:
+            for estr in self["extra_strings"]:
+                print "X-Extra-String: %s" % estr
         if self["content-type"].startswith("text/"):
             print "Content-Transfer-Encoding: 8bit"
             print "Content-Type: %s; charset=%s" % (self["content-type"], DEFAULT_ENCODING)
@@ -168,9 +174,15 @@ class Comment (LimitedAttrDict):
         assert element.tag == "comment", element.tag
         for field in element.getchildren():
             text = unescape(unicode(field.text).decode("unicode_escape").strip())
-            if field.tag == "body":
-                text+="\n"
-            self[field.tag] = text
+            if field.tag == "extra-string":
+                if "extra-strings" in self:
+                    self["extra-strings"].append(text)
+                else:
+                    self["extra-strings"] = [text]
+            else:
+                if field.tag == "body":
+                    text+="\n"
+                self[field.tag] = text
 
 def print_to_mbox(element):
     if element.tag == "bug":
@@ -181,16 +193,9 @@ def print_to_mbox(element):
         c = Comment()
         c.init_from_etree(element)
         c.print_to_mbox()
-    elif element.tag in ["bugs", "bug-list"]:
-        for b_elt in element.getchildren():
-            b = Bug()
-            b.init_from_etree(b_elt)
-            b.print_to_mbox()
-    elif element.tag in ["comments", "comment-list"]:
-        for c_elt in element.getchildren():
-            c = Comment()
-            c.init_from_etree(c_elt)
-            c.print_to_mbox()
+    elif element.tag in ["be-xml"]:
+        for elt in element.getchildren():
+            print_to_mbox(elt)
 
 if __name__ == "__main__":
     import sys