Added be-mbox-to-xml.

Reworked to allow "be comment" to handle unicode strings (see bug
e4ed63f6-9000-4d0b-98c3-487269140141).  The solution was to escape all
the unicode to produce and ASCII string before calling
ElementTree.XML, and then converting back to unicode afterwards.

Added a unicode-containing comment to the end of bug
f7ccd916-b5c7-4890-a2e3-8c8ace17ae3a so that there's a handy unicode
comment for testing.

XML headers (e.g. '<?xml version="1.0" encoding="UTF-8" ?>') are
now added to all xml output from be.

Switched non-text/* encoding library to base64 instead of
email.encoders, which makes that code in libbe/comment.py simpler.

Changed libbe/mapfile.py error encoding from string_escape to
unicode_escape so it can handle unicode.

Everything's still untested, and be-xml-to-mbox doesn't handle unicode
yet, but I felt this commit was getting a bit unwieldy ;).

1 files changed, 94 insertions, 0 deletions

diff --git a/xml/be-mbox-to-xml b/xml/be-mbox-to-xml
new file mode 100755
index 0000000..e9077b1
--- /dev/null
+++ b/xml/be-mbox-to-xml
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# Copyright (C) 2009 W. Trevor King <wking@drexel.edu>
+#
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+Convert an mbox into xml suitable for imput into be.
+  $ cat mbox | be-mbox-to-xml | be comment --xml <ID> -
+mbox is a flat-file format, consisting of a series of messages.
+Messages begin with a a From_ line, followed by RFC 822 email,
+followed by a blank line.
+"""
+
+from mailbox import mbox, Message  # the mailbox people really want an on-disk copy
+import email.utils
+import types
+
+import base64
+from libbe.encoding import get_encoding, set_IO_stream_encodings
+from time import asctime, gmtime
+from xml.sax import make_parser
+from xml.sax.handler import ContentHandler
+from xml.sax.saxutils import escape
+
+DEFAULT_ENCODING = get_encoding()
+set_IO_stream_encodings(DEFAULT_ENCODING)
+
+def comment_message_to_xml(message, fields=None):
+    if fields == None:
+        fields = {}
+    fields[u'alt-id'] = message[u'message-id']
+    fields[u'in-reply-to'] = message[u'in-reply-to']
+    fields[u'from'] = message[u'from']
+    fields[u'date'] = message[u'date']
+    fields[u'content-type'] = message.get_content_type()
+    for k,v in fields.items():
+        if v != None and type(v) != types.UnicodeType:
+            fields[k] = unicode(v, encoding=DEFAULT_ENCODING)
+
+    if message.is_multipart():
+        ret = []
+        alt_id = fields[u'alt-id']
+        for m in message.walk():
+            if m == message:
+                continue
+            if len(ret) >= 0:
+                fields.pop(u'alt-id')
+                fields[u'in-reply-to'] = alt_id
+            ret.append(comment_message_to_xml(m, fields))
+            return u'\n'.join(ret)
+
+    charset = message.get_content_charset(DEFAULT_ENCODING).lower()
+    #assert charset == DEFAULT_ENCODING.lower(), \
+    #    u"Unknown charset: %s" % charset
+
+    encoding = message[u'content-transfer-encoding'].lower()
+    body = message.get_payload(decode=True) # attempt to decode
+    assert body != None, "Unable to decode?"
+    if fields[u'content-type'].startswith(u"text/"):
+        body = unicode(body, encoding=charset).rstrip(u'\n')
+    else:
+        body = base64.encode(body)
+    fields[u'body'] = body
+    lines = [u"<comment>"]
+    for tag,body in fields.items():
+        if body != None:
+            ebody = escape(body)
+            lines.append(u"  <%s>%s</%s>" % (tag, ebody, tag))
+    lines.append(u"</comment>")
+    return u'\n'.join(lines)
+
+def main(mbox_filename):
+    mb = mbox(mbox_filename)
+    print u'<?xml version="1.0" encoding="%s" ?>' % DEFAULT_ENCODING
+    print u"<comment-list>"
+    for message in mb:
+        print comment_message_to_xml(message)
+    print u"</comment-list>"
+
+
+if __name__ == "__main__":
+    import sys
+    main(sys.argv[1])