diff options
Diffstat (limited to 'interfaces/xml')
| -rwxr-xr-x | interfaces/xml/be-mbox-to-xml | 104 | ||||
| -rwxr-xr-x | interfaces/xml/be-xml-to-mbox | 205 |
2 files changed, 309 insertions, 0 deletions
diff --git a/interfaces/xml/be-mbox-to-xml b/interfaces/xml/be-mbox-to-xml new file mode 100755 index 0000000..840a2a6 --- /dev/null +++ b/interfaces/xml/be-mbox-to-xml @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# Copyright (C) 2009 W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Convert an mbox into xml suitable for imput into be. + $ cat mbox | be-mbox-to-xml | be comment --xml <ID> - +mbox is a flat-file format, consisting of a series of messages. +Messages begin with a a From_ line, followed by RFC 822 email, +followed by a blank line. +""" + +import base64 +import email.utils +from libbe.encoding import get_encoding, set_IO_stream_encodings +from mailbox import mbox, Message # the mailbox people really want an on-disk copy +from time import asctime, gmtime +import types +from xml.sax import make_parser +from xml.sax.handler import ContentHandler +from xml.sax.saxutils import escape + +DEFAULT_ENCODING = get_encoding() +set_IO_stream_encodings(DEFAULT_ENCODING) + +def comment_message_to_xml(message, fields=None): + if fields == None: + fields = {} + new_fields = {} + new_fields[u'alt-id'] = message[u'message-id'] + new_fields[u'in-reply-to'] = message[u'in-reply-to'] + new_fields[u'from'] = message[u'from'] + new_fields[u'date'] = message[u'date'] + new_fields[u'content-type'] = message.get_content_type() + for k,v in new_fields.items(): + if v != None and type(v) != types.UnicodeType: + fields[k] = unicode(v, encoding=DEFAULT_ENCODING) + elif v == None and k in fields: + new_fields[k] = fields[k] + for k,v in fields.items(): + if k not in new_fields: + new_fields.k = fields[k] + fields = new_fields + + if message.is_multipart(): + ret = [] + alt_id = fields[u'alt-id'] + from_str = fields[u'from'] + date = fields[u'date'] + for m in message.walk(): + if m == message: + continue + fields[u'from'] = from_str + fields[u'date'] = date + if len(ret) >= 0: + fields.pop(u'alt-id') + fields[u'in-reply-to'] = alt_id + ret.append(comment_message_to_xml(m, fields)) + return u'\n'.join(ret) + + charset = message.get_content_charset(DEFAULT_ENCODING).lower() + #assert charset == DEFAULT_ENCODING.lower(), \ + # u"Unknown charset: %s" % charset + + encoding = message[u'content-transfer-encoding'].lower() + body = message.get_payload(decode=True) # attempt to decode + assert body != None, "Unable to decode?" + if fields[u'content-type'].startswith(u"text/"): + body = unicode(body, encoding=charset).rstrip(u'\n') + else: + body = base64.encode(body) + fields[u'body'] = body + lines = [u"<comment>"] + for tag,body in fields.items(): + if body != None: + ebody = escape(body) + lines.append(u" <%s>%s</%s>" % (tag, ebody, tag)) + lines.append(u"</comment>") + return u'\n'.join(lines) + +def main(mbox_filename): + mb = mbox(mbox_filename) + print u'<?xml version="1.0" encoding="%s" ?>' % DEFAULT_ENCODING + print u"<comment-list>" + for message in mb: + print comment_message_to_xml(message) + print u"</comment-list>" + + +if __name__ == "__main__": + import sys + main(sys.argv[1]) diff --git a/interfaces/xml/be-xml-to-mbox b/interfaces/xml/be-xml-to-mbox new file mode 100755 index 0000000..ea77c34 --- /dev/null +++ b/interfaces/xml/be-xml-to-mbox @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# Copyright (C) 2009 Chris Ball <cjb@laptop.org> +# W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Convert xml output of `be list --xml` into mbox format for browsing +with a mail reader. For example + $ be list --xml --status=all | be-xml-to-mbox | catmutt + +mbox is a flat-file format, consisting of a series of messages. +Messages begin with a a From_ line, followed by RFC 822 email, +followed by a blank line. +""" + +#from mailbox import mbox, Message # the mailbox people really want an on-disk copy +import codecs +import email.utils +from libbe.encoding import get_encoding, set_IO_stream_encodings +from libbe.utility import str_to_time as rfc2822_to_gmtime_integer +from time import asctime, gmtime +import types +try: # import core module, Python >= 2.5 + from xml.etree import ElementTree +except ImportError: # look for non-core module + from elementtree import ElementTree +from xml.sax.saxutils import unescape + + +DEFAULT_DOMAIN = "invalid.com" +DEFAULT_EMAIL = "dummy@" + DEFAULT_DOMAIN +DEFAULT_ENCODING = get_encoding() +set_IO_stream_encodings(DEFAULT_ENCODING) + +def rfc2822_to_asctime(rfc2822_string): + """Convert an RFC 2822-fomatted string into a asctime string. + >>> rfc2822_to_asctime("Thu, 01 Jan 1970 00:00:00 +0000") + "Thu Jan 01 00:00:00 1970" + """ + if rfc2822_string == "": + return asctime(gmtime(0)) + return asctime(gmtime(rfc2822_to_gmtime_integer(rfc2822_string))) + +class LimitedAttrDict (dict): + """ + Dict with error checking, to avoid invalid bug/comment fields. + """ + _attrs = [] # override with list of valid attribute names + def __init__(self, **kwargs): + dict.__init__(self) + for key,value in kwargs.items(): + self[key] = value + def __setitem__(self, key, item): + self._validate_key(key) + dict.__setitem__(self, key, item) + def _validate_key(self, key): + if key in self._attrs: + return + elif type(key) not in types.StringTypes: + raise TypeError, "Invalid attribute type %s for '%s'" % (type(key), key) + else: + raise ValueError, "Invalid attribute name '%s'" % key + +class Bug (LimitedAttrDict): + _attrs = [u"uuid", + u"short-name", + u"severity", + u"status", + u"assigned", + u"target", + u"reporter", + u"creator", + u"created", + u"summary", + u"comments", + u"extra-strings"] + def print_to_mbox(self): + name,addr = email.utils.parseaddr(self["creator"]) + print "From %s %s" % (addr, rfc2822_to_asctime(self["created"])) + print "Message-ID: <%s@%s>" % (self["uuid"], DEFAULT_DOMAIN) + print "Date: %s" % self["created"] + print "From: %s" % self["creator"] + print "Content-Type: %s; charset=%s" % ("text/plain", DEFAULT_ENCODING) + print "Content-Transfer-Encoding: 8bit" + print "Subject: %s: %s" % (self["short-name"], self["summary"]) + print "" + print self["summary"] + print "" + if "extra-strings" in self: + print "extra strings:\n ", + print '\n '.join(self["extra_strings"]) + print "" + if "comments" in self: + for comment in self["comments"]: + comment.print_to_mbox(self) + def init_from_etree(self, element): + assert element.tag == "bug", element.tag + for field in element.getchildren(): + text = unescape(unicode(field.text).decode("unicode_escape").strip()) + if field.tag == "comment": + comm = Comment() + comm.init_from_etree(field) + if "comments" in self: + self["comments"].append(comm) + else: + self["comments"] = [comm] + elif field.tag == "extra-string": + if "extra-strings" in self: + self["extra-strings"].append(text) + else: + self["extra-strings"] = [text] + else: + self[field.tag] = text + +class Comment (LimitedAttrDict): + _attrs = [u"uuid", + u"alt-id", + u"short-name", + u"in-reply-to", + u"from", + u"date", + u"content-type", + u"body"] + def print_to_mbox(self, bug=None): + if bug == None: + bug = Bug() + bug[u"uuid"] = u"no-uuid" + name,addr = email.utils.parseaddr(self["from"]) + print "From %s %s" % (addr, rfc2822_to_asctime(self["date"])) + if "uuid" in self: id = self["uuid"] + elif "alt-id" in self: id = self["alt-id"] + else: id = None + if id != None: + print "Message-ID: <%s@%s>" % (id, DEFAULT_DOMAIN) + print "Date: %s" % self["date"] + print "From: %s" % self["from"] + subject = "" + if "short-name" in self: + subject += self["short-name"]+u": " + if "summary" in bug: + subject += bug["summary"] + else: + subject += u"no-subject" + print "Subject: %s" % subject + if "in-reply-to" not in self.keys(): + self["in-reply-to"] = bug["uuid"] + print "In-Reply-To: <%s@%s>" % (self["in-reply-to"], DEFAULT_DOMAIN) + if self["content-type"].startswith("text/"): + print "Content-Transfer-Encoding: 8bit" + print "Content-Type: %s; charset=%s" % (self["content-type"], DEFAULT_ENCODING) + print "" + print self["body"] + else: # content type and transfer encoding already in XML MIME output + print self["body"] + print "" + def init_from_etree(self, element): + assert element.tag == "comment", element.tag + for field in element.getchildren(): + text = unescape(unicode(field.text).decode("unicode_escape").strip()) + if field.tag == "body": + text+="\n" + self[field.tag] = text + +def print_to_mbox(element): + if element.tag == "bug": + b = Bug() + b.init_from_etree(element) + b.print_to_mbox() + elif element.tag == "comment": + c = Comment() + c.init_from_etree(element) + c.print_to_mbox() + elif element.tag in ["bugs", "bug-list"]: + for b_elt in element.getchildren(): + b = Bug() + b.init_from_etree(b_elt) + b.print_to_mbox() + elif element.tag in ["comments", "comment-list"]: + for c_elt in element.getchildren(): + c = Comment() + c.init_from_etree(c_elt) + c.print_to_mbox() + +if __name__ == "__main__": + import sys + + if len(sys.argv) == 1: # no filename given, use stdin + xml_unicode = sys.stdin.read() + else: + xml_unicode = codecs.open(sys.argv[1], "r", DEFAULT_ENCODING).read() + xml_str = xml_unicode.encode("unicode_escape").replace(r"\n", "\n") + elist = ElementTree.XML(xml_str) + print_to_mbox(elist) |