From 072a46eefb66733ae570a9fb9abbc9570461a490 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 29 Dec 2009 21:53:58 -0500 Subject: Emptied interfaces directory Mostly throwing out a bunch of outdated GUIs. The email interface hasn't been moved over to the new 'Command' format yet... --- misc/xml/be-mbox-to-xml | 154 +++++++++++++++++++++++++++++++++++ misc/xml/be-xml-to-mbox | 208 ++++++++++++++++++++++++++++++++++++++++++++++++ misc/xml/catmutt | 59 ++++++++++++++ 3 files changed, 421 insertions(+) create mode 100755 misc/xml/be-mbox-to-xml create mode 100755 misc/xml/be-xml-to-mbox create mode 100755 misc/xml/catmutt (limited to 'misc') diff --git a/misc/xml/be-mbox-to-xml b/misc/xml/be-mbox-to-xml new file mode 100755 index 0000000..eda6d6e --- /dev/null +++ b/misc/xml/be-mbox-to-xml @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# Copyright (C) 2009 W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Convert an mbox into xml suitable for input into be. + $ be-mbox-to-xml file.mbox | be import-xml -c - +mbox is a flat-file format, consisting of a series of messages. +Messages begin with a a From_ line, followed by RFC 822 email, +followed by a blank line. +""" + +import base64 +import email.utils +from libbe.encoding import get_encoding, set_IO_stream_encodings +from libbe.utility import time_to_str +from mailbox import mbox, Message # the mailbox people really want an on-disk copy +from time import asctime, gmtime, mktime +import types +from xml.sax.saxutils import escape + +DEFAULT_ENCODING = get_encoding() +set_IO_stream_encodings(DEFAULT_ENCODING) + +KNOWN_IDS = [] + +def normalize_email_address(address): + """ + Standardize whitespace, etc. + """ + addr = email.utils.formataddr(email.utils.parseaddr(address)) + if len(addr) == 0: + return None + return addr + +def normalize_RFC_2822_date(date): + """ + Some email clients write non-RFC 2822-compliant date tags like: + Fri, 18 Sep 2009 08:49:02 -0400 (EDT) + with the non-standard (EDT) timezone name. This funtion attempts + to deal with such inconsistencies. + """ + time_tuple = email.utils.parsedate(date) + assert time_tuple != None, \ + 'unparsable date: "%s"' % date + return time_to_str(mktime(time_tuple)) + +def comment_message_to_xml(message, fields=None): + if fields == None: + fields = {} + new_fields = {} + new_fields[u'alt-id'] = message[u'message-id'] + new_fields[u'in-reply-to'] = message[u'in-reply-to'] + new_fields[u'author'] = normalize_email_address(message[u'from']) + new_fields[u'date'] = message[u'date'] + if new_fields[u'date'] != None: + new_fields[u'date'] = normalize_RFC_2822_date(new_fields[u'date']) + new_fields[u'content-type'] = message.get_content_type() + for k,v in new_fields.items(): + if v != None and type(v) != types.UnicodeType: + fields[k] = unicode(v, encoding=DEFAULT_ENCODING) + elif v == None and k in fields: + new_fields[k] = fields[k] + for k,v in fields.items(): + if k not in new_fields: + new_fields.k = fields[k] + fields = new_fields + + if fields[u'in-reply-to'] == None: + if message[u'references'] != None: + refs = message[u'references'].split() + for ref in refs: # search for a known reference id. + if ref in KNOWN_IDS: + fields[u'in-reply-to'] = ref + break + if fields[u'in-reply-to'] == None and len(refs) > 0: + fields[u'in-reply-to'] = refs[0] # default to the first + else: # check for mutliple in-reply-to references. + refs = fields[u'in-reply-to'].split() + found_ref = False + for ref in refs: # search for a known reference id. + if ref in KNOWN_IDS: + fields[u'in-reply-to'] = ref + found_ref = True + break + if found_ref == False and len(refs) > 0: + fields[u'in-reply-to'] = refs[0] # default to the first + + if fields[u'alt-id'] != None: + KNOWN_IDS.append(fields[u'alt-id']) + + if message.is_multipart(): + ret = [] + alt_id = fields[u'alt-id'] + from_str = fields[u'author'] + date = fields[u'date'] + for m in message.walk(): + if m == message: + continue + fields[u'author'] = from_str + fields[u'date'] = date + if len(ret) > 0: # we've added one part already + fields.pop(u'alt-id') # don't pass alt-id to other parts + fields[u'in-reply-to'] = alt_id # others respond to first + ret.append(comment_message_to_xml(m, fields)) + return u'\n'.join(ret) + + charset = message.get_content_charset(DEFAULT_ENCODING).lower() + #assert charset == DEFAULT_ENCODING.lower(), \ + # u"Unknown charset: %s" % charset + + if message[u'content-transfer-encoding'] == None: + encoding = DEFAULT_ENCODING + else: + encoding = message[u'content-transfer-encoding'].lower() + body = message.get_payload(decode=True) # attempt to decode + assert body != None, "Unable to decode?" + if fields[u'content-type'].startswith(u"text/"): + body = unicode(body, encoding=charset).rstrip(u'\n') + else: + body = base64.encode(body) + fields[u'body'] = body + lines = [u""] + for tag,body in fields.items(): + if body != None: + ebody = escape(body) + lines.append(u" <%s>%s" % (tag, ebody, tag)) + lines.append(u"") + return u'\n'.join(lines) + +def main(mbox_filename): + mb = mbox(mbox_filename) + print u'' % DEFAULT_ENCODING + print u"" + for message in mb: + print comment_message_to_xml(message) + print u"" + + +if __name__ == "__main__": + import sys + main(sys.argv[1]) diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox new file mode 100755 index 0000000..ef7b714 --- /dev/null +++ b/misc/xml/be-xml-to-mbox @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# Copyright (C) 2009 W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Convert xml output of `be list --xml` into mbox format for browsing +with a mail reader. For example + $ be list --xml --status=all | be-xml-to-mbox | catmutt + +mbox is a flat-file format, consisting of a series of messages. +Messages begin with a a From_ line, followed by RFC 822 email, +followed by a blank line. +""" + +#from mailbox import mbox, Message # the mailbox people really want an on-disk copy +import codecs +import email.utils +from libbe.encoding import get_encoding, set_IO_stream_encodings +from libbe.utility import str_to_time as rfc2822_to_gmtime_integer +from time import asctime, gmtime +import types +try: # import core module, Python >= 2.5 + from xml.etree import ElementTree +except ImportError: # look for non-core module + from elementtree import ElementTree +from xml.sax.saxutils import unescape + + +DEFAULT_DOMAIN = "invalid.com" +DEFAULT_EMAIL = "dummy@" + DEFAULT_DOMAIN +DEFAULT_ENCODING = get_encoding() +set_IO_stream_encodings(DEFAULT_ENCODING) + +def rfc2822_to_asctime(rfc2822_string): + """Convert an RFC 2822-fomatted string into a asctime string. + >>> rfc2822_to_asctime("Thu, 01 Jan 1970 00:00:00 +0000") + "Thu Jan 01 00:00:00 1970" + """ + if rfc2822_string == "": + return asctime(gmtime(0)) + return asctime(gmtime(rfc2822_to_gmtime_integer(rfc2822_string))) + +class LimitedAttrDict (dict): + """ + Dict with error checking, to avoid invalid bug/comment fields. + """ + _attrs = [] # override with list of valid attribute names + def __init__(self, **kwargs): + dict.__init__(self) + for key,value in kwargs.items(): + self[key] = value + def __setitem__(self, key, item): + self._validate_key(key) + dict.__setitem__(self, key, item) + def _validate_key(self, key): + if key in self._attrs: + return + elif type(key) not in types.StringTypes: + raise TypeError, "Invalid attribute type %s for '%s'" % (type(key), key) + else: + raise ValueError, "Invalid attribute name '%s'" % key + +class Bug (LimitedAttrDict): + _attrs = [u"uuid", + u"short-name", + u"severity", + u"status", + u"assigned", + u"reporter", + u"creator", + u"created", + u"summary", + u"comments", + u"extra-strings"] + def print_to_mbox(self): + if "creator" in self: + # otherwise, probably a `be show` uuid-only bug to avoid + # root comments. + name,addr = email.utils.parseaddr(self["creator"]) + print "From %s %s" % (addr, rfc2822_to_asctime(self["created"])) + print "Message-id: <%s@%s>" % (self["uuid"], DEFAULT_DOMAIN) + print "Date: %s" % self["created"] + print "From: %s" % self["creator"] + print "Content-Type: %s; charset=%s" % ("text/plain", DEFAULT_ENCODING) + print "Content-Transfer-Encoding: 8bit" + print "Subject: %s: %s" % (self["short-name"], self["summary"]) + if "extra-strings" in self: + for estr in self["extra_strings"]: + print "X-Extra-String: %s" % estr + print "" + print self["summary"] + print "" + if "comments" in self: + for comment in self["comments"]: + comment.print_to_mbox(self) + def init_from_etree(self, element): + assert element.tag == "bug", element.tag + for field in element.getchildren(): + text = unescape(unicode(field.text).decode("unicode_escape").strip()) + if field.tag == "comment": + comm = Comment() + comm.init_from_etree(field) + if "comments" in self: + self["comments"].append(comm) + else: + self["comments"] = [comm] + elif field.tag == "extra-string": + if "extra-strings" in self: + self["extra-strings"].append(text) + else: + self["extra-strings"] = [text] + else: + self[field.tag] = text + +class Comment (LimitedAttrDict): + _attrs = [u"uuid", + u"alt-id", + u"short-name", + u"in-reply-to", + u"author", + u"date", + u"content-type", + u"body", + u"extra-strings"] + def print_to_mbox(self, bug=None): + if bug == None: + bug = Bug() + bug[u"uuid"] = u"no-uuid" + name,addr = email.utils.parseaddr(self["author"]) + print "From %s %s" % (addr, rfc2822_to_asctime(self["date"])) + if "uuid" in self: id = self["uuid"] + elif "alt-id" in self: id = self["alt-id"] + else: id = None + if id != None: + print "Message-id: <%s@%s>" % (id, DEFAULT_DOMAIN) + print "Date: %s" % self["date"] + print "From: %s" % self["author"] + subject = "" + if "short-name" in self: + subject += self["short-name"]+u": " + if "summary" in bug: + subject += bug["summary"] + else: + subject += u"no-subject" + print "Subject: %s" % subject + if "in-reply-to" not in self.keys(): + self["in-reply-to"] = bug["uuid"] + print "In-Reply-To: <%s@%s>" % (self["in-reply-to"], DEFAULT_DOMAIN) + if "extra-strings" in self: + for estr in self["extra_strings"]: + print "X-Extra-String: %s" % estr + if self["content-type"].startswith("text/"): + print "Content-Transfer-Encoding: 8bit" + print "Content-Type: %s; charset=%s" % (self["content-type"], DEFAULT_ENCODING) + print "" + print self["body"] + else: # content type and transfer encoding already in XML MIME output + print self["body"] + print "" + def init_from_etree(self, element): + assert element.tag == "comment", element.tag + for field in element.getchildren(): + text = unescape(unicode(field.text).decode("unicode_escape").strip()) + if field.tag == "extra-string": + if "extra-strings" in self: + self["extra-strings"].append(text) + else: + self["extra-strings"] = [text] + else: + if field.tag == "body": + text+="\n" + self[field.tag] = text + +def print_to_mbox(element): + if element.tag == "bug": + b = Bug() + b.init_from_etree(element) + b.print_to_mbox() + elif element.tag == "comment": + c = Comment() + c.init_from_etree(element) + c.print_to_mbox() + elif element.tag in ["be-xml"]: + for elt in element.getchildren(): + print_to_mbox(elt) + +if __name__ == "__main__": + import sys + + if len(sys.argv) == 1: # no filename given, use stdin + xml_unicode = sys.stdin.read() + else: + xml_unicode = codecs.open(sys.argv[1], "r", DEFAULT_ENCODING).read() + xml_str = xml_unicode.encode("unicode_escape").replace(r"\n", "\n") + elist = ElementTree.XML(xml_str) + print_to_mbox(elist) diff --git a/misc/xml/catmutt b/misc/xml/catmutt new file mode 100755 index 0000000..601f14f --- /dev/null +++ b/misc/xml/catmutt @@ -0,0 +1,59 @@ +#!/bin/sh + +# catmutt - wrap mutt allowing mboxes read from stdin. +# +# Copyright (C) 1998-1999 Moritz Barsnick , +# 2009 William Trevor King +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# version 2 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# developed from grepm-0.6 +# http://www.barsnick.net/sw/grepm.html + +PROGNAME=`basename "$0"` +export TMPDIR="${TMPDIR-/tmp}" # used by mktemp +umask 077 + +if [ $# -gt 0 ] && [ "$1" = "--help" ]; then + echo 1>&2 "Usage: ${PROGNAME} [--help] mutt-arguments" + echo 1>&2 "" + echo 1>&2 "Read a mailbox file from stdin and opens it with mutt." + echo 1>&2 "For example: cat somefile.mbox | ${PROGNAME}" + exit 0 +fi + +# Note: the -t/-p options to mktemp are deprecated for mktemp (GNU +# coreutils) 7.1 in favor of --tmpdir but the --tmpdir option does not +# exist yet for my 6.10-3ubuntu2 coreutils +TMPFILE=`mktemp -t catmutt.XXXXXX` || exit 1 + +trap "rm -f ${TMPFILE}; exit 1" 1 2 3 13 15 + +cat > "${TMPFILE}" || exit 1 + +# Now that we've read in the mailbox file, reopen stdin for mutt/user +# interaction. When in a pipe we're not technically in a tty, so use +# a little hack from "greno" at +# http://www.linuxforums.org/forum/linux-programming-scripting/98607-bash-stdin-problem.html +tty="/dev/`ps -p$$ --no-heading | awk '{print $2}'`" +exec < ${tty} + +if [ `wc -c "${TMPFILE}" | awk '{print $1}'` -gt 0 ]; then + echo 1>&2 "Calling mutt on temporary mailbox file (${TMPFILE})." + mutt -R -f "${TMPFILE}" "$@" +else + echo 1>&2 "Empty mailbox input." +fi + +rm -f "${TMPFILE}" && echo 1>&2 "Deleted temporary mailbox file (${TMPFILE})." -- cgit From 4d4283ecd654f1efb058cd7f7dba6be88b70ee92 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 1 Jan 2010 08:11:08 -0500 Subject: Updated copyright information --- misc/xml/be-mbox-to-xml | 2 +- misc/xml/be-xml-to-mbox | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'misc') diff --git a/misc/xml/be-mbox-to-xml b/misc/xml/be-mbox-to-xml index eda6d6e..1fc41e0 100755 --- a/misc/xml/be-mbox-to-xml +++ b/misc/xml/be-mbox-to-xml @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2009 W. Trevor King +# Copyright (C) 2009-2010 W. Trevor King # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox index ef7b714..34d0aa3 100755 --- a/misc/xml/be-xml-to-mbox +++ b/misc/xml/be-xml-to-mbox @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2009 W. Trevor King +# Copyright (C) 2009-2010 W. Trevor King # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit From c8985785eb741ff646082879f1ca5e9cfe3873b0 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 20 Jan 2010 15:22:28 -0500 Subject: 'be-mbox-to-xml' -> 'be-mail-to-xml' + support for several formats. --- misc/xml/be-mail-to-xml | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ misc/xml/be-mbox-to-xml | 154 -------------------------------------------- 2 files changed, 166 insertions(+), 154 deletions(-) create mode 100755 misc/xml/be-mail-to-xml delete mode 100755 misc/xml/be-mbox-to-xml (limited to 'misc') diff --git a/misc/xml/be-mail-to-xml b/misc/xml/be-mail-to-xml new file mode 100755 index 0000000..2add065 --- /dev/null +++ b/misc/xml/be-mail-to-xml @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# Copyright (C) 2009-2010 W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Convert an mbox into xml suitable for input into be. + $ be-mbox-to-xml file.mbox | be import-xml -c - +mbox is a flat-file format, consisting of a series of messages. +Messages begin with a a From_ line, followed by RFC 822 email, +followed by a blank line. +""" + +import base64 +import codecs +import email.utils +from libbe.util.encoding import get_output_encoding +from libbe.util.utility import time_to_str +import mailbox # the mailbox people really want an on-disk copy +import optparse +import sys +from time import asctime, gmtime, mktime +import types +from xml.sax.saxutils import escape + +DEFAULT_ENCODING = get_output_encoding() +sys.stdout = codecs.getwriter(DEFAULT_ENCODING)(sys.stdout) + +KNOWN_IDS = [] + +def normalize_email_address(address): + """ + Standardize whitespace, etc. + """ + addr = email.utils.formataddr(email.utils.parseaddr(address)) + if len(addr) == 0: + return None + return addr + +def normalize_RFC_2822_date(date): + """ + Some email clients write non-RFC 2822-compliant date tags like: + Fri, 18 Sep 2009 08:49:02 -0400 (EDT) + with the non-standard (EDT) timezone name. This funtion attempts + to deal with such inconsistencies. + """ + time_tuple = email.utils.parsedate(date) + assert time_tuple != None, \ + 'unparsable date: "%s"' % date + return time_to_str(mktime(time_tuple)) + +def comment_message_to_xml(message, fields=None): + if fields == None: + fields = {} + new_fields = {} + new_fields[u'alt-id'] = message[u'message-id'] + new_fields[u'in-reply-to'] = message[u'in-reply-to'] + new_fields[u'author'] = normalize_email_address(message[u'from']) + new_fields[u'date'] = message[u'date'] + if new_fields[u'date'] != None: + new_fields[u'date'] = normalize_RFC_2822_date(new_fields[u'date']) + new_fields[u'content-type'] = message.get_content_type() + for k,v in new_fields.items(): + if v != None and type(v) != types.UnicodeType: + fields[k] = unicode(v, encoding=DEFAULT_ENCODING) + elif v == None and k in fields: + new_fields[k] = fields[k] + for k,v in fields.items(): + if k not in new_fields: + new_fields.k = fields[k] + fields = new_fields + + if fields[u'in-reply-to'] == None: + if message[u'references'] != None: + refs = message[u'references'].split() + for ref in refs: # search for a known reference id. + if ref in KNOWN_IDS: + fields[u'in-reply-to'] = ref + break + if fields[u'in-reply-to'] == None and len(refs) > 0: + fields[u'in-reply-to'] = refs[0] # default to the first + else: # check for mutliple in-reply-to references. + refs = fields[u'in-reply-to'].split() + found_ref = False + for ref in refs: # search for a known reference id. + if ref in KNOWN_IDS: + fields[u'in-reply-to'] = ref + found_ref = True + break + if found_ref == False and len(refs) > 0: + fields[u'in-reply-to'] = refs[0] # default to the first + + if fields[u'alt-id'] != None: + KNOWN_IDS.append(fields[u'alt-id']) + + if message.is_multipart(): + ret = [] + alt_id = fields[u'alt-id'] + from_str = fields[u'author'] + date = fields[u'date'] + for m in message.walk(): + if m == message: + continue + fields[u'author'] = from_str + fields[u'date'] = date + if len(ret) > 0: # we've added one part already + fields.pop(u'alt-id') # don't pass alt-id to other parts + fields[u'in-reply-to'] = alt_id # others respond to first + ret.append(comment_message_to_xml(m, fields)) + return u'\n'.join(ret) + + charset = message.get_content_charset(DEFAULT_ENCODING).lower() + #assert charset == DEFAULT_ENCODING.lower(), \ + # u"Unknown charset: %s" % charset + + if message[u'content-transfer-encoding'] == None: + encoding = DEFAULT_ENCODING + else: + encoding = message[u'content-transfer-encoding'].lower() + body = message.get_payload(decode=True) # attempt to decode + assert body != None, "Unable to decode?" + if fields[u'content-type'].startswith(u"text/"): + body = unicode(body, encoding=charset).rstrip(u'\n') + else: + body = base64.encode(body) + fields[u'body'] = body + lines = [u""] + for tag,body in fields.items(): + if body != None: + ebody = escape(body) + lines.append(u" <%s>%s" % (tag, ebody, tag)) + lines.append(u"") + return u'\n'.join(lines) + +def main(argv): + parser = optparse.OptionParser(usage='%prog [options] mailbox') + formats = ['mbox', 'Maildir', 'MH', 'Babyl', 'MMDF'] + parser.add_option('-f', '--format', type='choice', dest='format', + help="Select the mailbox format from %s. See the mailbox module's documention for descriptions of these formats." \ + % ', '.join(formats), + default='mbox', choices=formats) + options,args = parser.parse_args(argv) + mailbox_file = args[1] + reader = getattr(mailbox, options.format) + mb = reader(mailbox_file, factory=None) + print u'' % DEFAULT_ENCODING + print u"" + for message in mb: + print comment_message_to_xml(message) + print u"" + + +if __name__ == "__main__": + import sys + main(sys.argv) diff --git a/misc/xml/be-mbox-to-xml b/misc/xml/be-mbox-to-xml deleted file mode 100755 index 1fc41e0..0000000 --- a/misc/xml/be-mbox-to-xml +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2009-2010 W. Trevor King -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -""" -Convert an mbox into xml suitable for input into be. - $ be-mbox-to-xml file.mbox | be import-xml -c - -mbox is a flat-file format, consisting of a series of messages. -Messages begin with a a From_ line, followed by RFC 822 email, -followed by a blank line. -""" - -import base64 -import email.utils -from libbe.encoding import get_encoding, set_IO_stream_encodings -from libbe.utility import time_to_str -from mailbox import mbox, Message # the mailbox people really want an on-disk copy -from time import asctime, gmtime, mktime -import types -from xml.sax.saxutils import escape - -DEFAULT_ENCODING = get_encoding() -set_IO_stream_encodings(DEFAULT_ENCODING) - -KNOWN_IDS = [] - -def normalize_email_address(address): - """ - Standardize whitespace, etc. - """ - addr = email.utils.formataddr(email.utils.parseaddr(address)) - if len(addr) == 0: - return None - return addr - -def normalize_RFC_2822_date(date): - """ - Some email clients write non-RFC 2822-compliant date tags like: - Fri, 18 Sep 2009 08:49:02 -0400 (EDT) - with the non-standard (EDT) timezone name. This funtion attempts - to deal with such inconsistencies. - """ - time_tuple = email.utils.parsedate(date) - assert time_tuple != None, \ - 'unparsable date: "%s"' % date - return time_to_str(mktime(time_tuple)) - -def comment_message_to_xml(message, fields=None): - if fields == None: - fields = {} - new_fields = {} - new_fields[u'alt-id'] = message[u'message-id'] - new_fields[u'in-reply-to'] = message[u'in-reply-to'] - new_fields[u'author'] = normalize_email_address(message[u'from']) - new_fields[u'date'] = message[u'date'] - if new_fields[u'date'] != None: - new_fields[u'date'] = normalize_RFC_2822_date(new_fields[u'date']) - new_fields[u'content-type'] = message.get_content_type() - for k,v in new_fields.items(): - if v != None and type(v) != types.UnicodeType: - fields[k] = unicode(v, encoding=DEFAULT_ENCODING) - elif v == None and k in fields: - new_fields[k] = fields[k] - for k,v in fields.items(): - if k not in new_fields: - new_fields.k = fields[k] - fields = new_fields - - if fields[u'in-reply-to'] == None: - if message[u'references'] != None: - refs = message[u'references'].split() - for ref in refs: # search for a known reference id. - if ref in KNOWN_IDS: - fields[u'in-reply-to'] = ref - break - if fields[u'in-reply-to'] == None and len(refs) > 0: - fields[u'in-reply-to'] = refs[0] # default to the first - else: # check for mutliple in-reply-to references. - refs = fields[u'in-reply-to'].split() - found_ref = False - for ref in refs: # search for a known reference id. - if ref in KNOWN_IDS: - fields[u'in-reply-to'] = ref - found_ref = True - break - if found_ref == False and len(refs) > 0: - fields[u'in-reply-to'] = refs[0] # default to the first - - if fields[u'alt-id'] != None: - KNOWN_IDS.append(fields[u'alt-id']) - - if message.is_multipart(): - ret = [] - alt_id = fields[u'alt-id'] - from_str = fields[u'author'] - date = fields[u'date'] - for m in message.walk(): - if m == message: - continue - fields[u'author'] = from_str - fields[u'date'] = date - if len(ret) > 0: # we've added one part already - fields.pop(u'alt-id') # don't pass alt-id to other parts - fields[u'in-reply-to'] = alt_id # others respond to first - ret.append(comment_message_to_xml(m, fields)) - return u'\n'.join(ret) - - charset = message.get_content_charset(DEFAULT_ENCODING).lower() - #assert charset == DEFAULT_ENCODING.lower(), \ - # u"Unknown charset: %s" % charset - - if message[u'content-transfer-encoding'] == None: - encoding = DEFAULT_ENCODING - else: - encoding = message[u'content-transfer-encoding'].lower() - body = message.get_payload(decode=True) # attempt to decode - assert body != None, "Unable to decode?" - if fields[u'content-type'].startswith(u"text/"): - body = unicode(body, encoding=charset).rstrip(u'\n') - else: - body = base64.encode(body) - fields[u'body'] = body - lines = [u""] - for tag,body in fields.items(): - if body != None: - ebody = escape(body) - lines.append(u" <%s>%s" % (tag, ebody, tag)) - lines.append(u"") - return u'\n'.join(lines) - -def main(mbox_filename): - mb = mbox(mbox_filename) - print u'' % DEFAULT_ENCODING - print u"" - for message in mb: - print comment_message_to_xml(message) - print u"" - - -if __name__ == "__main__": - import sys - main(sys.argv[1]) -- cgit From 55f1e8588edc35410dd16b883e7cddd06ebc4ed6 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 20 Jan 2010 15:44:39 -0500 Subject: Strip footers (signatures) in be-mail-to-xml --- misc/xml/be-mail-to-xml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'misc') diff --git a/misc/xml/be-mail-to-xml b/misc/xml/be-mail-to-xml index 2add065..0b33465 100755 --- a/misc/xml/be-mail-to-xml +++ b/misc/xml/be-mail-to-xml @@ -34,6 +34,7 @@ from time import asctime, gmtime, mktime import types from xml.sax.saxutils import escape +BREAK = u'--' # signature separator DEFAULT_ENCODING = get_output_encoding() sys.stdout = codecs.getwriter(DEFAULT_ENCODING)(sys.stdout) @@ -60,6 +61,14 @@ def normalize_RFC_2822_date(date): 'unparsable date: "%s"' % date return time_to_str(mktime(time_tuple)) +def strip_footer(body): + body_lines = body.splitlines() + for i,line in enumerate(body_lines): + if line.startswith(BREAK): + break + i += 1 # increment past the current valid line. + return u'\n'.join(body_lines[:i]).strip() + def comment_message_to_xml(message, fields=None): if fields == None: fields = {} @@ -131,7 +140,7 @@ def comment_message_to_xml(message, fields=None): body = message.get_payload(decode=True) # attempt to decode assert body != None, "Unable to decode?" if fields[u'content-type'].startswith(u"text/"): - body = unicode(body, encoding=charset).rstrip(u'\n') + body = strip_footer(unicode(body, encoding=charset)) else: body = base64.encode(body) fields[u'body'] = body -- cgit From a7134ad51224e33df79d4589b677ad0718387f48 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 21 Jan 2010 09:17:40 -0500 Subject: Converted `be list --xml` to format. Fixed up be-xml-to-mbox following the recent libbe restructuring. Moved stdout manipulation in be-mail-to-xml into the if __name__ == '__main__' block, in case some other module wants to recycle some of its functions/methods. --- misc/xml/be-mail-to-xml | 6 +++--- misc/xml/be-xml-to-mbox | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'misc') diff --git a/misc/xml/be-mail-to-xml b/misc/xml/be-mail-to-xml index 0b33465..5a1a88f 100755 --- a/misc/xml/be-mail-to-xml +++ b/misc/xml/be-mail-to-xml @@ -23,20 +23,17 @@ followed by a blank line. """ import base64 -import codecs import email.utils from libbe.util.encoding import get_output_encoding from libbe.util.utility import time_to_str import mailbox # the mailbox people really want an on-disk copy import optparse -import sys from time import asctime, gmtime, mktime import types from xml.sax.saxutils import escape BREAK = u'--' # signature separator DEFAULT_ENCODING = get_output_encoding() -sys.stdout = codecs.getwriter(DEFAULT_ENCODING)(sys.stdout) KNOWN_IDS = [] @@ -172,4 +169,7 @@ def main(argv): if __name__ == "__main__": import sys + import codecs + + sys.stdout = codecs.getwriter(DEFAULT_ENCODING)(sys.stdout) main(sys.argv) diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox index 34d0aa3..015444e 100755 --- a/misc/xml/be-xml-to-mbox +++ b/misc/xml/be-xml-to-mbox @@ -25,10 +25,9 @@ followed by a blank line. """ #from mailbox import mbox, Message # the mailbox people really want an on-disk copy -import codecs import email.utils -from libbe.encoding import get_encoding, set_IO_stream_encodings -from libbe.utility import str_to_time as rfc2822_to_gmtime_integer +from libbe.util.encoding import get_output_encoding +from libbe.util.utility import str_to_time as rfc2822_to_gmtime_integer from time import asctime, gmtime import types try: # import core module, Python >= 2.5 @@ -40,8 +39,7 @@ from xml.sax.saxutils import unescape DEFAULT_DOMAIN = "invalid.com" DEFAULT_EMAIL = "dummy@" + DEFAULT_DOMAIN -DEFAULT_ENCODING = get_encoding() -set_IO_stream_encodings(DEFAULT_ENCODING) +DEFAULT_ENCODING = get_output_encoding() def rfc2822_to_asctime(rfc2822_string): """Convert an RFC 2822-fomatted string into a asctime string. @@ -197,7 +195,11 @@ def print_to_mbox(element): print_to_mbox(elt) if __name__ == "__main__": + import codecs import sys + + sys.stdin = codecs.getreader(DEFAULT_ENCODING)(sys.stdin) + sys.stdout = codecs.getwriter(DEFAULT_ENCODING)(sys.stdout) if len(sys.argv) == 1: # no filename given, use stdin xml_unicode = sys.stdin.read() -- cgit From e5c65b664a56fe63fa8b6a40680200cd47219618 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 21 Jan 2010 10:25:24 -0500 Subject: Add wrap_id to improve id handling in be-xml-to-mbox --- misc/xml/be-xml-to-mbox | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'misc') diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox index 015444e..81741bf 100755 --- a/misc/xml/be-xml-to-mbox +++ b/misc/xml/be-xml-to-mbox @@ -122,6 +122,11 @@ class Bug (LimitedAttrDict): else: self[field.tag] = text +def wrap_id(id): + if "@" not in id: + return "<%s@%s>" % (id, DEFAULT_DOMAIN) + return id + class Comment (LimitedAttrDict): _attrs = [u"uuid", u"alt-id", @@ -142,7 +147,9 @@ class Comment (LimitedAttrDict): elif "alt-id" in self: id = self["alt-id"] else: id = None if id != None: - print "Message-id: <%s@%s>" % (id, DEFAULT_DOMAIN) + print "Message-id: %s" % wrap_id(id) + if "alt-id" in self: + print "Alt-id: %s" % wrap_id(self["alt-id"]) print "Date: %s" % self["date"] print "From: %s" % self["author"] subject = "" @@ -155,7 +162,7 @@ class Comment (LimitedAttrDict): print "Subject: %s" % subject if "in-reply-to" not in self.keys(): self["in-reply-to"] = bug["uuid"] - print "In-Reply-To: <%s@%s>" % (self["in-reply-to"], DEFAULT_DOMAIN) + print "In-Reply-To: %s" % wrap_id(self["in-reply-to"]) if "extra-strings" in self: for estr in self["extra_strings"]: print "X-Extra-String: %s" % estr -- cgit From eaca6f4a93ad66caf9c4207e36471b9660481dc7 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 28 Jan 2010 17:50:17 -0500 Subject: Fix "extra_strings" -> "extra-strings" typos in be-xml-to-mbox --- misc/xml/be-xml-to-mbox | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'misc') diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox index 81741bf..e2c8700 100755 --- a/misc/xml/be-xml-to-mbox +++ b/misc/xml/be-xml-to-mbox @@ -95,7 +95,7 @@ class Bug (LimitedAttrDict): print "Content-Transfer-Encoding: 8bit" print "Subject: %s: %s" % (self["short-name"], self["summary"]) if "extra-strings" in self: - for estr in self["extra_strings"]: + for estr in self["extra-strings"]: print "X-Extra-String: %s" % estr print "" print self["summary"] @@ -164,7 +164,7 @@ class Comment (LimitedAttrDict): self["in-reply-to"] = bug["uuid"] print "In-Reply-To: %s" % wrap_id(self["in-reply-to"]) if "extra-strings" in self: - for estr in self["extra_strings"]: + for estr in self["extra-strings"]: print "X-Extra-String: %s" % estr if self["content-type"].startswith("text/"): print "Content-Transfer-Encoding: 8bit" -- cgit From 907d5f0887f52b6bb718d4ff2a58d3d214c480f8 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 2 Feb 2010 12:37:32 -0500 Subject: Fix be-xml-to-mbox handling of non-text/* content types --- misc/xml/be-xml-to-mbox | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'misc') diff --git a/misc/xml/be-xml-to-mbox b/misc/xml/be-xml-to-mbox index e2c8700..c8b7479 100755 --- a/misc/xml/be-xml-to-mbox +++ b/misc/xml/be-xml-to-mbox @@ -91,7 +91,8 @@ class Bug (LimitedAttrDict): print "Message-id: <%s@%s>" % (self["uuid"], DEFAULT_DOMAIN) print "Date: %s" % self["created"] print "From: %s" % self["creator"] - print "Content-Type: %s; charset=%s" % ("text/plain", DEFAULT_ENCODING) + print "Content-Type: %s; charset=%s" \ + % ("text/plain", DEFAULT_ENCODING) print "Content-Transfer-Encoding: 8bit" print "Subject: %s: %s" % (self["short-name"], self["summary"]) if "extra-strings" in self: @@ -168,11 +169,13 @@ class Comment (LimitedAttrDict): print "X-Extra-String: %s" % estr if self["content-type"].startswith("text/"): print "Content-Transfer-Encoding: 8bit" - print "Content-Type: %s; charset=%s" % (self["content-type"], DEFAULT_ENCODING) - print "" - print self["body"] - else: # content type and transfer encoding already in XML MIME output - print self["body"] + print "Content-Type: %s; charset=%s" \ + % (self["content-type"], DEFAULT_ENCODING) + else: + print "Content-Transfer-Encoding: base64" + print "Content-Type: %s;" % (self["content-type"]) + print "" + print self["body"] print "" def init_from_etree(self, element): assert element.tag == "comment", element.tag -- cgit