diff options
Diffstat (limited to 'interfaces/email')
18 files changed, 1877 insertions, 0 deletions
diff --git a/interfaces/email/interactive/README b/interfaces/email/interactive/README new file mode 100644 index 0000000..48bccdd --- /dev/null +++ b/interfaces/email/interactive/README @@ -0,0 +1,160 @@ +*************** +Email Interface +*************** + +Overview +======== + +The interactive email interface to Bugs Everywhere (BE) attempts to +provide a `Debian-bug-tracking-system-style`_ interface to a BE +repository. Users can mail in bug reports, comments, or control +requests, which will be committed to the served repository. +Developers can then pull the changes they approve of from the served +repository into their other repositories and push updates back onto +the served repository. + +.. _Debian-bug-tracking-system-style: http://www.debian.org/Bugs + +Architecture +============ + +In order to reduce setup costs, the entire interface can piggyback on +an existing email address, although from a security standpoint it's +probably best to create a dedicated user. Incoming email is filtered +by procmail, with matching emails being piped into ``be-handle-mail`` +for execution. + +Once ``be-handle-mail`` receives the email, the parsing method is +selected according to the subject tag that procmail used grab the +email in the first place. There are four parsing styles: + + +--------------------+----------------------------------+ + | Style | Subject | + +====================+==================================+ + | creating bugs | [be-bug:submit] new bug summary | + +--------------------+----------------------------------+ + | commenting on bugs | [be-bug:<bug-id>] commit message | + +--------------------+----------------------------------+ + | control | [be-bug] commit message | + +--------------------+----------------------------------+ + +These are analogous to ``submit@bugs.debian.org``, +``nnn@bugs.debian.org``, and ``control@bugs.debian.org`` respectively. + +Creating bugs +============= + +This interface creates a bug whose summary is given by the email's +post-tag subject. The body of the email must begin with a +pseudo-header containing at least the ``Version`` field. Anything after +the pseudo-header and before a line starting with ``--`` is, if present, +attached as the bug's first comment.:: + + From jdoe@example.com Fri Apr 18 12:00:00 2008 + From: John Doe <jdoe@example.com> + Date: Fri, 18 Apr 2008 12:00:00 +0000 + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + Subject: [be-bug:submit] Need tests for the email interface. + + Version: XYZ + Severity: minor + + Someone should write up a series of test emails to send into + be-handle-mail so we can test changes quickly without having to + use procmail. + + -- + Goofy tagline not included. + +Available pseudo-headers are ``Version``, ``Reporter``, ``Assign``, +``Depend``, ``Severity``, ``Status``, ``Tag``, and ``Target``. + +Commenting on bugs +================== + +This interface appends a comment to the bug specified in the subject +tag. The the first non-multipart body is attached with the +appropriate content-type. In the case of ``text/plain`` contents, +anything following a line starting with ``--`` is stripped.:: + + From jdoe@example.com Fri Apr 18 12:00:00 2008 + From: John Doe <jdoe@example.com> + Date: Fri, 18 Apr 2008 12:00:00 +0000 + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + Subject: [be-bug:XYZ] Isolated problem in baz() + + Finally tracked it down to the bar() call. Some sort of + string<->unicode conversion problem. Solution ideas? + + -- + Goofy tagline not included. + +Controlling bugs +================ + +This interface consists of a list of allowed be commands, with one +command per line. Blank lines and lines beginning with ``#`` are +ignored, as well anything following a line starting with ``--``. All +the listed commands are executed in order and their output returned. +The commands are split into arguments with the POSIX-compliant +shlex.split().:: + + From jdoe@example.com Fri Apr 18 12:00:00 2008 + From: John Doe <jdoe@example.com> + Date: Fri, 18 Apr 2008 12:00:00 +0000 + Content-Type: text/plain; charset=UTF-8 + Content-Transfer-Encoding: 8bit + Subject: [be-bug] I'll handle XYZ by release 1.2.3 + + assign XYZ "John Doe <jdoe@example.com>" + status XYZ assigned + severity XYZ critical + target XYZ 1.2.3 + + -- + Goofy tagline ignored. + +Example emails +============== + +Take a look at ``interfaces/email/interactive/examples`` for some +more examples. + +Procmail rules +============== + +The file ``_procmailrc`` as it stands is fairly appropriate for as a +dedicated user's ``~/.procmailrc``. It forwards matching mail to +``be-handle-mail``, which should be installed somewhere in the user's +path. All non-matching mail is dumped into ``/dev/null``. Everything +procmail does will be logged to ``~/be-mail/procmail.log``. + +If you're piggybacking the interface on top of an existing account, +you probably only need to add the ``be-handle-mail`` stanza to your +existing ``~/.procmailrc``, since you will still want to receive +non-bug emails. + +Note that you will probably have to add a:: + + --repo /path/to/served/repository + +option to the ``be-handle-mail`` invocation so it knows what repository to +serve. + +Multiple repositories may be served by the same email address by adding +multiple ``be-handle-mail`` stanzas, each matching a different tag, for +example the ``[be-bug`` portion of the stanza could be ``[projectX-bug``, +``[projectY-bug``, etc. If you change the base tag, be sure to add a:: + + --tag-base "projectX-bug" + +or equivalent to your ``be-handle-mail`` invocation. + +Testing +======= + +Send test emails in to ``be-handle-mail`` with something like:: + + cat examples/blank | ./be-handle-mail -o -l - -a diff --git a/interfaces/email/interactive/_procmailrc b/interfaces/email/interactive/_procmailrc new file mode 100644 index 0000000..d42c0cf --- /dev/null +++ b/interfaces/email/interactive/_procmailrc @@ -0,0 +1,22 @@ +# .procmailrc +# +# see man procmail, procmailrc, and procmailex +# +# If you already have a ~/.procmailrc file, you probably only need to +# insert the bug-email grabbing stanza in your ~/.procmailrc. +# +# This file is released to the Public Domain. + +MAILDIR=$HOME/be-mail +LOGFILE=$MAILDIR/procmail.log + +# Grab all incoming bug emails (but not replies). This rule eats +# matching emails (i.e. no further procmail processing). +:0 +* ^Subject: \[be-bug +* !^Subject:.*\[be-bug].*Re: +| be-handle-mail + +# Drop everything else +:0 +/dev/null diff --git a/interfaces/email/interactive/be-handle-mail b/interfaces/email/interactive/be-handle-mail new file mode 100755 index 0000000..c8343fc --- /dev/null +++ b/interfaces/email/interactive/be-handle-mail @@ -0,0 +1,909 @@ +#!/usr/bin/env python +# +# Copyright (C) 2009-2010 W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Provide and email interface to the distributed bugtracker Bugs +Everywhere. Recieves incoming email via procmail. Provides an +interface similar to the Debian Bug Tracker. There are currently +three distinct email types: submits, comments, and controls. The +email types are differentiated by tags in the email subject. See +SUBJECT_TAG* for the current values. + +Submit emails create a bug (and optionally add some intitial +comments). The post-tag subject is used as the bug summary, and the +email body is parsed for a pseudo-header. Any text after the +psuedo-header but before a possible line starting with BREAK is added +as the initial bug comment. + +Comment emails add comments to a bug. The first non-multipart portion +of the email is used as the comment body. If that portion has a +"text/plain" type, any text after and including a possible line +starting with BREAK is stripped to avoid lots of taglines cluttering +up the repository. + +Control emails preform any allowed BE commands. The first +non-multipart portion of the email is used as the comment body. If +that portion has a "text/plain" type, any text after and including a +possible line starting with BREAK is stripped. Each pre-BREAK line of +the portion should be a valid BE command, with the initial "be" +omitted, e.g. "be status XYZ fixed" --> "status XYZ fixed". + +Any changes made to the repository are commited after the email is +executed, with the email's post-tag subject as the commit message. +""" + +import codecs +import StringIO as StringIO +import email +from email.mime.multipart import MIMEMultipart +import email.utils +import os +import os.path +import re +import shlex +import sys +import time +import traceback +import types +import doctest +import unittest + +import libbe.bugdir +import libbe.bug +import libbe.comment +import libbe.diff +import libbe.command +import libbe.command.subscribe as subscribe +import libbe.storage +import libbe.ui.command_line +import libbe.util.encoding +import libbe.util.utility +import send_pgp_mime + +THIS_SERVER = u'thor.physics.drexel.edu' +THIS_ADDRESS = u'BE Bugs <wking@thor.physics.drexel.edu>' +UI = None +_THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +LOGPATH = os.path.join(_THIS_DIR, u'be-handle-mail.log') +LOGFILE = None + +# Tag strings generated by generate_global_tags() +SUBJECT_TAG_BASE = u'be-bug' +SUBJECT_TAG_RESPONSE = None +SUBJECT_TAG_START = None +SUBJECT_TAG_NEW = None +SUBJECT_TAG_COMMENT = None +SUBJECT_TAG_CONTROL = None + +BREAK = u'--' +NEW_REQUIRED_PSEUDOHEADERS = [u'Version'] +NEW_OPTIONAL_PSEUDOHEADERS = [u'Reporter', u'Assign', u'Depend', u'Severity', + u'Status', u'Tag', u'Target', + u'Confirm', u'Subscribe'] +CONTROL_COMMENT = u'#' +ALLOWED_COMMANDS = [u'assign', u'comment', u'commit', u'depend', u'diff', + u'due', u'help', u'list', u'merge', u'new', u'severity', + u'show', u'status', u'subscribe', u'tag', u'target'] + +AUTOCOMMIT = True + +ENCODING = u'utf-8' +libbe.util.encoding.ENCODING = ENCODING # force default encoding + +class InvalidEmail (ValueError): + def __init__(self, msg, message): + ValueError.__init__(self, message) + self.msg = msg + def response(self): + header = self.msg.response_header + body = [u'Error processing email:\n', + self.response_body(), u''] + response_generator = \ + send_pgp_mime.PGPMimeMessageFactory(u'\n'.join(body)) + response = MIMEMultipart() + response.attach(response_generator.plain()) + response.attach(self.msg.msg) + ret = send_pgp_mime.attach_root(header, response) + return ret + def response_body(self): + err_text = [unicode(self)] + return u'\n'.join(err_text) + +class InvalidSubject (InvalidEmail): + def __init__(self, msg, message=None): + if message == None: + message = u'Invalid subject' + InvalidEmail.__init__(self, msg, message) + def response_body(self): + err_text = u'\n'.join([unicode(self), u'', + u'full subject was:', + self.msg.subject()]) + return err_text + +class InvalidPseudoHeader (InvalidEmail): + def response_body(self): + err_text = [u'Invalid pseudo-header:\n', + unicode(self)] + return u'\n'.join(err_text) + +class InvalidCommand (InvalidEmail): + def __init__(self, msg, command, message=None): + bigmessage = u'Invalid execution command "%s"' % command + if message != None: + bigmessage += u'\n%s' % message + InvalidEmail.__init__(self, msg, bigmessage) + self.command = command + +class InvalidOption (InvalidCommand): + def __init__(self, msg, option, message=None): + bigmessage = u'Invalid option "%s"' % (option) + if message != None: + bigmessage += u'\n%s' % message + InvalidCommand.__init__(self, msg, info, command, bigmessage) + self.option = option + +class NotificationFailed (Exception): + def __init__(self, msg): + bigmessage = 'Notification failed: %s' % msg + Exception.__init__(self, bigmessage) + self.short_msg = msg + +class ID (object): + """ + Sometimes you want to reference the output of a command that + hasn't been executed yet. ID is there for situations like + > a = Command(msg, "new", ["create a bug"]) + > b = Command(msg, "comment", [ID(a), "and comment on it"]) + """ + def __init__(self, command): + self.command = command + def extract_id(self): + if hasattr(self, 'cached_id'): + return self._cached_id + assert self.command.ret == 0, self.command.ret + if self.command.command.name == u'new': + regexp = re.compile(u'Created bug with ID (.*)') + else: + raise NotImplementedError, self.command.command + match = regexp.match(self.command.stdout) + assert len(match.groups()) == 1, str(match.groups()) + self._cached_id = match.group(1) + return self._cached_id + def __str__(self): + if self.command.ret != 0: + return '<id for %s>' % repr(self.command) + return '<id %s>' % self.extract_id() + +class Command (object): + """ + A libbe.command.Command handler. + + Initialize with + Command(msg, command, args=None, stdin=None) + where + msg: the Message instance prompting this command + command: name of becommand to execute, e.g. "new" + args: list of arguments to pass to the command + stdin: if non-null, a string to pipe into the command's stdin + """ + def __init__(self, msg, command, args=None, stdin=None): + self.msg = msg + if args == None: + self.args = [] + else: + self.args = args + self.command = libbe.command.get_command_class(command_name=command)() + self.command._setup_io = lambda i_enc,o_enc : None + self.ret = None + self.stdin = stdin + self.stdout = None + def __str__(self): + return '<command: %s %s>' % (self.command, ' '.join([str(s) for s in self.args])) + def normalize_args(self): + """ + Expand any ID placeholders in self.args. + """ + for i,arg in enumerate(self.args): + if isinstance(arg, ID): + self.args[i] = arg.extract_id() + def run(self): + """ + Attempt to execute the command whose info is given in the dictionary + info. Returns the exit code, stdout, and stderr produced by the + command. + """ + if self.command.name in [None, u'']: # don't accept blank commands + raise InvalidCommand(self.msg, self, 'Blank') + elif self.command.name not in ALLOWED_COMMANDS: + raise InvalidCommand(self.msg, self, 'Not allowed') + assert self.ret == None, u'running %s twice!' % unicode(self) + self.normalize_args() + UI.io.set_stdin(self.stdin) + self.ret = libbe.ui.command_line.dispatch(UI, self.command, self.args) + self.stdout = UI.io.get_stdout() + return (self.ret, self.stdout) + def response_msg(self): + if self.ret == None: self.ret = -1 + response_body = [u'Results of running: (exit code %d)' % self.ret, + u' %s %s' % (self.command.name,u' '.join(self.args))] + if self.stdout != None and len(self.stdout) > 0: + response_body.extend([u'', u'output:', u'', self.stdout]) + response_body.append(u'') # trailing endline + response_generator = \ + send_pgp_mime.PGPMimeMessageFactory(u'\n'.join(response_body)) + return response_generator.plain() + +class DiffTree (libbe.diff.DiffTree): + """ + In order to avoid tons of tiny MIMEText attachments, bug-level + nodes set .add_child_text=True (in .join()), which is propogated + on to their descendents. Instead of creating their own + attachement, each of these descendents appends his data_part to + the end of the bug-level MIMEText attachment. + + For the example tree in the libbe.diff.Diff unittests: + bugdir + bugdir/settings + bugdir/bugs + bugdir/bugs/new + bugdir/bugs/new/c <- sets .add_child_text + bugdir/bugs/rem + bugdir/bugs/rem/b <- sets .add_child_text + bugdir/bugs/mod + bugdir/bugs/mod/a <- sets .add_child_text + bugdir/bugs/mod/a/settings + bugdir/bugs/mod/a/comments + bugdir/bugs/mod/a/comments/new + bugdir/bugs/mod/a/comments/new/acom + bugdir/bugs/mod/a/comments/rem + bugdir/bugs/mod/a/comments/mod + """ + def report_or_none(self): + report = self.report() + if report == None: + return None + payload = report.get_payload() + if payload == None or len(payload) == 0: + return None + return report + def report_string(self): + report = self.report_or_none() + if report == None: + return 'No changes' + else: + return send_pgp_mime.flatten(report, to_unicode=True) + def make_root(self): + return MIMEMultipart() + def join(self, root, parent, data_part): + if hasattr(parent, 'attach_child_text'): + self.attach_child_text = True + if data_part != None: + send_pgp_mime.append_text(parent.data_mime_part, u'\n\n%s' % (data_part)) + self.data_mime_part = parent.data_mime_part + else: + self.data_mime_part = None + if data_part != None: + self.data_mime_part = send_pgp_mime.encodedMIMEText(data_part) + if parent != None and parent.name in [u'new', u'rem', u'mod']: + self.attach_child_text = True + if data_part == None: # make blank data_mime_part for children's appends + self.data_mime_part = send_pgp_mime.encodedMIMEText(u'') + if self.data_mime_part != None: + self.data_mime_part[u'Content-Description'] = self.name + root.attach(self.data_mime_part) + def data_part(self, depth, indent=False): + return libbe.diff.DiffTree.data_part(self, depth, indent=indent) + +class Diff (libbe.diff.Diff): + def bug_add_string(self, bug): + return bug.string(show_comments=True) + def _comment_summary_string(self, comment): + return comment.string() + def comment_add_string(self, comment): + return self._comment_summary_string(comment) + def comment_rem_string(self, comment): + return self._comment_summary_string(comment) + +class Message (object): + def __init__(self, email_text=None, disable_parsing=False): + if disable_parsing == False: + self.text = email_text + p=email.Parser.Parser() + self.msg=p.parsestr(self.text) + if LOGFILE != None: + LOGFILE.write(u'handling %s\n' % self.author_addr()) + LOGFILE.write(u'\n%s\n\n' % self.text) + self.confirm = True # enable/disable confirmation email + def _yes_no(self, boolean): + if boolean == True: + return 'yes' + return 'no' + def author_tuple(self): + """ + Extract and normalize the sender's email address. Returns a + (name, email) tuple. + """ + if not hasattr(self, 'author_tuple_cache'): + self._author_tuple_cache = \ + send_pgp_mime.source_email(self.msg, return_realname=True) + return self._author_tuple_cache + def author_addr(self): + return email.utils.formataddr(self.author_tuple()) + def author_name(self): + return self.author_tuple()[0] + def author_email(self): + return self.author_tuple()[1] + def default_msg_attribute_access(self, attr_name, default=None): + if attr_name in self.msg: + return self.msg[attr_name] + return default + def message_id(self, default=None): + return self.default_msg_attribute_access('message-id', default=default) + def subject(self): + if 'subject' not in self.msg: + raise InvalidSubject(self, u'Email must contain a subject') + return self.msg['subject'] + def _split_subject(self): + """ + Returns (tag, subject), with missing values replaced by None. + """ + if hasattr(self, '_split_subject_cache'): + return self._split_subject_cache + args = self.subject().split(u']',1) + if len(args) < 1: + self._split_subject_cache = (None, None) + elif len(args) < 2: + self._split_subject_cache = (args[0]+u']', None) + else: + self._split_subject_cache = (args[0]+u']', args[1].strip()) + return self._split_subject_cache + def _subject_tag_type(self): + """ + Parse subject tag, return (type, value), where type is one of + None, "new", "comment", or "control"; and value is None except + in the case of "comment", in which case it's the bug + ID/shortname. + """ + tag,subject = self._split_subject() + type = None + value = None + if tag == SUBJECT_TAG_NEW: + type = u'new' + elif tag == SUBJECT_TAG_CONTROL: + type = u'control' + else: + match = SUBJECT_TAG_COMMENT.match(tag) + if len(match.groups()) == 1: + type = u'comment' + value = match.group(1) + return (type, value) + def validate_subject(self): + """ + Validate the subject line. + """ + tag,subject = self._split_subject() + if not tag.startswith(SUBJECT_TAG_START): + raise InvalidSubject( + self, u'Subject must start with "%s"' % SUBJECT_TAG_START) + tag_type,value = self._subject_tag_type() + if tag_type == None: + raise InvalidSubject(self, u'Invalid tag "%s"' % tag) + elif tag_type == u'new' and len(subject) == 0: + raise InvalidSubject(self, u'Cannot create a bug with blank title') + elif tag_type == u'comment' and len(value) == 0: + raise InvalidSubject(self, u'Must specify a bug ID to comment') + def _get_bodies_and_mime_types(self): + """ + Traverse the email message returning (body, mime_type) for + each non-mulitpart portion of the message. + """ + msg_charset = self.msg.get_content_charset(ENCODING).lower() + for part in self.msg.walk(): + if part.is_multipart(): + continue + body,mime_type=(part.get_payload(decode=True),part.get_content_type()) + charset = part.get_content_charset(msg_charset).lower() + if mime_type.startswith('text/'): + body = unicode(body, charset) # convert text types to unicode + yield (body, mime_type) + def _parse_body_pseudoheaders(self, body, required, optional, + dictionary=None): + """ + Grab any pseudo-headers from the beginning of body. Raise + InvalidPseudoHeader on errors. Returns the body text after + the pseudo-header and a dictionary of set options. If you + like, you can initialize the dictionary with some defaults + and pass your initialized dict in as dictionary. + """ + if dictionary == None: + dictionary = {} + body_lines = body.splitlines() + all = required+optional + for i,line in enumerate(body_lines): + line = line.strip() + if len(line) == 0: + break + if ':' not in line: + raise InvalidPseudoheader(self, line) + key,value = line.split(':', 1) + value = value.strip() + if key not in all: + raise InvalidPseudoHeader(self, key) + if len(value) == 0: + raise InvalidEmail( + self, u'Blank value for: %s' % key) + dictionary[key] = value + missing = [] + for key in required: + if key not in dictionary: + missing.append(key) + if len(missing) > 0: + raise InvalidPseudoHeader(self, + u'Missing required pseudo-headers:\n%s' + % u', '.join(missing)) + remaining_body = u'\n'.join(body_lines[i:]).strip() + return (remaining_body, dictionary) + def _strip_footer(self, body): + body_lines = body.splitlines() + for i,line in enumerate(body_lines): + if line.startswith(BREAK): + break + i += 1 # increment past the current valid line. + return u'\n'.join(body_lines[:i]).strip() + def parse(self): + """ + Parse the commands given in the email. Raises assorted + subclasses of InvalidEmail in the case of invalid messages, + otherwise returns a list of suggested commands to run. + """ + self.validate_subject() + tag_type,value = self._subject_tag_type() + if tag_type == u'new': + commands = self.parse_new() + elif tag_type == u'comment': + commands = self.parse_comment(value) + elif tag_type == u'control': + commands = self.parse_control() + else: + raise Exception, u'Unrecognized tag type "%s"' % tag_type + return commands + def parse_new(self): + command = u'new' + tag,subject = self._split_subject() + summary = subject + options = {u'Reporter': self.author_addr(), + u'Confirm': self._yes_no(self.confirm), + u'Subscribe': 'no', + } + body,mime_type = list(self._get_bodies_and_mime_types())[0] + comment_body,options = \ + self._parse_body_pseudoheaders(body, + NEW_REQUIRED_PSEUDOHEADERS, + NEW_OPTIONAL_PSEUDOHEADERS, + options) + if options[u'Confirm'].lower() == 'no': + self.confirm = False + if options[u'Subscribe'].lower() == 'yes' and self.confirm == True: + # respond with the subscription format rather than the + # normal command-output format, because the subscription + # format is more user-friendly. + self.confirm = False + args = [u'--reporter', options[u'Reporter']] + args.append(summary) + commands = [Command(self, command, args)] + id = ID(commands[0]) + comment_body = self._strip_footer(comment_body) + if len(comment_body) > 0: + command = u'comment' + comment = u'Version: %s\n\n'%options[u'Version'] + comment_body + args = [u'--author', self.author_addr(), + u'--alt-id', self.message_id(), + u'--content-type', mime_type] + args.append(id) + args.append(u'-') + commands.append(Command(self, u'comment', args, stdin=comment)) + for key,value in options.items(): + if key in [u'Version', u'Reporter', u'Confirm']: + continue # we've already handled these options + command = key.lower() + if key in [u'Depend', u'Tag', u'Target', u'Subscribe']: + args = [id, value] + else: + args = [value, id] + if key == u'Subscribe': + if value.lower() != 'yes': + continue + args = ['--subscriber', self.author_addr(), id] + commands.append(Command(self, command, args)) + return commands + def parse_comment(self, bug_uuid): + command = u'comment' + bug_id = bug_uuid + author = self.author_addr() + alt_id = self.message_id() + body,mime_type = list(self._get_bodies_and_mime_types())[0] + if mime_type == 'text/plain': + body = self._strip_footer(body) + content_type = mime_type + args = [u'--author', author] + if alt_id != None: + args.extend([u'--alt-id', alt_id]) + args.extend([u'--content-type', content_type, bug_id, u'-']) + commands = [Command(self, command, args, stdin=body)] + return commands + def parse_control(self): + body,mime_type = list(self._get_bodies_and_mime_types())[0] + commands = [] + for line in body.splitlines(): + line = line.strip() + if line.startswith(CONTROL_COMMENT) or len(line) == 0: + continue + if line.startswith(BREAK): + break + if type(line) == types.UnicodeType: + # work around http://bugs.python.org/issue1170 + line = line.encode('unicode escape') + fields = shlex.split(line) + if type(line) == types.UnicodeType: + # work around http://bugs.python.org/issue1170 + for field in fields: + field = unicode(field, 'unicode escape') + command,args = (fields[0], fields[1:]) + commands.append(Command(self, command, args)) + if len(commands) == 0: + raise InvalidEmail(self, u'No commands in control email.') + return commands + def run(self, repo='.'): + self._begin_response() + commands = self.parse() + try: + for i,command in enumerate(commands): + command.run() + self._add_response(command.response_msg()) + finally: + if AUTOCOMMIT == True: + tag,subject = self._split_subject() + self.commit_command = Command(self, 'commit', [subject]) + self.commit_command.run() + if LOGFILE != None: + LOGFILE.write(u'Autocommit:\n%s\n\n' % + send_pgp_mime.flatten(self.commit_command.response_msg(), + to_unicode=True)) + def _begin_response(self): + tag,subject = self._split_subject() + response_header = [u'From: %s' % THIS_ADDRESS, + u'To: %s' % self.author_addr(), + u'Date: %s' % libbe.util.utility.time_to_str(time.time()), + u'Subject: %s Re: %s'%(SUBJECT_TAG_RESPONSE,subject) + ] + if self.message_id() != None: + response_header.append(u'In-reply-to: %s' % self.message_id()) + self.response_header = \ + send_pgp_mime.header_from_text(text=u'\n'.join(response_header)) + self._response_messages = [] + def _add_response(self, response_message): + self._response_messages.append(response_message) + def response_email(self): + assert len(self._response_messages) > 0 + if len(self._response_messages) == 1: + response_body = self._response_messages[0] + else: + response_body = MIMEMultipart() + for message in self._response_messages: + response_body.attach(message) + return send_pgp_mime.attach_root(self.response_header, response_body) + def subscriber_emails(self, previous_revision=None): + if previous_revision == None: + if AUTOCOMMIT != True: # no way to tell what's changed + raise NotificationFailed('Autocommit dissabled') + if len(self._response_messages) == 0: + raise NotificationFailed('Initial email failed.') + if self.commit_command.ret != 0: + # commit failed. Error already logged. + raise NotificationFailed('Commit failed') + + bd = UI.storage_callbacks.get_bugdir() + writeable = bd.storage.writeable + bd.storage.writeable = False + if bd.storage.versioned == False: # no way to tell what's changed + bd.storage.writeable = writeable + raise NotificationFailed('Not versioned') + + bd.load_all_bugs() + subscribers = subscribe.get_bugdir_subscribers(bd, THIS_SERVER) + if len(subscribers) == 0: + bd.storage.writeable = writeable + return [] + for subscriber,subscriptions in subscribers.items(): + subscribers[subscriber] = [] + for id,types in subscriptions.items(): + for type in types: + subscribers[subscriber].append( + libbe.diff.Subscription(id,type)) + + before_bd, after_bd = self._get_before_and_after_bugdirs(bd, previous_revision) + diff = Diff(before_bd, after_bd) + diff.full_report(diff_tree=DiffTree) + header = self._subscriber_header(bd, previous_revision) + + emails = [] + for subscriber,subscriptions in subscribers.items(): + header.replace_header('to', subscriber) + report = diff.report_tree(subscriptions, diff_tree=DiffTree) + root = report.report_or_none() + if root != None: + emails.append(send_pgp_mime.attach_root(header, root)) + if LOGFILE != None: + LOGFILE.write(u'Preparing to notify %s of changes\n' % subscriber) + bd.storage.writeable = writeable + return emails + def _get_before_and_after_bugdirs(self, bd, previous_revision=None): + if previous_revision == None: + commit_msg = self.commit_command.stdout + assert commit_msg.startswith('Committed '), commit_msg + after_revision = commit_msg[len('Committed '):] + before_revision = bd.storage.revision_id(-2) + else: + before_revision = previous_revision + if before_revision == None: + # this commit was the initial commit + before_bd = libbe.bugdir.BugDir(from_disk=False, + manipulate_encodings=False) + else: + before_bd = libbe.bugdir.RevisionedBugDir(bd, before_revision) + #after_bd = bd.duplicate_bugdir(after_revision) + after_bd = bd # assume no changes since commit a few cycles ago + return (before_bd, after_bd) + def _subscriber_header(self, bd, previous_revision=None): + root_dir = os.path.basename(bd.storage.repo) + if previous_revision == None: + subject = 'Changes to %s on %s by %s' \ + % (root_dir, THIS_SERVER, self.author_addr()) + else: + subject = 'Changes to %s on %s since revision %s' \ + % (root_dir, THIS_SERVER, previous_revision) + header = [u'From: %s' % THIS_ADDRESS, + u'To: %s' % u'DUMMY-AUTHOR', + u'Date: %s' % libbe.util.utility.time_to_str(time.time()), + u'Subject: %s Re: %s' % (SUBJECT_TAG_RESPONSE, subject) + ] + return send_pgp_mime.header_from_text(text=u'\n'.join(header)) + +def generate_global_tags(tag_base=u'be-bug'): + """ + Generate a series of tags from a base tag string. + """ + global SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \ + SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL + SUBJECT_TAG_BASE = tag_base + SUBJECT_TAG_START = u'[%s' % tag_base + SUBJECT_TAG_RESPONSE = u'[%s]' % tag_base + SUBJECT_TAG_NEW = u'[%s:submit]' % tag_base + SUBJECT_TAG_COMMENT = re.compile(u'\[%s:([\-0-9a-z/]*)]' % tag_base) + SUBJECT_TAG_CONTROL = SUBJECT_TAG_RESPONSE + +def open_logfile(logpath=None): + """ + If logpath=None, default to global LOGPATH. + Special logpath strings: + "-" set LOGFILE to sys.stderr + "none" disable logging + Relative logpaths are expanded relative to _THIS_DIR + """ + global LOGPATH, LOGFILE + if logpath != None: + if logpath == u'-': + LOGPATH = u'stderr' + LOGFILE = sys.stderr + elif logpath == u'none': + LOGPATH = u'none' + LOGFILE = None + elif os.path.isabs(logpath): + LOGPATH = logpath + else: + LOGPATH = os.path.join(_THIS_DIR, logpath) + if LOGFILE == None and LOGPATH != u'none': + LOGFILE = codecs.open(LOGPATH, u'a+', + libbe.util.encoding.get_filesystem_encoding()) + +def close_logfile(): + if LOGFILE != None and LOGPATH not in [u'stderr', u'none']: + LOGFILE.close() + +def test(): + result = unittest.TextTestRunner(verbosity=2).run(suite) + num_errors = len(result.errors) + num_failures = len(result.failures) + num_bad = num_errors + num_failures + return num_bad + +def main(args): + from optparse import OptionParser + global AUTOCOMMIT, UI + + usage='be-handle-mail [options]\n\n%s' % (__doc__) + parser = OptionParser(usage=usage) + parser.add_option('-r', '--repo', dest='repo', default=_THIS_DIR, + metavar='REPO', + help='Select the BE repository to serve (%default).') + parser.add_option('-t', '--tag-base', dest='tag_base', + default=SUBJECT_TAG_BASE, metavar='TAG', + help='Set the subject tag base (%default).') + parser.add_option('-o', '--output', dest='output', action='store_true', + help="Don't mail the generated message, print it to stdout instead. Useful for testing be-handle-mail functionality without the whole mail transfer agent and procmail setup.") + parser.add_option('-l', '--logfile', dest='logfile', metavar='LOGFILE', + help='Set the logfile to LOGFILE. Relative paths are relative to the location of this be-handle-mail file (%s). The special value of "-" directs the log output to stderr, and "none" disables logging.' % _THIS_DIR) + parser.add_option('-a', '--disable-autocommit', dest='autocommit', + default=True, action='store_false', + help='Disable the autocommit after parsing the email.') + parser.add_option('-s', '--disable-subscribers', dest='subscribers', + default=True, action='store_false', + help='Disable subscriber notification emails.') + parser.add_option('--notify-since', dest='notify_since', metavar='REVISION', + help='Notify subscribers of all changes since REVISION. When this option is set, no input email parsing is done.') + parser.add_option('--test', dest='test', action='store_true', + help='Run internal unit-tests and exit.') + + pargs = args + options,args = parser.parse_args(args[1:]) + + if options.test == True: + num_bad = test() + if num_bad > 126: + num_bad = 1 + sys.exit(num_bad) + + AUTOCOMMIT = options.autocommit + + if options.notify_since == None: + msg_text = sys.stdin.read() + + open_logfile(options.logfile) + generate_global_tags(options.tag_base) + + io = libbe.command.StringInputOutput() + UI = libbe.command.UserInterface(io, location=options.repo) + + if options.notify_since != None: + if options.subscribers == True: + if LOGFILE != None: + LOGFILE.write(u'Checking for subscribers to notify since revision %s\n' + % options.notify_since) + try: + m = Message(disable_parsing=True) + emails = m.subscriber_emails(options.notify_since) + except NotificationFailed, e: + if LOGFILE != None: + LOGFILE.write(unicode(e) + u'\n') + else: + for msg in emails: + if options.output == True: + print send_pgp_mime.flatten(msg, to_unicode=True) + else: + send_pgp_mime.mail(msg, send_pgp_mime.sendmail) + close_logfile() + UI.cleanup() + sys.exit(0) + + if len(msg_text.strip()) == 0: # blank email!? + if LOGFILE != None: + LOGFILE.write(u'Blank email!\n') + close_logfile() + UI.cleanup() + sys.exit(1) + try: + m = Message(msg_text) + m.run() + except InvalidEmail, e: + response = e.response() + except Exception, e: + if LOGFILE != None: + LOGFILE.write(u'Uncaught exception:\n%s\n' % (e,)) + traceback.print_tb(sys.exc_traceback, file=LOGFILE) + close_logfile() + m.commit_command.cleanup() + UI.cleanup() + sys.exit(1) + else: + response = m.response_email() + if options.output == True: + print send_pgp_mime.flatten(response, to_unicode=True) + elif m.confirm == True: + if LOGFILE != None: + LOGFILE.write(u'Sending response to %s\n' % m.author_addr()) + LOGFILE.write(u'\n%s\n\n' % send_pgp_mime.flatten(response, + to_unicode=True)) + send_pgp_mime.mail(response, send_pgp_mime.sendmail) + else: + if LOGFILE != None: + LOGFILE.write(u'Response declined by %s\n' % m.author_addr()) + if options.subscribers == True: + if LOGFILE != None: + LOGFILE.write(u'Checking for subscribers\n') + try: + emails = m.subscriber_emails() + except NotificationFailed, e: + if LOGFILE != None: + LOGFILE.write(unicode(e) + u'\n') + else: + for msg in emails: + if options.output == True: + print send_pgp_mime.flatten(msg, to_unicode=True) + else: + send_pgp_mime.mail(msg, send_pgp_mime.sendmail) + + close_logfile() + m.commit_command.cleanup() + UI.cleanup() + +class GenerateGlobalTagsTestCase (unittest.TestCase): + def setUp(self): + super(GenerateGlobalTagsTestCase, self).setUp() + self.save_global_tags() + def tearDown(self): + self.restore_global_tags() + super(GenerateGlobalTagsTestCase, self).tearDown() + def save_global_tags(self): + self.saved_globals = [SUBJECT_TAG_BASE, SUBJECT_TAG_START, + SUBJECT_TAG_RESPONSE, SUBJECT_TAG_NEW, + SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL] + def restore_global_tags(self): + global SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \ + SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL + SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \ + SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL = \ + self.saved_globals + def test_restore_global_tags(self): + "Test global tag restoration by teardown function." + global SUBJECT_TAG_BASE + self.failUnlessEqual(SUBJECT_TAG_BASE, u'be-bug') + SUBJECT_TAG_BASE = 'projectX-bug' + self.failUnlessEqual(SUBJECT_TAG_BASE, u'projectX-bug') + self.restore_global_tags() + self.failUnlessEqual(SUBJECT_TAG_BASE, u'be-bug') + def test_subject_tag_base(self): + "Should set SUBJECT_TAG_BASE global correctly" + generate_global_tags(u'projectX-bug') + self.failUnlessEqual(SUBJECT_TAG_BASE, u'projectX-bug') + def test_subject_tag_start(self): + "Should set SUBJECT_TAG_START global correctly" + generate_global_tags(u'projectX-bug') + self.failUnlessEqual(SUBJECT_TAG_START, u'[projectX-bug') + def test_subject_tag_response(self): + "Should set SUBJECT_TAG_RESPONSE global correctly" + generate_global_tags(u'projectX-bug') + self.failUnlessEqual(SUBJECT_TAG_RESPONSE, u'[projectX-bug]') + def test_subject_tag_new(self): + "Should set SUBJECT_TAG_NEW global correctly" + generate_global_tags(u'projectX-bug') + self.failUnlessEqual(SUBJECT_TAG_NEW, u'[projectX-bug:submit]') + def test_subject_tag_control(self): + "Should set SUBJECT_TAG_CONTROL global correctly" + generate_global_tags(u'projectX-bug') + self.failUnlessEqual(SUBJECT_TAG_CONTROL, u'[projectX-bug]') + def test_subject_tag_comment(self): + "Should set SUBJECT_TAG_COMMENT global correctly" + generate_global_tags(u'projectX-bug') + m = SUBJECT_TAG_COMMENT.match('[projectX-bug:abc/xyz-123]') + self.failUnlessEqual(len(m.groups()), 1) + self.failUnlessEqual(m.group(1), u'abc/xyz-123') + +unitsuite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) +suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) + +if __name__ == "__main__": + main(sys.argv) diff --git a/interfaces/email/interactive/examples/blank b/interfaces/email/interactive/examples/blank new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/interfaces/email/interactive/examples/blank diff --git a/interfaces/email/interactive/examples/comment b/interfaces/email/interactive/examples/comment new file mode 100644 index 0000000..f22e4b2 --- /dev/null +++ b/interfaces/email/interactive/examples/comment @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <xyz@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug:a1d] Subject ignored + +We sure do. +-- +Goofy tagline ignored diff --git a/interfaces/email/interactive/examples/email_bugs b/interfaces/email/interactive/examples/email_bugs new file mode 100644 index 0000000..949e1c1 --- /dev/null +++ b/interfaces/email/interactive/examples/email_bugs @@ -0,0 +1,37 @@ +From jdoe@example.com Fri Apr 18 12:00:00 2008 +Content-Type: text/xml; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: quoted-printable +From: jdoe@example.com +To: a@b.com +Date: Fri, 18 Apr 2008 12:00:00 +0000 +Subject: [be-bug:xml] Updates to a, b + +<?xml version="1.0" encoding="utf-8" ?> +<be-xml> + <version> + <tag>1.0.0</tag> + <branch-nick>be</branch-nick> + <revno>446</revno> + <revision-id>wking@drexel.edu-20091119214553-iqyw2cpqluww3zna</revision-id> + </version> + <bug> + <uuid>a</uuid> + <short-name>a</short-name> + <severity>minor</severity> + <status>open</status> + <creator>John Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug A</summary> + </bug> + <bug> + <uuid>b</uuid> + <short-name>b</short-name> + <severity>minor</severity> + <status>closed</status> + <creator>Jane Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug B</summary> + </bug> +</be-xml> + diff --git a/interfaces/email/interactive/examples/failing_multiples b/interfaces/email/interactive/examples/failing_multiples new file mode 100644 index 0000000..cf50211 --- /dev/null +++ b/interfaces/email/interactive/examples/failing_multiples @@ -0,0 +1,16 @@ +From jdoe@example.com Fri Apr 18 12:00:00 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Commit message... + +new "test bug" +new "test bug 2" +failing-command +new "test bug 3" + +-- +This message fails partway through, but the partial changes should be +recorded in a commit... diff --git a/interfaces/email/interactive/examples/invalid_command b/interfaces/email/interactive/examples/invalid_command new file mode 100644 index 0000000..f2963c7 --- /dev/null +++ b/interfaces/email/interactive/examples/invalid_command @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] + +close +-- +Close is currently disabled for the email interface. diff --git a/interfaces/email/interactive/examples/invalid_subject b/interfaces/email/interactive/examples/invalid_subject new file mode 100644 index 0000000..1e2eb88 --- /dev/null +++ b/interfaces/email/interactive/examples/invalid_subject @@ -0,0 +1,9 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: Spam! + +This should elicit an "invalid subject" response email. diff --git a/interfaces/email/interactive/examples/list b/interfaces/email/interactive/examples/list new file mode 100644 index 0000000..acba424 --- /dev/null +++ b/interfaces/email/interactive/examples/list @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Subject ignored + +list --status all +-- +Dummy content diff --git a/interfaces/email/interactive/examples/missing_command b/interfaces/email/interactive/examples/missing_command new file mode 100644 index 0000000..bb390fc --- /dev/null +++ b/interfaces/email/interactive/examples/missing_command @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Subject ignored + +abcde +-- +This should elicit a "invalid command 'abcde'" response email. diff --git a/interfaces/email/interactive/examples/multiple_commands b/interfaces/email/interactive/examples/multiple_commands new file mode 100644 index 0000000..41ef730 --- /dev/null +++ b/interfaces/email/interactive/examples/multiple_commands @@ -0,0 +1,14 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Subject ignored + +help +list --status=all +list --status=fixed +show --xml 361 +-- +Goofy tagline ignored. diff --git a/interfaces/email/interactive/examples/new b/interfaces/email/interactive/examples/new new file mode 100644 index 0000000..c64db93 --- /dev/null +++ b/interfaces/email/interactive/examples/new @@ -0,0 +1,19 @@ +From jdoe@example.com Fri Apr 18 12:00:00 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug:submit] Need tests for the email interface. + +Version: XYZ +Reporter: Jane Doe +Assign: Dick Tracy +Depend: 00f +Severity: critical +Status: assigned +Tag: topsecret +Target: Law&Order + +-- +Goofy tagline not included, and no comment added. diff --git a/interfaces/email/interactive/examples/new_with_comment b/interfaces/email/interactive/examples/new_with_comment new file mode 100644 index 0000000..1077f0f --- /dev/null +++ b/interfaces/email/interactive/examples/new_with_comment @@ -0,0 +1,13 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug:submit] Need tests for the email interface. + +Version: XYZ + +I think so anyway. +-- +Goofy tagline not included. diff --git a/interfaces/email/interactive/examples/show b/interfaces/email/interactive/examples/show new file mode 100644 index 0000000..c5f8a4d --- /dev/null +++ b/interfaces/email/interactive/examples/show @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Subject ignored + +show --xml 361 +-- +Can we show a bug? diff --git a/interfaces/email/interactive/examples/unicode b/interfaces/email/interactive/examples/unicode new file mode 100644 index 0000000..f0e8001 --- /dev/null +++ b/interfaces/email/interactive/examples/unicode @@ -0,0 +1,11 @@ +From jdoe@example.com Fri Apr 18 11:18:58 2008 +Message-ID: <abcd@example.com> +Date: Fri, 18 Apr 2008 12:00:00 +0000 +From: John Doe <jdoe@example.com> +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Subject: [be-bug] Subject ignored + +show --xml f7ccd916-b5c7-4890-a2e3-8c8ace17ae3a +-- +Can we handle unicode output? diff --git a/interfaces/email/interactive/libbe b/interfaces/email/interactive/libbe new file mode 120000 index 0000000..7d18612 --- /dev/null +++ b/interfaces/email/interactive/libbe @@ -0,0 +1 @@ +../../../libbe
\ No newline at end of file diff --git a/interfaces/email/interactive/send_pgp_mime.py b/interfaces/email/interactive/send_pgp_mime.py new file mode 100644 index 0000000..517b1f0 --- /dev/null +++ b/interfaces/email/interactive/send_pgp_mime.py @@ -0,0 +1,611 @@ +#!/usr/bin/python +# +# Copyright (C) 2009-2010 W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Python module and command line tool for sending pgp/mime email. + +Mostly uses subprocess to call gpg and a sendmail-compatible mailer. +If you lack gpg, either don't use the encryption functions or adjust +the pgp_* commands. You may need to adjust the sendmail command to +point to whichever sendmail-compatible mailer you have on your system. +""" + +from cStringIO import StringIO +import os +import re +#import GnuPGInterface # Maybe should use this instead of subprocess +import smtplib +import subprocess +import sys +import tempfile +import types + +try: + from email import Message + from email.mime.text import MIMEText + from email.mime.multipart import MIMEMultipart + from email.mime.application import MIMEApplication + from email.encoders import encode_7or8bit + from email.generator import Generator + from email.parser import Parser + from email.utils import getaddress +except ImportError: + # adjust to old python 2.4 + from email import Message + from email.MIMEText import MIMEText + from email.MIMEMultipart import MIMEMultipart + from email.MIMENonMultipart import MIMENonMultipart + from email.Encoders import encode_7or8bit + from email.Generator import Generator + from email.Parser import Parser + from email.Utils import getaddresses + + getaddress = getaddresses + class MIMEApplication (MIMENonMultipart): + def __init__(self, _data, _subtype, _encoder, **params): + MIMENonMultipart.__init__(self, 'application', _subtype, **params) + self.set_payload(_data) + _encoder(self) + +usage="""usage: %prog [options] + +Scriptable PGP MIME email using gpg. + +You can use gpg-agent for passphrase caching if your key requires a +passphrase (it better!). Example usage would be to install gpg-agent, +and then run + export GPG_TTY=`tty` + eval $(gpg-agent --daemon) +in your shell before invoking this script. See gpg-agent(1) for more +details. Alternatively, you can send your passphrase in on stdin + echo 'passphrase' | %prog [options] +or use the --passphrase-file option + %prog [options] --passphrase-file FILE [more options] +Both of these alternatives are much less secure than gpg-agent. You +have been warned. +""" + +verboseInvoke = False +PGP_SIGN_AS = None +PASSPHRASE = None + +# The following commands are adapted from my .mutt/pgp configuration +# +# Printf-like sequences: +# %a The value of PGP_SIGN_AS. +# %f Expands to the name of a file with text to be signed/encrypted. +# %p Expands to the passphrase argument. +# %R A string with some number (0 on up) of pgp_reciepient_arg +# strings. +# %r One key ID (e.g. recipient email address) to build a +# pgp_reciepient_arg string. +# +# The above sequences can be used to optionally print a string if +# their length is nonzero. For example, you may only want to pass the +# -u/--local-user argument to gpg if PGP_SIGN_AS is defined. To +# optionally print a string based upon one of the above sequences, the +# following construct is used +# %?<sequence_char>?<optional_string>? +# where sequence_char is a character from the table above, and +# optional_string is the string you would like printed if status_char +# is nonzero. optional_string may contain other sequence as well as +# normal text, but it may not contain any question marks. +# +# see http://codesorcery.net/old/mutt/mutt-gnupg-howto +# http://www.mutt.org/doc/manual/manual-6.html#pgp_autosign +# http://tldp.org/HOWTO/Mutt-GnuPG-PGP-HOWTO-8.html +# for more details + +pgp_recipient_arg='-r "%r"' +pgp_stdin_passphrase_arg='--passphrase-fd 0' +pgp_sign_command='/usr/bin/gpg --no-verbose --quiet --batch %p --output - --detach-sign --armor --textmode %?a?-u "%a"? %f' +pgp_encrypt_only_command='/usr/bin/gpg --no-verbose --quiet --batch --output - --encrypt --armor --textmode --always-trust --encrypt-to "%a" %R -- %f' +pgp_encrypt_sign_command='/usr/bin/gpg --no-verbose --quiet --batch %p --output - --encrypt --sign %?a?-u "%a"? --armor --textmode --always-trust --encrypt-to "%a" %R -- %f' +sendmail='/usr/sbin/sendmail -t' + +def mail(msg, sendmail=None): + """ + Send an email Message instance on its merry way. + + We can shell out to the user specified sendmail in case + the local host doesn't have an SMTP server set up + for easy smtplib usage. + """ + if sendmail != None: + execute(sendmail, stdin=flatten(msg)) + return None + s = smtplib.SMTP() + s.connect() + s.sendmail(from_addr=source_email(msg), + to_addrs=target_emails(msg), + msg=flatten(msg)) + s.close() + +def header_from_text(text, encoding="us-ascii"): + """ + Simple wrapper for instantiating an email.Message from text. + >>> header = header_from_text('\\n'.join(['From: me@big.edu','To: you@big.edu','Subject: testing'])) + >>> print flatten(header) + From: me@big.edu + To: you@big.edu + Subject: testing + <BLANKLINE> + <BLANKLINE> + """ + text = text.strip() + if type(text) == types.UnicodeType: + text = text.encode(encoding) + # assume StringType arguments are already encoded + p = Parser() + return p.parsestr(text, headersonly=True) + +def guess_encoding(text): + if type(text) == types.StringType: + encoding = "us-ascii" + elif type(text) == types.UnicodeType: + for encoding in ["us-ascii", "iso-8859-1", "utf-8"]: + try: + text.encode(encoding) + except UnicodeError: + pass + else: + break + assert encoding != None + return encoding + +def encodedMIMEText(body, encoding=None): + if encoding == None: + encoding = guess_encoding(body) + if encoding == "us-ascii": + return MIMEText(body) + else: + # Create the message ('plain' stands for Content-Type: text/plain) + return MIMEText(body.encode(encoding), 'plain', encoding) + +def append_text(text_part, new_text): + original_payload = text_part.get_payload(decode=True) + new_payload = u"%s%s" % (original_payload, new_text) + new_encoding = guess_encoding(new_payload) + text_part.set_payload(new_payload.encode(new_encoding), new_encoding) + +def attach_root(header, root_part): + """ + Attach the email.Message root_part to the email.Message header + without generating a multi-part message. + """ + for k,v in header.items(): + root_part[k] = v + return root_part + +def execute(args, stdin=None, expect=(0,)): + """ + Execute a command (allows us to drive gpg). + """ + if verboseInvoke == True: + print >> sys.stderr, '$ '+args + try: + p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True) + except OSError, e: + strerror = '%s\nwhile executing %s' % (e.args[1], args) + raise Exception, strerror + output, error = p.communicate(input=stdin) + status = p.wait() + if verboseInvoke == True: + print >> sys.stderr, '(status: %d)\n%s%s' % (status, output, error) + if status not in expect: + strerror = '%s\nwhile executing %s\n%s\n%d' % (args[1], args, error, status) + raise Exception, strerror + return status, output, error + +def replace(template, format_char, replacement_text): + """ + >>> replace('--textmode %?a?-u %a? %f', 'f', 'file.in') + '--textmode %?a?-u %a? file.in' + >>> replace('--textmode %?a?-u %a? %f', 'a', '0xHEXKEY') + '--textmode -u 0xHEXKEY %f' + >>> replace('--textmode %?a?-u %a? %f', 'a', '') + '--textmode %f' + """ + if replacement_text == None: + replacement_text = "" + regexp = re.compile('%[?]'+format_char+'[?]([^?]*)[?]') + if len(replacement_text) > 0: + str = regexp.sub('\g<1>', template) + else: + str = regexp.sub('', template) + regexp = re.compile('%'+format_char) + str = regexp.sub(replacement_text, str) + return str + +def flatten(msg, to_unicode=False): + """ + Produce flat text output from an email Message instance. + """ + assert msg != None + fp = StringIO() + g = Generator(fp, mangle_from_=False) + g.flatten(msg) + text = fp.getvalue() + if to_unicode == True: + encoding = msg.get_content_charset() or "utf-8" + text = unicode(text, encoding=encoding) + return text + +def source_email(msg, return_realname=False): + """ + Search the header of an email Message instance to find the + sender's email address. + """ + froms = msg.get_all('from', []) + from_tuples = getaddresses(froms) # [(realname, email_address), ...] + assert len(from_tuples) == 1 + if return_realname == True: + return from_tuples[0] # (realname, email_address) + return from_tuples[0][1] # email_address + +def target_emails(msg): + """ + Search the header of an email Message instance to find a + list of recipient's email addresses. + """ + tos = msg.get_all('to', []) + ccs = msg.get_all('cc', []) + bccs = msg.get_all('bcc', []) + resent_tos = msg.get_all('resent-to', []) + resent_ccs = msg.get_all('resent-cc', []) + resent_bccs = msg.get_all('resent-bcc', []) + all_recipients = getaddresses(tos + ccs + bccs + resent_tos + + resent_ccs + resent_bccs) + return [addr[1] for addr in all_recipients] + +class PGPMimeMessageFactory (object): + """ + See http://www.ietf.org/rfc/rfc3156.txt for specification details. + >>> from_addr = "me@big.edu" + >>> to_addr = "you@you.edu" + >>> header = header_from_text('\\n'.join(['From: %s'%from_addr,'To: %s'%to_addr,'Subject: testing'])) + >>> source_email(header) == from_addr + True + >>> target_emails(header) == [to_addr] + True + >>> m = PGPMimeMessageFactory('check 1 2\\ncheck 1 2\\n') + >>> print flatten(m.clearBodyPart()) + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + check 1 2 + check 1 2 + <BLANKLINE> + >>> print flatten(m.plain()) + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + check 1 2 + check 1 2 + <BLANKLINE> + >>> signed = m.sign(header) + >>> signed.set_boundary('boundsep') + >>> print flatten(signed).replace('\\t', ' '*4) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + Content-Type: multipart/signed; protocol="application/pgp-signature"; + micalg="pgp-sha1"; boundary="boundsep" + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + --boundsep + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + check 1 2 + check 1 2 + <BLANKLINE> + --boundsep + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: signature + Content-Type: application/pgp-signature; name="signature.asc"; + charset="us-ascii" + <BLANKLINE> + -----BEGIN PGP SIGNATURE----- + ... + -----END PGP SIGNATURE----- + <BLANKLINE> + --boundsep-- + >>> encrypted = m.encrypt(header) + >>> encrypted.set_boundary('boundsep') + >>> print flatten(encrypted).replace('\\t', ' '*4) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + Content-Type: multipart/encrypted; + protocol="application/pgp-encrypted"; + micalg="pgp-sha1"; boundary="boundsep" + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + --boundsep + Content-Type: application/pgp-encrypted + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + Version: 1 + <BLANKLINE> + --boundsep + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Type: application/octet-stream; charset="us-ascii" + <BLANKLINE> + -----BEGIN PGP MESSAGE----- + ... + -----END PGP MESSAGE----- + <BLANKLINE> + --boundsep-- + >>> signedAndEncrypted = m.signAndEncrypt(header) + >>> signedAndEncrypted.set_boundary('boundsep') + >>> print flatten(signedAndEncrypted).replace('\\t', ' '*4) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + Content-Type: multipart/encrypted; + protocol="application/pgp-encrypted"; + micalg="pgp-sha1"; boundary="boundsep" + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + --boundsep + Content-Type: application/pgp-encrypted + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + Version: 1 + <BLANKLINE> + --boundsep + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Type: application/octet-stream; charset="us-ascii" + <BLANKLINE> + -----BEGIN PGP MESSAGE----- + ... + -----END PGP MESSAGE----- + <BLANKLINE> + --boundsep-- + """ + def __init__(self, body): + self.body = body + def clearBodyPart(self): + body = encodedMIMEText(self.body) + body.add_header('Content-Disposition', 'inline') + return body + def passphrase_arg(self, passphrase=None): + if passphrase == None and PASSPHRASE != None: + passphrase = PASSPHRASE + if passphrase == None: + return (None,'') + return (passphrase, pgp_stdin_passphrase_arg) + def plain(self): + """ + text/plain + """ + return encodedMIMEText(self.body) + def sign(self, header, passphrase=None): + """ + multipart/signed + +-> text/plain (body) + +-> application/pgp-signature (signature) + """ + passphrase,pass_arg = self.passphrase_arg(passphrase) + body = self.clearBodyPart() + bfile = tempfile.NamedTemporaryFile() + bfile.write(flatten(body)) + bfile.flush() + + args = replace(pgp_sign_command, 'f', bfile.name) + if PGP_SIGN_AS == None: + pgp_sign_as = '<%s>' % source_email(header) + else: + pgp_sign_as = PGP_SIGN_AS + args = replace(args, 'a', pgp_sign_as) + args = replace(args, 'p', pass_arg) + status,output,error = execute(args, stdin=passphrase) + signature = output + + sig = MIMEApplication(_data=signature, + _subtype='pgp-signature; name="signature.asc"', + _encoder=encode_7or8bit) + sig['Content-Description'] = 'signature' + sig.set_charset('us-ascii') + + msg = MIMEMultipart('signed', micalg='pgp-sha1', + protocol='application/pgp-signature') + msg.attach(body) + msg.attach(sig) + + msg['Content-Disposition'] = 'inline' + return msg + def encrypt(self, header, passphrase=None): + """ + multipart/encrypted + +-> application/pgp-encrypted (control information) + +-> application/octet-stream (body) + """ + body = self.clearBodyPart() + bfile = tempfile.NamedTemporaryFile() + bfile.write(flatten(body)) + bfile.flush() + + recipients = [replace(pgp_recipient_arg, 'r', recipient) + for recipient in target_emails(header)] + recipient_string = ' '.join(recipients) + args = replace(pgp_encrypt_only_command, 'R', recipient_string) + args = replace(args, 'f', bfile.name) + if PGP_SIGN_AS == None: + pgp_sign_as = '<%s>' % source_email(header) + else: + pgp_sign_as = PGP_SIGN_AS + args = replace(args, 'a', pgp_sign_as) + status,output,error = execute(args) + encrypted = output + + enc = MIMEApplication(_data=encrypted, _subtype='octet-stream', + _encoder=encode_7or8bit) + enc.set_charset('us-ascii') + + control = MIMEApplication(_data='Version: 1\n', _subtype='pgp-encrypted', + _encoder=encode_7or8bit) + + msg = MIMEMultipart('encrypted', micalg='pgp-sha1', + protocol='application/pgp-encrypted') + msg.attach(control) + msg.attach(enc) + + msg['Content-Disposition'] = 'inline' + return msg + def signAndEncrypt(self, header, passphrase=None): + """ + multipart/encrypted + +-> application/pgp-encrypted (control information) + +-> application/octet-stream (body) + """ + passphrase,pass_arg = self.passphrase_arg(passphrase) + body = self.sign(header, passphrase) + body.__delitem__('Bcc') + bfile = tempfile.NamedTemporaryFile() + bfile.write(flatten(body)) + bfile.flush() + + recipients = [replace(pgp_recipient_arg, 'r', recipient) + for recipient in target_emails(header)] + recipient_string = ' '.join(recipients) + args = replace(pgp_encrypt_only_command, 'R', recipient_string) + args = replace(args, 'f', bfile.name) + if PGP_SIGN_AS == None: + pgp_sign_as = '<%s>' % source_email(header) + else: + pgp_sign_as = PGP_SIGN_AS + args = replace(args, 'a', pgp_sign_as) + args = replace(args, 'p', pass_arg) + status,output,error = execute(args, stdin=passphrase) + encrypted = output + + enc = MIMEApplication(_data=encrypted, _subtype='octet-stream', + _encoder=encode_7or8bit) + enc.set_charset('us-ascii') + + control = MIMEApplication(_data='Version: 1\n', + _subtype='pgp-encrypted', + _encoder=encode_7or8bit) + + msg = MIMEMultipart('encrypted', micalg='pgp-sha1', + protocol='application/pgp-encrypted') + msg.attach(control) + msg.attach(enc) + + msg['Content-Disposition'] = 'inline' + return msg + +def test(): + import doctest + doctest.testmod() + + +if __name__ == '__main__': + from optparse import OptionParser + + parser = OptionParser(usage=usage) + parser.add_option('-t', '--test', dest='test', action='store_true', + help='Run doctests and exit') + + parser.add_option('-H', '--header-file', dest='header_filename', + help='file containing email header', metavar='FILE') + parser.add_option('-B', '--body-file', dest='body_filename', + help='file containing email body', metavar='FILE') + + parser.add_option('-P', '--passphrase-file', dest='passphrase_file', + help='file containing gpg passphrase', metavar='FILE') + parser.add_option('-p', '--passphrase-fd', dest='passphrase_fd', + help='file descriptor from which to read gpg passphrase (0 for stdin)', + type="int", metavar='DESCRIPTOR') + + parser.add_option('--mode', dest='mode', default='sign', + help="One of 'sign', 'encrypt', 'sign-encrypt', or 'plain'. Defaults to %default.", + metavar='MODE') + + parser.add_option('-a', '--sign-as', dest='sign_as', + help="The gpg key to sign with (gpg's -u/--local-user)", + metavar='KEY') + + parser.add_option('--output', dest='output', action='store_true', + help="Don't mail the generated message, print it to stdout instead.") + + (options, args) = parser.parse_args() + + stdin_used = False + + if options.passphrase_file != None: + PASSPHRASE = file(options.passphrase_file, 'r').read() + elif options.passphrase_fd != None: + if options.passphrase_fd == 0: + stdin_used = True + PASSPHRASE = sys.stdin.read() + else: + PASSPHRASE = os.read(options.passphrase_fd) + + if options.sign_as: + PGP_SIGN_AS = options.sign_as + + if options.test == True: + test() + sys.exit(0) + + header = None + if options.header_filename != None: + if options.header_filename == '-': + assert stdin_used == False + stdin_used = True + header = sys.stdin.read() + else: + header = file(options.header_filename, 'r').read() + if header == None: + raise Exception, "missing header" + headermsg = header_from_text(header) + body = None + if options.body_filename != None: + if options.body_filename == '-': + assert stdin_used == False + stdin_used = True + body = sys.stdin.read() + else: + body = file(options.body_filename, 'r').read() + if body == None: + raise Exception, "missing body" + + m = PGPMimeMessageFactory(body) + if options.mode == "sign": + bodymsg = m.sign(header) + elif options.mode == "encrypt": + bodymsg = m.encrypt(header) + elif options.mode == "sign-encrypt": + bodymsg = m.signAndEncrypt(header) + elif options.mode == "plain": + bodymsg = m.plain() + else: + print "Unrecognized mode '%s'" % options.mode + + message = attach_root(headermsg, bodymsg) + if options.output == True: + message = flatten(message) + print message + else: + mail(message, sendmail) |