aboutsummaryrefslogtreecommitdiffstats
path: root/interfaces/email/interactive/be-handle-mail
blob: fa8069888d4734cbe681a4a7c113a9a45691c5c2 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
#!/usr/bin/env python
#
# Copyright (C) 2009 W. Trevor King <wking@drexel.edu>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
Provide and email interface to the distributed bugtracker Bugs
Everywhere.  Recieves incoming email via procmail.  Provides an
interface similar to the Debian Bug Tracker.  There are currently
three distinct email types: submits, comments, and controls.  The
email types are differentiated by tags in the email subject.  See
SUBJECT_TAG* for the current values.

Submit emails create a bug (and optionally add some intitial
comments).  The post-tag subject is used as the bug summary, and the
email body is parsed for a pseudo-header.  Any text after the
psuedo-header but before a possible line starting with BREAK is added
as the initial bug comment.

Comment emails add comments to a bug.  The first non-multipart portion
of the email is used as the comment body.  If that portion has a
"text/plain" type, any text after and including a possible line
starting with BREAK is stripped to avoid lots of taglines cluttering
up the repository.

Control emails preform any allowed BE commands.  The first
non-multipart portion of the email is used as the comment body.  If
that portion has a "text/plain" type, any text after and including a
possible line starting with BREAK is stripped.  Each pre-BREAK line of
the portion should be a valid BE command, with the initial "be"
omitted, e.g. "be status XYZ fixed" --> "status XYZ fixed".

Any changes made to the repository are commited after the email is
executed, with the email's post-tag subject as the commit message.
"""

import codecs
import StringIO as StringIO
import email
from email.mime.multipart import MIMEMultipart
import email.utils
import os
import os.path
import re
import shlex
import sys
import time
import traceback
import doctest
import unittest

from becommands import subscribe
import libbe.cmdutil, libbe.encoding, libbe.utility, libbe.diff, \
    libbe.bugdir, libbe.bug, libbe.comment
import send_pgp_mime

THIS_SERVER = u"thor.physics.drexel.edu"
THIS_ADDRESS = u"BE Bugs <wking@thor.physics.drexel.edu>"

_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
BE_DIR = _THIS_DIR
LOGPATH = os.path.join(_THIS_DIR, u"be-handle-mail.log")
LOGFILE = None

# Tag strings generated by generate_global_tags()
SUBJECT_TAG_BASE = u"be-bug"
SUBJECT_TAG_RESPONSE = None
SUBJECT_TAG_START = None
SUBJECT_TAG_NEW = None
SUBJECT_TAG_COMMENT = None
SUBJECT_TAG_CONTROL = None

BREAK = u"--"
NEW_REQUIRED_PSEUDOHEADERS = [u"Version"]
NEW_OPTIONAL_PSEUDOHEADERS = [u"Reporter", u"Assign", u"Depend", u"Severity",
                              u"Status", u"Tag", u"Target",
                              u"Confirm", u"Subscribe"]
CONTROL_COMMENT = u"#"
ALLOWED_COMMANDS = [u"assign", u"comment", u"commit", u"depend", u"help",
                    u"list", u"merge", u"new", u"open", u"severity", u"show",
                    u"status", u"subscribe", u"tag", u"target"]

AUTOCOMMIT = True

libbe.encoding.ENCODING = u"utf-8" # force default encoding
ENCODING = libbe.encoding.get_encoding()

class InvalidEmail (ValueError):
    def __init__(self, msg, message):
        ValueError.__init__(self, message)
        self.msg = msg
    def response(self):
        header = self.msg.response_header
        body = [u"Error processing email:\n",
                self.response_body(), u""]
        response_generator = \
            send_pgp_mime.PGPMimeMessageFactory(u"\n".join(body))
        response = MIMEMultipart()
        response.attach(response_generator.plain())
        response.attach(self.msg.msg)
        ret = send_pgp_mime.attach_root(header, response)
        return ret
    def response_body(self):
        err_text = [unicode(self)]
        return u"\n".join(err_text)

class InvalidSubject (InvalidEmail):
    def __init__(self, msg, message=None):
        if message == None:
            message = u"Invalid subject"
        InvalidEmail.__init__(self, msg, message)
    def response_body(self):
        err_text = u"\n".join([unicode(self), u"",
                               u"full subject was:",
                               self.msg.subject()])
        return err_text

class InvalidPseudoHeader (InvalidEmail):
    def response_body(self):
        err_text = [u"Invalid pseudo-header:\n",
                    unicode(self)]
        return u"\n".join(err_text)

class InvalidCommand (InvalidEmail):
    def __init__(self, msg, command, message=None):
        bigmessage = u"Invalid execution command '%s'" % command
        if message != None:
            bigmessage += u"\n%s" % message
        InvalidEmail.__init__(self, msg, bigmessage)
        self.command = command

class InvalidOption (InvalidCommand):
    def __init__(self, msg, option, message=None):
        bigmessage = u"Invalid option '%s'" % (option)
        if message != None:
            bigmessage += u"\n%s" % message
        InvalidCommand.__init__(self, msg, info, command, bigmessage)
        self.option = option

class NotificationFailed (Exception):
    def __init__(self, msg):
        bigmessage = "Notification failed: %s" % msg
        Exception.__init__(self, bigmessage)
        self.short_msg = msg

class ID (object):
    """
    Sometimes you want to reference the output of a command that
    hasn't been executed yet.  ID is there for situations like
    > a = Command(msg, "new", ["create a bug"])
    > b = Command(msg, "comment", [ID(a), "and comment on it"])
    """
    def __init__(self, command):
        self.command = command
    def extract_id(self):
        if hasattr(self, "cached_id"):
            return self._cached_id
        assert self.command.ret == 0, self.command.ret
        if self.command.command == u"new":
            regexp = re.compile(u"Created bug with ID (.*)")
        else:
            raise NotImplementedError, self.command.command
        match = regexp.match(self.command.stdout)
        assert len(match.groups()) == 1, str(match.groups())
        self._cached_id = match.group(1)
        return self._cached_id
    def __str__(self):
        if self.command.ret != 0:
            return "<id for %s>" % repr(self.command)
        return "<id %s>" % self.extract_id()

class Command (object):
    """
    A becommands command wrapper.
    Doesn't validate input, so do that before initializing.

    Initialize with
      Command(msg, command, args=None, stdin=None)
    where
      msg:     the Message instance prompting this command
      command: name of becommand to execute, e.g. "new"
      args:    list of arguments to pass to the command
      stdin:   if non-null, a string to pipe into the command's stdin
    """
    def __init__(self, msg, command, args=None, stdin=None):
        self.msg = msg
        self.command = command
        if args == None:
            self.args = []
        else:
            self.args = args
        self.stdin = stdin
        self.ret = None
        self.stdout = None
        self.stderr = None
        self.err = None
    def __str__(self):
        return "<command: %s %s>" % (self.command, " ".join([str(s) for s in self.args]))
    def normalize_args(self):
        """
        Expand any ID placeholders in self.args.
        """
        for i,arg in enumerate(self.args):
            if isinstance(arg, ID):
                self.args[i] = arg.extract_id()
    def run(self):
        """
        Attempt to execute the command whose info is given in the dictionary
        info.  Returns the exit code, stdout, and stderr produced by the
        command.
        """
        if self.command in [None, u""]: # don't accept blank commands
            raise InvalidCommand(self.msg, self, "Blank")
        elif self.command not in ALLOWED_COMMANDS:
            raise InvalidCommand(self.msg, self, "Not allowed")
        assert self.ret == None, u"running %s twice!" % unicode(self)
        self.normalize_args()
        # set stdin and catch stdout and stderr
        if self.stdin != None:
            orig_stdin = sys.stdin
            sys.stdin = StringIO.StringIO(self.stdin)
        new_stdout = codecs.getwriter(ENCODING)(StringIO.StringIO())
        new_stderr = codecs.getwriter(ENCODING)(StringIO.StringIO())
        orig_stdout = sys.stdout
        orig_stderr = sys.stderr
        sys.stdout = new_stdout
        sys.stderr = new_stderr
        # run the command
        os.chdir(BE_DIR)
        try:
            self.ret = libbe.cmdutil.execute(self.command, self.args,
                                             manipulate_encodings=False)
        except libbe.cmdutil.GetHelp:
            print libbe.cmdutil.help(command)
        except libbe.cmdutil.GetCompletions:
            self.err = InvalidOption(self.msg, self.command, u"--complete")
        except libbe.cmdutil.UsageError, e:
            self.err = InvalidCommand(self.msg, self,
                                      "%s\n%s" % (type(e), unicode(e)))
        except libbe.cmdutil.UserError, e:
            self.err = InvalidCommand(self.msg, self,
                                      "%s\n%s" % (type(e), unicode(e)))
        # restore stdin, stdout, and stderr
        if self.stdin != None:
            sys.stdin = orig_stdin
        sys.stdout.flush()
        sys.stderr.flush()
        sys.stdout = orig_stdout
        sys.stderr = orig_stderr
        self.stdout = codecs.decode(new_stdout.getvalue(), ENCODING)
        self.stderr = codecs.decode(new_stderr.getvalue(), ENCODING)
        if self.err != None:
            raise self.err
        return (self.ret, self.stdout, self.stderr)
    def response_msg(self):
        if self.ret == None: self.ret = -1
        response_body = [u"Results of running: (exit code %d)" % self.ret,
                         u"  %s %s" % (self.command, u" ".join(self.args))]
        if self.stdout != None and len(self.stdout) > 0:
            response_body.extend([u"", u"stdout:", u"", self.stdout])
        if self.stderr != None and len(self.stderr) > 0:
            response_body.extend([u"", u"stderr:", u"", self.stderr])
        response_body.append(u"") # trailing endline
        response_generator = \
            send_pgp_mime.PGPMimeMessageFactory(u"\n".join(response_body))
        return response_generator.plain()

class DiffTree (libbe.diff.DiffTree):
    """
    In order to avoid tons of tiny MIMEText attachments, bug-level
    nodes set .add_child_text=True (in .join()), which is propogated
    on to their descendents.  Instead of creating their own
    attachement, each of these descendents appends his data_part to
    the end of the bug-level MIMEText attachment.

    For the example tree in the libbe.diff.Diff unittests:
      bugdir
      bugdir/settings
      bugdir/bugs
      bugdir/bugs/new
      bugdir/bugs/new/c          <- sets .add_child_text
      bugdir/bugs/rem
      bugdir/bugs/rem/b          <- sets .add_child_text
      bugdir/bugs/mod
      bugdir/bugs/mod/a          <- sets .add_child_text
      bugdir/bugs/mod/a/settings
      bugdir/bugs/mod/a/comments
      bugdir/bugs/mod/a/comments/new
      bugdir/bugs/mod/a/comments/new/acom
      bugdir/bugs/mod/a/comments/rem
      bugdir/bugs/mod/a/comments/mod
    """
    def report_or_none(self):
        report = self.report()
        payload = report.get_payload()
        if payload == None or len(payload) == 0:
            return None
        return report
    def report_string(self):
        report = self.report_or_none()
        if report == None:
            return "No changes"
        else:
            return send_pgp_mime.flatten(self.report(), to_unicode=True)
    def make_root(self):
        return MIMEMultipart()
    def join(self, root, parent, data_part):
        if hasattr(parent, "attach_child_text"):
            self.attach_child_text = True
            if data_part != None:
                send_pgp_mime.append_text(parent.data_mime_part, u"\n\n%s" % (data_part))
            self.data_mime_part = parent.data_mime_part
        else:
            self.data_mime_part = None
            if data_part != None:
                self.data_mime_part = send_pgp_mime.encodedMIMEText(data_part)
            if parent != None and parent.name in [u"new", u"rem", u"mod"]:
                self.attach_child_text = True
                if data_part == None: # make blank data_mime_part for children's appends
                    self.data_mime_part = send_pgp_mime.encodedMIMEText(u"")
            if self.data_mime_part != None:
                self.data_mime_part[u"Content-Description"] = self.name
                root.attach(self.data_mime_part)
    def data_part(self, depth, indent=False):
        return libbe.diff.DiffTree.data_part(self, depth, indent=indent)

class Diff (libbe.diff.Diff):
    def bug_add_string(self, bug):
        return bug.string(show_comments=True)
    def _comment_summary_string(self, comment):
        return comment.string()
    def comment_add_string(self, comment):
        return self._comment_summary_string(comment)
    def comment_rem_string(self, comment):
        return self._comment_summary_string(comment)

class Message (object):
    def __init__(self, email_text=None, disable_parsing=False):
        if disable_parsing == False:
            self.text = email_text
            p=email.Parser.Parser()
            self.msg=p.parsestr(self.text)
            if LOGFILE != None:
                LOGFILE.write(u"handling %s\n" % self.author_addr())
                LOGFILE.write(u"\n%s\n\n" % self.text)
        self.confirm = True # enable/disable confirmation email
    def _yes_no(self, boolean):
        if boolean == True:
            return "yes"
        return "no"
    def author_tuple(self):
        """
        Extract and normalize the sender's email address.  Returns a
        (name, email) tuple.
        """
        if not hasattr(self, "author_tuple_cache"):
            self._author_tuple_cache = \
                send_pgp_mime.source_email(self.msg, return_realname=True)
        return self._author_tuple_cache
    def author_addr(self):
        return email.utils.formataddr(self.author_tuple())
    def author_name(self):
        return self.author_tuple()[0]
    def author_email(self):
        return self.author_tuple()[1]
    def default_msg_attribute_access(self, attr_name, default=None):
        if attr_name in self.msg:
            return self.msg[attr_name]
        return default
    def message_id(self, default=None):
        return self.default_msg_attribute_access("message-id", default=default)
    def subject(self):
        if "subject" not in self.msg:
            raise InvalidSubject(self, u"Email must contain a subject")
        return self.msg["subject"]
    def _split_subject(self):
        """
        Returns (tag, subject), with missing values replaced by None.
        """
        if hasattr(self, "_split_subject_cache"):
            return self._split_subject_cache
        args = self.subject().split(u"]",1)
        if len(args) < 1:
            self._split_subject_cache = (None, None)
        elif len(args) < 2:
            self._split_subject_cache = (args[0]+u"]", None)
        else:
            self._split_subject_cache = (args[0]+u"]", args[1].strip())
        return self._split_subject_cache
    def _subject_tag_type(self):
        """
        Parse subject tag, return (type, value), where type is one of
        None, "new", "comment", or "control"; and value is None except
        in the case of "comment", in which case it's the bug
        ID/shortname.
        """
        tag,subject = self._split_subject()
        type = None
        value = None
        if tag == SUBJECT_TAG_NEW:
            type = u"new"
        elif tag == SUBJECT_TAG_CONTROL:
            type = u"control"
        else:
            match = SUBJECT_TAG_COMMENT.match(tag)
            if len(match.groups()) == 1:
                type = u"comment"
                value = match.group(1)
        return (type, value)
    def validate_subject(self):
        """
        Validate the subject line.
        """
        tag,subject = self._split_subject()
        if not tag.startswith(SUBJECT_TAG_START):
            raise InvalidSubject(
                self, u"Subject must start with '%s'" % SUBJECT_TAG_START)
        tag_type,value = self._subject_tag_type()
        if tag_type == None:
            raise InvalidSubject(self, u"Invalid tag '%s'" % tag)
        elif tag_type == u"new" and len(subject) == 0:
            raise InvalidSubject(self, u"Cannot create a bug with blank title")
        elif tag_type == u"comment" and len(value) == 0:
            raise InvalidSubject(self, u"Must specify a bug ID to comment")
    def _get_bodies_and_mime_types(self):
        """
        Traverse the email message returning (body, mime_type) for
        each non-mulitpart portion of the message.
        """
        msg_charset = self.msg.get_content_charset(ENCODING).lower()
        for part in self.msg.walk():
            if part.is_multipart():
                continue
            body,mime_type=(part.get_payload(decode=True),part.get_content_type())
            charset = part.get_content_charset(msg_charset).lower()
            if mime_type.startswith("text/"):
                body = unicode(body, charset) # convert text types to unicode
            yield (body, mime_type)
    def _parse_body_pseudoheaders(self, body, required, optional,
                                  dictionary=None):
        """
        Grab any pseudo-headers from the beginning of body.  Raise
        InvalidPseudoHeader on errors.  Returns the body text after
        the pseudo-header and a dictionary of set options.  If you
        like, you can initialize the dictionary with some defaults
        and pass your initialized dict in as dictionary.
        """
        if dictionary == None:
            dictionary = {}
        body_lines = body.splitlines()
        all = required+optional
        for i,line in enumerate(body_lines):
            line = line.strip()
            if len(line) == 0:
                break
            if ":" not in line:
                raise InvalidPseudoheader(self, line)
            key,value = line.split(":", 1)
            value = value.strip()
            if key not in all:
                raise InvalidPseudoHeader(self, key)
            if len(value) == 0:
                raise InvalidEmail(
                    self, u"Blank value for: %s" % key)
            dictionary[key] = value
        missing = []
        for key in required:
            if key not in dictionary:
                missing.append(key)
        if len(missing) > 0:
            raise InvalidPseudoHeader(self,
                                      u"Missing required pseudo-headers:\n%s"
                                      % u", ".join(missing))
        remaining_body = u"\n".join(body_lines[i:]).strip()
        return (remaining_body, dictionary)
    def _strip_footer(self, body):
        body_lines = body.splitlines()
        for i,line in enumerate(body_lines):
            if line.startswith(BREAK):
                break
            i += 1 # increment past the current valid line.
        return u"\n".join(body_lines[:i]).strip()
    def parse(self):
        """
        Parse the commands given in the email.  Raises assorted
        subclasses of InvalidEmail in the case of invalid messages,
        otherwise returns a list of suggested commands to run.
        """
        self.validate_subject()
        tag_type,value = self._subject_tag_type()
        if tag_type == u"new":
            commands = self.parse_new()
        elif tag_type == u"comment":
            commands = self.parse_comment(value)
        elif tag_type == u"control":
            commands = self.parse_control()
        else:
            raise Exception, u"Unrecognized tag type '%s'" % tag_type
        return commands
    def parse_new(self):
        command = u"new"
        tag,subject = self._split_subject()
        summary = subject
        options = {u"Reporter": self.author_addr(),
                   u"Confirm": self._yes_no(self.confirm),
                   u"Subscribe": "no",
                   }
        body,mime_type = list(self._get_bodies_and_mime_types())[0]
        comment_body,options = \
            self._parse_body_pseudoheaders(body,
                                           NEW_REQUIRED_PSEUDOHEADERS,
                                           NEW_OPTIONAL_PSEUDOHEADERS,
                                           options)
        if options[u"Confirm"].lower() == "no":
            self.confirm = False
        if options[u"Subscribe"].lower() == "yes" and self.confirm == True:
            # respond with the subscription format rather than the
            # normal command-output format, because the subscription
            # format is more user-friendly.
            self.confirm = False
        args = [u"--reporter", options[u"Reporter"]]
        args.append(summary)
        commands = [Command(self, command, args)]
        id = ID(commands[0])
        comment_body = self._strip_footer(comment_body)
        if len(comment_body) > 0:
            command = u"comment"
            comment = u"Version: %s\n\n"%options[u"Version"] + comment_body
            args = [u"--author", self.author_addr(),
                    u"--alt-id", self.message_id(),
                    u"--content-type", mime_type]
            args.append(id)
            args.append(u"-")
            commands.append(Command(self, u"comment", args, stdin=comment))
        for key,value in options.items():
            if key in [u"Version", u"Reporter", u"Confirm"]:
                continue # we've already handled these options
            command = key.lower()
            args = [id, value]
            if key == u"Subscribe":
                if value.lower() != "yes":
                    continue
                args = ["--subscriber", self.author_addr(), id]
            commands.append(Command(self, command, args))
        return commands
    def parse_comment(self, bug_uuid):
        command = u"comment"
        bug_id = bug_uuid
        author = self.author_addr()
        alt_id = self.message_id()
        body,mime_type = list(self._get_bodies_and_mime_types())[0]
        if mime_type == "text/plain":
            body = self._strip_footer(body)
        content_type = mime_type
        args = [u"--author", author, u"--alt-id", alt_id,
                u"--content-type", content_type, bug_id, u"-"]
        commands = [Command(self, command, args, stdin=body)]
        return commands
    def parse_control(self):
        body,mime_type = list(self._get_bodies_and_mime_types())[0]
        commands = []
        for line in body.splitlines():
            line = line.strip()
            if line.startswith(CONTROL_COMMENT) or len(line) == 0:
                continue
            if line.startswith(BREAK):
                break
            fields = shlex.split(line)
            command,args = (fields[0], fields[1:])
            commands.append(Command(self, command, args))
        if len(commands) == 0:
            raise InvalidEmail(self, u"No commands in control email.")
        return commands
    def run(self):
        self._begin_response()
        commands = self.parse()
        try:
            for command in commands:
                command.run()
                self._add_response(command.response_msg())
        finally:
            if AUTOCOMMIT == True:
                tag,subject = self._split_subject()
                self.commit_command = Command(self, "commit", [subject])
                self.commit_command.run()
                if LOGFILE != None:
                    LOGFILE.write(u"Autocommit:\n%s\n\n" %
                      send_pgp_mime.flatten(self.commit_command.response_msg(),
                                            to_unicode=True))
    def _begin_response(self):
        tag,subject = self._split_subject()
        response_header = [u"From: %s" % THIS_ADDRESS,
                           u"To: %s" % self.author_addr(),
                           u"Date: %s" % libbe.utility.time_to_str(time.time()),
                           u"Subject: %s Re: %s"%(SUBJECT_TAG_RESPONSE,subject)
                           ]
        if self.message_id() != None:
            response_header.append(u"In-reply-to: %s" % self.message_id())
        self.response_header = \
            send_pgp_mime.header_from_text(text=u"\n".join(response_header))
        self._response_messages = []
    def _add_response(self, response_message):
        self._response_messages.append(response_message)
    def response_email(self):
        assert len(self._response_messages) > 0
        if len(self._response_messages) == 1:
            response_body = self._response_messages[0]
        else:
            response_body = MIMEMultipart()
            for message in self._response_messages:
                response_body.attach(message)
        return send_pgp_mime.attach_root(self.response_header, response_body)
    def subscriber_emails(self, previous_revision=None):
        if previous_revision == None:
            if AUTOCOMMIT != True: # no way to tell what's changed
                raise NotificationFailed("Autocommit dissabled")
            if len(self._response_messages) == 0:
                raise NotificationFailed("Initial email failed.")
            if self.commit_command.ret != 0:
                # commit failed.  Error already logged.
                raise NotificationFailed("Commit failed")

        # read only bugdir.
        bd = libbe.bugdir.BugDir(from_disk=True,
                                 manipulate_encodings=False)
        if bd.vcs.versioned == False: # no way to tell what's changed
            raise NotificationFailed("Not versioned")

        bd.load_all_bugs()
        subscribers = subscribe.get_bugdir_subscribers(bd, THIS_SERVER)

        if len(subscribers) == 0:
            return [] 

        before_bd, after_bd = self._get_before_and_after_bugdirs(bd, previous_revision)
        diff = Diff(before_bd, after_bd)
        diff_tree = diff.report_tree(diff_tree=DiffTree)
        bug_index = {}
        for child in diff_tree.child_by_path("/bugs/new"):
            bug_index[child.name] = ("added", child)
        for child in diff_tree.child_by_path("/bugs/mod"):
            bug_index[child.name] = ("modified", child)
        for child in diff_tree.child_by_path("/bugs/rem"):
            bug_index[child.name] = ("removed", child)
        header = self._subscriber_header(bd, previous_revision)

        emails = []
        for subscriber,subscriptions in subscribers.items():
            header.replace_header("to", subscriber)
            parts = []
            if "DIR" in subscriptions: # make sure we check the DIR level first
                ordered_subscriptions = [("DIR", subscriptions.pop("DIR"))]
            else:
                ordered_subscriptions = []
            ordered_subscriptions.extend(subscriptions.items())
            for id,types in ordered_subscriptions:
                if id == "DIR":
                    if subscribe.BUGDIR_TYPE_ALL in types:
                        parts.append(diff_tree.report_or_none())
                        break # we've attached everything, so stop checking.
                    if subscribe.BUGDIR_TYPE_NEW in types:
                        new = diff_tree.child_by_path("/bugs/new")
                        parts.append(new.report_or_none())
                    continue # move on to next id
                # if we get this far, id refers to a bug.
                assert types == [subscribe.BUG_TYPE_ALL], types
                if id not in bug_index:
                    continue # no changes here, move on to next id
                type,bug_root = bug_index[id]
                if type == "added" \
                        and "DIR" in subscriptions \
                        and subscriptions["DIR"] == subscribe.BUGDIR_TYPE_NEW:
                    # this info already attached at the DIR level
                    continue # move on to next id
                parts.append(bug_root.report_or_none())
            parts = [p for p in parts if p != None]
            if len(parts) == 0:
                continue # no email to this subscriber
            elif len(parts) == 1:
                root = parts[0]
            else: # join subscription parts into a single body
                root = MIMEMultipart()
                root[u"Content-Description"] = u"Multiple subscription trees."
                for part in parts:
                    root.attach(part)
            emails.append(send_pgp_mime.attach_root(header, root))
            if LOGFILE != None:
                LOGFILE.write(u"Preparing to notify %s of changes\n" % subscriber)
        return emails
    def _get_before_and_after_bugdirs(self, bd, previous_revision=None):
        if previous_revision == None:
            commit_msg = self.commit_command.stdout
            assert commit_msg.startswith("Committed "), commit_msg
            after_revision = commit_msg[len("Committed "):]
            before_revision = bd.vcs.revision_id(-2)
        else:
            before_revision = previous_revision
        if before_revision == None:
            # this commit was the initial commit
            before_bd = libbe.bugdir.BugDir(from_disk=False,
                                            manipulate_encodings=False)
        else:
            before_bd = bd.duplicate_bugdir(before_revision)
        #after_bd = bd.duplicate_bugdir(after_revision)
        after_bd = bd # assume no changes since commit a few cycles ago
        return (before_bd, after_bd)
    def _subscriber_header(self, bd, previous_revision=None):
        root_dir = os.path.basename(bd.root)
        if previous_revision == None:
            subject = "Changes to %s on %s by %s" \
                % (root_dir, THIS_SERVER, self.author_addr())
        else:
            subject = "Changes to %s on %s since revision %s" \
                % (root_dir, THIS_SERVER, previous_revision)
        header = [u"From: %s" % THIS_ADDRESS,
                  u"To: %s" % u"DUMMY-AUTHOR",
                  u"Date: %s" % libbe.utility.time_to_str(time.time()),
                  u"Subject: %s Re: %s" % (SUBJECT_TAG_RESPONSE, subject)
                  ]
        return send_pgp_mime.header_from_text(text=u"\n".join(header))

def generate_global_tags(tag_base=u"be-bug"):
    """
    Generate a series of tags from a base tag string.
    """
    global SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \
        SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL
    SUBJECT_TAG_BASE = tag_base
    SUBJECT_TAG_START = u"[%s" % tag_base
    SUBJECT_TAG_RESPONSE = u"[%s]" % tag_base
    SUBJECT_TAG_NEW = u"[%s:submit]" % tag_base
    SUBJECT_TAG_COMMENT = re.compile(u"\[%s:([\-0-9a-z]*)]" % tag_base)
    SUBJECT_TAG_CONTROL = SUBJECT_TAG_RESPONSE

def open_logfile(logpath=None):
    """
    If logpath=None, default to global LOGPATH.
    Special logpath strings:
     "-"     set LOGFILE to sys.stderr
     "none"  disable logging
    Relative logpaths are expanded relative to _THIS_DIR
    """
    global LOGPATH, LOGFILE
    if logpath != None:
        if logpath == u"-":
            LOGPATH = u"stderr"
            LOGFILE = sys.stderr
        elif logpath == u"none":
            LOGPATH = u"none"
            LOGFILE = None
        elif os.path.isabs(logpath):
            LOGPATH = logpath
        else:
            LOGPATH = os.path.join(_THIS_DIR, logpath)
    if LOGFILE == None and LOGPATH != u"none":
        LOGFILE = codecs.open(LOGPATH, u"a+", ENCODING)
        LOGFILE.write(u"Default encoding: %s\n" % ENCODING)

def close_logfile():
    if LOGFILE != None and LOGPATH not in [u"stderr", u"none"]:
        LOGFILE.close()

def test():
    unitsuite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
    suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()])
    result = unittest.TextTestRunner(verbosity=2).run(suite)
    num_errors = len(result.errors)
    num_failures = len(result.failures)
    num_bad = num_errors + num_failures
    return num_bad

def main(args):
    from optparse import OptionParser
    global AUTOCOMMIT, BE_DIR

    usage="be-handle-mail [options]\n\n%s" % (__doc__)
    parser = OptionParser(usage=usage)
    parser.add_option('-b', '--be-dir', dest='be_dir', default=BE_DIR,
                      metavar="DIR",
                      help='Select the BE directory to serve (%default).')
    parser.add_option('-t', '--tag-base', dest='tag_base',
                      default=SUBJECT_TAG_BASE, metavar="TAG",
                      help='Set the subject tag base (%default).')
    parser.add_option('-o', '--output', dest='output', action='store_true',
                      help="Don't mail the generated message, print it to stdout instead.  Useful for testing be-handle-mail functionality without the whole mail transfer agent and procmail setup.")
    parser.add_option('-l', '--logfile', dest='logfile', metavar='LOGFILE',
                      help='Set the logfile to LOGFILE.  Relative paths are relative to the location of this be-handle-mail file (%s).  The special value of "-" directs the log output to stderr, and "none" disables logging.' % _THIS_DIR)
    parser.add_option('-a', '--disable-autocommit', dest='autocommit',
                      default=True, action='store_false',
                      help='Disable the autocommit after parsing the email.')
    parser.add_option('-s', '--disable-subscribers', dest='subscribers',
                      default=True, action='store_false',
                      help='Disable subscriber notification emails.')
    parser.add_option('--notify-since', dest='notify_since', metavar='REVISION',
                      help='Notify subscribers of all changes since REVISION.  When this option is set, no input email parsing is done.')
    parser.add_option('--test', dest='test', action='store_true',
                      help='Run internal unit-tests and exit.')

    pargs = args
    options,args = parser.parse_args(args[1:])

    if options.test == True:
        num_bad = test()
        if num_bad > 126:
            num_bad = 1
        sys.exit(num_bad)
    
    BE_DIR = options.be_dir
    AUTOCOMMIT = options.autocommit

    if options.notify_since == None:
        msg_text = sys.stdin.read()

    libbe.encoding.set_IO_stream_encodings(ENCODING) # _after_ reading message
    open_logfile(options.logfile)
    generate_global_tags(options.tag_base)

    if options.notify_since != None:
        if options.subscribers == True:
            if LOGFILE != None:
                LOGFILE.write(u"Checking for subscribers to notify since revision %s\n"
                              % options.notify_since)
            try:
                m = Message(disable_parsing=True)
                emails = m.subscriber_emails(options.notify_since)
            except NotificationFailed, e:
                if LOGFILE != None:
                    LOGFILE.write(unicode(e) + u"\n")
            else:
                for msg in emails:
                    if options.output == True:
                        print send_pgp_mime.flatten(msg, to_unicode=True)
                    else:
                        send_pgp_mime.mail(msg, send_pgp_mime.sendmail)
            close_logfile()
        sys.exit(0)

    if len(msg_text.strip()) == 0: # blank email!?
        if LOGFILE != None:
            LOGFILE.write(u"Blank email!\n")
            close_logfile()
        sys.exit(1)
    try:
        m = Message(msg_text)
        m.run()
    except InvalidEmail, e:
        response = e.response()
    except Exception, e:
        if LOGFILE != None:
            LOGFILE.write(u"Uncaught exception:\n%s\n" % (e,))
            traceback.print_tb(sys.exc_traceback, file=LOGFILE)
            close_logfile()
        sys.exit(1)
    else:
        response = m.response_email()
    if options.output == True:
        print send_pgp_mime.flatten(response, to_unicode=True)
    elif m.confirm == True:
        if LOGFILE != None:
            LOGFILE.write(u"Sending response to %s\n" % m.author_addr())
            LOGFILE.write(u"\n%s\n\n" % send_pgp_mime.flatten(response,
                                                              to_unicode=True))
        send_pgp_mime.mail(response, send_pgp_mime.sendmail)
    else:
        if LOGFILE != None:
            LOGFILE.write(u"Response declined by %s\n" % m.author_addr())
    if options.subscribers == True:
        if LOGFILE != None:
            LOGFILE.write(u"Checking for subscribers\n")
        try:
            emails = m.subscriber_emails()
        except NotificationFailed, e:
            if LOGFILE != None:
                LOGFILE.write(unicode(e) + u"\n")
        else:
            for msg in emails:
                if options.output == True:
                    print send_pgp_mime.flatten(msg, to_unicode=True)
                else:
                    send_pgp_mime.mail(msg, send_pgp_mime.sendmail)

    close_logfile()


class GenerateGlobalTagsTestCase (unittest.TestCase):
    def setUp(self):
        super(GenerateGlobalTagsTestCase, self).setUp()
        self.save_global_tags()
    def tearDown(self):
        self.restore_global_tags()
        super(GenerateGlobalTagsTestCase, self).tearDown()
    def save_global_tags(self):
        self.saved_globals = [SUBJECT_TAG_BASE, SUBJECT_TAG_START,
                              SUBJECT_TAG_RESPONSE, SUBJECT_TAG_NEW,
                              SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL]
    def restore_global_tags(self):
        global SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \
            SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL
        SUBJECT_TAG_BASE, SUBJECT_TAG_START, SUBJECT_TAG_RESPONSE, \
            SUBJECT_TAG_NEW, SUBJECT_TAG_COMMENT, SUBJECT_TAG_CONTROL = \
            self.saved_globals
    def test_restore_global_tags(self):
        "Test global tag restoration by teardown function."
        global SUBJECT_TAG_BASE
        self.failUnlessEqual(SUBJECT_TAG_BASE, u"be-bug")
        SUBJECT_TAG_BASE = "projectX-bug"
        self.failUnlessEqual(SUBJECT_TAG_BASE, u"projectX-bug")
        self.restore_global_tags()
        self.failUnlessEqual(SUBJECT_TAG_BASE, u"be-bug")
    def test_subject_tag_base(self):
        "Should set SUBJECT_TAG_BASE global correctly"
        generate_global_tags(u"projectX-bug")
        self.failUnlessEqual(SUBJECT_TAG_BASE, u"projectX-bug")
    def test_subject_tag_start(self):
        "Should set SUBJECT_TAG_START global correctly"
        generate_global_tags(u"projectX-bug")
        self.failUnlessEqual(SUBJECT_TAG_START, u"[projectX-bug")
    def test_subject_tag_response(self):
        "Should set SUBJECT_TAG_RESPONSE global correctly"
        generate_global_tags(u"projectX-bug")
        self.failUnlessEqual(SUBJECT_TAG_RESPONSE, u"[projectX-bug]")
    def test_subject_tag_new(self):
        "Should set SUBJECT_TAG_NEW global correctly"
        generate_global_tags(u"projectX-bug")
        self.failUnlessEqual(SUBJECT_TAG_NEW, u"[projectX-bug:submit]")
    def test_subject_tag_control(self):
        "Should set SUBJECT_TAG_CONTROL global correctly"
        generate_global_tags(u"projectX-bug")
        self.failUnlessEqual(SUBJECT_TAG_CONTROL, u"[projectX-bug]")
    def test_subject_tag_comment(self):
        "Should set SUBJECT_TAG_COMMENT global correctly"
        generate_global_tags(u"projectX-bug")
        m = SUBJECT_TAG_COMMENT.match("[projectX-bug:xyz-123]")
        self.failUnlessEqual(len(m.groups()), 1)
        self.failUnlessEqual(m.group(1), u"xyz-123")

if __name__ == "__main__":
    main(sys.argv)