Added patch comparing two branches ... cleanup abd cleanup_failling.

By this method, I hope we can approach cleanup_failling slowly.

1 files changed, 620 insertions, 0 deletions

diff --git a/cleanup_failing.patch b/cleanup_failing.patch
new file mode 100644
index 0000000..334cc82
--- /dev/null
+++ b/cleanup_failing.patch
@@ -0,0 +1,620 @@
+diff --git b/mail2news.py a/mail2news.py
+index e7cbcf9..412e4ee 100644
+--- b/mail2news.py
++++ a/mail2news.py
+@@ -1,9 +1,11 @@
++# -*- coding: utf-8 -*-
+ """Mail to news gateway script. Copyright 2000 Cosimo Alfarano
+ 
+ Author: Cosimo Alfarano
+ Date: September 16 2000
+ 
+-mail2news.py - (C) 2000 by Cosimo Alfarano <Alfarano@Students.CS.UniBo.It>
++mail2news.py - Copyright 2000 by
++    Cosimo Alfarano <Alfarano@Students.CS.UniBo.It>
+ You can use this software under the terms of the GPL. If we meet some day,
+ and you think this stuff is worth it, you can buy me a beer in return.
+ 
+@@ -14,6 +16,8 @@ Gets news email and sends it via SMTP.
+ class mail2news is  hopefully conform to rfc850.
+ 
+ """
++
++import email
+ import sys
+ from os import getpid
+ from socket import gethostbyaddr, gethostname
+@@ -24,80 +28,41 @@ import pyginfo
+ 
+ class mail2news:
+     """news to mail gateway class"""
+-
+     reader = None                # mode reader
+-#    newsgroups = None            # Newsgroups: local.test,local.moderated...
+-#    approved = None                # Approved: kame@aragorn.lorien.org
+     newsserver = 'localhost'    # no comment :)
+     port = 119
+     user = None
+     password = None
+ 
+-    hostname = gethostbyaddr(gethostname())[0]
+-
+-    heads_dict, smtpheads, nntpheads = {}, {}, {}
+-    email, headers, body = [], [], []
+-
+-    def readfile(self, opt):
+-
+-        for line in sys.stdin.readlines():
+-            self.email.append(line)
+-
+-        if(len(self.email) == 1 and self.email[0][0] == '/'):
+-            file = self.email[0][:-1]
+-            del self.email[0]
+-            for line in open(file, 'r').readlines():
+-                self.email.append(line)
+-
+-        # introduce nntpheads
+-        if opt.newsgroup != '':
+-            # TODO put it directly to self.message when we have it
+-            self.nntpheads['Newsgroups:'] = opt.newsgroup + '\n'
+-        if opt.approver != '':
+-            self.nntpheads['Approved:'] = opt.approver + '\n'
++    def __init__(self):
++        self.hostname = gethostbyaddr(gethostname())[0]
++        self.message = email.message_from_file(sys.stdin)
+ 
+-        return 1
++# """phase 3:
++#     format rfc 822 headers from input article
++#     """
++#
++#     m2n.mergeheads()  # make unique dict from NNTP and SMTP dicts
++#
++#     m2n.addheads()  # add some important heads
++#     m2n.renameheads()  # rename useless heads
++#     m2n.removeheads()  # remove other heads
++#
++#     m2n.sortheads()  # sort remaining heads :)
++#
++#     if opt.verbose:
++#         for line in m2n.headers:
++#             print line[:-1]
+ 
+-    def parseemail(self):
++    def x_parseemail(self):
+         """get news email from file or stdin and separate heads from body
+ 
+         REMEBER: headers value has '\n' as last char.
+             Use string[:-1] to ignore newline.
+         """
+-
+-        try:
+-            body = 0        # are we in body or in headers?
+-
+-            for line in self.email:
+-                if not body and len(line) == 1:
+-                    body = 1    # starts email body section
+-
+-                if not body:
+-                    try:
+-                        # if it is a multi-line header like Received:
+-                        if line[0] not in [' ', '\t']:
+-                            try:
+-                                head, value = line.split(' ', 1)
+-                            except ValueError:
+-                                value = ''
+-                            self.smtpheads[head] = value
+-                        else:
+-                            self.smtpheads[head] = '%s%s' % \
+-                                (self.smtpheads[head], line)
+-                    except (ValueError), message:
+-                        print('line: %s' % line)
+-                        print('(probably missing couple "Header: value" in %s)'
+-                              % line)
+-                        sys.exit(1)
+-
+-                elif len(line) > 0 and body:
+-                    self.body.append(line)
+-
+-        except (ValueError), message:
+-            print message
+-            sys.exit(1)
+-
+-        return self.smtpheads, self.body
++        # return self.smtpheads, self.body
++        # self.smtpheads contains headers, self.body body
++        pass
+ 
+     def puthead(self, dict, list, key):
+         """private, transform dict entries in list entries
+@@ -105,33 +70,34 @@ class mail2news:
+         """
+ 
+         if key in dict:
+-            list.append(key + ' ' + dict.get(key))
++            list.append(key + ' ' + dict[key])  # FIXME: Message instance doesn't have append
+             del dict[key]
+         else:
+             return 0
+         return 1
+ 
+     def sortheads(self):
+-        """make list sorted by heads: From: To: Subject: first,
+-           others, X-*, X-Resent-* last"""
++        """make list sorted by heads:
++           From: To: Subject: first, others, X-*, X-Resent-* last"""
+ 
+         # put at top
+         set = ('Newsgroups:', 'From:', 'To:', 'X-To:', 'Cc:', 'Subject:',
+                'Date:', 'Approved:', 'References:', 'Message-Id:')
+ 
+         for k in set:
+-            self.puthead(self.heads_dict, self.headers, k)
++            self.puthead(self.heads_dict, self.message, k)
+ 
+-        for k in self.heads_dict.keys():
+-            if k[:2] != 'X-' and k[:9] != 'X-Resent-' and k not in set:
++        for k in self.message:
++            if not k.startswith('X-') and not k.startswith('X-Resent-') and \
++                    k not in set:
+                         self.puthead(self.heads_dict, self.headers, k)
+ 
+-        for k in self.heads_dict.keys():
+-            if k[:2] == 'X-':
++        for k in self.message:
++            if k.startswith('X-'):
+                 self.puthead(self.heads_dict, self.headers, k)
+ 
+-        for k in self.heads_dict.keys():
+-            if k[:9] == 'X-Resent-':
++        for k in self.message:
++            if k.startswith('X-Resent-'):
+                 self.puthead(self.heads_dict, self.headers, k)
+ 
+         return self.headers
+@@ -142,14 +108,11 @@ class mail2news:
+ 
+         self.heads_dict = {}
+ 
+-        try:
+-            for header in self.smtpheads.keys():  # fill it w/ smtp old heads
+-                self.heads_dict[header] = self.smtpheads[header]
+-
+-            # and replace them w/ nntp new heads
+-            for header in self.nntpheads.keys():
+-                self.heads_dict[header] = self.nntpheads[header]
++        # FIXME something like a = set() ; a = self.message.items() ?
+ 
++        try:
++            for header in self.message.keys():  # fill it w/ smtp old heads
++                self.heads_dict[header] = self.message[header]
+         except KeyError, message:
+             print message
+ 
+@@ -160,22 +123,8 @@ class mail2news:
+         """
+ 
+         info = pyginfo.pygsinfo()
+-
+-        try:
+-            self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \
+-                info.PROGDESC + ' - Mail to News\n'
+-
+-# it is nntpheads stuff
+-#            if(self.newsgroups):
+-#                self.heads_dict['Newsgroups:'] = self.newsgroups
+-
+-#            if(self.approved):
+-#                self.heads_dict['Approved:'] = self.approved
+-
+-        except KeyError, message:
+-            print message
+-
+-        return self.heads_dict
++        self.message.add_header('X-Gateway', info.PROGNAME + ' ' +
++                                info.PROGDESC + ' - Mail to News')
+ 
+     def renameheads(self):
+         """rename headers such as Resent-*: to X-Resent-*:
+@@ -184,50 +133,31 @@ class mail2news:
+         handles References/In-Reply-To headers
+         """
+         try:
+-
+-### test
+-#   if(post):
+-#       if(post in self.heads_dict):
+-#           self.heads_dict['X-Original-' + post] = self.heads_dict[post]
+-#
+-#       self.heads_dict[post] = self.heads_dict[pre]
+-#       del(self.heads_dict[pre])
+-#
+-#   else:
+-#       if(pre[0:2] == 'X-' and pre in self.heads_dict):
+-#           self.heads_dict['X-Original-' + pre] = self.heads_dict[pre]
+-#       elif(not pre[0:2] == 'X-' and 'X-' + pre in self.heads_dict):
+-#           self.heads_dict['X-' + pre] = self.heads_dict[pre]
+-#       del(self.heads_dict[pre])
+-### end test
+-
+-            for key in self.heads_dict.keys():
+-                if(key[:7] in ['Resent-']):
+-                    if ('X-' + key) in self.heads_dict:
+-                        self.heads_dict['X-Original-' + key] = \
+-                            self.heads_dict['X-' + key]
+-                    self.heads_dict['X-' + key] = self.heads_dict[key]
+-                    del self.heads_dict[key]
++            for key in self.message.keys():
++                if key.startswith('Resent-'):
++                    if ('X-' + key) in self.message:
++                        self.message['X-Original-' + key] = \
++                            self.message['X-' + key]
++                    self.message['X-' + key] = self.message[key]
++                    del self.message[key]
+ 
+             # In rfc822 References: is considered, but many MUA doen't put it.
+-            if ('References:' not in self.heads_dict) and \
+-                    ('In-Reply-To:' in self.heads_dict):
+-                print self.heads_dict['In-Reply-To:']
++            if 'References:' not in self.message and \
++                    'In-Reply-To:' in self.message:
++                print self.message['In-Reply-To:']
+ 
++                # FIXME !!! Do konce metody je to dost zmatené!
+                 # some MUA uses msgid without '<' '>'
+ #                ref = findall('([^\s<>\']+@[^\s<>;:\']+)', \
+                 # but I prefer use RFC standards
++                # FIXME isn't In-Reply-To supposed to be unique???
+                 ref = findall('(<[^<>]+@[^<>]+>)',
+-                              self.heads_dict['In-Reply-To:'])
++                              self.message.get_all('In-Reply-To:'))
+ 
+                 # if found, keep first element that seems a Msg-ID.
+                 if(ref and len(ref)):
+                     self.heads_dict['References:'] = '%s\n' % ref[0]
+ 
+-#            if('To:' in self.heads_dict):
+-#                self.heads_dict['X-To:'] = self.heads_dict['To:']
+-#                del self.heads_dict['To:']
+-
+         except KeyError, message:
+             print message
+ 
+@@ -236,51 +166,32 @@ class mail2news:
+     def removeheads(self, heads=None):
+         """remove headers like Xref: Path: Lines:
+         """
+-
+-        try:
+         # removing some others useless headers .... (From is not From:)
+ 
+         rmheads = ['Received:', 'From', 'NNTP-Posting-Host:',
+                    'X-Trace:', 'X-Compliants-To:', 'NNTP-Posting-Date:']
+-            if(heads):
++        if heads:
+             rmheads.append(heads)
+ 
+         for head in rmheads:
+-                if head in self.heads_dict:
+-                    del self.heads_dict[head]
+-
+-#            if 'From' in self.heads_dict: # neither 'From ' nor 'From:'
+-#                del self.heads_dict['From']
+-
+-#            # neither 'From ' nor 'From:'
+-#            if 'NNTP-Posting-Host:' in self.heads_dict:
+-#                del self.heads_dict['']
+-
+-#            if 'Lines:' in self.heads_dict:
+-#                del self.heads_dict['Lines:']
++            if head in self.message:
++                del self.message[head]
+ 
+-            # it is usually set by INN, if ng is moderated...
+-#            if 'Sender:' in self.heads_dict:
+-#                del self.heads_dict['Sender:']
+-
+-            if 'Message-id:' in self.heads_dict:
++        # FIXME the following lines are buggy, because they don't expect
++        # the header 'Message-Id' being already present
++        if 'Message-id:' in self.message:
+             self.heads_dict['Message-Id:'] = self.heads_dict['Message-id:']
+             del(self.heads_dict['Message-id:'])
+ 
+-            if 'Message-ID:' in self.heads_dict:
++        if 'Message-ID:' in self.message:
+             self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID:']
+             del(self.heads_dict['Message-ID:'])
+ 
+         # If message-id is not present, I generate it
+-            if 'Message-Id:' not in self.heads_dict:
++        if 'Message-Id:' not in self.message:
+             msgid = '<pyg.%d@tuchailepuppapera.org>\n' % (getpid())
+             self.heads_dict['Message-Id:'] = msgid
+ 
+-        except KeyError, message:
+-            print message
+-
+-        return self.heads_dict
+-
+     def sendemail(self):
+         """Talk to NNTP server and try to send email."""
+         try:
+diff --git b/news2mail.py a/news2mail.py
+index bfd6fd1..a74655a 100644
+--- b/news2mail.py
++++ a/news2mail.py
+@@ -1,9 +1,11 @@
++# -*- coding: utf-8 -*-
+ """News to mail gateway script. Copyright 2000 Cosimo Alfarano
+ 
+ Author: Cosimo Alfarano
+ Date: June 11 2000
+ 
+-news2mail.py - (C) 2000 by Cosimo Alfarano <Alfarano@Students.CS.UniBo.It>
++news2mail.py - Copyright 2000 by
++    Cosimo Alfarano <Alfarano@Students.CS.UniBo.It>
+ You can use this software under the terms of the GPL. If we meet some day,
+ and you think this stuff is worth it, you can buy me a beer in return.
+ 
+@@ -23,9 +25,13 @@ normal (what pygs does) operations flow is:
+     Date:, normal headers ending with X-* and Resent-* headers.
+ 
+ """
++
++import email
+ import sys
+ import smtplib
++import string
+ import time
++#import rfc822
+ from socket import gethostbyaddr, gethostname
+ import tempfile
+ import pyginfo
+@@ -37,6 +43,7 @@ class news2mail:
+     TMPFILE = tempfile.mktemp()
+     wlfile = None
+     logfile = None
++    debug = 1
+ 
+     sender = ''
+     rcpt = ''
+@@ -44,60 +51,9 @@ class news2mail:
+ 
+     smtpserver = 'localhost'
+ 
+-    hostname = gethostbyaddr(gethostname())[0]
+-
+-    heads_dict, smtpheads, nntpheads = {}, {}, {}
+-    article, headers, body = [], [], []
+-
+-    debug = 1
+-
+-    def readfile(self):
+-
+-        for line in sys.stdin.readlines():
+-            self.article.append(line)
+-
+-        if (len(self.article) == 1 and self.article[0][0] == '/'):
+-            file = self.article[0][:-1]
+-            del self.article[0]
+-            for line in open(file, 'r').readlines():
+-                self.article.append(line)
+-
+-    def parsearticle(self):
+-        """get news article from file or stdin and separate heads from body
+-
+-        REMEBER: headers value has '\n' as last char.
+-            Use string[:-1] to ignore newline.
+-        """
+-
+-        try:
+-            body = 0        # are we in body or in headers?
+-
+-#            voidline = compile('^$')     # I need '\n', ^$ matches anything
+-#            spaceending = compile('\s*\n$')
+-
+-            for line in self.article:
+-                if not body and len(line) == 1:
+-                    body = 1    # starts article body section
+-
+-                if not body:
+-                    try:
+-                        head, value = line.split(' ', 1)
+-                        self.nntpheads[head] = value
+-                    except (ValueError), message:
+-                        print('string error: %s' % message)
+-                        print('(probably missing couple "Header: value" in %s)'
+-                              % line)
+-                        sys.exit(1)
+-
+-                elif len(line) > 0 and body:
+-                    self.body.append(line)
+-
+-            return self.nntpheads, self.body
+-
+-#        except (re.error, ValueError), message:
+-        except (ValueError), message:
+-            print message
+-            sys.exit(1)
++    def __init__(self):
++        self.hostname = gethostbyaddr(gethostname())[0]  # FQDN
++        self.message = email.message_from_file(sys.stdin)
+ 
+     def puthead(self, dict, list, key):
+         """private, x-form dict entries in list entries"""
+@@ -138,7 +94,8 @@ class news2mail:
+         self.heads_dict = {}
+ 
+         try:
+-            for header in self.nntpheads.keys():  # fill it w/ nntp old heads
++            # fill it w/ nntp old heads
++            for header in self.nntpheads.keys():
+                 self.heads_dict[header] = self.nntpheads[header]
+ 
+             # and replace them w/ smtp new heads
+@@ -160,9 +117,6 @@ class news2mail:
+             self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \
+                 info.__doc__ + '\n'
+ 
+-            ##self.heads_dict['X-Gateway:'] = '%s %s\n' %
+-            ##    (info.PROGNAME, info.__doc__)
+-
+             # to make Received: header
+             t = time.ctime(time.time())
+ 
+@@ -194,12 +148,12 @@ class news2mail:
+         headers renamed are useless or not rfc 822 copliant
+         """
+         try:
+-            if 'Newsgroups:' in self.heads_dict:
++            if 'Newsgroups:' in self.message:
+                 self.heads_dict['X-Newsgroups:'] = \
+                     self.heads_dict['Newsgroups:']
+                 del self.heads_dict['Newsgroups:']
+ 
+-            if 'NNTP-Posting-Host:' in self.heads_dict:
++            if 'NNTP-Posting-Host:' in self.message:
+                 self.heads_dict['X-NNTP-Posting-Host:'] = \
+                     self.heads_dict['NNTP-Posting-Host:']
+                 del self.heads_dict['NNTP-Posting-Host:']
+@@ -215,37 +169,37 @@ class news2mail:
+         try:
+             # removing some others useless headers ....
+ 
+-            if 'Approved:' in self.heads_dict:
++            if 'Approved:' in self.message:
+                 del self.heads_dict['Approved:']
+ 
+-            if 'From' in self.heads_dict:  # neither 'From ' nor 'From:'
++            if 'From' in self.message:  # neither 'From ' nor 'From:'
+                 del self.heads_dict['From']
+ 
+-            if 'Xref:' in self.heads_dict:
++            if 'Xref:' in self.message:
+                 del self.heads_dict['Xref:']
+ 
+-            if 'Path:' in self.heads_dict:
++            if 'Path:' in self.message:
+                 del self.heads_dict['Path:']
+ 
+-            if 'Lines:' in self.heads_dict:
++            if 'Lines:' in self.message:
+                 del self.heads_dict['Lines:']
+ 
+             # it is usually set by INN, if ng is moderated...
+-            if 'Sender:' in self.heads_dict:
++            if 'Sender:' in self.message:
+                 del self.heads_dict['Sender:']
+ 
+-            if 'Message-id:' in self.heads_dict:
++            if 'Message-id:' in self.message:
+                 self.heads_dict['Message-Id:'] = self.heads_dict['Message-id']
+                 del(self.heads_dict['Message-id'])
+ 
+-            if 'Message-ID:' in self.heads_dict:
++            if 'Message-ID:' in self.message:
+                 self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID']
+                 del(self.heads_dict['Message-ID'])
+ 
+             # If message-id is not present, I generate it
+-            if 'Message-Id:' not in self.heads_dict:
++            if 'Message-Id:' not in self.message:
+                 # It should put a real user@domain
+-                msgid = 'pyg@puppapera.org'  # FIXME unused variable
++                self.message['Message-Id'] = 'pyg@puppapera.org'
+ 
+         except KeyError, message:
+             print message
+diff --git b/pygm2n a/pygm2n
+index 49d83c3..9601c23 100755
+--- b/pygm2n
++++ a/pygm2n
+@@ -1,5 +1,5 @@
+ #!/usr/bin/env python
+-# -*- coding: utf-8 -*-
++
+ """News to mail gateway script. Copyright 2000 Cosimo Alfarano
+ 
+ Author: Cosimo Alfarano
+@@ -13,34 +13,37 @@ Thanks to md for this useful formula. Beer is beer.
+ 
+ 
+ def parse_cmdline():
++    """
++    set a dictionary with smtp new header in gw parameter (gw.smtpheads)
++    return (test,verbose) boolean tuple
++    """
+ 
+@@ -86,14 +89,6 @@ try:
+ 
+     opt = parse_cmdline()
+ 
+-    # reads stdin and parses article separating head from body
+-    m2n.readfile(opt)
+-    m2n.parseemail()
+-
+-    """phase 2:
+-    check whitelist for user's permission
+-    """
+-
+     """phase 3:
+     format rfc 822 headers from input article
+     """
+diff --git b/pygn2m a/pygn2m
+index 0b958f1..b46b40f 100755
+--- b/pygn2m
++++ a/pygn2m
+@@ -1,5 +1,6 @@
+ #!/usr/bin/env python
+ # -*- coding: utf-8 -*-
++
+ """News to mail gateway script. Copyright 2000 Cosimo Alfarano
+ 
+ Author: Cosimo Alfarano
+@@ -13,16 +14,19 @@ Thanks to md for this useful formula. Beer is beer.
++
++
++sys.path.append('/usr/lib/pyg')
++
++# import mail2news
+ 
+ 
+ def parse_cmdline():
+@@ -82,28 +84,24 @@ try:
+     n2m = news2mail.news2mail()
+     owner = None
+ 
+-    # it returns only test, other parms are set directly in the actual
+-    # parameter
+     args = parse_cmdline()
+ 
+     # check if n2m has some file prefercences set on commandline
+-    if n2m.wlfile is None:
+-        wl = os.environ['HOME'] + '/pyg.whitelist'
++    if args.wlfile is None:
++        wl = os.path.expanduser('~/pyg.whitelist')
+     else:
+-        wl = n2m.wlfile
++        wl = args.wlfile
+ 
+-    if n2m.logfile is None:
+-        log = os.environ['HOME'] + '/pyg.log'
++    if args.logfile is None:
++        log = os.path.expanduser('~/pyg.log')
+     else:
+-        log = n2m.logfile
+-
+-#     print 'using %s %s\n' % (wl,log)
++        log = args.logfile
+ 
+     wl = whitelist.whitelist(wl, log)
+ 
+     # reads stdin and parses article separating head from body
+-    n2m.readfile()
+-    n2m.parsearticle()
++    ## n2m.readfile()
++    ## n2m.parsearticle()
+ 
+     """phase 2:
+     check whitelist for user's permission