pygm2n and mail2news also use email.Parser.

6 files changed, 100 insertions, 337 deletions

diff --git a/cleanup_failing.patch b/cleanup_failing.patch
index 480d2d3..d7d049f 100644
--- a/cleanup_failing.patch
+++ b/cleanup_failing.patch
@@ -2,15 +2,6 @@ diff --git b/mail2news.py a/mail2news.py
 index e7cbcf9..412e4ee 100644
 --- b/mail2news.py
 +++ a/mail2news.py
-@@ -14,6 +16,8 @@ Gets news email and sends it via SMTP.
- class mail2news is  hopefully conform to rfc850.
- 
- """
-+
-+import email
- import sys
- from os import getpid
- from socket import gethostbyaddr, gethostname
 @@ -24,80 +28,41 @@ import pyginfo
  
  class mail2news:
@@ -24,33 +15,6 @@ index e7cbcf9..412e4ee 100644
      user = None
      password = None
  
--    hostname = gethostbyaddr(gethostname())[0]
--
--    heads_dict, smtpheads, nntpheads = {}, {}, {}
--    email, headers, body = [], [], []
--
--    def readfile(self, opt):
--
--        for line in sys.stdin.readlines():
--            self.email.append(line)
--
--        if(len(self.email) == 1 and self.email[0][0] == '/'):
--            file = self.email[0][:-1]
--            del self.email[0]
--            for line in open(file, 'r').readlines():
--                self.email.append(line)
--
--        # introduce nntpheads
--        if opt.newsgroup != '':
--            # TODO put it directly to self.message when we have it
--            self.nntpheads['Newsgroups:'] = opt.newsgroup + '\n'
--        if opt.approver != '':
--            self.nntpheads['Approved:'] = opt.approver + '\n'
-+    def __init__(self):
-+        self.hostname = gethostbyaddr(gethostname())[0]
-+        self.message = email.message_from_file(sys.stdin)
- 
--        return 1
 +# """phase 3:
 +#     format rfc 822 headers from input article
 +#     """
@@ -67,98 +31,6 @@ index e7cbcf9..412e4ee 100644
 +#         for line in m2n.headers:
 +#             print line[:-1]
  
--    def parseemail(self):
-+    def x_parseemail(self):
-         """get news email from file or stdin and separate heads from body
- 
-         REMEBER: headers value has '\n' as last char.
-             Use string[:-1] to ignore newline.
-         """
--
--        try:
--            body = 0        # are we in body or in headers?
--
--            for line in self.email:
--                if not body and len(line) == 1:
--                    body = 1    # starts email body section
--
--                if not body:
--                    try:
--                        # if it is a multi-line header like Received:
--                        if line[0] not in [' ', '\t']:
--                            try:
--                                head, value = line.split(' ', 1)
--                            except ValueError:
--                                value = ''
--                            self.smtpheads[head] = value
--                        else:
--                            self.smtpheads[head] = '%s%s' % \
--                                (self.smtpheads[head], line)
--                    except (ValueError), message:
--                        print('line: %s' % line)
--                        print('(probably missing couple "Header: value" in %s)'
--                              % line)
--                        sys.exit(1)
--
--                elif len(line) > 0 and body:
--                    self.body.append(line)
--
--        except (ValueError), message:
--            print message
--            sys.exit(1)
--
--        return self.smtpheads, self.body
-+        # return self.smtpheads, self.body
-+        # self.smtpheads contains headers, self.body body
-+        pass
- 
-     def puthead(self, dict, list, key):
-         """private, transform dict entries in list entries
-@@ -105,33 +70,34 @@ class mail2news:
-         """
- 
-         if key in dict:
--            list.append(key + ' ' + dict.get(key))
-+            list.append(key + ' ' + dict[key])  # FIXME: Message instance doesn't have append
-             del dict[key]
-         else:
-             return 0
-         return 1
- 
-     def sortheads(self):
--        """make list sorted by heads: From: To: Subject: first,
--           others, X-*, X-Resent-* last"""
-+        """make list sorted by heads:
-+           From: To: Subject: first, others, X-*, X-Resent-* last"""
- 
-         # put at top
-         set = ('Newsgroups:', 'From:', 'To:', 'X-To:', 'Cc:', 'Subject:',
-                'Date:', 'Approved:', 'References:', 'Message-Id:')
- 
-         for k in set:
--            self.puthead(self.heads_dict, self.headers, k)
-+            self.puthead(self.heads_dict, self.message, k)
- 
--        for k in self.heads_dict.keys():
--            if k[:2] != 'X-' and k[:9] != 'X-Resent-' and k not in set:
-+        for k in self.message:
-+            if not k.startswith('X-') and not k.startswith('X-Resent-') and \
-+                    k not in set:
-                         self.puthead(self.heads_dict, self.headers, k)
- 
--        for k in self.heads_dict.keys():
--            if k[:2] == 'X-':
-+        for k in self.message:
-+            if k.startswith('X-'):
-                 self.puthead(self.heads_dict, self.headers, k)
- 
--        for k in self.heads_dict.keys():
--            if k[:9] == 'X-Resent-':
-+        for k in self.message:
-+            if k.startswith('X-Resent-'):
-                 self.puthead(self.heads_dict, self.headers, k)
- 
-         return self.headers
 @@ -142,14 +108,11 @@ class mail2news:
  
          self.heads_dict = {}
@@ -178,94 +50,6 @@ index e7cbcf9..412e4ee 100644
          except KeyError, message:
              print message
  
-@@ -160,22 +123,8 @@ class mail2news:
-         """
- 
-         info = pyginfo.pygsinfo()
--
--        try:
--            self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \
--                info.PROGDESC + ' - Mail to News\n'
--
--# it is nntpheads stuff
--#            if(self.newsgroups):
--#                self.heads_dict['Newsgroups:'] = self.newsgroups
--
--#            if(self.approved):
--#                self.heads_dict['Approved:'] = self.approved
--
--        except KeyError, message:
--            print message
--
--        return self.heads_dict
-+        self.message.add_header('X-Gateway', info.PROGNAME + ' ' +
-+                                info.PROGDESC + ' - Mail to News')
- 
-     def renameheads(self):
-         """rename headers such as Resent-*: to X-Resent-*:
-@@ -184,50 +133,31 @@ class mail2news:
-         handles References/In-Reply-To headers
-         """
-         try:
--
--### test
--#   if(post):
--#       if(post in self.heads_dict):
--#           self.heads_dict['X-Original-' + post] = self.heads_dict[post]
--#
--#       self.heads_dict[post] = self.heads_dict[pre]
--#       del(self.heads_dict[pre])
--#
--#   else:
--#       if(pre[0:2] == 'X-' and pre in self.heads_dict):
--#           self.heads_dict['X-Original-' + pre] = self.heads_dict[pre]
--#       elif(not pre[0:2] == 'X-' and 'X-' + pre in self.heads_dict):
--#           self.heads_dict['X-' + pre] = self.heads_dict[pre]
--#       del(self.heads_dict[pre])
--### end test
--
--            for key in self.heads_dict.keys():
--                if(key[:7] in ['Resent-']):
--                    if ('X-' + key) in self.heads_dict:
--                        self.heads_dict['X-Original-' + key] = \
--                            self.heads_dict['X-' + key]
--                    self.heads_dict['X-' + key] = self.heads_dict[key]
--                    del self.heads_dict[key]
-+            for key in self.message.keys():
-+                if key.startswith('Resent-'):
-+                    if ('X-' + key) in self.message:
-+                        self.message['X-Original-' + key] = \
-+                            self.message['X-' + key]
-+                    self.message['X-' + key] = self.message[key]
-+                    del self.message[key]
- 
-             # In rfc822 References: is considered, but many MUA doen't put it.
--            if ('References:' not in self.heads_dict) and \
--                    ('In-Reply-To:' in self.heads_dict):
--                print self.heads_dict['In-Reply-To:']
-+            if 'References:' not in self.message and \
-+                    'In-Reply-To:' in self.message:
-+                print self.message['In-Reply-To:']
- 
-+                # FIXME !!! Do konce metody je to dost zmatené!
-                 # some MUA uses msgid without '<' '>'
- #                ref = findall('([^\s<>\']+@[^\s<>;:\']+)', \
-                 # but I prefer use RFC standards
-+                # FIXME isn't In-Reply-To supposed to be unique???
-                 ref = findall('(<[^<>]+@[^<>]+>)',
--                              self.heads_dict['In-Reply-To:'])
-+                              self.message.get_all('In-Reply-To:'))
- 
-                 # if found, keep first element that seems a Msg-ID.
-                 if(ref and len(ref)):
-                     self.heads_dict['References:'] = '%s\n' % ref[0]
- 
--#            if('To:' in self.heads_dict):
--#                self.heads_dict['X-To:'] = self.heads_dict['To:']
--#                del self.heads_dict['To:']
--
-         except KeyError, message:
-             print message
  
 @@ -236,51 +166,32 @@ class mail2news:
      def removeheads(self, heads=None):
diff --git a/mail2news.py b/mail2news.py
index 0b97b92..f802277 100644
--- a/mail2news.py
+++ b/mail2news.py
@@ -14,12 +14,16 @@ Gets news email and sends it via SMTP.
 class mail2news is  hopefully conform to rfc850.
 
 """
-import sys
+import email
+import logging
+#logging.basicConfig(level=logging.DEBUG)
+import nntplib
 from os import getpid
-from socket import gethostbyaddr, gethostname
 from re import findall
-from news2mail import news2mail
-import nntplib
+from collections import OrderedDict
+from socket import gethostbyaddr, gethostname
+import sys
+
 import pyginfo
 
 
@@ -38,106 +42,47 @@ class mail2news:
 
     heads_dict, smtpheads, nntpheads = {}, {}, {}
     email, headers, body = [], [], []
+    message = None
 
     def readfile(self, opt):
 
-        for line in sys.stdin.readlines():
-            self.email.append(line)
+        self.message = email.message_from_file(sys.stdin)
 
-        if(len(self.email) == 1 and self.email[0][0] == '/'):
-            file = self.email[0][:-1]
-            del self.email[0]
-            for line in open(file, 'r').readlines():
-                self.email.append(line)
+        if (len(self.message) == 0) \
+                and self.message.get_payload().startswith('/'):
+            msg_file_name = self.message.get_payload().strip()
+            del self.message
+            with open(msg_file_name, 'r') as msg_file:
+                self.message = email.message_from_file(msg_file)
 
         # introduce nntpheads
         if opt.newsgroup != '':
             # TODO put it directly to self.message when we have it
-            self.nntpheads['Newsgroups:'] = opt.newsgroup + '\n'
+            self.nntpheads['Newsgroups'] = opt.newsgroup
         if opt.approver != '':
-            self.nntpheads['Approved:'] = opt.approver + '\n'
+            self.nntpheads['Approved'] = opt.approver
 
         return 1
 
-    def parseemail(self):
-        """get news email from file or stdin and separate heads from body
-
-        REMEBER: headers value has '\n' as last char.
-            Use string[:-1] to ignore newline.
-        """
-
-        try:
-            body = 0        # are we in body or in headers?
-
-            for line in self.email:
-                if not body and len(line) == 1:
-                    body = 1    # starts email body section
-
-                if not body:
-                    try:
-                        # if it is a multi-line header like Received:
-                        if line[0] not in [' ', '\t']:
-                            try:
-                                head, value = line.split(' ', 1)
-                            except ValueError:
-                                value = ''
-                            self.smtpheads[head] = value
-                        else:
-                            self.smtpheads[head] = '%s%s' % \
-                                (self.smtpheads[head], line)
-                    except (ValueError), message:
-                        print('line: %s' % line)
-                        print('(probably missing couple "Header: value" in %s)'
-                              % line)
-                        sys.exit(1)
-
-                elif len(line) > 0 and body:
-                    self.body.append(line)
-
-        except (ValueError), message:
-            print message
-            sys.exit(1)
-
-        return self.smtpheads, self.body
-
     @staticmethod
-    def puthead(*args, **kwargs):
-        news2mail.puthead(*args, **kwargs)
-
-    def sortheads(self):
-        """make list sorted by heads: From: To: Subject: first,
-           others, X-*, X-Resent-* last"""
-
-        # put at top
-        set = ('Newsgroups:', 'From:', 'To:', 'X-To:', 'Cc:', 'Subject:',
-               'Date:', 'Approved:', 'References:', 'Message-Id:')
-
-        for k in set:
-            self.puthead(self.heads_dict, self.headers, k)
-
-        for k in self.heads_dict.keys():
-            if k[:2] != 'X-' and k[:9] != 'X-Resent-' and k not in set:
-                self.puthead(self.heads_dict, self.headers, k)
-
-        for k in self.heads_dict.keys():
-            if k[:2] == 'X-':
-                self.puthead(self.heads_dict, self.headers, k)
-
-        for k in self.heads_dict.keys():
-            if k[:9] == 'X-Resent-':
-                self.puthead(self.heads_dict, self.headers, k)
-
-        return self.headers
+    def puthead(from_dict, out_list, key):
+        """private, x-form dict entries in out_list entries"""
+        if key in from_dict:
+            out_list.append(key + ': ' + from_dict.get(key))
+        else:
+            return 0
+        return 1
 
     def mergeheads(self):
         """make a unique headers dictionary from NNTP and SMTP
         single headers dictionaries."""
 
-        self.heads_dict = {}
+        self.heads_dict = OrderedDict()
+        logging.debug('self.message.keys() = %s', self.message.keys())
 
         try:
-            for header in self.smtpheads.keys():  # fill it w/ smtp old heads
-                self.heads_dict[header] = self.smtpheads[header]
+            for header in self.message.keys():  # fill it w/ smtp old heads
+                self.heads_dict[header] = self.message[header]
 
             # and replace them w/ nntp new heads
             for header in self.nntpheads.keys():
@@ -155,8 +100,8 @@ class mail2news:
         info = pyginfo.pygsinfo()
 
         try:
-            self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \
-                info.PROGDESC + ' - Mail to News\n'
+            self.heads_dict['X-Gateway'] = info.PROGNAME + ' ' + \
+                info.PROGDESC + ' - Mail to News'
 
         except KeyError, message:
             print message
@@ -180,19 +125,19 @@ class mail2news:
                     del self.heads_dict[key]
 
             # In rfc822 References: is considered, but many MUA doen't put it.
-            if ('References:' not in self.heads_dict) and \
-                    ('In-Reply-To:' in self.heads_dict):
-                print self.heads_dict['In-Reply-To:']
+            if ('References' not in self.heads_dict) and \
+                    ('In-Reply-To' in self.heads_dict):
+                print self.heads_dict['In-Reply-To']
 
                 # some MUA uses msgid without '<' '>'
 #                ref = findall('([^\s<>\']+@[^\s<>;:\']+)', \
                 # but I prefer use RFC standards
                 ref = findall('(<[^<>]+@[^<>]+>)',
-                              self.heads_dict['In-Reply-To:'])
+                              self.heads_dict['In-Reply-To'])
 
                 # if found, keep first element that seems a Msg-ID.
                 if(ref and len(ref)):
-                    self.heads_dict['References:'] = '%s\n' % ref[0]
+                    self.heads_dict['References'] = '%s\n' % ref[0]
 
         except KeyError, message:
             print message
@@ -206,8 +151,8 @@ class mail2news:
         try:
             # removing some others useless headers .... (From is not From:)
 
-            rmheads = ['Received:', 'From', 'NNTP-Posting-Host:',
-                       'X-Trace:', 'X-Compliants-To:', 'NNTP-Posting-Date:']
+            rmheads = ['Received', 'From ', 'NNTP-Posting-Host',
+                       'X-Trace', 'X-Compliants-To', 'NNTP-Posting-Date']
             if(heads):
                 rmheads.append(heads)
 
@@ -215,24 +160,54 @@ class mail2news:
                 if head in self.heads_dict:
                     del self.heads_dict[head]
 
-            if 'Message-id:' in self.heads_dict:
-                self.heads_dict['Message-Id:'] = self.heads_dict['Message-id:']
-                del(self.heads_dict['Message-id:'])
+            if 'Message-id' in self.heads_dict:
+                self.heads_dict['Message-Id'] = self.heads_dict['Message-id']
+                del(self.heads_dict['Message-id'])
 
-            if 'Message-ID:' in self.heads_dict:
-                self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID:']
-                del(self.heads_dict['Message-ID:'])
+            if 'Message-ID' in self.heads_dict:
+                self.heads_dict['Message-Id'] = self.heads_dict['Message-ID']
+                del(self.heads_dict['Message-ID'])
 
             # If message-id is not present, I generate it
-            if 'Message-Id:' not in self.heads_dict:
+            if 'Message-Id' not in self.heads_dict:
                 msgid = '<pyg.%d@tuchailepuppapera.org>\n' % (getpid())
-                self.heads_dict['Message-Id:'] = msgid
+                self.heads_dict['Message-Id'] = msgid
 
         except KeyError, message:
             print message
 
         return self.heads_dict
 
+    def sortheads(self):
+        """make list sorted by heads: From: To: Subject: first,
+           others, X-*, X-Resent-* last"""
+
+        # put at top
+        head_set = ('Newsgroups', 'From', 'To', 'X-To', 'Cc', 'Subject',
+                    'Date', 'Approved', 'References', 'Message-Id')
+
+        logging.debug('self.heads_dict = %s', self.heads_dict)
+
+        for k in head_set:
+            self.puthead(self.heads_dict, self.headers, k)
+
+        for k in self.heads_dict.keys():
+            if not k.startswith('X-') and not k.startswith('X-Resent-') \
+                    and k not in head_set:
+                self.puthead(self.heads_dict, self.headers, k)
+
+        for k in self.heads_dict.keys():
+            if k.startswith('X-'):
+                self.puthead(self.heads_dict, self.headers, k)
+
+        for k in self.heads_dict.keys():
+            if k.startswith('X-Resent-'):
+                self.puthead(self.heads_dict, self.headers, k)
+
+        logging.debug('self.headers = %s', self.headers)
+
+        return self.headers
+
     def sendemail(self):
         """Talk to NNTP server and try to send email."""
         try:
diff --git a/news2mail.py b/news2mail.py
index 4dd83fc..e1909a1 100644
--- a/news2mail.py
+++ b/news2mail.py
@@ -23,6 +23,9 @@ normal (what pygs does) operations flow is:
 
 """
 import email
+import logging
+from collections import OrderedDict
+# logging.basicConfig(level=logging.DEBUG)
 import sys
 import smtplib
 import time
@@ -58,7 +61,6 @@ class news2mail(object):
     @staticmethod
     def puthead(from_dict, out_list, key):
         """private, x-form dict entries in out_list entries"""
-
         if key in from_dict:
             out_list.append(key + ' ' + from_dict.get(key))
         else:
@@ -76,24 +78,26 @@ class news2mail(object):
             self.puthead(self.heads_dict, self.headers, k)
 
         for k in self.heads_dict.keys():
-            if k[:2] != 'X-' and k[:7] != 'Resent-' and k not in header_set:
+            if not k.startswith('X-') and not k.startswith('Resent-') \
+                    and k not in header_set:
                 self.puthead(self.heads_dict, self.headers, k)
 
         for k in self.heads_dict.keys():
-            if k[:2] == 'X-':
+            if k.startswith('X-'):
                 self.puthead(self.heads_dict, self.headers, k)
 
         for k in self.heads_dict.keys():
-            if k[:7] == 'Resent-':
+            if k.startswith('Resent-'):
                 self.puthead(self.heads_dict, self.headers, k)
 
+        logging.debug('self.headers = %s', self.headers)
         return self.headers
 
     def mergeheads(self):
         """make a unique headers dictionary from NNTP and SMTP
         single headers dictionaries."""
 
-        self.heads_dict = {}
+        self.heads_dict = OrderedDict()
 
         try:
             for header in self.nntpheads.keys():  # fill it w/ nntp old heads
diff --git a/pygm2n b/pygm2n
index 49d83c3..51869b3 100755
--- a/pygm2n
+++ b/pygm2n
@@ -14,18 +14,15 @@ Thanks to md for this useful formula. Beer is beer.
 Gets news article and sends it via SMTP.
 """
 from __future__ import print_function
+
 import logging
 import sys
-
 import argparse
 
 import mail2news
 import pyginfo
 
 
-logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
-                    level=logging.DEBUG)
-
 sys.path.append('/usr/lib/pyg')
 
 
@@ -88,7 +85,7 @@ try:
 
     # reads stdin and parses article separating head from body
     m2n.readfile(opt)
-    m2n.parseemail()
+    # m2n.parseemail()
 
     """phase 2:
     check whitelist for user's permission
@@ -108,7 +105,7 @@ try:
 
     if opt.verbose:
         for line in m2n.headers:
-            print(line[:-1])
+            print(line)
 
     """phase 4:
     open smtp connection and send e-mail
diff --git a/pygn2m b/pygn2m
index 03f5ee7..14bb1e9 100755
--- a/pygn2m
+++ b/pygn2m
@@ -137,7 +137,7 @@ try:
     # prints formatted email message only (without send) if user wants
     if args.verbose:
         for line in n2m.headers:
-            print(line[:-1])
+            print(line)
 
     if owner is None:
         sys.exit(1)
diff --git a/test/test_wlp.py b/test/test_wlp.py
index dcb2b78..486fb42 100755
--- a/test/test_wlp.py
+++ b/test/test_wlp.py
@@ -41,30 +41,31 @@ class TestWLP(unittest.TestCase):
 
 
 class TestM2N(unittest.TestCase):
-    def test_m2n(self):
-        expected_output = """Newsgroups: pyg.test
+    expected_output = """Newsgroups: pyg.test
 From: Pyg <pyg@localhost.com>
-To:  this header probably broke RFC, but is frequent.
+To: User <user@localhost.com>
 Subject: test
 Date: Sun, 1 Feb 2002 16:40:40 +0200
 Message-Id: <20001001164040.Aa8326@localhost>
-Content-Type: text/plain; charset=us-ascii
-Mime-Version: 1.0
 Return-Path: <pyg@localhost>
+Mime-Version: 1.0
+Content-Type: text/plain; charset=us-ascii
 User-Agent: Mutt/1.2.5i
+X-Multiline:  this header probably broke RFC, but is frequent.
 X-Gateway: pyg The Python Gateway - Mail to News
 """
+
+    def test_m2n(self):
         with open('examples/mail') as in_mail:
             pid = subprocess.Popen(['./pygm2n', '-TV', '-n', 'pyg.test'],
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
             out, err = pid.communicate(in_mail.read())
-        self.assertEqual(out, expected_output)
+        self.assertEqual(out, self.expected_output)
 
 
 class TestN2M(unittest.TestCase):
-    def test_n2m(self):
-        expected_output = """Received: from GATEWAY by mitmanek.ceplovi.cz with pyg
+    expected_output = """Received: from GATEWAY by mitmanek.ceplovi.cz with pyg
     for <test@example.com> ; Mon Dec 15 17:13:30 2014 (CEST)
 From: kame@inwind.it (PYG)
 To: test@example.com
@@ -82,6 +83,8 @@ X-NNTP-Posting-Host: pyg.server.tld
 Resent-From: sender@example.com
 Resent-Sender: sender@example.com
 """
+
+    def test_n2m(self):
         env = os.environ
         env['PYTHONPATH'] = wlp_lib_path