diff options
| -rw-r--r-- | cleanup_failing.patch | 216 | ||||
| -rw-r--r-- | mail2news.py | 177 | ||||
| -rw-r--r-- | news2mail.py | 14 | ||||
| -rwxr-xr-x | pygm2n | 9 | ||||
| -rwxr-xr-x | pygn2m | 2 | ||||
| -rwxr-xr-x | test/test_wlp.py | 19 |
6 files changed, 100 insertions, 337 deletions
diff --git a/cleanup_failing.patch b/cleanup_failing.patch index 480d2d3..d7d049f 100644 --- a/cleanup_failing.patch +++ b/cleanup_failing.patch @@ -2,15 +2,6 @@ diff --git b/mail2news.py a/mail2news.py index e7cbcf9..412e4ee 100644 --- b/mail2news.py +++ a/mail2news.py -@@ -14,6 +16,8 @@ Gets news email and sends it via SMTP. - class mail2news is hopefully conform to rfc850. - - """ -+ -+import email - import sys - from os import getpid - from socket import gethostbyaddr, gethostname @@ -24,80 +28,41 @@ import pyginfo class mail2news: @@ -24,33 +15,6 @@ index e7cbcf9..412e4ee 100644 user = None password = None -- hostname = gethostbyaddr(gethostname())[0] -- -- heads_dict, smtpheads, nntpheads = {}, {}, {} -- email, headers, body = [], [], [] -- -- def readfile(self, opt): -- -- for line in sys.stdin.readlines(): -- self.email.append(line) -- -- if(len(self.email) == 1 and self.email[0][0] == '/'): -- file = self.email[0][:-1] -- del self.email[0] -- for line in open(file, 'r').readlines(): -- self.email.append(line) -- -- # introduce nntpheads -- if opt.newsgroup != '': -- # TODO put it directly to self.message when we have it -- self.nntpheads['Newsgroups:'] = opt.newsgroup + '\n' -- if opt.approver != '': -- self.nntpheads['Approved:'] = opt.approver + '\n' -+ def __init__(self): -+ self.hostname = gethostbyaddr(gethostname())[0] -+ self.message = email.message_from_file(sys.stdin) - -- return 1 +# """phase 3: +# format rfc 822 headers from input article +# """ @@ -67,98 +31,6 @@ index e7cbcf9..412e4ee 100644 +# for line in m2n.headers: +# print line[:-1] -- def parseemail(self): -+ def x_parseemail(self): - """get news email from file or stdin and separate heads from body - - REMEBER: headers value has '\n' as last char. - Use string[:-1] to ignore newline. - """ -- -- try: -- body = 0 # are we in body or in headers? -- -- for line in self.email: -- if not body and len(line) == 1: -- body = 1 # starts email body section -- -- if not body: -- try: -- # if it is a multi-line header like Received: -- if line[0] not in [' ', '\t']: -- try: -- head, value = line.split(' ', 1) -- except ValueError: -- value = '' -- self.smtpheads[head] = value -- else: -- self.smtpheads[head] = '%s%s' % \ -- (self.smtpheads[head], line) -- except (ValueError), message: -- print('line: %s' % line) -- print('(probably missing couple "Header: value" in %s)' -- % line) -- sys.exit(1) -- -- elif len(line) > 0 and body: -- self.body.append(line) -- -- except (ValueError), message: -- print message -- sys.exit(1) -- -- return self.smtpheads, self.body -+ # return self.smtpheads, self.body -+ # self.smtpheads contains headers, self.body body -+ pass - - def puthead(self, dict, list, key): - """private, transform dict entries in list entries -@@ -105,33 +70,34 @@ class mail2news: - """ - - if key in dict: -- list.append(key + ' ' + dict.get(key)) -+ list.append(key + ' ' + dict[key]) # FIXME: Message instance doesn't have append - del dict[key] - else: - return 0 - return 1 - - def sortheads(self): -- """make list sorted by heads: From: To: Subject: first, -- others, X-*, X-Resent-* last""" -+ """make list sorted by heads: -+ From: To: Subject: first, others, X-*, X-Resent-* last""" - - # put at top - set = ('Newsgroups:', 'From:', 'To:', 'X-To:', 'Cc:', 'Subject:', - 'Date:', 'Approved:', 'References:', 'Message-Id:') - - for k in set: -- self.puthead(self.heads_dict, self.headers, k) -+ self.puthead(self.heads_dict, self.message, k) - -- for k in self.heads_dict.keys(): -- if k[:2] != 'X-' and k[:9] != 'X-Resent-' and k not in set: -+ for k in self.message: -+ if not k.startswith('X-') and not k.startswith('X-Resent-') and \ -+ k not in set: - self.puthead(self.heads_dict, self.headers, k) - -- for k in self.heads_dict.keys(): -- if k[:2] == 'X-': -+ for k in self.message: -+ if k.startswith('X-'): - self.puthead(self.heads_dict, self.headers, k) - -- for k in self.heads_dict.keys(): -- if k[:9] == 'X-Resent-': -+ for k in self.message: -+ if k.startswith('X-Resent-'): - self.puthead(self.heads_dict, self.headers, k) - - return self.headers @@ -142,14 +108,11 @@ class mail2news: self.heads_dict = {} @@ -178,94 +50,6 @@ index e7cbcf9..412e4ee 100644 except KeyError, message: print message -@@ -160,22 +123,8 @@ class mail2news: - """ - - info = pyginfo.pygsinfo() -- -- try: -- self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \ -- info.PROGDESC + ' - Mail to News\n' -- --# it is nntpheads stuff --# if(self.newsgroups): --# self.heads_dict['Newsgroups:'] = self.newsgroups -- --# if(self.approved): --# self.heads_dict['Approved:'] = self.approved -- -- except KeyError, message: -- print message -- -- return self.heads_dict -+ self.message.add_header('X-Gateway', info.PROGNAME + ' ' + -+ info.PROGDESC + ' - Mail to News') - - def renameheads(self): - """rename headers such as Resent-*: to X-Resent-*: -@@ -184,50 +133,31 @@ class mail2news: - handles References/In-Reply-To headers - """ - try: -- --### test --# if(post): --# if(post in self.heads_dict): --# self.heads_dict['X-Original-' + post] = self.heads_dict[post] --# --# self.heads_dict[post] = self.heads_dict[pre] --# del(self.heads_dict[pre]) --# --# else: --# if(pre[0:2] == 'X-' and pre in self.heads_dict): --# self.heads_dict['X-Original-' + pre] = self.heads_dict[pre] --# elif(not pre[0:2] == 'X-' and 'X-' + pre in self.heads_dict): --# self.heads_dict['X-' + pre] = self.heads_dict[pre] --# del(self.heads_dict[pre]) --### end test -- -- for key in self.heads_dict.keys(): -- if(key[:7] in ['Resent-']): -- if ('X-' + key) in self.heads_dict: -- self.heads_dict['X-Original-' + key] = \ -- self.heads_dict['X-' + key] -- self.heads_dict['X-' + key] = self.heads_dict[key] -- del self.heads_dict[key] -+ for key in self.message.keys(): -+ if key.startswith('Resent-'): -+ if ('X-' + key) in self.message: -+ self.message['X-Original-' + key] = \ -+ self.message['X-' + key] -+ self.message['X-' + key] = self.message[key] -+ del self.message[key] - - # In rfc822 References: is considered, but many MUA doen't put it. -- if ('References:' not in self.heads_dict) and \ -- ('In-Reply-To:' in self.heads_dict): -- print self.heads_dict['In-Reply-To:'] -+ if 'References:' not in self.message and \ -+ 'In-Reply-To:' in self.message: -+ print self.message['In-Reply-To:'] - -+ # FIXME !!! Do konce metody je to dost zmatené! - # some MUA uses msgid without '<' '>' - # ref = findall('([^\s<>\']+@[^\s<>;:\']+)', \ - # but I prefer use RFC standards -+ # FIXME isn't In-Reply-To supposed to be unique??? - ref = findall('(<[^<>]+@[^<>]+>)', -- self.heads_dict['In-Reply-To:']) -+ self.message.get_all('In-Reply-To:')) - - # if found, keep first element that seems a Msg-ID. - if(ref and len(ref)): - self.heads_dict['References:'] = '%s\n' % ref[0] - --# if('To:' in self.heads_dict): --# self.heads_dict['X-To:'] = self.heads_dict['To:'] --# del self.heads_dict['To:'] -- - except KeyError, message: - print message @@ -236,51 +166,32 @@ class mail2news: def removeheads(self, heads=None): diff --git a/mail2news.py b/mail2news.py index 0b97b92..f802277 100644 --- a/mail2news.py +++ b/mail2news.py @@ -14,12 +14,16 @@ Gets news email and sends it via SMTP. class mail2news is hopefully conform to rfc850. """ -import sys +import email +import logging +#logging.basicConfig(level=logging.DEBUG) +import nntplib from os import getpid -from socket import gethostbyaddr, gethostname from re import findall -from news2mail import news2mail -import nntplib +from collections import OrderedDict +from socket import gethostbyaddr, gethostname +import sys + import pyginfo @@ -38,106 +42,47 @@ class mail2news: heads_dict, smtpheads, nntpheads = {}, {}, {} email, headers, body = [], [], [] + message = None def readfile(self, opt): - for line in sys.stdin.readlines(): - self.email.append(line) + self.message = email.message_from_file(sys.stdin) - if(len(self.email) == 1 and self.email[0][0] == '/'): - file = self.email[0][:-1] - del self.email[0] - for line in open(file, 'r').readlines(): - self.email.append(line) + if (len(self.message) == 0) \ + and self.message.get_payload().startswith('/'): + msg_file_name = self.message.get_payload().strip() + del self.message + with open(msg_file_name, 'r') as msg_file: + self.message = email.message_from_file(msg_file) # introduce nntpheads if opt.newsgroup != '': # TODO put it directly to self.message when we have it - self.nntpheads['Newsgroups:'] = opt.newsgroup + '\n' + self.nntpheads['Newsgroups'] = opt.newsgroup if opt.approver != '': - self.nntpheads['Approved:'] = opt.approver + '\n' + self.nntpheads['Approved'] = opt.approver return 1 - def parseemail(self): - """get news email from file or stdin and separate heads from body - - REMEBER: headers value has '\n' as last char. - Use string[:-1] to ignore newline. - """ - - try: - body = 0 # are we in body or in headers? - - for line in self.email: - if not body and len(line) == 1: - body = 1 # starts email body section - - if not body: - try: - # if it is a multi-line header like Received: - if line[0] not in [' ', '\t']: - try: - head, value = line.split(' ', 1) - except ValueError: - value = '' - self.smtpheads[head] = value - else: - self.smtpheads[head] = '%s%s' % \ - (self.smtpheads[head], line) - except (ValueError), message: - print('line: %s' % line) - print('(probably missing couple "Header: value" in %s)' - % line) - sys.exit(1) - - elif len(line) > 0 and body: - self.body.append(line) - - except (ValueError), message: - print message - sys.exit(1) - - return self.smtpheads, self.body - @staticmethod - def puthead(*args, **kwargs): - news2mail.puthead(*args, **kwargs) - - def sortheads(self): - """make list sorted by heads: From: To: Subject: first, - others, X-*, X-Resent-* last""" - - # put at top - set = ('Newsgroups:', 'From:', 'To:', 'X-To:', 'Cc:', 'Subject:', - 'Date:', 'Approved:', 'References:', 'Message-Id:') - - for k in set: - self.puthead(self.heads_dict, self.headers, k) - - for k in self.heads_dict.keys(): - if k[:2] != 'X-' and k[:9] != 'X-Resent-' and k not in set: - self.puthead(self.heads_dict, self.headers, k) - - for k in self.heads_dict.keys(): - if k[:2] == 'X-': - self.puthead(self.heads_dict, self.headers, k) - - for k in self.heads_dict.keys(): - if k[:9] == 'X-Resent-': - self.puthead(self.heads_dict, self.headers, k) - - return self.headers + def puthead(from_dict, out_list, key): + """private, x-form dict entries in out_list entries""" + if key in from_dict: + out_list.append(key + ': ' + from_dict.get(key)) + else: + return 0 + return 1 def mergeheads(self): """make a unique headers dictionary from NNTP and SMTP single headers dictionaries.""" - self.heads_dict = {} + self.heads_dict = OrderedDict() + logging.debug('self.message.keys() = %s', self.message.keys()) try: - for header in self.smtpheads.keys(): # fill it w/ smtp old heads - self.heads_dict[header] = self.smtpheads[header] + for header in self.message.keys(): # fill it w/ smtp old heads + self.heads_dict[header] = self.message[header] # and replace them w/ nntp new heads for header in self.nntpheads.keys(): @@ -155,8 +100,8 @@ class mail2news: info = pyginfo.pygsinfo() try: - self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \ - info.PROGDESC + ' - Mail to News\n' + self.heads_dict['X-Gateway'] = info.PROGNAME + ' ' + \ + info.PROGDESC + ' - Mail to News' except KeyError, message: print message @@ -180,19 +125,19 @@ class mail2news: del self.heads_dict[key] # In rfc822 References: is considered, but many MUA doen't put it. - if ('References:' not in self.heads_dict) and \ - ('In-Reply-To:' in self.heads_dict): - print self.heads_dict['In-Reply-To:'] + if ('References' not in self.heads_dict) and \ + ('In-Reply-To' in self.heads_dict): + print self.heads_dict['In-Reply-To'] # some MUA uses msgid without '<' '>' # ref = findall('([^\s<>\']+@[^\s<>;:\']+)', \ # but I prefer use RFC standards ref = findall('(<[^<>]+@[^<>]+>)', - self.heads_dict['In-Reply-To:']) + self.heads_dict['In-Reply-To']) # if found, keep first element that seems a Msg-ID. if(ref and len(ref)): - self.heads_dict['References:'] = '%s\n' % ref[0] + self.heads_dict['References'] = '%s\n' % ref[0] except KeyError, message: print message @@ -206,8 +151,8 @@ class mail2news: try: # removing some others useless headers .... (From is not From:) - rmheads = ['Received:', 'From', 'NNTP-Posting-Host:', - 'X-Trace:', 'X-Compliants-To:', 'NNTP-Posting-Date:'] + rmheads = ['Received', 'From ', 'NNTP-Posting-Host', + 'X-Trace', 'X-Compliants-To', 'NNTP-Posting-Date'] if(heads): rmheads.append(heads) @@ -215,24 +160,54 @@ class mail2news: if head in self.heads_dict: del self.heads_dict[head] - if 'Message-id:' in self.heads_dict: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-id:'] - del(self.heads_dict['Message-id:']) + if 'Message-id' in self.heads_dict: + self.heads_dict['Message-Id'] = self.heads_dict['Message-id'] + del(self.heads_dict['Message-id']) - if 'Message-ID:' in self.heads_dict: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID:'] - del(self.heads_dict['Message-ID:']) + if 'Message-ID' in self.heads_dict: + self.heads_dict['Message-Id'] = self.heads_dict['Message-ID'] + del(self.heads_dict['Message-ID']) # If message-id is not present, I generate it - if 'Message-Id:' not in self.heads_dict: + if 'Message-Id' not in self.heads_dict: msgid = '<pyg.%d@tuchailepuppapera.org>\n' % (getpid()) - self.heads_dict['Message-Id:'] = msgid + self.heads_dict['Message-Id'] = msgid except KeyError, message: print message return self.heads_dict + def sortheads(self): + """make list sorted by heads: From: To: Subject: first, + others, X-*, X-Resent-* last""" + + # put at top + head_set = ('Newsgroups', 'From', 'To', 'X-To', 'Cc', 'Subject', + 'Date', 'Approved', 'References', 'Message-Id') + + logging.debug('self.heads_dict = %s', self.heads_dict) + + for k in head_set: + self.puthead(self.heads_dict, self.headers, k) + + for k in self.heads_dict.keys(): + if not k.startswith('X-') and not k.startswith('X-Resent-') \ + and k not in head_set: + self.puthead(self.heads_dict, self.headers, k) + + for k in self.heads_dict.keys(): + if k.startswith('X-'): + self.puthead(self.heads_dict, self.headers, k) + + for k in self.heads_dict.keys(): + if k.startswith('X-Resent-'): + self.puthead(self.heads_dict, self.headers, k) + + logging.debug('self.headers = %s', self.headers) + + return self.headers + def sendemail(self): """Talk to NNTP server and try to send email.""" try: diff --git a/news2mail.py b/news2mail.py index 4dd83fc..e1909a1 100644 --- a/news2mail.py +++ b/news2mail.py @@ -23,6 +23,9 @@ normal (what pygs does) operations flow is: """ import email +import logging +from collections import OrderedDict +# logging.basicConfig(level=logging.DEBUG) import sys import smtplib import time @@ -58,7 +61,6 @@ class news2mail(object): @staticmethod def puthead(from_dict, out_list, key): """private, x-form dict entries in out_list entries""" - if key in from_dict: out_list.append(key + ' ' + from_dict.get(key)) else: @@ -76,24 +78,26 @@ class news2mail(object): self.puthead(self.heads_dict, self.headers, k) for k in self.heads_dict.keys(): - if k[:2] != 'X-' and k[:7] != 'Resent-' and k not in header_set: + if not k.startswith('X-') and not k.startswith('Resent-') \ + and k not in header_set: self.puthead(self.heads_dict, self.headers, k) for k in self.heads_dict.keys(): - if k[:2] == 'X-': + if k.startswith('X-'): self.puthead(self.heads_dict, self.headers, k) for k in self.heads_dict.keys(): - if k[:7] == 'Resent-': + if k.startswith('Resent-'): self.puthead(self.heads_dict, self.headers, k) + logging.debug('self.headers = %s', self.headers) return self.headers def mergeheads(self): """make a unique headers dictionary from NNTP and SMTP single headers dictionaries.""" - self.heads_dict = {} + self.heads_dict = OrderedDict() try: for header in self.nntpheads.keys(): # fill it w/ nntp old heads @@ -14,18 +14,15 @@ Thanks to md for this useful formula. Beer is beer. Gets news article and sends it via SMTP. """ from __future__ import print_function + import logging import sys - import argparse import mail2news import pyginfo -logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s', - level=logging.DEBUG) - sys.path.append('/usr/lib/pyg') @@ -88,7 +85,7 @@ try: # reads stdin and parses article separating head from body m2n.readfile(opt) - m2n.parseemail() + # m2n.parseemail() """phase 2: check whitelist for user's permission @@ -108,7 +105,7 @@ try: if opt.verbose: for line in m2n.headers: - print(line[:-1]) + print(line) """phase 4: open smtp connection and send e-mail @@ -137,7 +137,7 @@ try: # prints formatted email message only (without send) if user wants if args.verbose: for line in n2m.headers: - print(line[:-1]) + print(line) if owner is None: sys.exit(1) diff --git a/test/test_wlp.py b/test/test_wlp.py index dcb2b78..486fb42 100755 --- a/test/test_wlp.py +++ b/test/test_wlp.py @@ -41,30 +41,31 @@ class TestWLP(unittest.TestCase): class TestM2N(unittest.TestCase): - def test_m2n(self): - expected_output = """Newsgroups: pyg.test + expected_output = """Newsgroups: pyg.test From: Pyg <pyg@localhost.com> -To: this header probably broke RFC, but is frequent. +To: User <user@localhost.com> Subject: test Date: Sun, 1 Feb 2002 16:40:40 +0200 Message-Id: <20001001164040.Aa8326@localhost> -Content-Type: text/plain; charset=us-ascii -Mime-Version: 1.0 Return-Path: <pyg@localhost> +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii User-Agent: Mutt/1.2.5i +X-Multiline: this header probably broke RFC, but is frequent. X-Gateway: pyg The Python Gateway - Mail to News """ + + def test_m2n(self): with open('examples/mail') as in_mail: pid = subprocess.Popen(['./pygm2n', '-TV', '-n', 'pyg.test'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) out, err = pid.communicate(in_mail.read()) - self.assertEqual(out, expected_output) + self.assertEqual(out, self.expected_output) class TestN2M(unittest.TestCase): - def test_n2m(self): - expected_output = """Received: from GATEWAY by mitmanek.ceplovi.cz with pyg + expected_output = """Received: from GATEWAY by mitmanek.ceplovi.cz with pyg for <test@example.com> ; Mon Dec 15 17:13:30 2014 (CEST) From: kame@inwind.it (PYG) To: test@example.com @@ -82,6 +83,8 @@ X-NNTP-Posting-Host: pyg.server.tld Resent-From: sender@example.com Resent-Sender: sender@example.com """ + + def test_n2m(self): env = os.environ env['PYTHONPATH'] = wlp_lib_path |