From 5b25c51d3a02d49db2c7e33f9b65fad5432219f4 Mon Sep 17 00:00:00 2001 From: Matěj Cepl Date: Mon, 22 Dec 2014 00:26:30 +0100 Subject: pygn2m and news2mail.py use stdlib email parser. Instead of doing it on their own (poorly). --- cleanup_failing.patch | 291 -------------------------------------------------- mail2news.py | 56 +--------- news2mail.py | 139 ++++++++---------------- pygn2m | 20 ++-- whitelist.py | 29 ++--- 5 files changed, 73 insertions(+), 462 deletions(-) diff --git a/cleanup_failing.patch b/cleanup_failing.patch index 334cc82..480d2d3 100644 --- a/cleanup_failing.patch +++ b/cleanup_failing.patch @@ -2,19 +2,6 @@ diff --git b/mail2news.py a/mail2news.py index e7cbcf9..412e4ee 100644 --- b/mail2news.py +++ a/mail2news.py -@@ -1,9 +1,11 @@ -+# -*- coding: utf-8 -*- - """Mail to news gateway script. Copyright 2000 Cosimo Alfarano - - Author: Cosimo Alfarano - Date: September 16 2000 - --mail2news.py - (C) 2000 by Cosimo Alfarano -+mail2news.py - Copyright 2000 by -+ Cosimo Alfarano - You can use this software under the terms of the GPL. If we meet some day, - and you think this stuff is worth it, you can buy me a beer in return. - @@ -14,6 +16,8 @@ Gets news email and sends it via SMTP. class mail2news is hopefully conform to rfc850. @@ -340,281 +327,3 @@ index e7cbcf9..412e4ee 100644 def sendemail(self): """Talk to NNTP server and try to send email.""" try: -diff --git b/news2mail.py a/news2mail.py -index bfd6fd1..a74655a 100644 ---- b/news2mail.py -+++ a/news2mail.py -@@ -1,9 +1,11 @@ -+# -*- coding: utf-8 -*- - """News to mail gateway script. Copyright 2000 Cosimo Alfarano - - Author: Cosimo Alfarano - Date: June 11 2000 - --news2mail.py - (C) 2000 by Cosimo Alfarano -+news2mail.py - Copyright 2000 by -+ Cosimo Alfarano - You can use this software under the terms of the GPL. If we meet some day, - and you think this stuff is worth it, you can buy me a beer in return. - -@@ -23,9 +25,13 @@ normal (what pygs does) operations flow is: - Date:, normal headers ending with X-* and Resent-* headers. - - """ -+ -+import email - import sys - import smtplib -+import string - import time -+#import rfc822 - from socket import gethostbyaddr, gethostname - import tempfile - import pyginfo -@@ -37,6 +43,7 @@ class news2mail: - TMPFILE = tempfile.mktemp() - wlfile = None - logfile = None -+ debug = 1 - - sender = '' - rcpt = '' -@@ -44,60 +51,9 @@ class news2mail: - - smtpserver = 'localhost' - -- hostname = gethostbyaddr(gethostname())[0] -- -- heads_dict, smtpheads, nntpheads = {}, {}, {} -- article, headers, body = [], [], [] -- -- debug = 1 -- -- def readfile(self): -- -- for line in sys.stdin.readlines(): -- self.article.append(line) -- -- if (len(self.article) == 1 and self.article[0][0] == '/'): -- file = self.article[0][:-1] -- del self.article[0] -- for line in open(file, 'r').readlines(): -- self.article.append(line) -- -- def parsearticle(self): -- """get news article from file or stdin and separate heads from body -- -- REMEBER: headers value has '\n' as last char. -- Use string[:-1] to ignore newline. -- """ -- -- try: -- body = 0 # are we in body or in headers? -- --# voidline = compile('^$') # I need '\n', ^$ matches anything --# spaceending = compile('\s*\n$') -- -- for line in self.article: -- if not body and len(line) == 1: -- body = 1 # starts article body section -- -- if not body: -- try: -- head, value = line.split(' ', 1) -- self.nntpheads[head] = value -- except (ValueError), message: -- print('string error: %s' % message) -- print('(probably missing couple "Header: value" in %s)' -- % line) -- sys.exit(1) -- -- elif len(line) > 0 and body: -- self.body.append(line) -- -- return self.nntpheads, self.body -- --# except (re.error, ValueError), message: -- except (ValueError), message: -- print message -- sys.exit(1) -+ def __init__(self): -+ self.hostname = gethostbyaddr(gethostname())[0] # FQDN -+ self.message = email.message_from_file(sys.stdin) - - def puthead(self, dict, list, key): - """private, x-form dict entries in list entries""" -@@ -138,7 +94,8 @@ class news2mail: - self.heads_dict = {} - - try: -- for header in self.nntpheads.keys(): # fill it w/ nntp old heads -+ # fill it w/ nntp old heads -+ for header in self.nntpheads.keys(): - self.heads_dict[header] = self.nntpheads[header] - - # and replace them w/ smtp new heads -@@ -160,9 +117,6 @@ class news2mail: - self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \ - info.__doc__ + '\n' - -- ##self.heads_dict['X-Gateway:'] = '%s %s\n' % -- ## (info.PROGNAME, info.__doc__) -- - # to make Received: header - t = time.ctime(time.time()) - -@@ -194,12 +148,12 @@ class news2mail: - headers renamed are useless or not rfc 822 copliant - """ - try: -- if 'Newsgroups:' in self.heads_dict: -+ if 'Newsgroups:' in self.message: - self.heads_dict['X-Newsgroups:'] = \ - self.heads_dict['Newsgroups:'] - del self.heads_dict['Newsgroups:'] - -- if 'NNTP-Posting-Host:' in self.heads_dict: -+ if 'NNTP-Posting-Host:' in self.message: - self.heads_dict['X-NNTP-Posting-Host:'] = \ - self.heads_dict['NNTP-Posting-Host:'] - del self.heads_dict['NNTP-Posting-Host:'] -@@ -215,37 +169,37 @@ class news2mail: - try: - # removing some others useless headers .... - -- if 'Approved:' in self.heads_dict: -+ if 'Approved:' in self.message: - del self.heads_dict['Approved:'] - -- if 'From' in self.heads_dict: # neither 'From ' nor 'From:' -+ if 'From' in self.message: # neither 'From ' nor 'From:' - del self.heads_dict['From'] - -- if 'Xref:' in self.heads_dict: -+ if 'Xref:' in self.message: - del self.heads_dict['Xref:'] - -- if 'Path:' in self.heads_dict: -+ if 'Path:' in self.message: - del self.heads_dict['Path:'] - -- if 'Lines:' in self.heads_dict: -+ if 'Lines:' in self.message: - del self.heads_dict['Lines:'] - - # it is usually set by INN, if ng is moderated... -- if 'Sender:' in self.heads_dict: -+ if 'Sender:' in self.message: - del self.heads_dict['Sender:'] - -- if 'Message-id:' in self.heads_dict: -+ if 'Message-id:' in self.message: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-id'] - del(self.heads_dict['Message-id']) - -- if 'Message-ID:' in self.heads_dict: -+ if 'Message-ID:' in self.message: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID'] - del(self.heads_dict['Message-ID']) - - # If message-id is not present, I generate it -- if 'Message-Id:' not in self.heads_dict: -+ if 'Message-Id:' not in self.message: - # It should put a real user@domain -- msgid = 'pyg@puppapera.org' # FIXME unused variable -+ self.message['Message-Id'] = 'pyg@puppapera.org' - - except KeyError, message: - print message -diff --git b/pygm2n a/pygm2n -index 49d83c3..9601c23 100755 ---- b/pygm2n -+++ a/pygm2n -@@ -1,5 +1,5 @@ - #!/usr/bin/env python --# -*- coding: utf-8 -*- -+ - """News to mail gateway script. Copyright 2000 Cosimo Alfarano - - Author: Cosimo Alfarano -@@ -13,34 +13,37 @@ Thanks to md for this useful formula. Beer is beer. - - - def parse_cmdline(): -+ """ -+ set a dictionary with smtp new header in gw parameter (gw.smtpheads) -+ return (test,verbose) boolean tuple -+ """ - -@@ -86,14 +89,6 @@ try: - - opt = parse_cmdline() - -- # reads stdin and parses article separating head from body -- m2n.readfile(opt) -- m2n.parseemail() -- -- """phase 2: -- check whitelist for user's permission -- """ -- - """phase 3: - format rfc 822 headers from input article - """ -diff --git b/pygn2m a/pygn2m -index 0b958f1..b46b40f 100755 ---- b/pygn2m -+++ a/pygn2m -@@ -1,5 +1,6 @@ - #!/usr/bin/env python - # -*- coding: utf-8 -*- -+ - """News to mail gateway script. Copyright 2000 Cosimo Alfarano - - Author: Cosimo Alfarano -@@ -13,16 +14,19 @@ Thanks to md for this useful formula. Beer is beer. -+ -+ -+sys.path.append('/usr/lib/pyg') -+ -+# import mail2news - - - def parse_cmdline(): -@@ -82,28 +84,24 @@ try: - n2m = news2mail.news2mail() - owner = None - -- # it returns only test, other parms are set directly in the actual -- # parameter - args = parse_cmdline() - - # check if n2m has some file prefercences set on commandline -- if n2m.wlfile is None: -- wl = os.environ['HOME'] + '/pyg.whitelist' -+ if args.wlfile is None: -+ wl = os.path.expanduser('~/pyg.whitelist') - else: -- wl = n2m.wlfile -+ wl = args.wlfile - -- if n2m.logfile is None: -- log = os.environ['HOME'] + '/pyg.log' -+ if args.logfile is None: -+ log = os.path.expanduser('~/pyg.log') - else: -- log = n2m.logfile -- --# print 'using %s %s\n' % (wl,log) -+ log = args.logfile - - wl = whitelist.whitelist(wl, log) - - # reads stdin and parses article separating head from body -- n2m.readfile() -- n2m.parsearticle() -+ ## n2m.readfile() -+ ## n2m.parsearticle() - - """phase 2: - check whitelist for user's permission diff --git a/mail2news.py b/mail2news.py index e7cbcf9..0b97b92 100644 --- a/mail2news.py +++ b/mail2news.py @@ -18,6 +18,7 @@ import sys from os import getpid from socket import gethostbyaddr, gethostname from re import findall +from news2mail import news2mail import nntplib import pyginfo @@ -99,17 +100,9 @@ class mail2news: return self.smtpheads, self.body - def puthead(self, dict, list, key): - """private, transform dict entries in list entries - Appends key of dict to list, deleting it from dict. - """ - - if key in dict: - list.append(key + ' ' + dict.get(key)) - del dict[key] - else: - return 0 - return 1 + @staticmethod + def puthead(*args, **kwargs): + news2mail.puthead(*args, **kwargs) def sortheads(self): """make list sorted by heads: From: To: Subject: first, @@ -165,13 +158,6 @@ class mail2news: self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \ info.PROGDESC + ' - Mail to News\n' -# it is nntpheads stuff -# if(self.newsgroups): -# self.heads_dict['Newsgroups:'] = self.newsgroups - -# if(self.approved): -# self.heads_dict['Approved:'] = self.approved - except KeyError, message: print message @@ -185,22 +171,6 @@ class mail2news: """ try: -### test -# if(post): -# if(post in self.heads_dict): -# self.heads_dict['X-Original-' + post] = self.heads_dict[post] -# -# self.heads_dict[post] = self.heads_dict[pre] -# del(self.heads_dict[pre]) -# -# else: -# if(pre[0:2] == 'X-' and pre in self.heads_dict): -# self.heads_dict['X-Original-' + pre] = self.heads_dict[pre] -# elif(not pre[0:2] == 'X-' and 'X-' + pre in self.heads_dict): -# self.heads_dict['X-' + pre] = self.heads_dict[pre] -# del(self.heads_dict[pre]) -### end test - for key in self.heads_dict.keys(): if(key[:7] in ['Resent-']): if ('X-' + key) in self.heads_dict: @@ -224,10 +194,6 @@ class mail2news: if(ref and len(ref)): self.heads_dict['References:'] = '%s\n' % ref[0] -# if('To:' in self.heads_dict): -# self.heads_dict['X-To:'] = self.heads_dict['To:'] -# del self.heads_dict['To:'] - except KeyError, message: print message @@ -249,20 +215,6 @@ class mail2news: if head in self.heads_dict: del self.heads_dict[head] -# if 'From' in self.heads_dict: # neither 'From ' nor 'From:' -# del self.heads_dict['From'] - -# # neither 'From ' nor 'From:' -# if 'NNTP-Posting-Host:' in self.heads_dict: -# del self.heads_dict[''] - -# if 'Lines:' in self.heads_dict: -# del self.heads_dict['Lines:'] - - # it is usually set by INN, if ng is moderated... -# if 'Sender:' in self.heads_dict: -# del self.heads_dict['Sender:'] - if 'Message-id:' in self.heads_dict: self.heads_dict['Message-Id:'] = self.heads_dict['Message-id:'] del(self.heads_dict['Message-id:']) diff --git a/news2mail.py b/news2mail.py index bfd6fd1..4dd83fc 100644 --- a/news2mail.py +++ b/news2mail.py @@ -9,7 +9,6 @@ and you think this stuff is worth it, you can buy me a beer in return. Thanks to md for this useful formula. Beer is beer. - Gets news article and sends it via SMTP. class news2mail is hopefully conform to rfc822. @@ -23,18 +22,17 @@ normal (what pygs does) operations flow is: Date:, normal headers ending with X-* and Resent-* headers. """ +import email import sys import smtplib import time from socket import gethostbyaddr, gethostname -import tempfile import pyginfo -class news2mail: +class news2mail(object): """news to mail gateway class""" - TMPFILE = tempfile.mktemp() wlfile = None logfile = None @@ -48,62 +46,21 @@ class news2mail: heads_dict, smtpheads, nntpheads = {}, {}, {} article, headers, body = [], [], [] + message = None debug = 1 def readfile(self): + self.message = email.message_from_file(sys.stdin) + self.nntpheads = dict(((key, value) + for (key, value) in self.message.items())) - for line in sys.stdin.readlines(): - self.article.append(line) - - if (len(self.article) == 1 and self.article[0][0] == '/'): - file = self.article[0][:-1] - del self.article[0] - for line in open(file, 'r').readlines(): - self.article.append(line) - - def parsearticle(self): - """get news article from file or stdin and separate heads from body - - REMEBER: headers value has '\n' as last char. - Use string[:-1] to ignore newline. - """ - - try: - body = 0 # are we in body or in headers? - -# voidline = compile('^$') # I need '\n', ^$ matches anything -# spaceending = compile('\s*\n$') - - for line in self.article: - if not body and len(line) == 1: - body = 1 # starts article body section - - if not body: - try: - head, value = line.split(' ', 1) - self.nntpheads[head] = value - except (ValueError), message: - print('string error: %s' % message) - print('(probably missing couple "Header: value" in %s)' - % line) - sys.exit(1) - - elif len(line) > 0 and body: - self.body.append(line) - - return self.nntpheads, self.body + @staticmethod + def puthead(from_dict, out_list, key): + """private, x-form dict entries in out_list entries""" -# except (re.error, ValueError), message: - except (ValueError), message: - print message - sys.exit(1) - - def puthead(self, dict, list, key): - """private, x-form dict entries in list entries""" - - if key in dict: - list.append(key + ' ' + dict.get(key)) + if key in from_dict: + out_list.append(key + ' ' + from_dict.get(key)) else: return 0 return 1 @@ -112,13 +69,14 @@ class news2mail: """make list sorting heads, Received: From: To: Subject: first, others, X-*, Resent-* last""" - set = ('Received:', 'From:', 'To:', 'Subject:', 'Date:') # put at top + # put at top + header_set = ('Received', 'From', 'To', 'Subject', 'Date') - for k in set: + for k in header_set: self.puthead(self.heads_dict, self.headers, k) for k in self.heads_dict.keys(): - if k[:2] != 'X-' and k[:7] != 'Resent-' and k not in set: + if k[:2] != 'X-' and k[:7] != 'Resent-' and k not in header_set: self.puthead(self.heads_dict, self.headers, k) for k in self.heads_dict.keys(): @@ -157,10 +115,10 @@ class news2mail: info = pyginfo.pygsinfo() try: - self.heads_dict['X-Gateway:'] = info.PROGNAME + ' ' + \ + self.heads_dict['X-Gateway'] = info.PROGNAME + ' ' + \ info.__doc__ + '\n' - ##self.heads_dict['X-Gateway:'] = '%s %s\n' % + ##self.heads_dict['X-Gateway'] = '%s %s\n' % ## (info.PROGNAME, info.__doc__) # to make Received: header @@ -182,7 +140,7 @@ class news2mail: '\n\tfor <' + self.rcpt + '> ; ' + \ t + ' (' + tzone + ')\n' - self.heads_dict['Received:'] = tmp + self.heads_dict['Received'] = tmp except KeyError, message: print message @@ -194,15 +152,15 @@ class news2mail: headers renamed are useless or not rfc 822 copliant """ try: - if 'Newsgroups:' in self.heads_dict: - self.heads_dict['X-Newsgroups:'] = \ - self.heads_dict['Newsgroups:'] - del self.heads_dict['Newsgroups:'] - - if 'NNTP-Posting-Host:' in self.heads_dict: - self.heads_dict['X-NNTP-Posting-Host:'] = \ - self.heads_dict['NNTP-Posting-Host:'] - del self.heads_dict['NNTP-Posting-Host:'] + if 'Newsgroups' in self.heads_dict: + self.heads_dict['X-Newsgroups'] = \ + self.heads_dict['Newsgroups'] + del self.heads_dict['Newsgroups'] + + if 'NNTP-Posting-Host' in self.heads_dict: + self.heads_dict['X-NNTP-Posting-Host'] = \ + self.heads_dict['NNTP-Posting-Host'] + del self.heads_dict['NNTP-Posting-Host'] except KeyError, message: print message @@ -214,38 +172,24 @@ class news2mail: try: # removing some others useless headers .... + # that includes BOTH 'From ' and 'From' + # 'Sender is usually set by INN, if ng is moderated... + for key in ('Approved', 'From', 'Xref', 'Path', 'Lines', 'Sender'): + if key in self.heads_dict: + del self.heads_dict[key] - if 'Approved:' in self.heads_dict: - del self.heads_dict['Approved:'] - - if 'From' in self.heads_dict: # neither 'From ' nor 'From:' - del self.heads_dict['From'] - - if 'Xref:' in self.heads_dict: - del self.heads_dict['Xref:'] + if 'Message-id' in self.heads_dict: + self.heads_dict['Message-Id'] = self.heads_dict['Message-id'] + del self.heads_dict['Message-id'] - if 'Path:' in self.heads_dict: - del self.heads_dict['Path:'] - - if 'Lines:' in self.heads_dict: - del self.heads_dict['Lines:'] - - # it is usually set by INN, if ng is moderated... - if 'Sender:' in self.heads_dict: - del self.heads_dict['Sender:'] - - if 'Message-id:' in self.heads_dict: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-id'] - del(self.heads_dict['Message-id']) - - if 'Message-ID:' in self.heads_dict: - self.heads_dict['Message-Id:'] = self.heads_dict['Message-ID'] - del(self.heads_dict['Message-ID']) + if 'Message-ID' in self.heads_dict: + self.heads_dict['Message-Id'] = self.heads_dict['Message-ID'] + del self.heads_dict['Message-ID'] # If message-id is not present, I generate it - if 'Message-Id:' not in self.heads_dict: + if 'Message-Id' not in self.heads_dict: # It should put a real user@domain - msgid = 'pyg@puppapera.org' # FIXME unused variable + self.heads_dict['Message-Id'] = 'pyg@puppapera.org' except KeyError, message: print message @@ -255,6 +199,9 @@ class news2mail: def sendarticle(self): """Talk to SMTP server and try to send email.""" try: + raise NotImplementedError( + 'sendarticle does not use email.Message yet') + msglist = [] s = smtplib.SMTP(self.smtpserver) diff --git a/pygn2m b/pygn2m index 0b958f1..03f5ee7 100755 --- a/pygn2m +++ b/pygn2m @@ -87,15 +87,15 @@ try: args = parse_cmdline() # check if n2m has some file prefercences set on commandline - if n2m.wlfile is None: - wl = os.environ['HOME'] + '/pyg.whitelist' + if args.wlfile is None: + wl = os.path.expanduser('~/pyg.whitelist') else: - wl = n2m.wlfile + wl = args.wlfile - if n2m.logfile is None: - log = os.environ['HOME'] + '/pyg.log' + if args.logfile is None: + log = os.path.expanduser('~/pyg.log') else: - log = n2m.logfile + log = args.logfile # print 'using %s %s\n' % (wl,log) @@ -103,17 +103,17 @@ try: # reads stdin and parses article separating head from body n2m.readfile() - n2m.parsearticle() +# n2m.parsearticle() """phase 2: check whitelist for user's permission """ # make a first check of From: address - owner = wl.checkfrom(n2m.nntpheads['From:']) + owner = wl.checkfrom(n2m.nntpheads['From']) if owner is None: if sys.stdin.isatty() == 1 or args.test: - print ('"%s" is not in whitelist!' % (n2m.nntpheads['From:'][:-1])) + print ('"%s" is not in whitelist!' % (n2m.nntpheads['From'][:-1])) else: wl.logmsg(n2m.nntpheads, wl.DENY) @@ -139,7 +139,7 @@ try: for line in n2m.headers: print(line[:-1]) - if args.owner is None: + if owner is None: sys.exit(1) """phase 4: diff --git a/whitelist.py b/whitelist.py index fab0824..c017402 100644 --- a/whitelist.py +++ b/whitelist.py @@ -67,9 +67,12 @@ class whitelist(object): def checkfrom(self, fromhead): """have you permission to be here, sir?""" - for owner in self.wl.keys(): -# if(self.wl[owner]['From:'] == fromhead[:-1]): # remove '\n' +# if(self.wl[owner]['From'] == fromhead[:-1]): # remove '\n' + # here colon after 'From' IS required, because binary module wl + # expects it. + # TODO: when switching to the python lexxing, remove this + # limitation. if fromhead[:-1].find(self.wl[owner]['From:']) >= 0: return owner else: @@ -104,26 +107,26 @@ class whitelist(object): self.log.write('at %s (%s)\n' % (ltime, tzone)) if owner is not None: self.log.write('\tWLOwner: ' + owner + '\n') - self.log.write('\tFrom: ' + heads.get('From:', 'NOT PRESENT\n')) - self.log.write('\tSubject: ' + heads.get('Subject:', 'NOT PRESENT\n')) - self.log.write('\tSender: ' + heads.get('Sender:', 'NOT PRESENT\n')) - self.log.write('\tDate: ' + heads.get('Date:', 'NOT PRESENT\n')) + self.log.write('\tFrom: ' + heads.get('From', 'NOT PRESENT\n')) + self.log.write('\tSubject: ' + heads.get('Subject', 'NOT PRESENT\n')) + self.log.write('\tSender: ' + heads.get('Sender', 'NOT PRESENT\n')) + self.log.write('\tDate: ' + heads.get('Date', 'NOT PRESENT\n')) # some client create Message-Id other Message-ID. - if 'Message-ID:' in heads: - self.log.write('\tMessage-ID: ' + heads.get('Message-ID:')) + if 'Message-ID' in heads: + self.log.write('\tMessage-ID: ' + heads.get('Message-ID')) else: - self.log.write('\tMessage-Id: ' + heads.get('Message-Id:', + self.log.write('\tMessage-Id: ' + heads.get('Message-Id', 'NOT PRESENT\n')) # X-Newsgroups: and To: are present if user is trusted, else # Newsgroup: exists since no changes on nntp headers are done. - if 'X-Newsgroups:' in heads: - self.log.write('\tTo: ' + heads.get('To:', 'NOT PRESENT\n')) - self.log.write('\tX-Newsgroups: ' + heads.get('X-Newsgroups:', + if 'X-Newsgroups' in heads: + self.log.write('\tTo: ' + heads.get('To', 'NOT PRESENT\n')) + self.log.write('\tX-Newsgroups: ' + heads.get('X-Newsgroups', 'NOT PRESENT\n')) else: self.log.write('\tNewsgroups: ' + - heads.get('Newsgroups:', 'NOT PRESENT\n')) + heads.get('Newsgroups', 'NOT PRESENT\n')) self.log.write('\n') -- cgit