Groundwork for parsing Received: header (and generating Path: not to beparse_received

forgotten)

2 files changed, 286 insertions, 0 deletions

diff --git a/test/samples/testing-email-01.eml b/test/samples/testing-email-01.eml
new file mode 100644
index 0000000..9c00523
--- /dev/null
+++ b/test/samples/testing-email-01.eml
@@ -0,0 +1,129 @@
+From mcepl@redhat.com Wed Jan  7 17:36:15 2015
+Return-Path: mcepl@redhat.com
+Received: from zmta02.collab.prod.int.phx2.redhat.com (LHLO
+ zmta02.collab.prod.int.phx2.redhat.com) (10.5.81.9) by
+ zmail13.collab.prod.int.phx2.redhat.com with LMTP; Wed, 7 Jan 2015 11:36:14
+ -0500 (EST)
+Received: from int-mx14.intmail.prod.int.phx2.redhat.com (int-mx14.intmail.prod.int.phx2.redhat.com [10.5.11.27])
+	by zmta02.collab.prod.int.phx2.redhat.com (Postfix) with ESMTP id D959F121409
+	for <mcepl@mail.corp.redhat.com>; Wed,  7 Jan 2015 11:36:14 -0500 (EST)
+Received: from mx1.redhat.com (ext-mx12.extmail.prod.ext.phx2.redhat.com [10.5.110.17])
+	by int-mx14.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id t07GaEPV007184
+	for <mcepl@redhat.com>; Wed, 7 Jan 2015 11:36:14 -0500
+Received: from d1080.master.cz (p-lab.cz [89.185.245.149] (may be forged))
+	by mx1.redhat.com (8.14.4/8.14.4) with ESMTP id t07GaC1j031854
+	for <mcepl@redhat.com>; Wed, 7 Jan 2015 11:36:13 -0500
+Received: by d1080.master.cz (Postfix)
+	id 1932D3218192; Wed,  7 Jan 2015 17:36:12 +0100 (CET)
+Delivered-To: virtuser_566@d1080.master.cz
+Received: by d1080.master.cz (Postfix, from userid 557)
+	id 113733218161; Wed,  7 Jan 2015 17:36:12 +0100 (CET)
+X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on p-lab.cz
+X-Spam-Level: 
+X-Spam-Status: No, score=-105.0 required=2.0 tests=RCVD_IN_DNSWL_HI,
+	SPF_HELO_PASS,URIBL_BLOCKED,USER_IN_ALL_SPAM_TO autolearn=disabled
+	version=3.3.1
+Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28])
+	by d1080.master.cz (Postfix) with ESMTP id 80EB93218161
+	for <mcepl@cepl.eu>; Wed,  7 Jan 2015 17:36:10 +0100 (CET)
+Received: from int-mx11.intmail.prod.int.phx2.redhat.com (int-mx11.intmail.prod.int.phx2.redhat.com [10.5.11.24])
+	by mx1.redhat.com (8.14.4/8.14.4) with ESMTP id t07Ga8gQ031700
+	(version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=FAIL);
+	Wed, 7 Jan 2015 11:36:09 -0500
+Received: from mitmanek.localdomain (ovpn-116-62.ams2.redhat.com [10.36.116.62])
+	by int-mx11.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with ESMTP id t07Ga7QL024180;
+	Wed, 7 Jan 2015 11:36:08 -0500
+Received: from mitmanek.redhat.com (mitmanek.ceplovi.cz [127.0.0.1])
+	by mitmanek.localdomain (Postfix) with ESMTP id 44EA710093A06;
+	Wed,  7 Jan 2015 17:36:06 +0100 (CET)
+From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= <mcepl@redhat.com>
+To: piglit@lists.freedesktop.org
+Cc: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= <mcepl@cepl.eu>
+Subject: [PATCH] Do we really need python 2.7?
+Date: Wed,  7 Jan 2015 17:36:00 +0100
+Message-Id: <1420648560-24144-1-git-send-email-mcepl@redhat.com>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+X-Scanned-By: MIMEDefang 2.68 on 10.5.11.27
+X-Scanned-By: MIMEDefang 2.68 on 10.5.110.17
+X-Scanned-By: MIMEDefang 2.68 on 10.5.11.24
+Content-Transfer-Encoding: quoted-printable
+X-RedHat-Spam-Score: -1.106  (BAYES_00,RDNS_NONE,URIBL_BLOCKED) 89.185.245.149 [89.185.245.149] 89.185.245.149 [89.185.245.149] <mcepl@redhat.com>
+Status: RO
+Content-Length: 2410
+Lines: 73
+
+From: Mat=C4=9Bj Cepl <mcepl@cepl.eu>
+
+Hi,
+
+I am in the process of struggle to build piglit from the master
+(commit 4adb082) on RHEL-6 (we would like to continue to use it
+for testing).
+
+When I tried just naively build our RHEL-7/Fedora 20+ package on
+EL-6 configuration failed because it complained it is missing
+python 2.6. Which is a bit strange, because of course, RHEL-6
+does contain python 2.6. So, I went digging to find out where
+this message comes from and I have discovered this interesting
+piece of code:
+
+    # Check for presence of Python 2.6 or greater.
+    foreach(python_cmd python2 python)
+            execute_process(
+                    COMMAND ${python_cmd} -c \
+                        "import sys; assert '2.7' <=3D sys.version < '3'"
+                    OUTPUT_QUIET
+                    ERROR_QUIET
+                    RESULT_VARIABLE python_version_check_error_code)
+            if(python_version_check_error_code EQUAL 0)
+                    set(python ${python_cmd})
+                    break()
+            endif(python_version_check_error_code EQUAL 0)
+    endforeach(python_cmd)
+
+    if(NOT DEFINED python)
+            message(FATAL_ERROR "python version 2.x (where x >=3D 6) requ=
+ired")
+    endif(NOT DEFINED python)
+
+First obviously this code lies. Either we really care about
+python 2.7 and we should declare our loyalities openly, or piglit
+can be working with python 2.6 (which I hope) and then that '2.7'
+is just a typo.
+
+Which one it is? How difficult it would be to switch piglit to
+python 2.6? I hope not that difficult (as there were not that
+many changes between 2.6 and 2.7). Does anybody know?
+
+And of course, when I apply just the attached patch I get to yet
+another problem: freeglut (RHEL-6 has freeglut-2.6.0). But let's
+walk this minefield one mine at the time.
+
+Best,
+
+Mat=C4=9Bj
+
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 6fb6c8a..d2f2f0e 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -190,7 +190,7 @@ ENDIF()
+ # Check for presence of Python 2.6 or greater.
+ foreach(python_cmd python2 python)
+ 	execute_process(
+-		COMMAND ${python_cmd} -c "import sys; assert '2.7' <=3D sys.version < =
+'3'"
++		COMMAND ${python_cmd} -c "import sys; assert '2.6' <=3D sys.version < =
+'3'"
+ 		OUTPUT_QUIET
+ 		ERROR_QUIET
+ 		RESULT_VARIABLE python_version_check_error_code)
+--=20
+1.8.3.1
+
+
diff --git a/test/test_parse_received.py b/test/test_parse_received.py
new file mode 100644
index 0000000..8fa1945
--- /dev/null
+++ b/test/test_parse_received.py
@@ -0,0 +1,157 @@
+import datetime
+import email
+import unittest
+
+from dateutil.tz import tzoffset
+from rply import Token
+
+import parse_received
+
+
+#logging.basicConfig(level=logging.DEBUG)
+
+
+INPUT_01 = \
+    """from server.mymailhost.com
+    (mail.mymailhost.com [126.43.75.123])
+    by pilot01.cl.msu.edu (8.10.2/8.10.2) with ESMTP id NAA23597;
+    Fri, 12 Jul 2002 16:11:20 -0400 (EDT)"""
+INPUT_02 = \
+    """from d1080.master.cz (p-lab.cz [89.185.245.149] (may be forged))
+    by mx1.redhat.com (8.14.4/8.14.4) with ESMTP id t07GaC1j031854
+    for <mcepl@redhat.com>; Wed, 7 Jan 2015 11:36:13 -0500"""
+
+
+class TestReceivedLexer(unittest.TestCase):
+    maxDiff = None
+
+    def test_simple_01(self):
+        expected = [
+            Token('FROMSEP', 'from'),
+            Token('DOMAIN', 'server.mymailhost.com'),
+            Token('DOMAIN', 'mail.mymailhost.com'),
+            Token('IPV4ADDRESS', '[126.43.75.123]'),
+            Token('BYSEP', 'by'),
+            Token('DOMAIN', 'pilot01.cl.msu.edu'),
+            Token('SMTPVERSION', '8.10.2/8.10.2'),
+            Token('WITHSEP', 'with'),
+            Token('SMTPSEP', 'ESMTP'),
+            Token('IDSEP', 'id'),
+            Token('STRING', 'NAA23597;'),
+            Token('DATETIME', 'Fri, 12 Jul 2002 16:11:20 -0400 (EDT)')
+        ]
+        parsed = list(parse_received.lexer.lex(INPUT_01))
+        self.assertEqual(parsed, expected)
+
+    def test_simple_02(self):
+        expected = [
+            Token('FROMSEP', 'from'),
+            Token('DOMAIN', 'd1080.master.cz'),
+            Token('DOMAIN', 'p-lab.cz'),
+            Token('IPV4ADDRESS', '[89.185.245.149]'),
+            Token('IGNORABLESTR', 'may be forged'),
+            Token('BYSEP', 'by'),
+            Token('DOMAIN', 'mx1.redhat.com'),
+            Token('SMTPVERSION', '8.14.4/8.14.4'),
+            Token('WITHSEP', 'with'),
+            Token('SMTPSEP', 'ESMTP'),
+            Token('IDSEP', 'id'),
+            Token('STRING', 't07GaC1j031854'),
+            Token('FORSEP', 'for'),
+            Token('EMAILADDR', '<mcepl@redhat.com>'),
+            Token('DATETIME', 'Wed, 7 Jan 2015 11:36:13 -0500')
+        ]
+        parsed = list(parse_received.lexer.lex(INPUT_02))
+        self.assertEqual(parsed, expected)
+
+
+class TestReceivedParser(unittest.TestCase):
+    maxDiff = None
+
+    def test_simple_01(self):
+        expected = [
+            [Token('FROMSEP', 'from'),
+             [Token('DOMAIN', 'server.mymailhost.com'),
+              [Token('DOMAIN', 'mail.mymailhost.com'),
+               Token('IPV4ADDRESS', '[126.43.75.123]')]]
+             ],
+            [Token('BYSEP', 'by'),
+             [Token('DOMAIN', 'pilot01.cl.msu.edu'),
+              [Token('SMTPVERSION', '8.10.2/8.10.2')]]
+             ],
+            [[[],
+             [Token('SMTPSEP', 'ESMTP')],
+             [Token('STRING', 'NAA23597;')], []]
+             ],
+            Token('DATETIME', 'Fri, 12 Jul 2002 16:11:20 -0400 (EDT)')
+        ]
+        stream = parse_received.lexer.lex(INPUT_01)
+        stream.idx = 0
+        parsed = parse_received.parser.parse(stream)
+        self.assertEqual(parsed, expected)
+
+    def test_simple_02(self):
+        expected = [
+            [Token('FROMSEP', 'from'),
+             [Token('DOMAIN', 'd1080.master.cz'),
+              [Token('DOMAIN', 'p-lab.cz'),
+               Token('IPV4ADDRESS', '[89.185.245.149]')]]
+             ],
+            [Token('BYSEP', 'by'),
+             [Token('DOMAIN', 'mx1.redhat.com'),
+              [Token('SMTPVERSION', '8.14.4/8.14.4')]]
+             ],
+            [[[],
+             [Token('SMTPSEP', 'ESMTP')],
+             [Token('STRING', 't07GaC1j031854')],
+             [Token('FORSEP', 'for')]]
+             ],
+            Token('DATETIME', 'Wed, 7 Jan 2015 11:36:13 -0500')
+        ]
+        stream = parse_received.lexer.lex(INPUT_02)
+        stream.idx = 0
+        parsed = parse_received.parser.parse(stream)
+        self.assertEqual(parsed, expected)
+
+    def test_parse_01(self):
+        expected = {
+            'by': {'server': 'pilot01.cl.msu.edu'},
+            'date': datetime.datetime(
+                2002, 7, 12, 16, 11, 20, tzinfo=tzoffset(u'EDT', -14400)),
+            'from': {
+                'halo': 'server.mymailhost.com',
+                'ipaddr': '[126.43.75.123]',
+                'reveresed': 'mail.mymailhost.com'
+            }
+        }
+
+        observed = parse_received.parse_header(INPUT_01)
+        self.assertEqual(observed, expected)
+
+    def test_parse_02(self):
+        expected = {
+            'by': {'server': 'mx1.redhat.com'},
+            'date': datetime.datetime(
+                2015, 1, 7, 11, 36, 13, tzinfo=tzoffset(None, -18000)),
+            'from': {
+                'halo': 'd1080.master.cz',
+                'ipaddr': '[89.185.245.149]',
+                'reveresed': 'p-lab.cz'
+            }
+        }
+
+        observed = parse_received.parse_header(INPUT_02)
+        self.assertEqual(observed, expected)
+
+    def test_email_01(self):
+        with open('examples/mail', 'r') as inmail:
+            msg = email.message_from_file(inmail)
+            received_hdrs = msg.get_all('Received')
+            expected = []
+            parsed_headers = []
+            for hdr in received_hdrs:
+                parsed_headers.append(parse_received.parse_header(hdr))
+            self.assertEqual(parsed_headers, expected)
+
+if __name__ == '__main__':
+    unittest.main()