# This file is part of the sos project: https://github.com/sosreport/sos
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# version 2 of the GNU General Public License.
#
# See the LICENSE file in the source distribution for further information.
import unittest
from ipaddress import ip_interface
from sos.cleaner.parsers.ip_parser import SoSIPParser
from sos.cleaner.parsers.mac_parser import SoSMacParser
from sos.cleaner.parsers.hostname_parser import SoSHostnameParser
from sos.cleaner.parsers.keyword_parser import SoSKeywordParser
from sos.cleaner.parsers.ipv6_parser import SoSIPv6Parser
from sos.cleaner.mappings.ip_map import SoSIPMap
from sos.cleaner.mappings.mac_map import SoSMacMap
from sos.cleaner.mappings.hostname_map import SoSHostnameMap
from sos.cleaner.mappings.keyword_map import SoSKeywordMap
from sos.cleaner.mappings.ipv6_map import SoSIPv6Map
class CleanerMapTests(unittest.TestCase):
def setUp(self):
self.mac_map = SoSMacMap()
self.ip_map = SoSIPMap()
self.host_map = SoSHostnameMap()
self.host_map.load_domains_from_options(['redhat.com'])
self.kw_map = SoSKeywordMap()
self.ipv6_map = SoSIPv6Map()
def test_mac_map_obfuscate_valid_v4(self):
_test = self.mac_map.get('12:34:56:78:90:ab')
self.assertNotEqual(_test, '12:34:56:78:90:ab')
def test_mac_map_obfuscate_valid_v6(self):
_test = self.mac_map.get('12:34:56:ff:fe:78:90:ab')
self.assertNotEqual(_test, '12:34:56:ff:fe:78:90:ab')
def test_mac_map_obfuscate_valid_v6_quad(self):
_test = self.mac_map.get('1234:56ff:fe78:90ab')
self.assertNotEqual(_test, '1234:56ff:fe78:90ab')
def test_mac_map_skip_ignores(self):
_test = self.mac_map.get('ff:ff:ff:ff:ff:ff')
self.assertEquals(_test, 'ff:ff:ff:ff:ff:ff')
def test_mac_map_avoid_duplicate_obfuscation(self):
_test = self.mac_map.get('ab:cd:ef:fe:dc:ba')
_dup = self.mac_map.get(_test)
self.assertEquals(_test, _dup)
def test_ip_map_obfuscate_v4_with_cidr(self):
_test = self.ip_map.get('192.168.1.0/24')
self.assertNotEqual(_test, '192.168.1.0/24')
def test_ip_map_obfuscate_no_cidr(self):
_test = self.ip_map.get('192.168.2.2')
self.assertNotEqual(_test, '192.168.2.2')
def test_ip_map_obfuscate_same_subnet(self):
_net = ip_interface(self.ip_map.get('192.168.3.0/24'))
_test = ip_interface(self.ip_map.get('192.168.3.1'))
self.assertTrue(_test.ip in _net.network)
def test_ip_map_get_same_with_or_without_cidr(self):
_hostwsub = self.ip_map.get('192.168.4.1/24')
_hostnosub = self.ip_map.get('192.168.4.1')
self.assertEqual(_hostwsub.split('/')[0], _hostnosub)
def test_ip_skip_ignores(self):
_test = self.ip_map.get('127.0.0.1')
self.assertEquals(_test, '127.0.0.1')
def test_hostname_obfuscate_domain_options(self):
_test = self.host_map.get('www.redhat.com')
self.assertNotEqual(_test, 'www.redhat.com')
def test_hostname_obfuscate_same_item(self):
_test1 = self.host_map.get('example.redhat.com')
_test2 = self.host_map.get('example.redhat.com')
self.assertEqual(_test1, _test2)
def test_hostname_obfuscate_just_domain(self):
_test = self.host_map.get('redhat.com')
self.assertEqual(_test, 'obfuscateddomain0.com')
def test_hostname_no_obfuscate_non_loaded_domain(self):
_test = self.host_map.get('foobar.com')
self.assertEqual(_test, 'foobar.com')
def test_hostname_no_obfuscate_non_loaded_fqdn(self):
_test = self.host_map.get('example.foobar.com')
self.assertEqual(_test, 'example.foobar.com')
def test_keyword_single(self):
_test = self.kw_map.get('foobar')
self.assertEqual(_test, 'obfuscatedword0')
def test_ipv6_obfuscate_global(self):
_net = '2022:1104:abcd::'
_ob_net = self.ipv6_map.get(_net)
self.assertNotEqual(_net, _ob_net, 'Address was unchanged')
self.assertTrue(_ob_net.startswith('534f'), 'Global address does not start with identifier')
_host = '2022:1104:abcd::1234'
_ob_host = self.ipv6_map.get(_host)
self.assertNotEqual(_host, _ob_host, 'Host address was unchanged')
self.assertTrue(_host.startswith(_net), 'Host address not in network')
def test_ipv6_link_local(self):
_test = 'fe80::1234'
_ob_test = self.ipv6_map.get(_test)
self.assertTrue(_ob_test.startswith('fe80'), 'Link-local identifier not maintained')
self.assertNotEqual(_test, _ob_test, 'Device address was unchanged')
def test_ipv6_private(self):
_net = 'fd00:abcd::'
_host = 'fd00:abcd::1234'
_ob_net = self.ipv6_map.get(_net).split('/')[0]
_ob_host = self.ipv6_map.get(_host)
self.assertTrue(_ob_net.startswith('fd53'), 'Private network does not start with identifier')
self.assertTrue(_ob_host.startswith(_ob_net), 'Private address not in same network')
self.assertNotEqual(_net, _ob_net, 'Private network was unchanged')
def test_ipv6_short_network(self):
_net = 'ff02::'
_ob_net = self.ipv6_map.get(_net)
self.assertTrue(_ob_net.startswith(('53', '54')), f'Short network does not start with identifier: {_ob_net}')
def test_ipv6_consistent_obfuscation(self):
_test = '2022:1104:abcd::ef09'
_new = self.ipv6_map.get(_test)
_second = self.ipv6_map.get(_test)
self.assertEqual(_new, _second, "Same address produced two different results")
def test_ipv6_global_no_collision(self):
"""Tests that generating more than 256 global network obfuscations does
not produce any repeats"""
_nets = []
for i in range(1, 300):
_nets.append(self.ipv6_map.get(f"f{i:03}::abcd").split('::')[0])
# if there are any duplicates, then the length of the set will not match
self.assertTrue(len(set(_nets)) == len(_nets), "Duplicate global network obfuscations produced")
self.assertTrue(_nets[-1].startswith('54'), "First hextet of global network obfuscation over 256 not expected '54'")
class CleanerParserTests(unittest.TestCase):
def setUp(self):
self.ip_parser = SoSIPParser(config={})
self.ipv6_parser = SoSIPv6Parser(config={})
self.mac_parser = SoSMacParser(config={})
self.host_parser = SoSHostnameParser(config={},
opt_domains=['foobar.com'])
self.kw_parser = SoSKeywordParser(config={}, keywords=['foobar'])
self.kw_parser_none = SoSKeywordParser(config={})
self.kw_parser.generate_item_regexes()
def test_ip_parser_valid_ipv4_line(self):
line = 'foobar foo 10.0.0.1/24 barfoo bar'
_test = self.ip_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_ip_parser_invalid_ipv4_line(self):
line = 'foobar foo 10.1.2.350 barfoo bar'
self.assertRaises(ValueError, self.ip_parser.parse_line, line)
def test_ip_parser_package_version_line(self):
line = 'mycoolpackage-1.2.3.4.5'
_test = self.ip_parser.parse_line(line)[0]
self.assertEqual(line, _test)
def test_mac_parser_valid_ipv4_line(self):
line = 'foobar foo 13:24:35:46:57:68 bar barfoo'
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_mac_parser_valid_ipv6_line(self):
line = 'foobar foo AA:BB:CC:FF:FE:DD:EE:FF bar barfoo'
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_mac_parser_with_quotes(self):
line = "foobar foo '12:34:56:78:90:AA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "aa:12:bb:34:cc:56" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_mac_parser_with_quotes_ipv6(self):
line = "foobar foo 'FF:EE:DD:FF:FE:CC:BB:AA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "DD:EE:FF:FF:FE:BB:CC:AA" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_mac_parser_with_quotes_ipv6_quad(self):
line = "foobar foo 'AABB:CCDD:EEFF:FFAA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "AAFF:FFEE:DDCC:BBAA" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_hostname_load_hostname_string(self):
fqdn = 'myhost.subnet.example.com'
self.host_parser.load_hostname_into_map(fqdn)
def test_hostname_valid_domain_line(self):
self.host_parser.load_hostname_into_map('myhost.subnet.example.com')
line = 'testing myhost.subnet.example.com in a string'
_test = self.host_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_hostname_short_name_in_line(self):
self.host_parser.load_hostname_into_map('myhost.subnet.example.com')
line = 'testing just myhost in a line'
_test = self.host_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_obfuscate_whole_fqdn_for_given_domainname(self):
self.host_parser.load_hostname_into_map('sostestdomain.domain')
line = 'let obfuscate soshost.sostestdomain.domain'
_test = self.host_parser.parse_line(line)[0]
self.assertFalse('soshost' in _test)
self.assertFalse('sostestdomain' in _test)
def test_hostname_no_obfuscate_underscore(self):
line = 'pam_env.so _why.not_'
_test = self.host_parser.parse_line(line)[0]
self.assertEqual(line, _test)
def test_keyword_parser_valid_line(self):
line = 'this is my foobar test line'
_test = self.kw_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_keyword_parser_no_change_by_default(self):
line = 'this is my foobar test line'
_test = self.kw_parser_none.parse_line(line)[0]
self.assertEqual(line, _test)
def test_ipv6_parser_strings(self):
t1 = 'testing abcd:ef01::1234 as a compressed address'
t2 = 'testing abcd:ef01::5678:1234 as a separate address'
t3 = 'testing 2607:c540:8c00:3318::34/64 as another address'
t4 = 'testing 2007:1234:5678:90ab:0987:6543:21fe:dcba as a full address'
t1_test = self.ipv6_parser.parse_line(t1)[0]
t2_test = self.ipv6_parser.parse_line(t2)[0]
t3_test = self.ipv6_parser.parse_line(t3)[0]
t4_test = self.ipv6_parser.parse_line(t4)[0]
self.assertNotEqual(t1, t1_test, f"Parser did not match and obfuscate '{t1}'")
self.assertNotEqual(t2, t2_test, f"Parser did not match and obfuscate '{t2}'")
self.assertNotEqual(t3, t3_test, f"Parser did not match and obfuscate '{t3}'")
self.assertNotEqual(t4, t4_test, f"Parser did not match and obfuscate '{t4}'")
def test_ipv6_no_match_signature(self):
modstr = '2D:4F:6E:55:4F:E8:5E:D2:D2:A3:73:62:AB:FD:F9:C5:A5:53:31:93'
mod_test = self.ipv6_parser.parse_line(modstr)[0]
self.assertEqual(modstr, mod_test, "Parser matched module signature, and should not")
def test_ipv6_no_match_log_false_positive(self):
logln = 'Automatically imported trusted_ca::ca from trusted_ca/ca into production'
log_test = self.ipv6_parser.parse_line(logln)[0]
self.assertEqual(logln, log_test, "IPv6 parser incorrectly matched a log line of 'trusted_ca::ca'")
