# This file is part of the sos project: https://github.com/sosreport/sos
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# version 2 of the GNU General Public License.
#
# See the LICENSE file in the source distribution for further information.
import unittest
from ipaddress import ip_interface
from sos.cleaner.parsers.ip_parser import SoSIPParser
from sos.cleaner.parsers.mac_parser import SoSMacParser
from sos.cleaner.parsers.hostname_parser import SoSHostnameParser
from sos.cleaner.parsers.keyword_parser import SoSKeywordParser
from sos.cleaner.parsers.ipv6_parser import SoSIPv6Parser
from sos.cleaner.parsers.username_parser import SoSUsernameParser
from sos.cleaner.mappings.ip_map import SoSIPMap
from sos.cleaner.mappings.mac_map import SoSMacMap
from sos.cleaner.mappings.hostname_map import SoSHostnameMap
from sos.cleaner.mappings.keyword_map import SoSKeywordMap
from sos.cleaner.mappings.ipv6_map import SoSIPv6Map
from sos.cleaner.preppers import SoSPrepper
from sos.cleaner.preppers.hostname import HostnamePrepper
from sos.cleaner.preppers.ip import IPPrepper
from sos.cleaner.archives.sos import SoSReportArchive
from sos.options import SoSOptions
class CleanerMapTests(unittest.TestCase):
def setUp(self):
self.mac_map = SoSMacMap()
self.ip_map = SoSIPMap()
self.host_map = SoSHostnameMap()
self.host_map.sanitize_item('redhat.com')
self.kw_map = SoSKeywordMap()
self.ipv6_map = SoSIPv6Map()
def test_mac_map_obfuscate_valid_v4(self):
_test = self.mac_map.get('12:34:56:78:90:ab')
self.assertNotEqual(_test, '12:34:56:78:90:ab')
def test_mac_map_obfuscate_valid_v6(self):
_test = self.mac_map.get('12:34:56:ff:fe:78:90:ab')
self.assertNotEqual(_test, '12:34:56:ff:fe:78:90:ab')
def test_mac_map_obfuscate_valid_v6_quad(self):
_test = self.mac_map.get('1234:56ff:fe78:90ab')
self.assertNotEqual(_test, '1234:56ff:fe78:90ab')
def test_mac_map_skip_ignores(self):
_test = self.mac_map.get('ff:ff:ff:ff:ff:ff')
self.assertEqual(_test, 'ff:ff:ff:ff:ff:ff')
def test_mac_map_avoid_duplicate_obfuscation(self):
_test = self.mac_map.get('ab:cd:ef:fe:dc:ba')
_dup = self.mac_map.get(_test)
self.assertEqual(_test, _dup)
def test_ip_map_obfuscate_v4_with_cidr(self):
_test = self.ip_map.get('192.168.1.0/24')
self.assertNotEqual(_test, '192.168.1.0/24')
def test_ip_map_obfuscate_no_cidr(self):
_test = self.ip_map.get('192.168.2.2')
self.assertNotEqual(_test, '192.168.2.2')
def test_ip_map_obfuscate_same_subnet(self):
_net = ip_interface(self.ip_map.get('192.168.3.0/24'))
_test = ip_interface(self.ip_map.get('192.168.3.1'))
self.assertTrue(_test.ip in _net.network)
def test_ip_map_get_same_with_or_without_cidr(self):
_hostwsub = self.ip_map.get('192.168.4.1/24')
_hostnosub = self.ip_map.get('192.168.4.1')
self.assertEqual(_hostwsub.split('/')[0], _hostnosub)
def test_ip_skip_ignores(self):
_test = self.ip_map.get('127.0.0.1')
self.assertEqual(_test, '127.0.0.1')
def test_hostname_obfuscate_domain_options(self):
_test = self.host_map.get('www.redhat.com')
self.assertNotEqual(_test, 'www.redhat.com')
def test_hostname_obfuscate_same_item(self):
_test1 = self.host_map.get('example.redhat.com')
_test2 = self.host_map.get('example.redhat.com')
self.assertEqual(_test1, _test2)
def test_hostname_obfuscate_just_domain(self):
_test = self.host_map.get('redhat.com')
self.assertEqual(_test, 'obfuscateddomain0.com')
def test_hostname_no_obfuscate_non_loaded_domain(self):
_test = self.host_map.get('foobar.com')
self.assertEqual(_test, 'foobar.com')
def test_hostname_no_obfuscate_non_loaded_fqdn(self):
_test = self.host_map.get('example.foobar.com')
self.assertEqual(_test, 'example.foobar.com')
def test_keyword_single(self):
_test = self.kw_map.get('foobar')
self.assertEqual(_test, 'obfuscatedword0')
def test_ipv6_obfuscate_global(self):
_net = '2022:1104:abcd::'
_ob_net = self.ipv6_map.get(_net)
self.assertNotEqual(_net, _ob_net, 'Address was unchanged')
self.assertTrue(_ob_net.startswith('534f'),
'Global address does not start with identifier')
_host = '2022:1104:abcd::1234'
_ob_host = self.ipv6_map.get(_host)
self.assertNotEqual(_host, _ob_host, 'Host address was unchanged')
self.assertTrue(_host.startswith(_net), 'Host address not in network')
def test_ipv6_link_local(self):
_test = 'fe80::1234'
_ob_test = self.ipv6_map.get(_test)
self.assertTrue(_ob_test.startswith('fe80'),
'Link-local identifier not maintained')
self.assertNotEqual(_test, _ob_test, 'Device address was unchanged')
def test_ipv6_private(self):
_net = 'fd00:abcd::'
_host = 'fd00:abcd::1234'
_ob_net = self.ipv6_map.get(_net).split('/')[0]
_ob_host = self.ipv6_map.get(_host)
self.assertTrue(_ob_net.startswith('fd53'),
'Private network does not start with identifier')
self.assertTrue(_ob_host.startswith(_ob_net),
'Private address not in same network')
self.assertNotEqual(_net, _ob_net, 'Private network was unchanged')
def test_ipv6_short_network(self):
_net = 'ff02::'
_ob_net = self.ipv6_map.get(_net)
self.assertTrue(_ob_net.startswith(('53', '54')),
f'Short network does not start with identifier: '
f'{_ob_net}')
def test_ipv6_consistent_obfuscation(self):
_test = '2022:1104:abcd::ef09'
_new = self.ipv6_map.get(_test)
_second = self.ipv6_map.get(_test)
self.assertEqual(_new, _second,
"Same address produced two different results")
def test_ipv6_global_no_collision(self):
"""Tests that generating more than 256 global network obfuscations does
not produce any repeats"""
_nets = []
for i in range(1, 300):
_nets.append(self.ipv6_map.get(f"f{i:03}::abcd").split('::')[0])
# if there are any duplicates, then the length of the set will not
# match
self.assertTrue(len(set(_nets)) == len(_nets),
"Duplicate global network obfuscations produced")
self.assertTrue(_nets[-1].startswith('54'),
"First hextet of global network obfuscation over 256"
" not expected '54'")
class CleanerParserTests(unittest.TestCase):
def setUp(self):
self.ip_parser = SoSIPParser(config={})
self.ipv6_parser = SoSIPv6Parser(config={})
self.mac_parser = SoSMacParser(config={})
self.host_parser = SoSHostnameParser(config={})
self.host_parser.mapping.add('foobar.com')
self.kw_parser = SoSKeywordParser(config={})
self.kw_parser.mapping.add('foobar')
self.kw_parser_none = SoSKeywordParser(config={})
self.kw_parser.generate_item_regexes()
self.uname_parser = SoSUsernameParser(config={})
self.uname_parser.mapping.add('DOMAIN\\myusername')
self.uname_parser.mapping.add('foo')
def test_ip_parser_valid_ipv4_line(self):
line = 'foobar foo 10.0.0.1/24 barfoo bar'
_test = self.ip_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_ip_parser_invalid_ipv4_line(self):
line = 'foobar foo 10.1.2.350 barfoo bar'
self.assertRaises(ValueError, self.ip_parser.parse_line, line)
def test_ip_parser_package_version_line(self):
line = 'mycoolpackage-1.2.3.4.5'
_test = self.ip_parser.parse_line(line)[0]
self.assertEqual(line, _test)
def test_mac_parser_valid_ipv4_line(self):
line = 'foobar foo 13:24:35:46:57:68 bar barfoo'
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_mac_parser_valid_ipv6_line(self):
line = 'foobar foo AA:BB:CC:FF:FE:DD:EE:FF bar barfoo'
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_mac_parser_with_quotes(self):
line = "foobar foo '12:34:56:78:90:AA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "aa:12:bb:34:cc:56" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_mac_parser_with_quotes_ipv6(self):
line = "foobar foo 'FF:EE:DD:FF:FE:CC:BB:AA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "DD:EE:FF:FF:FE:BB:CC:AA" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_mac_parser_with_quotes_ipv6_quad(self):
line = "foobar foo 'AABB:CCDD:EEFF:FFAA' bar barfoo"
_test = self.mac_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
dline = 'foobar foo "AAFF:FFEE:DDCC:BBAA" bar barfoo'
_dtest = self.mac_parser.parse_line(dline)[0]
self.assertNotEqual(dline, _dtest)
def test_hostname_load_hostname_string(self):
fqdn = 'myhost.subnet.example.com'
self.host_parser.mapping.add(fqdn)
def test_hostname_valid_domain_line(self):
self.host_parser.mapping.add('myhost.subnet.example.com')
line = 'testing myhost.subnet.example.com in a string'
_test = self.host_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_hostname_short_name_in_line(self):
self.host_parser.mapping.add('myhost.subnet.example.com')
line = 'testing just myhost in a line'
_test = self.host_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_obfuscate_whole_fqdn_for_given_domainname(self):
self.host_parser.mapping.add('sostestdomain.domain')
line = 'let obfuscate soshost.sostestdomain.domain'
_test = self.host_parser.parse_line(line)[0]
self.assertFalse('soshost' in _test)
self.assertFalse('sostestdomain' in _test)
def test_hostname_no_obfuscate_underscore(self):
line = 'pam_env.so _why.not_'
_test = self.host_parser.parse_line(line)[0]
self.assertEqual(line, _test)
def test_keyword_parser_valid_line(self):
line = 'this is my foobar test line'
_test = self.kw_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_keyword_parser_fullword_only(self):
line = 'notfoobar and foobars line'
_test = self.kw_parser.parse_line(line)[0]
self.assertEqual(line, _test)
def test_keyword_parser_no_change_by_default(self):
line = 'this is my foobar test line'
_test = self.kw_parser_none.parse_line(line)[0]
self.assertEqual(line, _test)
def test_ipv6_parser_strings(self):
t1 = 'testing abcd:ef01::1234 as a compressed address'
t2 = 'testing abcd:ef01::5678:1234 as a separate address'
t3 = 'testing 2607:c540:8c00:3318::34/64 as another address'
t4 = ('testing 2007:1234:5678:90ab:0987:6543:21fe:dcba as a full '
'address')
t1_test = self.ipv6_parser.parse_line(t1)[0]
t2_test = self.ipv6_parser.parse_line(t2)[0]
t3_test = self.ipv6_parser.parse_line(t3)[0]
t4_test = self.ipv6_parser.parse_line(t4)[0]
self.assertNotEqual(t1, t1_test,
f"Parser did not match and obfuscate '{t1}'")
self.assertNotEqual(t2, t2_test,
f"Parser did not match and obfuscate '{t2}'")
self.assertNotEqual(t3, t3_test,
f"Parser did not match and obfuscate '{t3}'")
self.assertNotEqual(t4, t4_test,
f"Parser did not match and obfuscate '{t4}'")
def test_ipv6_no_match_signature(self):
modstr = '2D:4F:6E:55:4F:E8:5E:D2:D2:A3:73:62:AB:FD:F9:C5:A5:53:31:93'
mod_test = self.ipv6_parser.parse_line(modstr)[0]
self.assertEqual(modstr, mod_test,
"Parser matched module signature, and should not")
def test_ipv6_no_match_log_false_positive(self):
logln = ('Automatically imported trusted_ca::ca from trusted_ca/ca'
' into production')
log_test = self.ipv6_parser.parse_line(logln)[0]
self.assertEqual(logln, log_test,
"IPv6 parser incorrectly matched a log line of"
"'trusted_ca::ca'")
def test_ad_username(self):
line = "DOMAIN\\myusername"
_test = self.uname_parser.parse_line(line)[0]
self.assertNotEqual(line, _test)
def test_too_short_username(self):
line = "but foo is too short username"
_test = self.uname_parser.parse_line(line)[0]
self.assertEqual(line, _test)
class PrepperTests(unittest.TestCase):
"""
Ensure that the translations for different parser/mapping methods are
working
"""
def setUp(self):
self.prepper = SoSPrepper(SoSOptions())
self.archive = SoSReportArchive(
archive_path='tests/test_data/'
'sosreport-cleanertest-2021-08-03-qpkxdid.tar.xz',
tmpdir='/tmp'
)
self.host_prepper = HostnamePrepper(SoSOptions(domains=[]))
self.ipv4_prepper = IPPrepper(SoSOptions())
def test_parser_method_translation(self):
self.assertEqual([],
self.prepper.get_parser_file_list('hostname', None))
def test_mapping_method_translation(self):
self.assertEqual([], self.prepper.get_items_for_map('foobar', None))
def test_hostname_prepper_map_items(self):
self.assertEqual(
['cleanertest'],
self.host_prepper.get_items_for_map('hostname', self.archive)
)
def test_ipv4_prepper_parser_files(self):
self.assertEqual(
['sos_commands/networking/ip_-o_addr'],
self.ipv4_prepper.get_parser_file_list('ip', self.archive)
)
def test_ipv4_prepper_invalid_parser_files(self):
self.assertEqual(
[],
self.ipv4_prepper.get_parser_file_list('foobar', self.archive)
)
