[tests] Add test for cleaning and existing archive

Adds a test that ensures we are properly extracting and cleaning an
already existing archive (one is being included under tests/test_data/
now), much the same as we test for a proper `sos report --clean` run.

As part of this, some aspects of the `full_report_run.py` test are moved
into the base test classes, and the new `existing_archive.py` test will
explicitly run `sos clean` instead of `sos report`.

The included archive for obfuscation testing is taken from a stock
Fedora 34 VM with a known configuration which is reflected in the items
being tested for.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

4 files changed, 110 insertions, 25 deletions

diff --git a/tests/cleaner_tests/existing_archive.py b/tests/cleaner_tests/existing_archive.py
new file mode 100644
index 00000000..0eaf6c8d
--- /dev/null
+++ b/tests/cleaner_tests/existing_archive.py
@@ -0,0 +1,84 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import os
+import re
+import json
+from sos_tests import StageTwoReportTest
+
+ARCHIVE = 'sosreport-cleanertest-2021-08-03-qpkxdid'
+
+class ExistingArchiveCleanTest(StageTwoReportTest):
+    """Ensure that we can extract an already created archive and clean it the
+    same as we would an in-line run of `report --clean`.
+
+    Note that this copies heavily from the full_report_run test.
+
+    :avocado: tags=stagetwo
+    """
+
+    sos_cmd = '-v tests/test_data/%s.tar.xz' % ARCHIVE
+    sos_component = 'clean'
+
+    def test_obfuscation_log_created(self):
+        self.assertFileExists(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE))
+
+    def test_from_cmdline_logged(self):
+        with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log:
+            for line in log:
+                if 'From cmdline' in line:
+                    assert 'From cmdline: True' in line, "Did not properly log cmdline run"
+                    break
+
+    def test_extraction_completed_successfully(self):
+        with open(os.path.join(self.tmpdir, '%s-obfuscation.log' % ARCHIVE), 'r') as log:
+            for line in log:
+                if 'Extracted path is' in line:
+                    path = line.split('Extracted path is')[-1].strip()
+                    assert path.startswith(self.tmpdir), "Extracted path appears wrong: %s (tmpdir: %s)" % (path, self.tmpdir)
+                    return
+            self.fail("Extracted path not logged")
+
+    def test_private_map_was_generated(self):
+        self.assertOutputContains('A mapping of obfuscated elements is available at')
+        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        self.assertFileExists(map_file)
+
+    def test_tarball_named_obfuscated(self):
+        self.assertTrue('obfuscated' in self.archive)
+
+    def test_hostname_not_in_any_file(self):
+        # much faster to just use grep here
+        content = self.grep_for_content('cleanertest')
+        if not content:
+            assert True
+        else:
+            self.fail("Hostname appears in files: %s"
+                      % "\n".join(f for f in content))
+
+    def test_no_empty_obfuscations(self):
+        # get the private map file name
+        map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
+        with open(map_file, 'r') as mf:
+            map_json = json.load(mf)
+        for mapping in map_json:
+            for key, val in map_json[mapping].items():
+                assert key, "Empty key found in %s" % mapping
+                assert val, "%s mapping for '%s' empty" % (mapping, key)
+
+    def test_ip_not_in_any_file(self):
+        content = self.grep_for_content('10.0.0.15')
+        if not content:
+            assert True
+        else:
+            self.fail("IP appears in files: %s" % "\n".join(f for f in content))
+
+    def test_user_is_obfuscated(self):
+        """Ensure that the 'testuser1' user created at install is obfuscated
+        """
+        self.assertFileNotHasContent('var/log/anaconda/journal.log', 'testuser1')
diff --git a/tests/cleaner_tests/full_report_run.py b/tests/cleaner_tests/full_report_run.py
index 21059602..3b28e7a2 100644
--- a/tests/cleaner_tests/full_report_run.py
+++ b/tests/cleaner_tests/full_report_run.py
@@ -27,28 +27,6 @@ class FullCleanTest(StageTwoReportTest):
     # influenced by previous clean runs
     files = ['/etc/sos/cleaner/default_mapping']
 
-    def _grep_for_content(self, search):
-        """Call out to grep for finding a specific string 'search' in any place
-        in the archive
-        """
-        cmd = "grep -ril '%s' %s" % (search, self.archive_path)
-        try:
-            out = process.run(cmd)
-            rc = out.exit_status
-        except process.CmdError as err:
-            out = err.result
-            rc = err.result.exit_status
-
-        if rc == 1:
-            # grep will return an exit code of 1 if no matches are found,
-            # which is what we want
-            return False
-        else:
-            flist = []
-            for ln in out.stdout.decode('utf-8').splitlines():
-                flist.append(ln.split(self.tmpdir)[-1])
-            return flist
-
     def test_private_map_was_generated(self):
         self.assertOutputContains('A mapping of obfuscated elements is available at')
         map_file = re.findall('/.*sosreport-.*-private_map', self.cmd_output.stdout)[-1]
@@ -60,7 +38,7 @@ class FullCleanTest(StageTwoReportTest):
     def test_hostname_not_in_any_file(self):
         host = self.sysinfo['pre']['networking']['hostname']
         # much faster to just use grep here
-        content = self._grep_for_content(host)
+        content = self.grep_for_content(host)
         if not content:
             assert True
         else:
@@ -79,7 +57,7 @@ class FullCleanTest(StageTwoReportTest):
 
     def test_ip_not_in_any_file(self):
         ip = self.sysinfo['pre']['networking']['ip_addr']
-        content = self._grep_for_content(ip)
+        content = self.grep_for_content(ip)
         if not content:
             assert True
         else:
diff --git a/tests/sos_tests.py b/tests/sos_tests.py
index cae65431..56c3f0b0 100644
--- a/tests/sos_tests.py
+++ b/tests/sos_tests.py
@@ -287,6 +287,7 @@ class BaseSoSReportTest(BaseSoSTest):
     _manifest = None
     _exception_expected = False
     encrypt_pass = None
+    sos_component = 'report'
 
     @property
     def manifest(self):
@@ -317,6 +318,28 @@ class BaseSoSReportTest(BaseSoSTest):
             raise
         return _archive
 
+    def grep_for_content(self, search):
+        """Call out to grep for finding a specific string 'search' in any place
+        in the archive
+        """
+        cmd = "grep -ril '%s' %s" % (search, self.archive_path)
+        try:
+            out = process.run(cmd)
+            rc = out.exit_status
+        except process.CmdError as err:
+            out = err.result
+            rc = err.result.exit_status
+
+        if rc == 1:
+            # grep will return an exit code of 1 if no matches are found,
+            # which is what we want
+            return False
+        else:
+            flist = []
+            for ln in out.stdout.decode('utf-8').splitlines():
+                flist.append(ln.split(self.tmpdir)[-1])
+            return flist
+
     def get_encrypted_path(self):
         """Since avocado re-instantiates a new object for every test_ method,
         we need to be able to retrieve the original path for the encrypted
@@ -347,7 +370,7 @@ class BaseSoSReportTest(BaseSoSTest):
         return os.path.join(self.tmpdir, "sosreport-%s" % self.__class__.__name__)
         
     def _generate_sos_command(self):
-        return "%s report --batch --tmp-dir %s %s" % (SOS_BIN, self.tmpdir, self.sos_cmd)
+        return "%s %s --batch --tmp-dir %s %s" % (SOS_BIN, self.sos_component, self.tmpdir, self.sos_cmd)
 
     def _execute_sos_cmd(self):
         super(BaseSoSReportTest, self)._execute_sos_cmd()
diff --git a/tests/test_data/sosreport-cleanertest-2021-08-03-qpkxdid.tar.xz b/tests/test_data/sosreport-cleanertest-2021-08-03-qpkxdid.tar.xz
new file mode 100644
index 00000000..7d5fc6c2
--- /dev/null
+++ b/tests/test_data/sosreport-cleanertest-2021-08-03-qpkxdid.tar.xz