[tests] Add integration testing for Foreman

Adds an integration test for the upstream Foreman project. After
stageone tests have passed, and if the PR has changes relevant to the
Foreman plugins (or the cirrus config), launch a pre-built GCE image
that has a Foreman deployment on it.

Per request of the Foreman devel team, these tests will be run against
both a CentOS and Debian based Foreman image. Currently, testing is
performed against Foreman 2.4.

Resolves: #2531

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

3 files changed, 143 insertions, 30 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 9b57e5c1..34f6b10c 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -20,6 +20,8 @@ env:
     # These are generated images pushed to GCP from Red Hat
     FEDORA_IMAGE_NAME: "f${FEDORA_VER}-server-sos-testing"
     FEDORA_PRIOR_IMAGE_NAME: "f${FEDORA_PRIOR_VER}-server-sos-testing"
+    FOREMAN_CENTOS_IMAGE_NAME: "foreman-24-centos-8-sos-testing"
+    FOREMAN_DEBIAN_IMAGE_NAME: "foreman-24-debian-sos-testing"
 
     # Images exist on GCP already
     CENTOS_IMAGE_NAME: "centos-stream-8-v20210316"
@@ -97,16 +99,16 @@ report_stageone_task:
             BUILD_NAME: ${UBUNTU_PRIOR_NAME}
             VM_IMAGE_NAME: ${UBUNTU_PRIOR_IMAGE_NAME}
     remove_sos_script: &remove_sos |
-        if [ $(command -v apt) ] && [ $(command -v sosreport) ]; then
+        if [ $(command -v apt) ]; then
             apt -y purge sosreport
             apt update
             apt -y install python3-pip
         fi
-        if [ $(command -v dnf) ] && [ $(command -v sos) ]; then
+        if [ $(command -v dnf) ]; then
             dnf -y remove sos
         fi
     setup_script: &setup 'pip3 install avocado-framework'
-    main_script: PYTHONPATH=tests/ avocado run -t stageone tests/
+    main_script: PYTHONPATH=tests/ avocado run -t stageone tests/report_tests tests/vendor_tests/
 
 # IFF the stage one tests all pass, then run stage two for latest distros
 report_stagetwo_task:
@@ -129,4 +131,25 @@ report_stagetwo_task:
             VM_IMAGE_NAME: ${UBUNTU_IMAGE_NAME}
     remove_sos_script: *remove_sos
     setup_script: *setup
-    main_script: PYTHONPATH=tests/ avocado run -t stagetwo tests/
+    main_script: PYTHONPATH=tests/ avocado run -t stagetwo tests/report_tests tests/vendor_tests/
+
+report_foreman_task:
+    skip: "!changesInclude('.cirrus.yml', '**/{__init__,apache,foreman,foreman_tests,candlepin,pulp,pulpcore}.py')"
+    alias: "foreman_integration"
+    name: "Integration Test - Foreman ${FOREMAN_VER}"
+    depends_on: stageone_report
+    gce_instance: &bigvm
+        <<: *standardvm
+        memory: "8Gb"
+    matrix:
+        - env:
+            PROJECT: ${SOS_PROJECT}
+            VM_IMAGE_NAME: ${FOREMAN_CENTOS_IMAGE_NAME}
+            FOREMAN_VER: "2.4 - CentOS Stream 8"
+        - env:
+            PROJECT: ${SOS_PROJECT}
+            VM_IMAGE_NAME: ${FOREMAN_DEBIAN_IMAGE_NAME}
+            FOREMAN_VER: "2.4 - Debian 10"
+    remove_sos_script: *remove_sos
+    setup_script: *setup
+    main_script: PYTHONPATH=tests/ avocado run -t foreman tests/product_tests/foreman/
diff --git a/tests/product_tests/foreman/foreman_tests.py b/tests/product_tests/foreman/foreman_tests.py
new file mode 100644
index 00000000..747d2ae5
--- /dev/null
+++ b/tests/product_tests/foreman/foreman_tests.py
@@ -0,0 +1,106 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+
+from sos_tests import StageOneReportTest, redhat_only, ubuntu_only
+
+# known values in our CI test images
+FOREMAN_DB_PASSWORD = r'S0Sdb=p@ssw0rd!'
+FOREMAN_ADMIN_PASSWORD = r'S0S@dmin\\p@ssw0rd!'
+CANDLEPIN_DB_PASSWORD = r'S0SKatello%sp@ssw0rd!'
+
+FOREMAN_PASSWORDS = [FOREMAN_DB_PASSWORD, FOREMAN_ADMIN_PASSWORD, CANDLEPIN_DB_PASSWORD]
+
+
+class ForemanBasicTest(StageOneReportTest):
+    """Ensure that a basic execution runs as expected with all TFM related
+    plugins. For the Red Hat tests, it assumes Foreman has been deployed on a
+    Katello system. On Debian systems, these tests are skipped.
+
+    :avocado: tags=foreman
+    """
+
+    sos_cmd = '-v'
+
+    def test_tfm_plugins_ran(self):
+        self.assertPluginIncluded([
+            'apache',
+            'foreman',
+            'postgresql',
+            'puppet',
+            'ruby'
+        ])
+
+    @redhat_only
+    def test_candlepin_plugin_ran(self):
+        self.assertPluginIncluded('candlepin')
+
+    def test_foreman_keys_skipped(self):
+        self.assertFileGlobNotInArchive("/etc/foreman*/*key.pem")
+
+    def test_foreman_database_sizes_collected(self):
+        self.assertFileCollected('sos_commands/foreman/foreman_db_tables_sizes')
+
+    def test_foreman_installer_dirs_collected(self):
+        self.assertFileGlobInArchive("/etc/foreman-installer/*")
+        self.assertFileGlobInArchive("/var/log/foreman-installer/*")
+
+    def test_foreman_production_log_collected(self):
+        self.assertFileCollected('/var/log/foreman/production.log')
+
+    def test_foreman_database_dump_collected(self):
+        self.assertFileCollected('sos_commands/foreman/foreman_settings_table')
+
+    def test_foreman_tasks_csv_collected(self):
+        self.assertFileCollected('sos_commands/foreman/foreman_tasks_tasks')
+
+    def test_proxyfeatures_not_collected(self):
+        self.assertFileGlobNotInArchive("sos_commands/foreman/smart_proxies_features/*")
+
+    def test_foreman_config_postproc_worked(self):
+        self.assertFileNotHasContent('/etc/foreman/database.yml', FOREMAN_DB_PASSWORD)
+
+    def test_foreman_password_postproc_worked(self):
+        for _check in ['/var/log/foreman-installer/foreman.log', '/etc/foreman-installer/scenarios.d/foreman-answers.yaml']:
+            for passwd in FOREMAN_PASSWORDS:
+                self.assertFileNotHasContent(_check, passwd)
+
+    @redhat_only
+    def test_candlepin_table_sizes_collected(self):
+        self.assertFileCollected('sos_commands/candlepin/candlepin_db_tables_sizes')
+
+    @redhat_only
+    def test_katello_password_postproc_worked(self):
+        for _check in ['/var/log/foreman-installer/katello.log', '/etc/foreman-installer/scenarios.d/katello-answers.yaml']:
+            for passwd in FOREMAN_PASSWORDS:
+                self.assertFileNotHasContent(_check, passwd)
+
+    @redhat_only
+    def test_foreman_httpd_collected(self):
+        self.assertFileGlobInArchive("/var/log/httpd*/foreman-ssl_*_ssl*log*")
+
+    @ubuntu_only
+    def test_foreman_apache_collected(self):
+        self.assertFileGlobInArchive("/var/log/apache2/foreman-ssl_*_ssl*log*")
+
+    def test_ruby_gems_collected(self):
+        self.assertFileCollected('sos_commands/ruby/gem_list')
+
+
+class ForemanWithOptionsTest(StageOneReportTest):
+    """Enable Foreman/Katello/Candlepin specific options and ensure they are
+    working
+
+    :avocado: tags=foreman
+    """
+
+    sos_cmd = '-v -k foreman.proxyfeatures=on'
+
+    @redhat_only
+    def test_proxyfeatures_collected(self):
+        self.assertFileGlobInArchive("sos_commands/foreman/smart_proxies_features/*")
diff --git a/tests/sos_tests.py b/tests/sos_tests.py
index 4f50f800..5c7480f4 100644
--- a/tests/sos_tests.py
+++ b/tests/sos_tests.py
@@ -313,7 +313,7 @@ class BaseSoSReportTest(BaseSoSTest):
         :param fname:  The name of the file within the archive
         :type fname:  ``str``
         """
-        if os.path.exists(fname):
+        if fname.startswith(('sos_', '/sos_')) or os.path.exists(fname):
             self.assertFileExists(self.get_name_in_archive(fname))
         else:
             assert True
@@ -333,11 +333,15 @@ class BaseSoSReportTest(BaseSoSTest):
         :param fname:  The glob to match filenames of
         :type fname:  ``str``
         """
-        if not glob.glob(fname):
-            assert True
+        if fname.startswith(('sos_', '/sos_')):
+            files = glob.glob(os.path.join(self.archive_path, fname.lstrip('/')))
+        elif not glob.glob(fname):
+            # force the test to pass since the file glob could not have been
+            # collected
+            files = True
         else:
             files = glob.glob(os.path.join(self.archive_path, fname.lstrip('/')))
-            assert files, "No files matching %s found" % fname
+        assert files, "No files matching %s found" % fname
 
     def assertFileGlobNotInArchive(self, fname):
         """Ensure that there are NO files in the archive matching a given fname
@@ -490,15 +494,13 @@ class StageOneReportTest(BaseSoSReportTest):
     ':avocado: tags=stageone' to ensure the base tests run with new test cases
 
     :avocado: disable
-    :avocado: tags=stageone
+    :avocado: tags=stageone,foreman
     """
 
     sos_cmd = ''
 
     def test_archive_created(self):
         """Ensure that the archive tarball was created and has the right owner
-
-        :avocado: tags=stageone
         """
         self.assertFileExists(self.archive)
         self.assertTrue(os.stat(self.archive).st_uid == 0)
@@ -515,8 +517,6 @@ class StageOneReportTest(BaseSoSReportTest):
     def test_no_new_kmods_loaded(self):
         """Ensure that no additional kernel modules have been loaded during an
         execution of a test
-
-        :avocado: tags=stageone
         """
         self.assertCountEqual(self.sysinfo['pre']['modules'],
                               self.sysinfo['post']['modules'])
@@ -524,36 +524,22 @@ class StageOneReportTest(BaseSoSReportTest):
     def test_archive_has_sos_dirs(self):
         """Ensure that we have the expected directory layout with in the
         archive
-
-        :avocado: tags=stageone
         """
         self.assertFileCollected('sos_commands')
         self.assertFileCollected('sos_logs')
 
     def test_manifest_created(self):
-        """
-        :avocado: tags=stageone
-        """
         self.assertFileCollected('sos_reports/manifest.json')
 
     @skipIf(lambda x: '--no-report' in x.sos_cmd, '--no-report used in command')
     def test_html_reports_created(self):
-        """
-        :avocado: tags=stageone
-        """
         self.assertFileCollected('sos_reports/sos.html')
 
     def test_no_exceptions_during_execution(self):
-        """
-        :avocado: tags=stageone
-        """
         self.assertSosLogNotContains('caught exception in plugin')
         self.assertFileGlobNotInArchive('sos_logs/*-plugin-errors.txt')
 
     def test_no_ip_changes(self):
-        """
-        :avocado: tags=stageone
-        """
         # I.E. make sure we didn't cause any NIC flaps that for some reason
         # resulted in a new primary IP address. TODO: build this out to make
         # sure this IP is still bound to the same NIC
@@ -593,7 +579,7 @@ class StageTwoReportTest(BaseSoSReportTest):
                       tests/test_data/fake_plugins
 
     :avocado: disable
-    :avocado: tags=stagetwo
+    :avocado: tags=stagetwo,foreman2
     """
 
     sos_cmd = ''
@@ -740,8 +726,6 @@ class StageTwoReportTest(BaseSoSReportTest):
 
     def test_archive_created(self):
         """Ensure that the archive tarball was created and has the right owner
-
-        :avocado: tags=stagetwo
         """
         # kind of a hack, but since avocado test order is predicatable, we can
         # use this to avoid calling setUp() and tearDown() at each test_ method