[cirrus] Upload logs from failed tests to GCE storage bucket

Adds failure handling to cirrus tasks that actually run sos so that if
an error is encountered in the test suite, the logs from the tests are
uploaded to the GCE cloud storage bucket associated with the GCE sos
project.

This will also make those logs available from the cirrus task page.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

2 files changed, 37 insertions, 4 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index cf7f77ba..c77289d4 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -124,6 +124,11 @@ report_stageone_task:
         fi
     setup_script: &setup 'pip3 install avocado-framework'
     main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t stageone tests/{cleaner,collect,report,vendor}_tests
+    on_failure:
+        fail_script: &faillogs |
+            tar cf sos-fail-logs.tar /var/tmp/avocado*
+        log_artifacts:
+            path: "sos-fail-logs.tar"
 
 # IFF the stage one tests all pass, then run stage two for latest distros
 report_stagetwo_task:
@@ -154,6 +159,10 @@ report_stagetwo_task:
         fi
     setup_script: *setup
     main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t stagetwo tests/{cleaner,collect,report,vendor}_tests
+    on_failure:
+        fail_script: *faillogs
+        log_artifacts:
+            path: "sos-fail-logs.tar"
 
 report_foreman_task:
     skip: "!changesInclude('.cirrus.yml', '**/{__init__,apache,foreman,foreman_tests,candlepin,pulp,pulpcore}.py')"
@@ -175,3 +184,7 @@ report_foreman_task:
     remove_sos_script: *remove_sos
     setup_script: *setup
     main_script: PYTHONPATH=tests/ avocado run --test-runner=runner -t foreman tests/product_tests/foreman/
+    on_failure:
+        fail_script: *faillogs
+        log_artifacts:
+            path: "sos-fail-logs.tar"
diff --git a/tests/sos_tests.py b/tests/sos_tests.py
index 6b817d56..eeab1c2a 100644
--- a/tests/sos_tests.py
+++ b/tests/sos_tests.py
@@ -68,6 +68,7 @@ class BaseSoSTest(Test):
     sos_timeout = 300
     redhat_only = False
     ubuntu_only = False
+    end_of_test_case = False
 
     @property
     def klass_name(self):
@@ -255,6 +256,23 @@ class BaseSoSTest(Test):
                     break
         self.sysinfo = self.get_sysinfo()
 
+    def tearDown(self):
+        """If a test being run is the last one defined for a test case, then
+        we should remove the extracted tarball directory so that we can upload
+        a reasonably sized artifact to the GCE storage bucket if any tests
+        fail during the test execution.
+
+        The use of `end_of_test_case` is a bit wonky because we don't have a
+        different/more reliable way to identify that a test class has completed
+        all tests - mainly because avocado re-initializes the entire class
+        for each `test_*` method defined within it.
+        """
+        if self.end_of_test_case:
+            # remove the extracted directory only if we have the tarball
+            if self.archive and os.path.exists(self.archive):
+                if os.path.exists(self.archive_path):
+                    shutil.rmtree(self.archive_path)
+
     def setup_mocking(self):
         """Since we need to use setUp() in our overrides of avocado.Test,
         provide an alternate method for test cases that subclass BaseSoSTest
@@ -654,10 +672,6 @@ class StageOneReportTest(BaseSoSReportTest):
     def test_html_reports_created(self):
         self.assertFileCollected('sos_reports/sos.html')
 
-    def test_no_exceptions_during_execution(self):
-        self.assertSosLogNotContains('caught exception in plugin')
-        self.assertFileGlobNotInArchive('sos_logs/*-plugin-errors.txt')
-
     def test_no_ip_changes(self):
         # I.E. make sure we didn't cause any NIC flaps that for some reason
         # resulted in a new primary IP address. TODO: build this out to make
@@ -665,6 +679,11 @@ class StageOneReportTest(BaseSoSReportTest):
         self.assertEqual(self.sysinfo['pre']['networking']['ip_addr'],
                          self.sysinfo['post']['networking']['ip_addr'])
 
+    def test_no_exceptions_during_execution(self):
+        self.end_of_test_case = True
+        self.assertSosLogNotContains('caught exception in plugin')
+        self.assertFileGlobNotInArchive('sos_logs/*-plugin-errors.txt')
+
 
 class StageTwoReportTest(BaseSoSReportTest):
     """This is the testing class to subclass when light mocking is needed to
@@ -736,6 +755,7 @@ class StageTwoReportTest(BaseSoSReportTest):
     def tearDown(self):
         if self.end_of_test_case:
             self.teardown_mocking()
+            super(StageTwoReportTest, self).tearDown()
 
     def teardown_mocking(self):
         """Undo any and all mocked setup that we did for tests