[tests] Add tests to ensure help output

In the past, there have been otherwise trivial typos and the like that
have caused `--help` output to be unreliable. In the case of "help
options" such as `--list-plugins` this also includes potentially
unavailable detailed information about whatever is being listed.

These tests are sanity checks to ensure that changes aren't regressing
this behavior and that the informational output options, and `--help`
directly, continue to provide the right output.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

4 files changed, 217 insertions, 0 deletions

diff --git a/tests/cleaner_tests/help_output_tests.py b/tests/cleaner_tests/help_output_tests.py
new file mode 100644
index 00000000..2f14438c
--- /dev/null
+++ b/tests/cleaner_tests/help_output_tests.py
@@ -0,0 +1,35 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+
+from sos_tests import StageOneOutputTest
+
+
+class CleanHelpTest(StageOneOutputTest):
+    """Basic check to make sure --help works with clean
+
+    :avocado: tags=stageone
+    """
+
+    sos_cmd = 'clean --help'
+
+    def test_all_help_sections_present(self):
+        self.assertOutputContains('Global Options:')
+        self.assertOutputContains('Cleaner/Masking Options:')
+        self.assertOutputContains('TARGET                The directory or archive to obfuscate')
+
+
+class MaskHelpTest(CleanHelpTest):
+    """The same test, but ensuring the use of the 'mask' alias works. In
+    reality this is more testing argparse rather than anything else, but it
+    is still good to ensure the aliases remain working
+
+    :avocado: tags=stageone
+    """
+
+    sos_cmd = 'mask --help'
diff --git a/tests/collect_tests/help_output_tests.py b/tests/collect_tests/help_output_tests.py
new file mode 100644
index 00000000..258bb8d7
--- /dev/null
+++ b/tests/collect_tests/help_output_tests.py
@@ -0,0 +1,104 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import glob
+import os
+import re
+
+from avocado.utils import software_manager
+from sos_tests import StageOneOutputTest, SOS_REPO_ROOT, skipIf
+
+installer = software_manager
+sm = installer.SoftwareManager()
+
+PEXPECT_PRESENT = sm.check_installed('python3-pexpect')
+
+
+class CollectHelpOutputTest(StageOneOutputTest):
+    """Ensure that --help is behaving as expected, based on if the placeholder
+    component is being used or not
+
+    Note that this is tagged for both stageone and stagetwo due to the config
+    of the GCE images used - pexpect is not normally installed, but we will
+    install it as part of the CirrusCI setup for stagetwo. This allows us to
+    perform checks for both the real and placeholder components in CI, without
+    creating a output tests for stagetwo, which would be silly for tests run on
+    contributor workstations
+
+    :avocado: tags=stageone,stagetwo
+    """
+
+    sos_cmd = 'collect --help'
+
+    @skipIf(PEXPECT_PRESENT, "python3-pexpect is installed, placeholder will not be used")
+    def test_placeholder_warning_shown(self):
+        self.assertOutputContains("WARNING: `collect` is not available with this installation!")
+
+    @skipIf(PEXPECT_PRESENT is False, "python3-pexpect not installed locally")
+    def test_help_sections_present(self):
+        self.assertOutputNotContains("WARNING: `collect` is not available with this installation!")
+        self.assertOutputContains("Global Options:")
+        self.assertOutputContains("Report Passthru Options:")
+        self.assertOutputContains("Collector Options:")
+
+
+class CollectOptionsHelpTest(StageOneOutputTest):
+    """Ensure that available cluster profiles and their options are printed
+
+    Note that this is tagged for both stageone and stagetwo due to the config
+    of the GCE images used - pexpect is not normally installed, but we will
+    install it as part of the CirrusCI setup for stagetwo. This allows us to
+    perform checks for both the real and placeholder components in CI, without
+    creating a output tests for stagetwo, which would be silly for tests run on
+    contributor workstations
+
+    :avocado: tags=stageone,stagetwo
+    """
+
+    _exception_expected = not PEXPECT_PRESENT
+    sos_cmd = 'collect --list-options'
+
+    @skipIf(PEXPECT_PRESENT is False, "python3-pexpect not installed locally")
+    def test_cluster_profiles_shown(self):
+        _out = re.search("Use the short name with --cluster-type or cluster options \(-c\)(.*?)The following cluster options are available:",
+                         self.cmd_output.stdout, re.S).group(1).splitlines()
+        _profs = {}
+        for ln in _out:
+            if not ln:
+                continue
+            ln = [c for c in ln.lstrip().split('  ') if c]
+            _profs[ln[0]] = ln[1]
+
+        clusters = []
+        # get a list of names of profile pyfiles 
+        for pyfile in glob.glob(os.path.join(SOS_REPO_ROOT, 'sos/collector/clusters/*.py')):
+            pyf = os.path.basename(pyfile)
+            if pyf.startswith('__'):
+                continue
+            clusters.append(pyf.split('.py')[0])
+
+        assert len(_profs.keys()) > 0, "No profiles detected in output"
+
+        # make sure each pyfile is reported for supported cluster types
+        # this has the secondary effect of enforcing a stylistic requirement
+        # where at least one profile must match the pyfile name
+        for clus in clusters:
+            assert clus in _profs, "Cluster '%s' not displayed in --list-options" % clus
+
+    @skipIf(PEXPECT_PRESENT is False, "python3-pexpect not installed locally")
+    def test_cluster_options_shown(self):
+        _opts = re.search(" Cluster                   Option Name     Type       Default    Description(.*?)Options take the form of cluster.name=value",
+                          self.cmd_output.stdout, re.S).group(1).splitlines()
+
+        _opts = [o for o in _opts if o]
+
+        assert _opts, "No option output detected"
+
+    @skipIf(PEXPECT_PRESENT, "python3-pexpect is installed, placeholder will be unused")
+    def test_placeholder_component_used(self):
+        self.assertOutputContains("(unavailable) Collect an sos report from multiple nodes")
diff --git a/tests/report_tests/help_output_tests.py b/tests/report_tests/help_output_tests.py
new file mode 100644
index 00000000..f1a4104b
--- /dev/null
+++ b/tests/report_tests/help_output_tests.py
@@ -0,0 +1,51 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import re
+
+from sos_tests import StageOneOutputTest
+
+
+class ReportHelpTest(StageOneOutputTest):
+    """Ensure that --help gives the expected output in the expected format
+
+    :avocado: tags=stageone
+    """
+
+    sos_cmd = 'report --help'
+
+    def test_all_help_sections_present(self):
+        self.assertOutputContains('Global Options:')
+        self.assertOutputContains('Report Options:')
+        self.assertOutputContains('Cleaner/Masking Options:')
+
+
+class ReportListPluginsTest(StageOneOutputTest):
+    """Ensure that --list-plugins gives the expected output
+
+    :avocado: tags=stageone
+    """
+
+    sos_cmd = 'report --list-plugins'
+
+    def test_all_plugin_sections_present(self):
+        self.assertOutputContains('plugins are currently enabled:')
+        self.assertOutputContains('plugins are currently disabled:')
+        self.assertOutputContains('options are available for ALL plugins:')
+        self.assertOutputContains('plugin options are available:')
+        self.assertOutputContains('Profiles:')
+
+    def test_no_missing_plugin_descriptions(self):
+        _out = re.search("The following plugins are currently enabled:(.*?)The following plugins are currently disabled:",
+                         self.cmd_output.stdout, re.S).group(1).splitlines()
+        for ln in _out:
+            # Ignore newlines
+            if not ln:
+                continue
+            assert len(ln) > 1, "Plugin '%s' missing description" % ln[0]
+
diff --git a/tests/sos_tests.py b/tests/sos_tests.py
index 9fc4ceef..84173cf8 100644
--- a/tests/sos_tests.py
+++ b/tests/sos_tests.py
@@ -840,3 +840,30 @@ class StageOneReportExceptionTest(BaseSoSReportTest):
     @skipIf(lambda x: x.archive_still_expected, "Output expected in test")
     def test_no_archive_generated(self):
         self.assertTrue(self.archive is None)
+
+
+class StageOneOutputTest(BaseSoSTest):
+    """This test class should be used for tests that are only checking or
+    validating output from a specific command or option, such as --help or
+    --list.
+
+    :avocado: disable
+    :avocado: tags=stageone
+    """
+
+    sos_cmd = ''
+
+    def _generate_sos_command(self):
+        return "%s %s" % (SOS_BIN, self.sos_cmd)
+
+    @skipIf(lambda x: x._exception_expected, "Non-zero exit code expected")
+    def test_help_output_successful(self):
+        self.assertTrue(self.cmd_output.exit_status == 0)
+        assert self.cmd_output.stdout, "No stdout output generated"
+        assert not self.cmd_output.stderr, "stderr received, but not expected: %s" % self.cmd_output.stderr
+
+    @skipIf(lambda x: not x._exception_expected, "Not anticipating stderr output")
+    def test_help_error_reported(self):
+        self.assertTrue(self.cmd_output.exit_status != 0)
+        assert not self.cmd_output.stdout, "stdout received, but not expected: %s" % self.cmd_output.stdout
+        assert self.cmd_output.stderr, "No stderr output generated"