[testing] Add most testing via tox.ini

Consolidate testing, so that things can be run from CLI, CI and
various other means in a consistent way by using tox.

* Updated all the flake8 tests in the tests folder.
* Added pylint as a neutral test so that we can work on this in stages
  and have some collaboration on what we test and don't
* The tox tests for unit, stageone, stagetwo testing makes is easier
  for users to know how to run tests, and not have to do things
  manually
* Using tox for CI doesn't make sense, as that will create virtual
  envs and will disregard system/snap based python modules so may not
  work

Signed-off-by: Arif Ali <arif.ali@canonical.com>

18 files changed, 90 insertions, 43 deletions

diff --git a/tests/report_tests/__init__.py b/tests/report_tests/__init__.py
index 8b137891..e69de29b 100644
--- a/tests/report_tests/__init__.py
+++ b/tests/report_tests/__init__.py
@@ -1 +0,0 @@
-
diff --git a/tests/report_tests/basic_report_tests.py b/tests/report_tests/basic_report_tests.py
index 58618cbc..acdf5e76 100644
--- a/tests/report_tests/basic_report_tests.py
+++ b/tests/report_tests/basic_report_tests.py
@@ -40,7 +40,10 @@ class NormalSoSReport(StageOneReportTest):
             "No tag summary generated in report"
         )
         self.assertTrue(
-            isinstance(self.manifest['components']['report']['tag_summary'], dict),
+            isinstance(
+                self.manifest['components']['report']['tag_summary'],
+                dict
+            ),
             "Tag summary malformed"
         )
 
@@ -73,7 +76,8 @@ class RestrictedSoSReport(StageOneReportTest):
     :avocado: tags=stageone
     """
 
-    sos_cmd = '-o kernel,host,sudo,hardware,dbus,x11 --no-env-var --no-report -t1 --no-postproc'
+    sos_cmd = ('-o kernel,host,sudo,hardware,dbus,x11 --no-env-var '
+               '--no-report -t1 --no-postproc')
 
     def test_no_env_vars_collected(self):
         self.assertFileNotCollected('environment')
@@ -90,7 +94,8 @@ class RestrictedSoSReport(StageOneReportTest):
         self.assertOutputNotContains('substituting')
 
     def test_only_selected_plugins_run(self):
-        self.assertOnlyPluginsIncluded(['kernel', 'host', 'sudo', 'hardware', 'dbus', 'x11'])
+        self.assertOnlyPluginsIncluded(['kernel', 'host', 'sudo',
+                                        'hardware', 'dbus', 'x11'])
 
 
 class DisabledCollectionsReport(StageOneReportTest):
@@ -98,7 +103,8 @@ class DisabledCollectionsReport(StageOneReportTest):
     :avocado: tags=stageone
     """
 
-    sos_cmd = "-n networking,system,logs --skip-files=/etc/fstab --skip-commands='journalctl*'"
+    sos_cmd = ("-n networking,system,logs --skip-files=/etc/fstab "
+               "--skip-commands='journalctl*'")
 
     def test_plugins_disabled(self):
         self.assertPluginNotIncluded('networking')
@@ -115,4 +121,3 @@ class DisabledCollectionsReport(StageOneReportTest):
 
     def test_skip_commands_working(self):
         self.assertFileGlobNotInArchive('sos_commands/*/journalctl*')
-
diff --git a/tests/report_tests/command_priority_tests.py b/tests/report_tests/command_priority_tests.py
index ea6b38fa..f5a601de 100644
--- a/tests/report_tests/command_priority_tests.py
+++ b/tests/report_tests/command_priority_tests.py
@@ -33,7 +33,9 @@ class CommandPriorityTest(StageOneReportTest):
     def test_process_correct_priorities(self):
         cmds = self.get_plugin_manifest('process')['commands']
         # ensure root symlinked ps ran first
-        self.assertTrue(cmds[0]['priority'] == 1 and 'ps_aux' in cmds[0]['tags'])
+        self.assertTrue(
+            cmds[0]['priority'] == 1 and 'ps_aux' in cmds[0]['tags']
+        )
 
         # get lsof and iotop command entries
         _lsof = None
@@ -49,6 +51,3 @@ class CommandPriorityTest(StageOneReportTest):
         self.assertEqual(_lsof['priority'], 50)
         self.assertEqual(_iotop['priority'], 100)
         self.assertTrue(_lsof['start_time'] < _iotop['start_time'])
-
-            
-        
diff --git a/tests/report_tests/encryption_tests.py b/tests/report_tests/encryption_tests.py
index d60d9d78..64b01fde 100644
--- a/tests/report_tests/encryption_tests.py
+++ b/tests/report_tests/encryption_tests.py
@@ -25,8 +25,9 @@ class EncryptedReportTest(StageOneReportTest):
         self.assertOutputContains(r'/.*sosreport-.*tar.*\.gpg')
         _cmd = "file %s" % self.encrypted_path
         res = process.run(_cmd)
-        self.assertTrue(("GPG symmetrically encrypted data" in res.stdout.decode())
-                        or ("PGP symmetric key encrypted data" in res.stdout.decode()))
+        self.assertTrue(
+            ("GPG symmetrically encrypted data" in res.stdout.decode())
+            or ("PGP symmetric key encrypted data" in res.stdout.decode()))
 
     def test_tarball_named_secure(self):
         self.assertTrue('secured-' in self.encrypted_path)
@@ -49,4 +50,7 @@ class EncryptedCleanedReportTest(EncryptedReportTest):
         self.assertTrue('obfuscated' in self.archive)
 
     def test_ip_address_was_obfuscated(self):
-        self.assertFileNotHasContent('ip_addr', self.sysinfo['pre']['networking']['ip_addr'])
+        self.assertFileNotHasContent(
+            'ip_addr',
+            self.sysinfo['pre']['networking']['ip_addr']
+        )
diff --git a/tests/report_tests/exception_tests.py b/tests/report_tests/exception_tests.py
index 678c06d6..a5f1b8ab 100644
--- a/tests/report_tests/exception_tests.py
+++ b/tests/report_tests/exception_tests.py
@@ -28,7 +28,8 @@ class InvalidPluginOptionTest(StageOneReportExceptionTest):
     sos_cmd = '-o kernel -k kernel.colonel=on'
 
     def test_caught_invalid_plugin_option(self):
-        self.assertOutputContains(r'no such option "colonel" for plugin \(kernel\)')
+        self.assertOutputContains('no such option "colonel" for plugin '
+                                  r'\(kernel\)')
 
 
 class InvalidReportOptionTest(StageOneReportExceptionTest):
@@ -51,4 +52,5 @@ class InvalidPluginDisableTest(StageOneReportTest):
     sos_cmd = '-n logs,foobar,networking'
 
     def test_caught_invalid_plugin_name(self):
-        self.assertOutputContains("Requested to skip non-existing plugin 'foobar'")
+        self.assertOutputContains("Requested to skip non-existing plugin "
+                                  "'foobar'")
diff --git a/tests/report_tests/help_output_tests.py b/tests/report_tests/help_output_tests.py
index f1a4104b..8321f193 100644
--- a/tests/report_tests/help_output_tests.py
+++ b/tests/report_tests/help_output_tests.py
@@ -41,11 +41,11 @@ class ReportListPluginsTest(StageOneOutputTest):
         self.assertOutputContains('Profiles:')
 
     def test_no_missing_plugin_descriptions(self):
-        _out = re.search("The following plugins are currently enabled:(.*?)The following plugins are currently disabled:",
+        _out = re.search("The following plugins are currently enabled:(.*?)"
+                         "The following plugins are currently disabled:",
                          self.cmd_output.stdout, re.S).group(1).splitlines()
         for ln in _out:
             # Ignore newlines
             if not ln:
                 continue
             assert len(ln) > 1, "Plugin '%s' missing description" % ln[0]
-
diff --git a/tests/report_tests/low_priority_tests.py b/tests/report_tests/low_priority_tests.py
index 97bf0a28..0859cf73 100644
--- a/tests/report_tests/low_priority_tests.py
+++ b/tests/report_tests/low_priority_tests.py
@@ -30,4 +30,7 @@ class LowPrioTest(StageOneReportTest):
 
     def test_niceness_set(self):
         self.assertSosLogContains('Set niceness of report to 19')
-        self.assertEqual(self.manifest['components']['report']['priority']['niceness'], 19)
+        self.assertEqual(
+            self.manifest['components']['report']['priority']['niceness'],
+            19
+        )
diff --git a/tests/report_tests/options_tests/options_tests.py b/tests/report_tests/options_tests/options_tests.py
index e912da8d..15f0b572 100644
--- a/tests/report_tests/options_tests/options_tests.py
+++ b/tests/report_tests/options_tests/options_tests.py
@@ -27,10 +27,14 @@ class OptionsFromConfigTest(StageTwoReportTest):
 
     def test_plugopts_logged_from_config(self):
         self.assertSosLogContains(
-            r"Set kernel plugin option to \(name=with-timer, desc='gather /proc/timer\* statistics', value=True, default=False\)"
+            r"Set kernel plugin option to \(name=with-timer, "
+            r"desc='gather /proc/timer\* statistics', value=True, "
+            r"default=False\)"
         )
         self.assertSosLogContains(
-            r"Set kernel plugin option to \(name=trace, desc='gather /sys/kernel/debug/tracing/trace file', value=True, default=False\)"
+            r"Set kernel plugin option to \(name=trace, "
+            "desc='gather /sys/kernel/debug/tracing/trace file', "
+            r"value=True, default=False\)"
         )
 
     def test_disabled_plugopts_not_loaded(self):
@@ -41,5 +45,7 @@ class OptionsFromConfigTest(StageTwoReportTest):
 
     def test_effective_options_logged_correctly(self):
         self.assertSosLogContains(
-            "effective options now: --batch --case-id 8675309 --only-plugins host,kernel --plugopts kernel.with-timer=on,kernel.trace=yes"
+            "effective options now: --batch --case-id 8675309 "
+            "--only-plugins host,kernel "
+            "--plugopts kernel.with-timer=on,kernel.trace=yes"
         )
diff --git a/tests/report_tests/plugin_tests/collect_manual_tests.py b/tests/report_tests/plugin_tests/collect_manual_tests.py
index f63a074e..b0c92333 100644
--- a/tests/report_tests/plugin_tests/collect_manual_tests.py
+++ b/tests/report_tests/plugin_tests/collect_manual_tests.py
@@ -32,6 +32,10 @@ class CollectManualTest(StageOneReportTest):
 
     def test_manifest_collections_correct(self):
         pkgman = self.get_plugin_manifest('unpackaged')
-        self.assertTrue(any(c['name'] == 'unpackaged' for c in pkgman['collections']))
+        self.assertTrue(
+            any(c['name'] == 'unpackaged' for c in pkgman['collections'])
+        )
         pyman = self.get_plugin_manifest('python')
-        self.assertTrue(any(c['name'] == 'digests.json' for c in pyman['collections']))
+        self.assertTrue(
+            any(c['name'] == 'digests.json' for c in pyman['collections'])
+        )
diff --git a/tests/report_tests/plugin_tests/defaults.py b/tests/report_tests/plugin_tests/defaults.py
index 4cdaabd4..4d790114 100644
--- a/tests/report_tests/plugin_tests/defaults.py
+++ b/tests/report_tests/plugin_tests/defaults.py
@@ -31,7 +31,8 @@ class DefaultCollectionsTest(StageTwoReportTest):
         assert ent, "No manifest entry for systemctl status cups"
 
     def test_journal_collected(self):
-        self.assertFileCollected('sos_commands/cups/journalctl_--no-pager_--unit_cups')
+        self.assertFileCollected('sos_commands/cups/journalctl_--no-pager_'
+                                 '--unit_cups')
         _m = self.get_plugin_manifest('cups')
         ent = None
         for cmd in _m['commands']:
@@ -39,4 +40,6 @@ class DefaultCollectionsTest(StageTwoReportTest):
                 ent = cmd
         assert ent, "No manifest entry for journalctl cups"
 
-        assert 'journal_cups' in ent['tags'], "Journal tags not correct: %s" % ent['tags']
+        assert \
+            'journal_cups' in ent['tags'], \
+            f"Journal tags not correct: {ent['tags']}"
diff --git a/tests/report_tests/plugin_tests/krb5.py b/tests/report_tests/plugin_tests/krb5.py
index ab6b2379..260e5736 100644
--- a/tests/report_tests/plugin_tests/krb5.py
+++ b/tests/report_tests/plugin_tests/krb5.py
@@ -9,6 +9,7 @@
 
 from sos_tests import StageTwoReportTest, redhat_only, ubuntu_only
 
+
 class Krb5PluginTest(StageTwoReportTest):
     """Ensure that the krb5 plugin activates for the distros that we support it
     on.
diff --git a/tests/report_tests/plugin_tests/logs.py b/tests/report_tests/plugin_tests/logs.py
index 49f1c592..c3811b3f 100644
--- a/tests/report_tests/plugin_tests/logs.py
+++ b/tests/report_tests/plugin_tests/logs.py
@@ -14,6 +14,7 @@ from sos_tests import StageOneReportTest, StageTwoReportTest
 from string import ascii_uppercase, digits
 from time import sleep
 
+
 class LogsPluginTest(StageOneReportTest):
     """Ensure common collections from the `logs` plugin are properly collected
 
@@ -24,7 +25,8 @@ class LogsPluginTest(StageOneReportTest):
 
     def test_journalctl_collections(self):
         self.assertFileCollected('sos_commands/logs/journalctl_--disk-usage')
-        self.assertFileCollected('sos_commands/logs/journalctl_--no-pager_--boot')
+        self.assertFileCollected('sos_commands/logs/journalctl_--no-pager_'
+                                 '--boot')
 
     def test_journal_runtime_collected(self):
         self.assertFileGlobInArchive('/var/log/journal/*')
@@ -63,7 +65,8 @@ class JournalSizeLimitTest(StageTwoReportTest):
         for i in range(2):
             # generate 10MB, write it, then write it in reverse.
             # Spend less time generating new strings
-            rand = ''.join(random.choice(ascii_uppercase + digits) for _ in range(rsize))
+            rand = ''.join(
+                random.choice(ascii_uppercase + digits) for _ in range(rsize))
             sosfd.write(rand + '\n')
             # sleep to avoid burst rate-limiting
             sleep(5)
@@ -73,10 +76,16 @@ class JournalSizeLimitTest(StageTwoReportTest):
         journ = 'sos_commands/logs/journalctl_--no-pager'
         self.assertFileCollected(journ)
         jsize = os.stat(self.get_name_in_archive(journ)).st_size
-        assert jsize <= 20971520, "Collected journal is larger than 20MB (size: %s)" % jsize
+        assert \
+            jsize <= 20971520, \
+            f"Collected journal is larger than 20MB (size: {jsize})"
 
     def test_journal_tailed_and_linked(self):
-        tailed = self.get_name_in_archive('sos_strings/logs/journalctl_--no-pager.tailed')
+        tailed = self.get_name_in_archive('sos_strings/logs/'
+                                          'journalctl_--no-pager.tailed')
         self.assertFileExists(tailed)
-        journ = self.get_name_in_archive('sos_commands/logs/journalctl_--no-pager')
-        assert os.path.islink(journ), "Journal in sos_commands/logs is not a symlink"
+        journ = self.get_name_in_archive('sos_commands/logs/'
+                                         'journalctl_--no-pager')
+        assert \
+            os.path.islink(journ), \
+            "Journal in sos_commands/logs is not a symlink"
diff --git a/tests/report_tests/plugin_tests/sos_extras/sos_extras.py b/tests/report_tests/plugin_tests/sos_extras/sos_extras.py
index ae5c347a..4c7f9783 100644
--- a/tests/report_tests/plugin_tests/sos_extras/sos_extras.py
+++ b/tests/report_tests/plugin_tests/sos_extras/sos_extras.py
@@ -25,8 +25,10 @@ class SosExtrasPluginTest(StageTwoReportTest):
         self.assertPluginIncluded('sos_extras')
 
     def test_setup_message_displayed(self):
-        self.assertOutputContains('Collecting data from extras file /etc/sos/extras.d/sos_testing.conf')
+        self.assertOutputContains('Collecting data from extras file '
+                                  '/etc/sos/extras.d/sos_testing.conf')
 
     def test_extras_config_parsed(self):
         self.assertFileCollected('/etc/fstab')
-        self.assertFileCollected('sos_commands/sos_extras/sos_testing.conf/echo_sos_test')
+        self.assertFileCollected('sos_commands/sos_extras/sos_testing.conf'
+                                 '/echo_sos_test')
diff --git a/tests/report_tests/plugin_tests/sudo/sudo.py b/tests/report_tests/plugin_tests/sudo/sudo.py
index e04a73a7..1539294c 100644
--- a/tests/report_tests/plugin_tests/sudo/sudo.py
+++ b/tests/report_tests/plugin_tests/sudo/sudo.py
@@ -33,4 +33,3 @@ class SudoLdapScrubbedTest(StageTwoReportTest):
 
     def test_bindpw_scrubbed(self):
         self.assertFileNotHasContent('/etc/sudo-ldap.conf', 'sostestpassword')
-
diff --git a/tests/report_tests/plugin_tests/teamd.py b/tests/report_tests/plugin_tests/teamd.py
index 1c64e457..6b1b9337 100644
--- a/tests/report_tests/plugin_tests/teamd.py
+++ b/tests/report_tests/plugin_tests/teamd.py
@@ -32,11 +32,15 @@ class TeamdPluginTest(StageTwoReportTest):
         # create the team device
         res = process.run('nmcli con add type team ifname sostesting',
                           timeout=30)
-        assert res.exit_status == 0, "Failed creating team device: %s" % res.stdout_text
+        assert \
+            res.exit_status == 0, \
+            f"Failed creating team device: {res.stdout_text}"
 
     def post_test_tear_down(self):
         res = process.run('nmcli con delete team-sostesting', timeout=30)
-        assert res.exit_status == 0, "Failed to delete temp team device: %s" % res.stdout_text
+        assert \
+            res.exit_status == 0, \
+            f"Failed to delete temp team device: {res.stdout_text}"
 
     def test_teamd_plugin_executed(self):
         self.assertPluginIncluded('teamd')
diff --git a/tests/report_tests/smoke_tests.py b/tests/report_tests/smoke_tests.py
index 2d5c41e3..593e9a0c 100644
--- a/tests/report_tests/smoke_tests.py
+++ b/tests/report_tests/smoke_tests.py
@@ -52,9 +52,14 @@ class AllPluginSmokeTest(StageOneReportTest):
 
         Make sure our warnings are displayed
         """
-        self.assertOutputContains('Not logged in to OCP API, and no login token provided. Will not collect `oc` commands')
-        self.assertOutputContains('Source the environment file for the user intended to connect to the OpenStack environment.')
-        self.assertOutputContains('Some or all of the skydive params are not set properly.')
+        self.assertOutputContains('Not logged in to OCP API, and no login '
+                                  'token provided. Will not collect `oc` '
+                                  'commands')
+        self.assertOutputContains('Source the environment file for the user '
+                                  'intended to connect to the OpenStack '
+                                  'environment.')
+        self.assertOutputContains('Some or all of the skydive params are not '
+                                  'set properly.')
 
 
 class ExpectedDefaultPluginsTest(StageOneReportTest):
@@ -109,4 +114,3 @@ class ExpectedDefaultPluginsTest(StageOneReportTest):
             'apt',
             'ubuntu'
         ])
-
diff --git a/tests/report_tests/timeout/timeout_test.py b/tests/report_tests/timeout/timeout_test.py
index cfb148d7..8bc5ae61 100644
--- a/tests/report_tests/timeout/timeout_test.py
+++ b/tests/report_tests/timeout/timeout_test.py
@@ -15,7 +15,6 @@ class TimeoutTest(Plugin, IndependentPlugin):
     short_desc = 'Tests timeout functionality in test suite'
     plugin_timeout = 100
 
-
     def setup(self):
         self.add_cmd_output('sleep 15')
         self.add_cmd_output('echo I slept great', suggest_filename='echo_good')
diff --git a/tests/report_tests/timeout/timeout_tests.py b/tests/report_tests/timeout/timeout_tests.py
index 94117e8e..f18b7b38 100644
--- a/tests/report_tests/timeout/timeout_tests.py
+++ b/tests/report_tests/timeout/timeout_tests.py
@@ -50,9 +50,12 @@ class NativeCmdTimeoutTest(StageTwoReportTest):
         self.assertFileCollected('sos_commands/timeout_test/echo_good')
 
     def test_command_timed_out(self):
-        self.assertSosLogContains(r"\[plugin:timeout_test\] command 'sleep 30' timed out after 10s")
+        self.assertSosLogContains(
+            r"\[plugin:timeout_test\] command 'sleep 30' timed out after 10s"
+        )
         self.assertFileCollected('sos_commands/timeout_test/sleep_30')
 
+
 class MultipleTimeoutValues(NativeCmdTimeoutTest):
     """Test that our plugin timeout option priority is functioning correctly
 
@@ -60,7 +63,8 @@ class MultipleTimeoutValues(NativeCmdTimeoutTest):
     """
 
     install_plugins = ['timeout_test']
-    sos_cmd = '-o timeout_test,host --plugin-timeout=30 -k timeout_test.timeout=60'
+    sos_cmd = ('-o timeout_test,host --plugin-timeout=30 -k '
+               'timeout_test.timeout=60')
 
     def test_correct_plugin_timeout(self):
         man = self.get_plugin_manifest('timeout_test')