[report,plugin] Control journal size separate from log-size

Historically, journal sizes have been limited to the *higher* of 100MB
or `--log-size`. While this had the benefit of potentially controlling
both logs and journals with the same option, it was not immediately
intuitive to end users and downright prevented collecting less than
100MB of journals.

Address this by separating journal size limiting from `--log-size` by
adding a new `--journal-size` option (default 100). This will allow
users to individually control journal sizes without any "gotcha"
scenarios with relation to general log size limiting.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

6 files changed, 45 insertions, 25 deletions

diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
index 5b2b1d58..4facd556 100644
--- a/man/en/sos-report.1
+++ b/man/en/sos-report.1
@@ -27,6 +27,7 @@ sos report \- Collect and package diagnostic and support data
           [--list-profiles]\fR
           [--verify]\fR
           [--log-size]\fR
+          [--journal-size]\fR
           [--all-logs]\fR
           [--since YYYYMMDD[HHMMSS]]\fR
           [--skip-commands commands]\fR
@@ -185,14 +186,23 @@ Places a limit on the size of collected logs and output in MiB. Note that this
 causes sos to capture the last X amount of the file or command output collected.
 
 By default, this is set to 25 MiB and applies to all files and command output collected
-with the exception of journal collections, which are limited to 100 MiB.
+with the exception of journal collections, which are limited by the \fB--journal-size\fR
+option instead.
 
 Setting this value to 0 removes all size limitations, and any files or commands
 collected will be collected in their entirety, which may drastically increase the
 size of the final sos report tarball and the memory usage of sos during collection
-of commands, such as very large journals that may be several GiB in size.
+of commands.
 
 .TP
+.B \--journal-size
+Places a limit on the size of journals collected in MiB. Note that this causes sos
+to capture the last X amount of the journal.
+
+By default, this is set to 100 MiB. Setting this value to 0 removes all size limitations,
+as does the use of the \fB--all-logs\fR option. This may drastically increase the size
+of the final sos report tarball.
+.TP
 .B \--all-logs
 Tell plugins to collect all possible log data ignoring any size limits
 and including logs in non-default locations. This option may significantly
diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py
index 233d0895..78628761 100644
--- a/sos/collector/__init__.py
+++ b/sos/collector/__init__.py
@@ -88,6 +88,7 @@ class SoSCollector(SoSComponent):
         'image': '',
         'force_pull_image': True,
         'jobs': 4,
+        'journal_size': 0,
         'keywords': [],
         'keyword_file': None,
         'keep_binary_files': False,
@@ -300,11 +301,14 @@ class SoSCollector(SoSComponent):
                                   "collections. 'auto' for policy control.")
         sos_grp.add_argument('-e', '--enable-plugins', action="extend",
                              help='Enable specific plugins for sosreport')
+        sos_grp.add_argument('--journal-size', type=int, default=0,
+                             help='Limit the size of journals in MiB')
         sos_grp.add_argument('-k', '--plugin-option', '--plugopts',
                              action="extend", dest='plugopts',
                              help='Plugin option as plugname.option=value')
         sos_grp.add_argument('--log-size', default=0, type=int,
-                             help='Limit the size of individual logs (in MiB)')
+                             help='Limit the size of individual logs '
+                                  '(not journals) in MiB')
         sos_grp.add_argument('-n', '--skip-plugins', action="extend",
                              help='Skip these plugins')
         sos_grp.add_argument('-o', '--only-plugins', action="extend",
diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py
index 1562db9a..19609d98 100644
--- a/sos/collector/sosnode.py
+++ b/sos/collector/sosnode.py
@@ -645,6 +645,10 @@ class SosNode():
                     "--namespaces=%s" % self.opts.namespaces
                 )
 
+        if self.check_sos_version('4.5.2'):
+            if self.opts.journal_size:
+                sos_opts.append(f"--journal-size={self.opts.journal_size}")
+
         self.update_cmd_from_cluster()
 
         sos_cmd = sos_cmd.replace(
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 927c4ebc..c7dd858b 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -92,6 +92,7 @@ class SoSReport(SoSComponent):
         'estimate_only': False,
         'experimental': False,
         'enable_plugins': [],
+        'journal_size': 100,
         'keywords': [],
         'keyword_file': None,
         'plugopts': [],
@@ -241,6 +242,10 @@ class SoSReport(SoSComponent):
         report_grp.add_argument("-e", "--enable-plugins", action="extend",
                                 dest="enable_plugins", type=str,
                                 help="enable these plugins", default=[])
+        report_grp.add_argument("--journal-size", type=int, default=100,
+                                dest="journal_size",
+                                help="limit the size of collected journals "
+                                     "in MiB")
         report_grp.add_argument("-k", "--plugin-option", "--plugopts",
                                 action="extend",
                                 dest="plugopts", type=str,
@@ -262,7 +267,7 @@ class SoSReport(SoSComponent):
         report_grp.add_argument("--log-size", action="store", dest="log_size",
                                 type=int, default=25,
                                 help="limit the size of collected logs "
-                                     "(in MiB)")
+                                     "(not journals) in MiB")
         report_grp.add_argument("--namespaces", default=None,
                                 help="limit number of namespaces to collect "
                                      "output for - 0 means unlimited")
diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py
index b75053bf..dbb741f0 100644
--- a/sos/report/plugins/__init__.py
+++ b/sos/report/plugins/__init__.py
@@ -1561,8 +1561,8 @@ class Plugin():
         """
 
         global_options = (
-            'all_logs', 'allow_system_changes', 'cmd_timeout', 'log_size',
-            'plugin_timeout', 'since', 'verify'
+            'all_logs', 'allow_system_changes', 'cmd_timeout', 'journal_size',
+            'log_size', 'plugin_timeout', 'since', 'verify'
         )
 
         if optionname in global_options:
@@ -2932,13 +2932,11 @@ class Plugin():
         identifier_opt = " --identifier %s"
         catalog_opt = " --catalog"
 
-        journal_size = 100
-        all_logs = self.get_option("all_logs")
-        log_size = sizelimit or self.get_option("log_size")
-        log_size = max(log_size, journal_size) if not all_logs else 0
-        if sizelimit == 0:
+        if sizelimit == 0 or self.get_option("all_logs"):
             # allow for specific sizelimit overrides in plugins
             log_size = 0
+        else:
+            log_size = sizelimit or self.get_option('journal_size')
 
         if isinstance(units, str):
             units = [units]
diff --git a/tests/report_tests/plugin_tests/logs.py b/tests/report_tests/plugin_tests/logs.py
index d62ab8e2..49f1c592 100644
--- a/tests/report_tests/plugin_tests/logs.py
+++ b/tests/report_tests/plugin_tests/logs.py
@@ -30,7 +30,7 @@ class LogsPluginTest(StageOneReportTest):
         self.assertFileGlobInArchive('/var/log/journal/*')
 
 
-class LogsSizeLimitTest(StageTwoReportTest):
+class JournalSizeLimitTest(StageTwoReportTest):
     """Test that journal size limiting is working and is independent of
     --log-size
 
@@ -40,7 +40,7 @@ class LogsSizeLimitTest(StageTwoReportTest):
     :avocado: tags=stagetwo
     """
 
-    sos_cmd = '-o logs'
+    sos_cmd = '-o logs --journal-size=20 --log-size=10'
     sos_timeout = 500
     packages = {
         'rhel': ['python3-systemd'],
@@ -48,36 +48,35 @@ class LogsSizeLimitTest(StageTwoReportTest):
     }
 
     def pre_sos_setup(self):
+        # if the journal is already over our size limit, don't write anything
+        # new to it
+        from systemd import journal
+        _reader = journal.Reader()
+        _size = _reader.get_usage() / 1024 / 1024
+        if _size > 20:
+            return
         # write 20MB at a time to side-step rate/size limiting on some distros
-        # write over 100MB to ensure we will actually size limit inside sos,
+        # write over 20MB to ensure we will actually size limit inside sos,
         # allowing for any compression or de-dupe systemd does
-        from systemd import journal
         sosfd = journal.stream('sos-testing')
         rsize = 10 * 1048576
-        for i in range(6):
+        for i in range(2):
             # generate 10MB, write it, then write it in reverse.
             # Spend less time generating new strings
             rand = ''.join(random.choice(ascii_uppercase + digits) for _ in range(rsize))
             sosfd.write(rand + '\n')
             # sleep to avoid burst rate-limiting
-            sleep(10)
+            sleep(5)
             sosfd.write(rand[::-1] + '\n')
 
     def test_journal_size_limit(self):
         journ = 'sos_commands/logs/journalctl_--no-pager'
         self.assertFileCollected(journ)
         jsize = os.stat(self.get_name_in_archive(journ)).st_size
-        assert jsize <= 105906176, "Collected journal is larger than 100MB (size: %s)" % jsize
-        assert jsize > 27262976, "Collected journal limited by --log-size (size: %s)" % jsize
+        assert jsize <= 20971520, "Collected journal is larger than 20MB (size: %s)" % jsize
 
     def test_journal_tailed_and_linked(self):
         tailed = self.get_name_in_archive('sos_strings/logs/journalctl_--no-pager.tailed')
         self.assertFileExists(tailed)
         journ = self.get_name_in_archive('sos_commands/logs/journalctl_--no-pager')
         assert os.path.islink(journ), "Journal in sos_commands/logs is not a symlink"
-
-    def test_string_not_in_manifest(self):
-        # we don't want truncated collections appearing in the strings section
-        # of the manifest for the plugin
-        manifest = self.get_plugin_manifest('logs')
-        self.assertFalse(manifest['strings'])