[sosreport] grab logs since date

Adds a --since switch that takes a date as an argument. This switch
will skip the archive files with a mtime older than the date.

Also, --since affects journalctl execution for --all.

Resolves: #1678

Signed-off-by: David Vallee Delisle <dvd@redhat.com>
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

6 files changed, 65 insertions, 12 deletions

diff --git a/man/en/sosreport.1 b/man/en/sosreport.1
index 271d0935..379aa92d 100644
--- a/man/en/sosreport.1
+++ b/man/en/sosreport.1
@@ -24,6 +24,7 @@ sosreport \- Collect and package diagnostic and support data
           [--verify]\fR
           [--log-size]\fR
           [--all-logs]\fR
+          [--since YYYYMMDD[HHMMSS]]\fR
           [-z|--compression-type method]\fR
           [--encrypt-key KEY]\fR
           [--encrypt-pass PASS]\fR
@@ -153,6 +154,10 @@ Tell plugins to collect all possible log data ignoring any size limits
 and including logs in non-default locations. This option may significantly
 increase the size of reports.
 .TP
+.B \--since YYYYMMDD[HHMMSS]
+Limits the collection to logs newer than this date.
+This also affects \--all-logs. Will pad with 0s if HHMMSS isn't specified.
+.TP
 .B \-z, \--compression-type METHOD
 Override the default compression type specified by the active policy.
 .TP
diff --git a/sos/__init__.py b/sos/__init__.py
index 725c003a..c24316e9 100644
--- a/sos/__init__.py
+++ b/sos/__init__.py
@@ -56,7 +56,7 @@ _arg_names = [
     'encrypt_pass', 'experimental', 'label', 'list_plugins', 'list_presets',
     'list_profiles', 'log_size', 'noplugins', 'noreport', 'no_env_vars',
     'note', 'onlyplugins', 'plugin_timeout', 'plugopts', 'preset', 'profiles',
-    'quiet', 'sysroot', 'threads', 'tmp_dir', 'verbosity', 'verify'
+    'quiet', 'since', 'sysroot', 'threads', 'tmp_dir', 'verbosity', 'verify'
 ]
 
 #: Arguments with non-zero default values
@@ -158,6 +158,7 @@ class SoSOptions(object):
         self.add_preset = ""
         self.alloptions = False
         self.all_logs = False
+        self.since = None
         self.batch = False
         self.build = False
         self.case_id = ""
diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py
index 533fa3c4..e0d56d91 100644
--- a/sos/plugins/__init__.py
+++ b/sos/plugins/__init__.py
@@ -26,6 +26,7 @@ import errno
 # PYCOMPAT
 import six
 from six.moves import zip, filter
+from datetime import datetime
 
 # FileNotFoundError does not exist in 2.7, so map it to IOError
 try:
@@ -778,7 +779,7 @@ class Plugin(object):
 
         global_options = (
             'all_logs', 'allow_system_changes', 'log_size', 'plugin_timeout',
-            'verify'
+            'since', 'verify'
         )
 
         if optionname in global_options:
@@ -806,11 +807,20 @@ class Plugin(object):
     def _add_copy_paths(self, copy_paths):
         self.copy_paths.update(copy_paths)
 
-    def add_copy_spec(self, copyspecs, sizelimit=None, tailit=True, pred=None):
-        """Add a file or glob but limit it to sizelimit megabytes. If fname is
-        a single file the file will be tailed to meet sizelimit. If the first
-        file in a glob is too large it will be tailed to meet the sizelimit.
+    def add_copy_spec(self, copyspecs, sizelimit=None, maxage=None,
+                      tailit=True, pred=None):
+        """Add a file or glob but limit it to sizelimit megabytes. Collect
+        files with mtime not older than maxage hours.
+        If fname is a single file the file will be tailed to meet sizelimit.
+        If the first file in a glob is too large it will be tailed to meet
+        the sizelimit.
         """
+        since = None
+        if self.get_option('since'):
+            since = self.get_option('since')
+
+        logarchive_pattern = re.compile(r'.*((\.(zip|gz|bz2|xz))|[-.][\d]+)$')
+
         if not self.test_predicate(pred=pred):
             self._log_info("skipped copy spec '%s' due to predicate (%s)" %
                            (copyspecs, self.get_predicate(pred=pred)))
@@ -851,6 +861,21 @@ class Plugin(object):
                 except OSError:
                     return 0
 
+            def time_filter(path):
+                """ When --since is passed, or maxage is coming from the
+                plugin, we need to filter out older files """
+
+                if logarchive_pattern.search(path) is None:
+                    return True
+                filetime = datetime.fromtimestamp(getmtime(path))
+                if ((since and filetime < since) or
+                   (maxage and (time()-filetime < maxage*3600))):
+                    return False
+                return True
+
+            if since or maxage:
+                files = list(filter(lambda f: time_filter(f), files))
+
             files.sort(key=getmtime, reverse=True)
             current_size = 0
             limit_reached = False
@@ -861,12 +886,16 @@ class Plugin(object):
                     self._log_debug("skipping forbidden path '%s'" % _file)
                     continue
                 try:
-                    current_size += os.stat(_file)[stat.ST_SIZE]
+                    filestat = os.stat(_file)
                 except OSError:
                     self._log_info("failed to stat '%s'" % _file)
+                    continue
+                current_size += filestat[stat.ST_SIZE]
+
                 if sizelimit and current_size > sizelimit:
                     limit_reached = True
                     break
+
                 self._add_copy_paths([_file])
 
             if limit_reached and tailit and not _file_is_compressed(_file):
@@ -1163,7 +1192,7 @@ class Plugin(object):
         journal_cmd = "journalctl --no-pager "
         unit_opt = " --unit %s"
         boot_opt = " --boot %s"
-        since_opt = " --since %s"
+        since_opt = " --since '%s'"
         until_opt = " --until %s"
         lines_opt = " --lines %s"
         output_opt = " --output %s"
diff --git a/sos/plugins/logs.py b/sos/plugins/logs.py
index dec01f3a..040529e8 100644
--- a/sos/plugins/logs.py
+++ b/sos/plugins/logs.py
@@ -52,7 +52,8 @@ class Logs(Plugin):
                 self.add_copy_spec(i)
 
         if self.get_option('all_logs'):
-            self.add_journal(boot="this", allfields=True, output="verbose")
+            self.add_journal(boot="this", since=self.get_option("since"),
+                             allfields=True, output="verbose")
             self.add_journal(boot="last", allfields=True, output="verbose")
 
     def postproc(self):
@@ -95,8 +96,8 @@ class RedHatLogs(Logs, RedHatPlugin):
                 days = int(self.get_option("log_days"))
             except ValueError:
                 days = 3
-            if self.get_option("all_logs"):
-                since = ""
+            if self.get_option("all_logs") and self.get_options("since"):
+                since = self.get_options("since")
             else:
                 since = "-%ddays" % days
             self.add_journal(since=since)
@@ -122,7 +123,7 @@ class DebianLogs(Logs, DebianPlugin, UbuntuPlugin):
 
         if journal and self.is_installed("systemd"):
             if self.get_option("all_logs"):
-                since = ""
+                since = self.get_options("since")
             else:
                 since = "-%ddays" % days
             self.add_journal(since=since)
diff --git a/sos/sosreport.py b/sos/sosreport.py
index f186256a..ad826ddd 100644
--- a/sos/sosreport.py
+++ b/sos/sosreport.py
@@ -21,6 +21,7 @@ import os
 import errno
 import logging
 
+from datetime import datetime
 from argparse import ArgumentParser, Action
 from sos.plugins import import_plugin
 from sos.utilities import ImporterHelper, SoSTimeoutError
@@ -64,6 +65,14 @@ def _format_list(first_line, items, indent=False, sep=", "):
     return lines
 
 
+def _format_since(date):
+    """ This function will format --since arg to append 0s if enduser
+    didn't. It's used in the _get_parser.
+    This will also be a good place to add human readable and relative
+    date parsing (like '2 days ago') in the future """
+    return datetime.strptime('{:<014s}'.format(date), '%Y%m%d%H%M%S')
+
+
 class TempFileUtil(object):
 
     def __init__(self, tmp_dir):
@@ -126,6 +135,12 @@ def _get_parser():
                         dest="all_logs", default=False,
                         help="collect all available logs regardless "
                              "of size")
+    parser.add_argument("--since", action="store",
+                        dest="since", default=None,
+                        type=_format_since,
+                        help="Escapes archived files older than date. "
+                             "This will also affect --all-logs. "
+                             "Format: YYYYMMDD[HHMMSS]")
     parser.add_argument("--batch", action="store_true",
                         dest="batch", default=False,
                         help="batch mode - do not prompt interactively")
@@ -306,6 +321,7 @@ class SoSReport(object):
                 sys.stderr.write("Unknown preset: '%s'\n" % self.opts.preset)
                 self.preset = self.policy.probe_preset()
                 self.opts.list_presets = True
+
         # --preset=auto
         if not self.preset:
             self.preset = self.policy.probe_preset()
diff --git a/tests/plugin_tests.py b/tests/plugin_tests.py
index e81575cf..b8760429 100644
--- a/tests/plugin_tests.py
+++ b/tests/plugin_tests.py
@@ -93,6 +93,7 @@ class EnablerPlugin(Plugin):
 class MockOptions(object):
     all_logs = False
     dry_run = False
+    since = None
     log_size = 25
     allow_system_changes = False