From a0112b5a4b68b5bb88a01f5b33ff99ce075b8a87 Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Thu, 13 Oct 2022 15:38:58 -0400 Subject: [report,Plugin] Add a tag summary to report manifest After all plugins have run, sos will now generate a "tag summary" and add it to the report section of the manifest. This summary will be a json-formatted entry that details all collections within the report that have any tag associated with them at all. This should allow for easier parsing of the manifest by analyzers such as Red Hat Insights. As part of this change, commands will no longer be automatically tagged with the name of the binary used in the command collection. Additionally, manual collections performed by a plugin's `collect()` method will now be recorded in the manifest in the same manner as file and command output collections. Signed-off-by: Jake Hunsaker --- sos/component.py | 3 +- sos/report/__init__.py | 34 ++++++++++++++++++++++ sos/report/plugins/__init__.py | 33 ++++++++++++--------- tests/report_tests/basic_report_tests.py | 9 ++++++ .../plugin_tests/collect_manual_tests.py | 4 +-- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/sos/component.py b/sos/component.py index 91ccf91b..b2869748 100644 --- a/sos/component.py +++ b/sos/component.py @@ -440,7 +440,8 @@ class SoSMetadata(): Used to write manifest.json to the final archives. """ - return json.dumps(self, default=lambda o: getattr(o, '_values', str(o)), + return json.dumps(self, + default=lambda o: getattr(o, '_values', str(o)), indent=indent) # vim: set et ts=4 sw=4 : diff --git a/sos/report/__init__.py b/sos/report/__init__.py index bbf73d21..434b38b3 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -1489,6 +1489,8 @@ class SoSReport(SoSComponent): directory = None # report directory path (--build) map_file = None # path of the map file generated for the report + self.generate_manifest_tag_summary() + # use this instead of self.opts.clean beyond the initial check if # cleaning was requested in case SoSCleaner fails for some reason do_clean = False @@ -1703,6 +1705,38 @@ class SoSReport(SoSComponent): self.report_md.add_list('disabled_plugins', self.opts.skip_plugins) self.report_md.add_section('plugins') + def generate_manifest_tag_summary(self): + """Add a section to the manifest that contains a dict summarizing the + tags that were created and assigned during this report's creation. + + This summary dict can be used for easier inspection of tagged items by + inspection/analyzer projects such as Red Hat Insights. The format of + this dict is `{tag_name: [file_list]}`. + """ + def compile_tags(ent, key='filepath'): + for tag in ent['tags']: + if not ent[key] or not tag: + continue + try: + path = tag_summary[tag] + except KeyError: + path = [] + path.extend( + ent[key] if isinstance(ent[key], list) else [ent[key]] + ) + tag_summary[tag] = sorted(list(set(path))) + + tag_summary = {} + for plug in self.report_md.plugins: + for cmd in plug.commands: + compile_tags(cmd) + for _file in plug.files: + compile_tags(_file, 'files_copied') + for collection in plug.collections: + compile_tags(collection) + self.report_md.add_field('tag_summary', + dict(sorted(tag_summary.items()))) + def _merge_preset_options(self): # Log command line options msg = "[%s:%s] executing 'sos %s'" diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py index 76341a04..d5521ead 100644 --- a/sos/report/plugins/__init__.py +++ b/sos/report/plugins/__init__.py @@ -616,7 +616,7 @@ class Plugin(): self.manifest.add_list('files', []) self.manifest.add_field('strings', {}) self.manifest.add_field('containers', {}) - self.manifest.add_field('collections', {}) + self.manifest.add_list('collections', []) def set_default_cmd_environment(self, env_vars): """ @@ -1605,10 +1605,11 @@ class Plugin(): :returns: The tag(s) associated with `fname` :rtype: ``list`` of strings """ + tags = [] for key, val in self.filetags.items(): if re.match(key, fname): - return val - return [] + tags.extend(val) + return tags def generate_copyspec_tags(self): """After file collections have completed, retroactively generate @@ -1623,7 +1624,7 @@ class Plugin(): matched_files = [] for cfile in self.copied_files: if re.match(file_regex, cfile['srcpath']): - matched_files.append(cfile['dstpath']) + matched_files.append(cfile['dstpath'].lstrip('/')) if matched_files: manifest_data['files_copied'] = matched_files self.manifest.files.append(manifest_data) @@ -1705,12 +1706,15 @@ class Plugin(): """Generate a tag to add for a single file copyspec This tag will be set to the filename, minus any extensions - except '.conf' which will be converted to '_conf' + except for special extensions like .conf or .log, which will be + mangled to _conf or similar. """ - fname = fname.replace('-', '_') - if fname.endswith('.conf'): - return fname.replace('.', '_') - return fname.split('.')[0] + if fname.startswith(('/proc', '/sys')): + return + _fname = fname.split('/')[-1] + _fname = _fname.replace('-', '_') + if _fname.endswith(('.conf', '.log', '.txt')): + return _fname.replace('.', '_') for copyspec in copyspecs: if not (copyspec and len(copyspec)): @@ -1727,7 +1731,10 @@ class Plugin(): _spec_tags = [] if len(files) == 1: - _spec_tags = [get_filename_tag(files[0].split('/')[-1])] + _spec = get_filename_tag(files[0]) + if _spec: + _spec_tags.append(_spec) + _spec_tags.extend(self.get_tags_for_file(files[0])) _spec_tags.extend(tags) _spec_tags = list(set(_spec_tags)) @@ -2312,7 +2319,6 @@ class Plugin(): tags = [tags] _tags.extend(tags) - _tags.append(cmd.split(' ')[0]) _tags.extend(self.get_tags_for_cmd(cmd)) if cmd_as_tag: @@ -3164,10 +3170,11 @@ class Plugin(): self._log_info(f"manual collection '{fname}' finished in {run}") if isinstance(tags, str): tags = [tags] - self.manifest.collections[fname] = { + self.manifest.collections.append({ + 'name': fname, 'filepath': _pfname, 'tags': tags - } + }) except Exception as err: self._log_info(f"Error with collection file '{fname}': {err}") diff --git a/tests/report_tests/basic_report_tests.py b/tests/report_tests/basic_report_tests.py index f98d0f6e..c5d7d3ae 100644 --- a/tests/report_tests/basic_report_tests.py +++ b/tests/report_tests/basic_report_tests.py @@ -33,6 +33,15 @@ class NormalSoSReport(StageOneReportTest): def test_free_symlink_created(self): self.assertFileCollected('free') + def test_tag_summary_created(self): + self.assertTrue( + 'tag_summary' in self.manifest['components']['report'], + "No tag summary generated in report" + ) + self.assertTrue( + isinstance(self.manifest['components']['report']['tag_summary'], dict), + "Tag summary malformed" + ) class LogLevelTest(StageOneReportTest): """ diff --git a/tests/report_tests/plugin_tests/collect_manual_tests.py b/tests/report_tests/plugin_tests/collect_manual_tests.py index 7000d5e0..f63a074e 100644 --- a/tests/report_tests/plugin_tests/collect_manual_tests.py +++ b/tests/report_tests/plugin_tests/collect_manual_tests.py @@ -32,6 +32,6 @@ class CollectManualTest(StageOneReportTest): def test_manifest_collections_correct(self): pkgman = self.get_plugin_manifest('unpackaged') - self.assertTrue(pkgman['collections']['unpackaged']) + self.assertTrue(any(c['name'] == 'unpackaged' for c in pkgman['collections'])) pyman = self.get_plugin_manifest('python') - self.assertTrue(pyman['collections']['digests.json']) + self.assertTrue(any(c['name'] == 'digests.json' for c in pyman['collections'])) -- cgit