From 31d8d182526bce894f7b9dea1dc02f2a98e652b3 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Wed, 3 Feb 2021 22:31:19 +0100 Subject: [report] add csv output format to plugins_overview.py - add CSV output format to the script - add distros and profiles info - fix some formatting issues Resolves: #2405 Signed-off-by: Pavel Moravec Signed-off-by: Jake Hunsaker --- plugins_overview.py | 84 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 21 deletions(-) (limited to 'plugins_overview.py') diff --git a/plugins_overview.py b/plugins_overview.py index 7258eb86..4eb2f694 100644 --- a/plugins_overview.py +++ b/plugins_overview.py @@ -1,9 +1,14 @@ # this script generates for each plugin: +# - URL to upstream code +# - list of distros +# - list of profiles # - list of paths it collects (add_copy_spec) # - list of paths it forbits to collect (add_forbidden_path) # - list of commands it calls (add_cmd_output) # -# Output of the script: a JSON object with plugins in keys +# Output of the script: +# - a JSON object with plugins in keys +# - or CSV format in case "csv" cmdline is provided # # TODO: # - improve parsing that will be never ideal :) @@ -11,39 +16,50 @@ # - add_blockdev_cmd # - add_string_as_file # - ?? -# - profile info? -# - distro info? -# - output in csv (multiline per plugin) import os import re import json +import sys PLUGDIR = 'sos/report/plugins' plugs_data = {} # the map of all plugins data to collect plugcontent = '' # content of plugin file just being processed -# method to parse items of a_s_c/a_c_o/.. methods to fetch list of -# copyspecs/commands without all the mess around. The method removes: -# 1) trailing/leading whitespace -# 2) trailing+leading quote marks -# 3) comments (anything after '#') -# 3) anything after ',' supposed to be an optional arguments of a_s_c/a_c_o -# methods like sizelimit or suggest_filename -def my_strip(item): - return item.strip().strip("\"'").split("#")[0].split(",")[0] +# method to parse an item of a_s_c/a_c_o/.. methods +# we work on an assumption the item is a string quoted by \" or optionally +# by \'. If we detect at least 2 such chars in the item, take what is between those. +def add_valid_item(dest, item): + for qoutemark in "\"\'": + split = item.split(qoutemark) + if len(split) > 2: + dest.append(split[1]) + return -# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) and -# to extend the `dest` list by the parsed items -def add_all_items(method, dest): - for match in re.findall("%s\((.*?)\)" % method, plugcontent, flags=re.MULTILINE|re.DOTALL): +# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) split +# by comma; add each valid item to the `dest` list +def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'): + regexp = "%s%s(.*?)%s" % (method, wrapopen, wrapclose) + for match in re.findall(regexp, plugcontent, flags=re.MULTILINE|re.DOTALL): + # tuple of distros ended by either (class|from|import) + if isinstance(match,tuple): + for item in list(match): + if item not in ['class', 'from', 'import']: + for it in item.split(','): + # dirty hack to remove spaces and "Plugin" + if "Plugin" not in it: + continue + it = it.strip(' ()')[0:-6] + if len(it): + dest.append(it) # list of specs separated by comma .. - if match.startswith('['): - dest.extend([my_strip(item) for item in match[1:-1].split(',')]) + elif match.startswith('[') or match.startswith('('): + for item in match.split(','): + add_valid_item(dest, item) # .. or a singleton spec else: - dest.append(my_strip(match)) + add_valid_item(dest, match) # main body: traverse report's plugins directory and for each plugin, grep for # add_copy_spec / add_forbidden_path / add_cmd_output there @@ -52,8 +68,12 @@ for plugfile in sorted(os.listdir(PLUGDIR)): if not plugfile.endswith('.py') or plugfile == '__init__.py': continue plugname = plugfile[:-3] +# if plugname != 'bcache': +# continue plugs_data[plugname] = { 'sourcecode': 'https://github.com/sosreport/sos/blob/master/sos/report/plugins/%s.py' % plugname, + 'distros': [], + 'profiles': [], 'copyspecs': [], 'forbidden': [], 'commands': [], @@ -62,6 +82,8 @@ for plugfile in sorted(os.listdir(PLUGDIR)): 'env': [], } plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n','') + add_all_items("from sos.report.plugins import ", plugs_data[plugname]['distros'], wrapopen='', wrapclose='(class|from|import)') + add_all_items("profiles = ", plugs_data[plugname]['profiles'], wrapopen='') add_all_items("add_copy_spec", plugs_data[plugname]['copyspecs']) add_all_items("add_forbidden_path", plugs_data[plugname]['forbidden']) add_all_items("add_cmd_output", plugs_data[plugname]['commands']) @@ -70,4 +92,24 @@ for plugfile in sorted(os.listdir(PLUGDIR)): add_all_items("add_journal", plugs_data[plugname]['journals']) add_all_items("add_env_var", plugs_data[plugname]['env']) -print(json.dumps(plugs_data)) +# print output; if "csv" is cmdline argument, print in CSV format, else JSON +if (len(sys.argv) > 1) and (sys.argv[1] == "csv"): + print("plugin;url;distros;profiles;copyspecs;forbidden;commands;service_status;journals;env_vars") + for plugname in plugs_data.keys(): + plugin = plugs_data[plugname] + # determine max number of lines - usually "max(len(copyspec),len(commands))" + # ignore 'sourcecode' key as it + maxline = 1 + plugkeys = list(plugin.keys()) + plugkeys.remove('sourcecode') + for key in plugkeys: + maxline = max(maxline, len(plugin[key])) + for line in range(maxline): + out = ";" if line>0 else ("%s;%s" % (plugname, plugin['sourcecode'])) + for key in plugkeys: + out += ";" + if line