[report] add csv output format to plugins_overview.py

- add CSV output format to the script
- add distros and profiles info
- fix some formatting issues

Resolves: #2405

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

1 files changed, 63 insertions, 21 deletions

diff --git a/plugins_overview.py b/plugins_overview.py
index 7258eb86..4eb2f694 100644
--- a/plugins_overview.py
+++ b/plugins_overview.py
@@ -1,9 +1,14 @@
 # this script generates for each plugin:
+# - URL to upstream code
+# - list of distros
+# - list of profiles
 # - list of paths it collects (add_copy_spec)
 # - list of paths it forbits to collect (add_forbidden_path)
 # - list of commands it calls (add_cmd_output)
 #
-# Output of the script: a JSON object with plugins in keys
+# Output of the script: 
+# - a JSON object with plugins in keys
+# - or CSV format in case "csv" cmdline is provided
 #
 # TODO:
 # - improve parsing that will be never ideal :)
@@ -11,39 +16,50 @@
 #   - add_blockdev_cmd
 #   - add_string_as_file
 #   - ??
-# - profile info?
-# - distro info?
-# - output in csv (multiline per plugin)
 
 import os
 import re
 import json
+import sys
 
 PLUGDIR = 'sos/report/plugins'
 
 plugs_data = {}     # the map of all plugins data to collect
 plugcontent = ''    # content of plugin file just being processed
 
-# method to parse items of a_s_c/a_c_o/.. methods to fetch list of
-# copyspecs/commands without all the mess around. The method removes:
-# 1) trailing/leading whitespace
-# 2) trailing+leading quote marks
-# 3) comments (anything after '#')
-# 3) anything after ',' supposed to be an optional arguments of a_s_c/a_c_o
-#    methods like sizelimit or suggest_filename
-def my_strip(item):
-    return item.strip().strip("\"'").split("#")[0].split(",")[0]
+# method to parse an item of a_s_c/a_c_o/.. methods
+# we work on an assumption the item is a string quoted by \" or optionally
+# by \'. If we detect at least 2 such chars in the item, take what is between those.
+def add_valid_item(dest, item):
+    for qoutemark in "\"\'":
+        split = item.split(qoutemark)
+        if len(split) > 2:
+            dest.append(split[1])
+            return
 
-# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) and
-# to extend the `dest` list by the parsed items
-def add_all_items(method, dest):
-    for match in re.findall("%s\((.*?)\)" % method, plugcontent, flags=re.MULTILINE|re.DOTALL):
+# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) split
+# by comma; add each valid item to the `dest` list
+def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
+    regexp = "%s%s(.*?)%s" % (method, wrapopen, wrapclose)
+    for match in re.findall(regexp, plugcontent, flags=re.MULTILINE|re.DOTALL):
+        # tuple of distros ended by either (class|from|import)
+        if isinstance(match,tuple):
+            for item in list(match):
+                if item not in ['class', 'from', 'import']:
+                    for it in item.split(','):
+                        # dirty hack to remove spaces and "Plugin"
+                        if "Plugin" not in it:
+                            continue
+                        it = it.strip(' ()')[0:-6]
+                        if len(it):
+                            dest.append(it)
         # list of specs separated by comma ..
-        if match.startswith('['):
-            dest.extend([my_strip(item) for item in match[1:-1].split(',')])
+        elif match.startswith('[') or match.startswith('('):
+            for item in match.split(','):
+                add_valid_item(dest, item)
         # .. or a singleton spec
         else:
-            dest.append(my_strip(match))
+            add_valid_item(dest, match)
 
 # main body: traverse report's plugins directory and for each plugin, grep for 
 # add_copy_spec / add_forbidden_path / add_cmd_output there
@@ -52,8 +68,12 @@ for plugfile in sorted(os.listdir(PLUGDIR)):
     if not plugfile.endswith('.py') or plugfile == '__init__.py':
         continue
     plugname = plugfile[:-3]
+#    if plugname != 'bcache':
+#        continue
     plugs_data[plugname] = {
             'sourcecode': 'https://github.com/sosreport/sos/blob/master/sos/report/plugins/%s.py' % plugname,
+            'distros': [],
+            'profiles': [],
             'copyspecs': [],
             'forbidden': [],
             'commands': [],
@@ -62,6 +82,8 @@ for plugfile in sorted(os.listdir(PLUGDIR)):
             'env': [],
     }
     plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n','')
+    add_all_items("from sos.report.plugins import ", plugs_data[plugname]['distros'], wrapopen='', wrapclose='(class|from|import)')
+    add_all_items("profiles = ", plugs_data[plugname]['profiles'], wrapopen='')
     add_all_items("add_copy_spec", plugs_data[plugname]['copyspecs'])
     add_all_items("add_forbidden_path", plugs_data[plugname]['forbidden'])
     add_all_items("add_cmd_output", plugs_data[plugname]['commands'])
@@ -70,4 +92,24 @@ for plugfile in sorted(os.listdir(PLUGDIR)):
     add_all_items("add_journal", plugs_data[plugname]['journals'])
     add_all_items("add_env_var", plugs_data[plugname]['env'])
 
-print(json.dumps(plugs_data))
+# print output; if "csv" is cmdline argument, print in CSV format, else JSON
+if (len(sys.argv) > 1) and (sys.argv[1] == "csv"):
+    print("plugin;url;distros;profiles;copyspecs;forbidden;commands;service_status;journals;env_vars")
+    for plugname in plugs_data.keys():
+        plugin = plugs_data[plugname]
+        # determine max number of lines - usually "max(len(copyspec),len(commands))"
+        # ignore 'sourcecode' key as it
+        maxline = 1
+        plugkeys = list(plugin.keys())
+        plugkeys.remove('sourcecode')
+        for key in plugkeys:
+            maxline = max(maxline, len(plugin[key]))
+        for line in range(maxline):
+            out = ";" if line>0 else ("%s;%s" % (plugname, plugin['sourcecode']))
+            for key in plugkeys:
+                out += ";"
+                if line<len(plugin[key]):
+                    out += plugin[key][line]
+            print(out)
+else:
+    print(json.dumps(plugs_data))