[sosreport,reporting] replace HTML reports by Report subclass

Current HTML report generation is slow and its implementation is full
of scattered html code. We shall utilize Report class instead.

Additionally, add JSON report format for easy automated parsing.

Further, fixed decoding of unicode content of a report.

Fixed report_tests suite including some pycodestyle errors.

Resolves: #1713

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

4 files changed, 199 insertions, 186 deletions

diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py
index e0d56d91..4e9c4cb4 100644
--- a/sos/plugins/__init__.py
+++ b/sos/plugins/__init__.py
@@ -1417,59 +1417,6 @@ class Plugin(object):
         """
         pass
 
-    def report(self):
-        """ Present all information that was gathered in an html file that
-        allows browsing the results.
-        """
-        # make this prettier
-        html = u'<hr/><a name="%s"></a>\n' % self.name()
-
-        # Intro
-        html = html + "<h2> Plugin <em>" + self.name() + "</em></h2>\n"
-
-        # Files
-        if len(self.copied_files):
-            html = html + "<p>Files copied:<br><ul>\n"
-            for afile in self.copied_files:
-                html = html + '<li><a href="%s">%s</a>' % \
-                    (u'..' + _to_u(afile['dstpath']), _to_u(afile['srcpath']))
-                if afile['symlink'] == "yes":
-                    html = html + " (symlink to %s)" % _to_u(afile['pointsto'])
-                html = html + '</li>\n'
-            html = html + "</ul></p>\n"
-
-        # Command Output
-        if len(self.executed_commands):
-            html = html + "<p>Commands Executed:<br><ul>\n"
-            # convert file name to relative path from our root
-            # don't use relpath - these are HTML paths not OS paths.
-            for cmd in self.executed_commands:
-                if cmd["file"] and len(cmd["file"]):
-                    cmd_rel_path = u"../" + _to_u(self.commons['cmddir']) \
-                        + "/" + _to_u(cmd['file'])
-                    html = html + '<li><a href="%s">%s</a></li>\n' % \
-                        (cmd_rel_path, _to_u(cmd['exe']))
-                else:
-                    html = html + '<li>%s</li>\n' % (_to_u(cmd['exe']))
-            html = html + "</ul></p>\n"
-
-        # Alerts
-        if len(self.alerts):
-            html = html + "<p>Alerts:<br><ul>\n"
-            for alert in self.alerts:
-                html = html + '<li>%s</li>\n' % _to_u(alert)
-            html = html + "</ul></p>\n"
-
-        # Custom Text
-        if self.custom_text != "":
-            html = html + "<p>Additional Information:<br>\n"
-            html = html + _to_u(self.custom_text) + "</p>\n"
-
-        if six.PY2:
-            return html.encode('utf8')
-        else:
-            return html
-
     def check_process_by_name(self, process):
         """Checks if a named process is found in /proc/[0-9]*/cmdline.
         Returns either True or False."""
diff --git a/sos/reporting.py b/sos/reporting.py
index cb97e473..934aaa2f 100644
--- a/sos/reporting.py
+++ b/sos/reporting.py
@@ -51,12 +51,17 @@ class Report(Node):
                 self.data[node.name] = node.data
 
 
+def _decode(s):
+    """returns a string text for a given unicode/str input"""
+    return (s if isinstance(s, six.text_type) else s.decode('utf8', 'ignore'))
+
+
 class Section(Node):
     """A section is a container for leaf elements. Sections may be nested
     inside of Report objects only."""
 
     def __init__(self, name):
-        self.name = name
+        self.name = _decode(name)
         self.data = {}
 
     def can_add(self, node):
@@ -73,9 +78,9 @@ class Command(Leaf):
     ADDS_TO = "commands"
 
     def __init__(self, name, return_code, href):
-        self.data = {"name": name,
+        self.data = {"name": _decode(name),
                      "return_code": return_code,
-                     "href": href}
+                     "href": _decode(href)}
 
 
 class CopiedFile(Leaf):
@@ -83,16 +88,17 @@ class CopiedFile(Leaf):
     ADDS_TO = "copied_files"
 
     def __init__(self, name, href):
-        self.data = {"name": name,
-                     "href": href}
+        self.data = {"name": _decode(name),
+                     "href": _decode(href)}
 
 
 class CreatedFile(Leaf):
 
     ADDS_TO = "created_files"
 
-    def __init__(self, name):
-        self.data = {"name": name}
+    def __init__(self, name, href):
+        self.data = {"name": _decode(name),
+                     "href": _decode(href)}
 
 
 class Alert(Leaf):
@@ -100,7 +106,7 @@ class Alert(Leaf):
     ADDS_TO = "alerts"
 
     def __init__(self, content):
-        self.data = content
+        self.data = _decode(content)
 
 
 class Note(Leaf):
@@ -108,7 +114,7 @@ class Note(Leaf):
     ADDS_TO = "notes"
 
     def __init__(self, content):
-        self.data = content
+        self.data = _decode(content)
 
 
 def ends_bs(string):
@@ -125,32 +131,60 @@ def ends_bs(string):
 class PlainTextReport(object):
     """Will generate a plain text report from a top_level Report object"""
 
+    HEADER = ""
+    FOOTER = ""
     LEAF = "  * %(name)s"
     ALERT = "  ! %s"
     NOTE = "  * %s"
-    DIVIDER = "=" * 72
+    PLUGLISTHEADER = "Loaded Plugins:"
+    PLUGLISTITEM = "  {name}"
+    PLUGLISTSEP = "\n"
+    PLUGLISTMAXITEMS = 5
+    PLUGLISTFOOTER = ""
+    PLUGINFORMAT = "{name}"
+    PLUGDIVIDER = "=" * 72
 
     subsections = (
-        (Command, LEAF,      "-  commands executed:"),
-        (CopiedFile, LEAF,   "-  files copied:"),
-        (CreatedFile, LEAF,  "-  files created:"),
-        (Alert, ALERT,       "-  alerts:"),
-        (Note, NOTE,         "-  notes:"),
+        (Command, LEAF,      "-  commands executed:", ""),
+        (CopiedFile, LEAF,   "-  files copied:",      ""),
+        (CreatedFile, LEAF,  "-  files created:",     ""),
+        (Alert, ALERT,       "-  alerts:",            ""),
+        (Note, NOTE,         "-  notes:",             ""),
     )
 
     line_buf = []
 
     def __init__(self, report_node):
-        self.report_node = report_node
+        self.report_data = sorted(six.iteritems(report_node.data))
 
     def unicode(self):
         self.line_buf = line_buf = []
-        for section_name, section_contents in sorted(six.iteritems(
-                self.report_node.data)):
-            line_buf.append(section_name + "\n" + self.DIVIDER)
-            for type_, format_, header in self.subsections:
+
+        if (len(self.HEADER) > 0):
+            line_buf.append(self.HEADER)
+
+        # generate section/plugin list, split long list to multiple lines
+        line_buf.append(self.PLUGLISTHEADER)
+        line = ""
+        i = 0
+        plugcount = len(self.report_data)
+        for section_name, _ in self.report_data:
+            line += self.PLUGLISTITEM.format(name=section_name)
+            i += 1
+            if (i % self.PLUGLISTMAXITEMS == 0) and (i < plugcount):
+                line += self.PLUGLISTSEP
+        line += self.PLUGLISTFOOTER
+        line_buf.append(line)
+
+        for section_name, section_contents in self.report_data:
+            line_buf.append(self.PLUGDIVIDER)
+            line_buf.append(self.PLUGINFORMAT.format(name=section_name))
+            for type_, format_, header, footer in self.subsections:
                 self.process_subsection(section_contents, type_.ADDS_TO,
-                                        header, format_)
+                                        header, format_, footer)
+
+        if (len(self.FOOTER) > 0):
+            line_buf.append(self.FOOTER)
 
         # Workaround python.six mishandling of strings ending in '/' by
         # adding a single space following any '\' at end-of-line.
@@ -158,16 +192,70 @@ class PlainTextReport(object):
         line_buf = [line + " " if ends_bs(line) else line for line in line_buf]
 
         output = u'\n'.join(map(lambda i: (i if isinstance(i, six.text_type)
-                                           else six.u(i)), line_buf))
+                                           else i.decode('utf8', 'ignore')),
+                                line_buf))
         if six.PY3:
             return output
         else:
             return output.encode('utf8')
 
-    def process_subsection(self, section, key, header, format_):
+    def process_subsection(self, section, key, header, format_, footer):
         if key in section:
             self.line_buf.append(header)
             for item in section.get(key):
                 self.line_buf.append(format_ % item)
+            if (len(footer) > 0):
+                self.line_buf.append(footer)
+
+
+class HTMLReport(PlainTextReport):
+    """Will generate a HTML report from a top_level Report object"""
+
+    HEADER = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+         "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+        <head>
+            <link rel="stylesheet" type="text/css" media="screen"
+                  href="donot.css" />
+            <meta http-equiv="Content-Type" content="text/html;
+                  charset=utf-8" />
+            <title>Sos System Report</title>
+            <style>
+                td {
+                    padding: 0 5px;
+                   }
+            </style>
+        </head>
+        <body>\n"""
+    FOOTER = "</body></html>"
+    LEAF = '<li><a href="%(href)s">%(name)s</a></li>'
+    ALERT = "<li>%s</li>"
+    NOTE = "<li>%s</li>"
+    PLUGLISTHEADER = "<h3>Loaded Plugins:</h3><table><tr>"
+    PLUGLISTITEM = '<td><a href="#{name}">{name}</a></td>\n'
+    PLUGLISTSEP = "</tr>\n<tr>"
+    PLUGLISTMAXITEMS = 5
+    PLUGLISTFOOTER = "</tr></table>"
+    PLUGINFORMAT = '<a name="{name}"></a><h2>Plugin <em>{name}</em></h2>'
+    PLUGDIVIDER = "<hr/>\n"
+
+    subsections = (
+        (Command, LEAF,      "<p>Commands executed:<br><ul>", "</ul></p>"),
+        (CopiedFile, LEAF,   "<p>Files copied:<br><ul>",      "</ul></p>"),
+        (CreatedFile, LEAF,  "<p>Files created:<br><ul>",     "</ul></p>"),
+        (Alert, ALERT,       "<p>Alerts:<br><ul>",            "</ul></p>"),
+        (Note, NOTE,         "<p>Notes:<br><ul>",             "</ul></p>"),
+    )
+
+
+class JSONReport(PlainTextReport):
+    """Will generate a JSON report from a top_level Report object"""
+
+    def unicode(self):
+        output = json.dumps(self.report_data, indent=4, ensure_ascii=False)
+        if six.PY3:
+            return output
+        else:
+            return output.encode('utf8')
 
 # vim: set et ts=4 sw=4 :
diff --git a/sos/sosreport.py b/sos/sosreport.py
index ad826ddd..11cae530 100644
--- a/sos/sosreport.py
+++ b/sos/sosreport.py
@@ -37,7 +37,8 @@ from sos import _arg_defaults, SoSOptions
 import sos.policies
 from sos.archive import TarFileArchive
 from sos.reporting import (Report, Section, Command, CopiedFile, CreatedFile,
-                           Alert, Note, PlainTextReport)
+                           Alert, Note, PlainTextReport, JSONReport,
+                           HTMLReport)
 
 # PYCOMPAT
 import six
@@ -1103,9 +1104,10 @@ class SoSReport(object):
         ]) + '\n'
         self.archive.add_string(env, 'environment')
 
-    def plain_report(self):
+    def generate_reports(self):
         report = Report()
 
+        # generate report content
         for plugname, plug in self.loaded_plugins:
             section = Section(name=plugname)
 
@@ -1121,97 +1123,43 @@ class SoSReport(object):
 
             for cmd in plug.executed_commands:
                 section.add(Command(name=cmd['exe'], return_code=0,
-                                    href="../" + cmd['file']))
+                                    href=os.path.join(
+                                        "..",
+                                        self.get_commons()['cmddir'],
+                                        cmd['file']
+                                    )))
 
             for content, f in plug.copy_strings:
-                section.add(CreatedFile(name=f))
+                section.add(CreatedFile(name=f,
+                                        href=os.path.join(
+                                            "..",
+                                            "sos_strings",
+                                            plugname,
+                                            f)))
 
             report.add(section)
-        try:
-            fd = self.get_temp_file()
-            output = PlainTextReport(report).unicode()
-            fd.write(output)
-            fd.flush()
-            self.archive.add_file(fd, dest=os.path.join('sos_reports',
-                                                        'sos.txt'))
-        except (OSError, IOError) as e:
-            if e.errno in fatal_fs_errors:
-                self.ui_log.error("")
-                self.ui_log.error(" %s while writing text report"
-                                  % e.strerror)
-                self.ui_log.error("")
-                self._exit(1)
 
-    def html_report(self):
-        try:
-            self._html_report()
-        except (OSError, IOError) as e:
-            if e.errno in fatal_fs_errors:
-                self.ui_log.error("")
-                self.ui_log.error(" %s while writing HTML report"
-                                  % e.strerror)
-                self.ui_log.error("")
-                self._exit(1)
-
-    def _html_report(self):
-        # Generate the header for the html output file
-        rfd = self.get_temp_file()
-        rfd.write("""
-        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-         "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-        <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
-        <head>
-            <link rel="stylesheet" type="text/css" media="screen"
-                  href="donot.css" />
-            <meta http-equiv="Content-Type" content="text/html;
-                  charset=utf-8" />
-            <title>Sos System Report</title>
-        </head>
-        <body>
-        """)
-
-        # Make a pass to gather Alerts and a list of module names
-        allAlerts = []
-        plugNames = []
-        for plugname, plug in self.loaded_plugins:
-            for alert in plug.alerts:
-                allAlerts.append('<a href="#%s">%s</a>: %s' % (plugname,
-                                                               plugname,
-                                                               alert))
-            plugNames.append(plugname)
-
-        # Create a table of links to the module info
-        rfd.write("<hr/><h3>Loaded Plugins:</h3>")
-        rfd.write("<table><tr>\n")
-        rr = 0
-        for i in range(len(plugNames)):
-            rfd.write('<td><a href="#%s">%s</a></td>\n' % (plugNames[i],
-                                                           plugNames[i]))
-            rr = divmod(i, 4)[1]
-            if (rr == 3):
-                rfd.write('</tr>')
-        if not (rr == 3):
-            rfd.write('</tr>')
-        rfd.write('</table>\n')
-
-        rfd.write('<hr/><h3>Alerts:</h3>')
-        rfd.write('<ul>')
-        for alert in allAlerts:
-            rfd.write('<li>%s</li>' % alert)
-        rfd.write('</ul>')
-
-        # Call the report method for each plugin
-        for plugname, plug in self.loaded_plugins:
+        # print it in text, JSON and HTML formats
+        formatlist = (
+            (PlainTextReport, "sos.txt",  "text"),
+            (JSONReport,      "sos.json", "JSON"),
+            (HTMLReport,      "sos.html", "HTML")
+        )
+        for class_, filename, type_ in formatlist:
             try:
-                html = plug.report()
-            except Exception:
-                self.handle_exception()
-            else:
-                rfd.write(html)
-        rfd.write("</body></html>")
-        rfd.flush()
-        self.archive.add_file(rfd, dest=os.path.join('sos_reports',
-                                                     'sos.html'))
+                fd = self.get_temp_file()
+                output = class_(report).unicode()
+                fd.write(output)
+                fd.flush()
+                self.archive.add_file(fd, dest=os.path.join('sos_reports',
+                                                            filename))
+            except (OSError, IOError) as e:
+                if e.errno in fatal_fs_errors:
+                    self.ui_log.error("")
+                    self.ui_log.error(" %s while writing %s report"
+                                      % (e.strerror, type_))
+                    self.ui_log.error("")
+                    self._exit(1)
 
     def postproc(self):
         for plugname, plug in self.loaded_plugins:
@@ -1399,8 +1347,7 @@ class SoSReport(object):
             if not self.opts.no_env_vars:
                 self.collect_env_vars()
             if not self.opts.noreport:
-                self.html_report()
-                self.plain_report()
+                self.generate_reports()
             self.postproc()
             self.version()
             return self.final_work()
diff --git a/tests/report_tests.py b/tests/report_tests.py
index dd390669..e18c4cf4 100644
--- a/tests/report_tests.py
+++ b/tests/report_tests.py
@@ -8,8 +8,9 @@ try:
 except ImportError:
     import simplejson as json
 
-from sos.reporting import Report, Section, Command, CopiedFile, CreatedFile, Alert
-from sos.reporting import PlainTextReport
+from sos.reporting import (Report, Section, Command, CopiedFile, CreatedFile,
+                           Alert, PlainTextReport)
+
 
 class ReportTest(unittest.TestCase):
 
@@ -38,22 +39,23 @@ class ReportTest(unittest.TestCase):
         report.add(section2)
 
         expected = json.dumps({"section": {},
-                               "section2": {},})
+                               "section2": {}, })
 
         self.assertEquals(expected, str(report))
 
-
     def test_deeply_nested(self):
         report = Report()
         section = Section(name="section")
-        command = Command(name="a command", return_code=0, href="does/not/matter")
+        command = Command(name="a command", return_code=0,
+                          href="does/not/matter")
 
         section.add(command)
         report.add(section)
 
-        expected = json.dumps({"section": {"commands": [{"name": "a command",
-                                                         "return_code": 0,
-                                                         "href": "does/not/matter"}]}})
+        expected = json.dumps({"section": {
+            "commands": [{"name": "a command",
+                          "return_code": 0,
+                          "href": "does/not/matter"}]}})
 
         self.assertEquals(expected, str(report))
 
@@ -63,22 +65,37 @@ class TestPlainReport(unittest.TestCase):
     def setUp(self):
         self.report = Report()
         self.section = Section(name="plugin")
-        self.div = PlainTextReport.DIVIDER
+        self.div = '\n' + PlainTextReport.PLUGDIVIDER
+        self.pluglist = "Loaded Plugins:\n{pluglist}"
+        self.defaultheader = u''.join([
+            self.pluglist.format(pluglist="  plugin"),
+            self.div,
+            "\nplugin\n"
+        ])
 
     def test_basic(self):
-        self.assertEquals("", PlainTextReport(self.report).unicode())
+        self.assertEquals(self.pluglist.format(pluglist=""),
+                          PlainTextReport(self.report).unicode())
 
     def test_one_section(self):
         self.report.add(self.section)
 
-        self.assertEquals("plugin\n" + self.div, PlainTextReport(self.report).unicode())
+        self.assertEquals(self.defaultheader,
+                          PlainTextReport(self.report).unicode() + '\n')
 
     def test_two_sections(self):
         section1 = Section(name="first")
         section2 = Section(name="second")
         self.report.add(section1, section2)
 
-        self.assertEquals("first\n" + self.div + "\nsecond\n" + self.div, PlainTextReport(self.report).unicode())
+        self.assertEquals(u''.join([
+                            self.pluglist.format(pluglist="  first  second"),
+                            self.div,
+                            "\nfirst",
+                            self.div,
+                            "\nsecond"
+                        ]),
+                        PlainTextReport(self.report).unicode())
 
     def test_command(self):
         cmd = Command(name="ls -al /foo/bar/baz",
@@ -87,32 +104,46 @@ class TestPlainReport(unittest.TestCase):
         self.section.add(cmd)
         self.report.add(self.section)
 
-        self.assertEquals("plugin\n" + self.div + "\n-  commands executed:\n  * ls -al /foo/bar/baz",
-                PlainTextReport(self.report).unicode())
+        self.assertEquals(u''.join([
+                            self.defaultheader,
+                            "-  commands executed:\n  * ls -al /foo/bar/baz"
+                        ]),
+                        PlainTextReport(self.report).unicode())
 
     def test_copied_file(self):
         cf = CopiedFile(name="/etc/hosts", href="etc/hosts")
         self.section.add(cf)
         self.report.add(self.section)
 
-        self.assertEquals("plugin\n" + self.div + "\n-  files copied:\n  * /etc/hosts",
-                PlainTextReport(self.report).unicode())
+        self.assertEquals(u''.join([
+                            self.defaultheader,
+                            "-  files copied:\n  * /etc/hosts"
+                        ]),
+                        PlainTextReport(self.report).unicode())
 
     def test_created_file(self):
-        crf = CreatedFile(name="sample.txt")
+        crf = CreatedFile(name="sample.txt",
+                          href="../sos_strings/sample/sample.txt")
         self.section.add(crf)
         self.report.add(self.section)
 
-        self.assertEquals("plugin\n" + self.div + "\n-  files created:\n  * sample.txt",
-                PlainTextReport(self.report).unicode())
+        self.assertEquals(u''.join([
+                            self.defaultheader,
+                            "-  files created:\n  * sample.txt"
+                        ]),
+                        PlainTextReport(self.report).unicode())
 
     def test_alert(self):
         alrt = Alert("this is an alert")
         self.section.add(alrt)
         self.report.add(self.section)
 
-        self.assertEquals("plugin\n" + self.div + "\n-  alerts:\n  ! this is an alert",
-                PlainTextReport(self.report).unicode())
+        self.assertEquals(u''.join([
+                            self.defaultheader,
+                            "-  alerts:\n  ! this is an alert"
+                        ]),
+                        PlainTextReport(self.report).unicode())
+
 
 if __name__ == "__main__":
     unittest.main()