[foreman] Natively implement foreman-debug in sos

Previously, the foreman plugin just served to collect the output of the
'foreman-debug' bash script, however even the 'foreman specific' items
that were collected when the generic items were skipped included
duplicate information sos already collects in other plugins.

Additionally, after discussions with senior members of the Red Hat
Satellite support team, it was decided that porting the collection of
these bits to sos was preferable given the infrastructure sos provides
that foreman-debug does not.

The truly foreman-specific bits are now collected by the foreman plugin.
This includes log files, config files, and command output for
Satellite/Foreman-specific rubygems. Additionally, database output that
was collected by foreman-debug is now collected by the foreman plugin
and the file names are preserved - e.g. 'foreman_settings_table' remains
the same.

The katello extension for foreman-debug has been split out into a new,
separate katello plugin.

Resolves: #1525

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
Signed-off-by: Bryn M. Reeves <bmr@redhat.com>

1 files changed, 207 insertions, 7 deletions

diff --git a/sos/plugins/foreman.py b/sos/plugins/foreman.py
index b9f39c88..1f6a7afb 100644
--- a/sos/plugins/foreman.py
+++ b/sos/plugins/foreman.py
@@ -1,3 +1,4 @@
+# Copyright (C) 2018 Red Hat, Inc., Jake Hunsaker  <jhunsake@redhat.com>
 # Copyright (C) 2013 Red Hat, Inc., Lukas Zapletal <lzap@redhat.com>
 
 # This file is part of the sos project: https://github.com/sosreport/sos
@@ -8,22 +9,221 @@
 #
 # See the LICENSE file in the source distribution for further information.
 
-from sos.plugins import Plugin, RedHatPlugin
+from sos.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin,\
+                        SCLPlugin
+from pipes import quote
 
 
-class Foreman(Plugin, RedHatPlugin):
+class Foreman(Plugin):
     """Foreman/Satellite 6 systems management
     """
 
     plugin_name = 'foreman'
     profiles = ('sysmgmt',)
-    packages = ('foreman-debug',)
+    packages = ('foreman', 'foreman-proxy')
+    option_list = [
+        ('months', 'number of months for dynflow output', 'fast', 6)
+    ]
 
     def setup(self):
-        cmd = "foreman-debug"
+        self.add_forbidden_path([
+            "/etc/foreman*/*key.pem",
+            "/etc/foreman*/encryption_key.rb"
+        ])
 
-        path = self.get_cmd_output_path(name="foreman-debug")
-        self.add_cmd_output("%s -g -q -a -d %s" % (cmd, path),
-                            chroot=self.tmp_in_sysroot(), timeout=900)
+        _hostname = self.get_command_output('hostname')['output'].strip()
+        _host_f = self.get_command_output('hostname -f')['output'].strip()
+
+        # Collect these completely everytime
+        self.add_copy_spec([
+            "/var/log/foreman/production.log",
+            "/var/log/{}*/foreman-ssl_access_ssl.log".format(self.apachepkg)
+        ], sizelimit=0)
+
+        # Allow limiting these
+        self.add_copy_spec([
+            "/etc/foreman/",
+            "/etc/foreman-proxy/",
+            "/etc/sysconfig/foreman",
+            "/etc/default/foreman",
+            "/etc/foreman-installer/",
+            "/var/log/foreman/dynflow_executor*log*",
+            "/var/log/foreman/dynflow_executor*.output*",
+            "/var/log/foreman/apipie_cache*.log*",
+            "/var/log/foreman/cron*.log*",
+            "/var/log/foreman/db_migrate*log*",
+            "/var/log/foreman/db_seed*log*",
+            "/var/log/foreman/production.log-*",
+            "/var/log/foreman-proxy/cron*log*",
+            "/var/log/foreman-proxy/migrate_settings*log*",
+            "/var/log/foreman-proxy/proxy*log*",
+            "/var/log/foreman-proxy/smart_proxy_dynflow_core*log*",
+            "/var/log/foreman-selinux-install.log",
+            "/var/log/foreman-proxy-certs-generate*",
+            "/var/log/foreman-installer/",
+            "/var/log/foreman-maintain/",
+            "/var/log/syslog*",
+            # Specific to TFM, _all_ catalina logs are relevant. Adding this
+            # here rather than the tomcat plugin to ease maintenance and not
+            # pollute non-Sat sosreports that enable the tomcat plugin
+            "/var/log/tomcat*/catalina*log*",
+            "/var/log/tomcat*/host-manager*log*",
+            "/var/log/tomcat*/localhost*log*",
+            "/var/log/tomcat*/manager*log*",
+            "/usr/share/foreman/Gemfile*",
+            "/var/lib/puppet/ssl/certs/ca.pem",
+            "/etc/puppetlabs/puppet/ssl/certs/ca.pem",
+            "/etc/puppetlabs/puppet/ssl/certs/{}.pem".format(_hostname),
+            "/var/lib/puppet/ssl/certs/{}.pem".format(_hostname),
+            "/var/log/{}*/foreman-ssl_access_ssl*log*".format(self.apachepkg),
+            "/var/log/{}*/error_log*".format(self.apachepkg),
+            "/etc/{}*/conf/".format(self.apachepkg),
+            "/etc/{}*/conf.d/".format(self.apachepkg)
+        ])
+
+        self.add_cmd_output([
+            'bundle --local --gemfile=/usr/share/foreman/Gemfile*',
+            'hammer ping',
+            'foreman-selinux-relabel -nv',
+            'foreman-maintain service status',
+            'passenger-status --show pool',
+            'passenger-status --show requests',
+            'passenger-status --show backtraces',
+            'passenger-memory-stats',
+            'ls -lanR /root/ssl-build',
+            'ls -lanR /usr/share/foreman/config/hooks',
+            'ping -c1 -W1 %s' % _hostname,
+            'ping -c1 -W1 %s' % _host_f,
+            'ping -c1 -W1 localhost'
+        ])
+
+        months = '%s months' % self.get_option('months')
+
+        # Construct the DB queries, using the months option to limit the range
+        # of entries returned
+
+        scmd = (
+            "select id,name,value from settings where name not similar to "
+            "'%(pass|key|secret)%'"
+        )
+
+        authcmd = (
+            'select type,name,host,port,account,base_dn,attr_login,'
+            'onthefly_register,tls from auth_sources'
+        )
+
+        dyncmd = (
+            'select dynflow_execution_plans.* from foreman_tasks_tasks join '
+            'dynflow_execution_plans on (foreman_tasks_tasks.external_id = '
+            'dynflow_execution_plans.uuid) where foreman_tasks_tasks.'
+            'started_at > NOW() - interval %s' % quote(months)
+        )
+
+        dactioncmd = (
+             'select dynflow_actions.* from foreman_tasks_tasks join '
+             'dynflow_actions on (foreman_tasks_tasks.external_id = '
+             'dynflow_actions.execution_plan_uuid) where foreman_tasks_tasks.'
+             'started_at > NOW() - interval %s' % quote(months)
+        )
+
+        dstepscmd = (
+            'select dynflow_steps.* from foreman_tasks_tasks join '
+            'dynflow_steps on (foreman_tasks_tasks.external_id = '
+            'dynflow_steps.execution_plan_uuid) where foreman_tasks_tasks.'
+            'started_at > NOW() - interval %s' % quote(months)
+        )
+
+        # Populate this dict with DB queries that should be saved directly as
+        # postgres formats them. The key will be the filename in the foreman
+        # plugin directory, with the value being the DB query to run
+
+        foremandb = {
+            'foreman_settings_table': scmd,
+            'foreman_auth_table': authcmd,
+            'dynflow_schema_info': 'select * from dynflow_schema_info',
+            'foreman_tasks_tasks': 'select * from foreman_tasks_tasks'
+        }
+
+        # Same as above, but for CSV output
+
+        foremancsv = {
+            'dynflow_execution_plans': dyncmd,
+            'dynflow_actions': dactioncmd,
+            'dynflow_steps': dstepscmd,
+        }
+
+        for table in foremandb:
+            _cmd = self.build_query_cmd(foremandb[table])
+            self.add_cmd_output(_cmd, suggest_filename=table)
+
+        for dyn in foremancsv:
+            _cmd = self.build_query_cmd(foremancsv[dyn], csv=True)
+            self.add_cmd_output(_cmd, suggest_filename=dyn)
+
+    def build_query_cmd(self, query, csv=False):
+        """
+        Builds the command needed to invoke the pgsql query as the postgres
+        user.
+        The query requires significant quoting work to satisfy both the
+        shell and postgres parsing requirements. Note that this will generate
+        a large amount of quoting in sos logs referencing the command being run
+        """
+        _cmd = "su postgres -c %s"
+        if not csv:
+            _dbcmd = "psql foreman -c %s"
+        else:
+            _dbcmd = "psql foreman -A -F , -X -c %s"
+        dbq = _dbcmd % quote(query)
+        return _cmd % quote(dbq)
+
+    def postproc(self):
+        satreg = r"((foreman.*)?(\"::(foreman(.*?)|katello).*)?(::(.*)::.*" \
+              r"(passw|cred|token|secret|key).*(\")?:))(.*)"
+        self.do_path_regex_sub(
+            "/var/log/foreman-installer/sat*",
+            satreg,
+            r"\1 ********")
+        # need to do two passes here, debug output has different formatting
+        sat_debug_reg = (r"(\s)* (Found key: (\"(foreman(.*?)|katello)"
+                         r"::(.*(token|secret|key|passw).*)\") value:) "
+                         r"(.*)")
+        self.do_path_regex_sub(
+            "/var/log/foreman-installer/sat*",
+            sat_debug_reg,
+            r"\1 \2 ********")
+        self.do_path_regex_sub(
+            "/var/log/foreman-installer/foreman-proxy*",
+            r"(\s*proxy_password\s=) (.*)",
+            r"\1 ********")
+        self.do_path_regex_sub(
+            "/etc/foreman(.*)((yaml|yml|conf)(.*)?)",
+            r"((\:|\s*)(passw|cred|token|secret|key).*(\:\s|=))(.*)",
+            r"\1********")
+        self.do_path_regex_sub(
+            "/var/log/foreman-maintain/foreman-maintain.log*",
+            r"((passw|cred|token|secret)=)(.*)",
+            r"\1********")
+        self.do_path_regex_sub(
+            "/var/log/%s*/foreman-ssl_access_ssl.log*" % self.apachepkg,
+            r"(.*\?(passw|cred|token|secret|key).*=)(.*) (HTTP.*(.*))",
+            r"\1******** \4")
+
+# Let the base Foreman class handle the string substitution of the apachepkg
+# attr so we can keep all log definitions centralized in the main class
+
+
+class RedHatForeman(Foreman, SCLPlugin, RedHatPlugin):
+
+    apachepkg = 'httpd'
+
+    def setup(self):
+        super(RedHatForeman, self).setup()
+        self.add_cmd_output_scl('tfm', 'gem list',
+                                suggest_filename='scl enable tfm gem list')
+
+
+class DebianForeman(Foreman, DebianPlugin, UbuntuPlugin):
+
+    apachepkg = 'apache'
 
 # vim: set et ts=4 sw=4 :