[containers_log] Fetch all container logs

Until now, we didn't have a central plugin for collecting
service container logs. Each service was managing its own
things, and this lead to missing log files in the past.

Ensuring we fetch the content of /var/log/containers on its
own ensures we won't forget to add possible new directories,
like the "stdouts" we recently added in OpenStack in order
to get the container STDOUT written on the disk.

Therefore, this commit does 2 main things:
- cleanup all the /var/logs/containers custom copy
- create a central plugin that takes care of this location on
  its own. It's active by default and ensure the location
  actually exists.

Resolves: #1668

Signed-off-by: Cédric Jeanneret <cjeanner@redhat.com>
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

24 files changed, 37 insertions, 89 deletions

diff --git a/sos/plugins/container_log.py b/sos/plugins/container_log.py
new file mode 100644
index 00000000..6208ecc8
--- /dev/null
+++ b/sos/plugins/container_log.py
@@ -0,0 +1,34 @@
+# Copyright (C) 2019 Red Hat, Inc., Cedric Jeanneret <cjeanner@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+from sos.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin
+import os
+
+
+class ContainerLog(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin):
+    '''All container logs
+    '''
+    plugin_name = 'container_log'
+    logdir = '/var/log/containers/'
+    files = (logdir, )
+
+    def setup(self):
+        if self.get_option('all_logs'):
+            self.add_copy_spec(logdir)
+        else:
+            self.collect_subdirs()
+
+    def collect_subdirs(self, root=logdir):
+        '''Collect *.log files from subdirs of passed root path
+        '''
+        for dirName, _, _ in os.walk(root):
+            self.add_copy_spec(os.path.join(dirName, '*.log'))
+
+# vim: set et ts=4 sw=4 :
diff --git a/sos/plugins/gnocchi.py b/sos/plugins/gnocchi.py
index bc7c8c4d..988cf89b 100644
--- a/sos/plugins/gnocchi.py
+++ b/sos/plugins/gnocchi.py
@@ -43,15 +43,11 @@ class GnocchiPlugin(Plugin, RedHatPlugin):
             self.add_copy_spec([
                 "/var/log/gnocchi/*",
                 "/var/log/httpd/gnocchi*",
-                "/var/log/containers/gnocchi/*",
-                "/var/log/containers/httpd/gnocchi-api/*"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/gnocchi/*.log",
                 "/var/log/httpd/gnocchi*.log",
-                "/var/log/containers/gnocchi/*.log",
-                "/var/log/containers/httpd/gnocchi-api/*log"
             ])
 
         vars_all = [p in os.environ for p in [
diff --git a/sos/plugins/mongodb.py b/sos/plugins/mongodb.py
index 56d04e4e..5cb327dc 100644
--- a/sos/plugins/mongodb.py
+++ b/sos/plugins/mongodb.py
@@ -32,7 +32,6 @@ class MongoDb(Plugin, DebianPlugin, UbuntuPlugin):
             self.var_puppet_gen + "/etc/",
             self.var_puppet_gen + "/etc/systemd/system/mongod.service.d/",
             "/var/log/mongodb/mongodb.log",
-            "/var/log/containers/mongodb/mongodb.log",
             "/var/lib/mongodb/mongodb.log*"
         ])
         self.add_cmd_output("du -sh /var/lib/mongodb/")
diff --git a/sos/plugins/mysql.py b/sos/plugins/mysql.py
index 411d90b8..9d2cf0c8 100644
--- a/sos/plugins/mysql.py
+++ b/sos/plugins/mysql.py
@@ -34,7 +34,6 @@ class Mysql(Plugin):
             # Required for MariaDB under pacemaker (MariaDB-Galera)
             "/var/log/mysqld.log",
             "/var/log/mysql/mysqld.log",
-            "/var/log/containers/mysql/mysqld.log",
             "/var/log/mariadb/mariadb.log",
             "/var/lib/mysql/grastate.dat"
         ])
@@ -42,7 +41,6 @@ class Mysql(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/mysql*",
-                "/var/log/containers/mysql*",
                 "/var/log/mariadb*"
             ])
 
diff --git a/sos/plugins/opendaylight.py b/sos/plugins/opendaylight.py
index 2dcb1b88..47a6624a 100644
--- a/sos/plugins/opendaylight.py
+++ b/sos/plugins/opendaylight.py
@@ -28,26 +28,20 @@ class OpenDaylight(Plugin, RedHatPlugin):
 
         if self.get_option("all_logs"):
 
-            # /var/log/containers/opendaylight/ path is specific to ODL
             # Oxygen-SR3 and earlier versions, and may be removed in a future
             # update.
 
             self.add_copy_spec([
                 "/opt/opendaylight/data/log/",
-                "/var/log/containers/opendaylight/",
-                "/var/log/containers/opendaylight/karaf/logs/",
             ])
 
         else:
 
-            # /var/log/containers/opendaylight/ path is specific to ODL
             # Oxygen-SR3 and earlier versions, and may be removed in a future
             # update.
 
             self.add_copy_spec([
                 "/opt/opendaylight/data/log/*.log*",
-                "/var/log/containers/opendaylight/*.log*",
-                "/var/log/containers/opendaylight/karaf/logs/*.log*",
             ])
 
         self.add_cmd_output("docker logs opendaylight_api")
diff --git a/sos/plugins/openstack_aodh.py b/sos/plugins/openstack_aodh.py
index 2c9057a6..0f432ae1 100644
--- a/sos/plugins/openstack_aodh.py
+++ b/sos/plugins/openstack_aodh.py
@@ -38,15 +38,11 @@ class OpenStackAodh(Plugin, RedHatPlugin):
             self.add_copy_spec([
                 "/var/log/aodh/*",
                 "/var/log/httpd/aodh*",
-                "/var/log/containers/aodh/*",
-                "/var/log/containers/httpd/aodh-api/*"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/aodh/*.log",
                 "/var/log/httpd/aodh*.log",
-                "/var/log/containers/aodh/*.log",
-                "/var/log/containers/httpd/aodh-api/*log"
             ])
 
         vars_all = [p in os.environ for p in [
diff --git a/sos/plugins/openstack_ceilometer.py b/sos/plugins/openstack_ceilometer.py
index 518d7f13..c2f14502 100644
--- a/sos/plugins/openstack_ceilometer.py
+++ b/sos/plugins/openstack_ceilometer.py
@@ -28,12 +28,10 @@ class OpenStackCeilometer(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/ceilometer/*",
-                "/var/log/containers/ceilometer/*"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/ceilometer/*.log",
-                "/var/log/containers/ceilometer/*.log"
             ])
         self.add_copy_spec([
             "/etc/ceilometer/*",
diff --git a/sos/plugins/openstack_cinder.py b/sos/plugins/openstack_cinder.py
index 247f1a12..bde94a55 100644
--- a/sos/plugins/openstack_cinder.py
+++ b/sos/plugins/openstack_cinder.py
@@ -61,15 +61,11 @@ class OpenStackCinder(Plugin):
             self.add_copy_spec([
                 "/var/log/cinder/",
                 "/var/log/httpd/cinder*",
-                "/var/log/containers/cinder/",
-                "/var/log/containers/httpd/cinder-api/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/cinder/*.log",
                 "/var/log/httpd/cinder*.log",
-                "/var/log/containers/cinder/*.log",
-                "/var/log/containers/httpd/cinder-api/*log"
             ])
 
     def running_in_container(self):
diff --git a/sos/plugins/openstack_glance.py b/sos/plugins/openstack_glance.py
index 74adf53f..665d97a7 100644
--- a/sos/plugins/openstack_glance.py
+++ b/sos/plugins/openstack_glance.py
@@ -28,14 +28,10 @@ class OpenStackGlance(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/glance/",
-                "/var/log/containers/glance/",
-                "/var/log/containers/httpd/glance-api/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/glance/*.log",
-                "/var/log/containers/glance/*.log",
-                "/var/log/containers/httpd/glance-api/*log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openstack_heat.py b/sos/plugins/openstack_heat.py
index 6fa85931..8a1bbb14 100644
--- a/sos/plugins/openstack_heat.py
+++ b/sos/plugins/openstack_heat.py
@@ -61,16 +61,10 @@ class OpenStackHeat(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/heat/",
-                "/var/log/containers/heat/",
-                "/var/log/containers/httpd/heat-api/",
-                "/var/log/containers/httpd/heat-api-cfn"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/heat/*.log",
-                "/var/log/containers/heat/*.log",
-                "/var/log/containers/httpd/heat-api/*log",
-                "/var/log/containers/httpd/heat-api-cfn/*log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openstack_horizon.py b/sos/plugins/openstack_horizon.py
index 0bc75053..68999aa2 100644
--- a/sos/plugins/openstack_horizon.py
+++ b/sos/plugins/openstack_horizon.py
@@ -27,14 +27,10 @@ class OpenStackHorizon(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/horizon/",
-                "/var/log/containers/horizon/",
-                "/var/log/containers/httpd/horizon/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/horizon/*.log",
-                "/var/log/containers/horizon/*.log",
-                "/var/log/containers/httpd/horizon/*log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openstack_instack.py b/sos/plugins/openstack_instack.py
index 0d1f84c1..891d4e21 100644
--- a/sos/plugins/openstack_instack.py
+++ b/sos/plugins/openstack_instack.py
@@ -42,16 +42,12 @@ class OpenStackInstack(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/mistral/",
-                "/var/log/containers/mistral/",
                 "/var/log/zaqar/",
-                "/var/log/containers/zaqar/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/mistral/*.log",
-                "/var/log/containers/mistral/*.log",
                 "/var/log/zaqar/*.log",
-                "/var/log/containers/zaqar/*.log"
             ])
 
         vars_all = [p in os.environ for p in [
diff --git a/sos/plugins/openstack_ironic.py b/sos/plugins/openstack_ironic.py
index dc76301c..f59f1b0d 100644
--- a/sos/plugins/openstack_ironic.py
+++ b/sos/plugins/openstack_ironic.py
@@ -46,14 +46,10 @@ class OpenStackIronic(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/ironic/",
-                "/var/log/containers/ironic/",
-                "/var/log/containers/httpd/ironic-api/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/ironic/*.log",
-                "/var/log/containers/ironic/*.log",
-                "/var/log/containers/httpd/ironic-api/*log"
             ])
 
         for path in ['/var/lib/ironic', '/httpboot', '/tftpboot']:
@@ -157,12 +153,9 @@ class RedHatIronic(OpenStackIronic, RedHatPlugin):
         self.add_copy_spec('/var/lib/ironic-inspector/')
         if self.get_option("all_logs"):
             self.add_copy_spec('/var/log/ironic-inspector/')
-            self.add_copy_spec('/var/log/containers/ironic-inspector/')
         else:
             self.add_copy_spec('/var/log/ironic-inspector/*.log')
             self.add_copy_spec('/var/log/ironic-inspector/ramdisk/')
-            self.add_copy_spec('/var/log/containers/ironic-inspector/*.log')
-            self.add_copy_spec('/var/log/containers/ironic-inspector/ramdisk/')
 
         self.add_journal(units="openstack-ironic-inspector-dnsmasq")
 
diff --git a/sos/plugins/openstack_keystone.py b/sos/plugins/openstack_keystone.py
index 8005577e..7b1c56a9 100644
--- a/sos/plugins/openstack_keystone.py
+++ b/sos/plugins/openstack_keystone.py
@@ -40,14 +40,10 @@ class OpenStackKeystone(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/keystone/",
-                "/var/log/containers/keystone/",
-                "/var/log/containers/httpd/keystone/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/keystone/*.log",
-                "/var/log/containers/keystone/*.log",
-                "/var/log/containers/httpd/keystone/*log"
             ])
 
         # collect domain config directory, if specified
diff --git a/sos/plugins/openstack_manila.py b/sos/plugins/openstack_manila.py
index e6409d00..2f61e78c 100644
--- a/sos/plugins/openstack_manila.py
+++ b/sos/plugins/openstack_manila.py
@@ -33,14 +33,10 @@ class OpenStackManila(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/manila/*",
-                "/var/log/containers/manila/*",
-                "/var/log/containers/httpd/manila-api/*"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/manila/*.log",
-                "/var/log/containers/manila/*.log",
-                "/var/log/containers/httpd/manila-api/*log"
             ])
 
     def apply_regex_sub(self, regexp, subst):
diff --git a/sos/plugins/openstack_neutron.py b/sos/plugins/openstack_neutron.py
index 9d74193f..3719770c 100644
--- a/sos/plugins/openstack_neutron.py
+++ b/sos/plugins/openstack_neutron.py
@@ -25,14 +25,10 @@ class OpenStackNeutron(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/neutron/",
-                "/var/log/containers/neutron/",
-                "/var/log/containers/httpd/neutron-api/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/neutron/*.log",
-                "/var/log/containers/neutron/*.log",
-                "/var/log/containers/httpd/neutron-api/*log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openstack_nova.py b/sos/plugins/openstack_nova.py
index 22ddeb55..e197d7a5 100644
--- a/sos/plugins/openstack_nova.py
+++ b/sos/plugins/openstack_nova.py
@@ -89,18 +89,9 @@ class OpenStackNova(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/nova/",
-                "/var/log/containers/nova/",
-                "/var/log/containers/httpd/nova-api/",
-                "/var/log/containers/httpd/nova-placement/"
             ])
         else:
-            # apply sizelimit to individual logs separately
-            novadirs = [
-                "/var/log/nova/",
-                "/var/log/containers/nova/",
-                "/var/log/containers/httpd/nova-api/",
-                "/var/log/containers/httpd/nova-placement/"
-            ]
+            novadir = '/var/log/nova/'
             novalogs = [
                 "nova-api.log*",
                 "nova-compute.log*",
@@ -109,9 +100,8 @@ class OpenStackNova(Plugin):
                 "nova-placement-api.log*",
                 "nova-scheduler.log*"
             ]
-            for novadir in novadirs:
-                for novalog in novalogs:
-                    self.add_copy_spec(os.path.join(novadir, novalog))
+            for novalog in novalogs:
+                self.add_copy_spec(os.path.join(novadir, novalog))
 
         self.add_copy_spec([
             "/etc/nova/",
diff --git a/sos/plugins/openstack_octavia.py b/sos/plugins/openstack_octavia.py
index 57ba041d..40c1d635 100644
--- a/sos/plugins/openstack_octavia.py
+++ b/sos/plugins/openstack_octavia.py
@@ -35,14 +35,10 @@ class OpenStackOctavia(Plugin):
         # logs
         if self.get_option("all_logs"):
             self.add_copy_spec([
-                "/var/log/containers/httpd/octavia-api/*",
-                "/var/log/containers/octavia/*",
                 "/var/log/octavia/*",
             ])
         else:
             self.add_copy_spec([
-                "/var/log/containers/httpd/octavia-api/*.log",
-                "/var/log/containers/octavia/*.log",
                 "/var/log/octavia/*.log",
             ])
 
diff --git a/sos/plugins/openstack_sahara.py b/sos/plugins/openstack_sahara.py
index 7b8e182a..1993b58c 100644
--- a/sos/plugins/openstack_sahara.py
+++ b/sos/plugins/openstack_sahara.py
@@ -31,12 +31,10 @@ class OpenStackSahara(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/sahara/",
-                "/var/log/containers/sahara/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/sahara/*.log",
-                "/var/log/containers/sahara/*.log"
             ])
 
     def apply_regex_sub(self, regexp, subst):
diff --git a/sos/plugins/openstack_swift.py b/sos/plugins/openstack_swift.py
index 681cbf57..22329681 100644
--- a/sos/plugins/openstack_swift.py
+++ b/sos/plugins/openstack_swift.py
@@ -27,14 +27,10 @@ class OpenStackSwift(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/swift/",
-                "/var/log/containers/swift/",
-                "/var/log/containers/httpd/swift-proxy/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/swift/*.log",
-                "/var/log/containers/swift/*.log",
-                "/var/log/containers/httpd/swift-proxy/*log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openstack_trove.py b/sos/plugins/openstack_trove.py
index 75068cc2..09bb1144 100644
--- a/sos/plugins/openstack_trove.py
+++ b/sos/plugins/openstack_trove.py
@@ -26,12 +26,10 @@ class OpenStackTrove(Plugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/trove/",
-                "/var/log/containers/trove/"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/trove/*.log",
-                "/var/log/containers/trove/*.log"
             ])
 
         self.add_copy_spec([
diff --git a/sos/plugins/openvswitch.py b/sos/plugins/openvswitch.py
index dd9817fd..6e484e5a 100644
--- a/sos/plugins/openvswitch.py
+++ b/sos/plugins/openvswitch.py
@@ -25,7 +25,6 @@ class OpenVSwitch(Plugin):
         all_logs = self.get_option("all_logs")
 
         log_dirs = [
-            '/var/log/containers/openvswitch/',
             '/var/log/openvswitch/',
             '/usr/local/var/log/openvswitch/',
         ]
diff --git a/sos/plugins/rabbitmq.py b/sos/plugins/rabbitmq.py
index f7e29a98..35fa3db1 100644
--- a/sos/plugins/rabbitmq.py
+++ b/sos/plugins/rabbitmq.py
@@ -51,7 +51,6 @@ class RabbitMQ(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin):
         ])
         self.add_copy_spec([
             "/var/log/rabbitmq/*",
-            "/var/log/containers/rabbitmq/*"
         ])
 
     def postproc(self):
diff --git a/sos/plugins/redis.py b/sos/plugins/redis.py
index bca6fa29..36d90521 100644
--- a/sos/plugins/redis.py
+++ b/sos/plugins/redis.py
@@ -39,12 +39,10 @@ class Redis(Plugin, RedHatPlugin):
         if self.get_option("all_logs"):
             self.add_copy_spec([
                 "/var/log/redis/redis.log*",
-                "/var/log/containers/redis/redis.log*"
             ])
         else:
             self.add_copy_spec([
                 "/var/log/redis/redis.log",
-                "/var/log/containers/redis/redis.log"
             ])
 
     def postproc(self):