[kubernetes] k8s improvements for Charmed Kubernetes

Add more resources to output when kubernetes.all option is true.
Remove openshift specific resources from base class.
Output json format if kubernetes.all option is true.
Export pod logs to an individual file for each container, so
the plugin can export logs if there are multiple containers in
a pod.

Signed-off-by: Shunde Zhang <shunde.zhang@canonical.com>

1 files changed, 124 insertions, 51 deletions

diff --git a/sos/report/plugins/kubernetes.py b/sos/report/plugins/kubernetes.py
index 12b8a776..41c7ddeb 100644
--- a/sos/report/plugins/kubernetes.py
+++ b/sos/report/plugins/kubernetes.py
@@ -13,6 +13,7 @@ from fnmatch import translate
 import re
 from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin,
                                 UbuntuPlugin, PluginOpt)
+import json
 
 
 class Kubernetes(Plugin):
@@ -21,6 +22,39 @@ class Kubernetes(Plugin):
 
     plugin_name = "kubernetes"
     profiles = ('container',)
+    plugin_timeout = 1200
+
+    config_files = [
+        "/etc/kubernetes",
+        "/run/flannel",
+    ]
+    resources = [
+        'events',
+        'deployments',
+        'ingresses',
+        'pods',
+        'pvc',
+        'services',
+        'daemonsets',
+        'replicasets',
+        'endpoints',
+        'statefulsets',
+        'configmaps',
+        'serviceaccounts',
+        'secrets',
+        'jobs',
+        'cronjobs',
+        'clusterroles',
+        'clusterrolebindings'
+    ]
+
+    # these are not namespaced, must pull separately.
+    global_resources = [
+        'sc',
+        'pv',
+        'roles',
+        'rolebindings'
+    ]
 
     option_list = [
         PluginOpt('all', default=False,
@@ -34,26 +68,13 @@ class Kubernetes(Plugin):
     ]
 
     kube_cmd = "kubectl"
-    resources = [
-        'deployments',
-        'ingresses',
-        'limitranges',
-        'pods',
-        'policies',
-        'pvc',
-        'rc',
-        'resourcequotas',
-        'routes',
-        'services'
-    ]
 
     def check_is_master(self):
         """ Check if this is the master node """
         return any(self.path_exists(f) for f in self.files)
 
     def setup(self):
-        self.add_copy_spec("/etc/kubernetes")
-        self.add_copy_spec("/run/flannel")
+        self.add_copy_spec(self.config_files)
 
         self.add_env_var([
             'KUBECONFIG',
@@ -83,21 +104,26 @@ class Kubernetes(Plugin):
             return
 
         for subcmd in ['version', 'config view']:
-            self.add_cmd_output('%s %s' % (self.kube_cmd, subcmd))
-
-        # these are not namespaced, must pull separately.
-        global_resources = [
-            'namespaces',
-            'projects',
-            'pvs'
-        ]
-        self.add_cmd_output([
-            "%s get %s" % (self.kube_cmd, res) for res in global_resources
-        ])
+            self.add_cmd_output(
+                f'{self.kube_cmd} {subcmd}',
+                subdir='cluster-info'
+            )
+
+        if self.get_option('all'):
+            self.add_cmd_output([
+                f"{self.kube_cmd} get -o json {res}"
+                for res in self.global_resources
+            ], subdir='cluster-info')
+        else:
+            self.add_cmd_output([
+                f"{self.kube_cmd} get {res}"
+                for res in self.global_resources
+            ], subdir='cluster-info')
 
         # Get detailed node information
-        nodes = self.collect_cmd_output("%s get nodes" % self.kube_cmd)
-        if nodes['status'] == 0:
+        nodes = self.collect_cmd_output(f"{self.kube_cmd} get nodes",
+                                        subdir='cluster-info')
+        if nodes['status'] == 0 and self.get_option('describe'):
             for line in nodes['output'].splitlines()[1:]:
                 # find first word in the line and ignore empty+blank lines
                 words = line.split()
@@ -105,12 +131,19 @@ class Kubernetes(Plugin):
                     continue
                 node = words[0]
                 self.add_cmd_output(
-                    "%s describe node %s" % (self.kube_cmd, node),
-                    subdir='nodes'
+                    f"{self.kube_cmd} describe node {node}",
+                    subdir='cluster-info'
                 )
 
+        self.add_cmd_output([
+            f"{self.kube_cmd} get -o json nodes",
+        ], subdir='cluster-info')
+
         # Also collect master metrics
-        self.add_cmd_output("%s get --raw /metrics" % self.kube_cmd)
+        self.add_cmd_output(
+            f"{self.kube_cmd} get --raw /metrics",
+            subdir='cluster-info'
+        )
 
         # CNV is not part of the base installation, but can be added
         if self.is_installed('kubevirt-virtctl'):
@@ -123,7 +156,8 @@ class Kubernetes(Plugin):
     def collect_per_resource_details(self):
         """ Collect details about each resource in all namespaces """
         # get all namespaces in use
-        kns = self.collect_cmd_output('%s get namespaces' % self.kube_cmd)
+        kns = self.collect_cmd_output(f'{self.kube_cmd} get namespaces',
+                                      subdir='cluster-info')
         # namespace is the 1st word on line, until the line has spaces only
         kn_output = kns['output'].splitlines()[1:]
         knsps = [n.split()[0] for n in kn_output if n and len(n.split())]
@@ -131,49 +165,73 @@ class Kubernetes(Plugin):
         for nspace in knsps:
             knsp = '--namespace=%s' % nspace
             if self.get_option('all'):
-                k_cmd = '%s %s %s' % (self.kube_cmd, "get -o json", knsp)
-
-                self.add_cmd_output('%s events' % k_cmd)
+                k_cmd = f'{self.kube_cmd} get -o json {knsp}'
 
                 for res in self.resources:
-                    self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res)
+                    self.add_cmd_output(
+                        f'{k_cmd} {res}',
+                        subdir=f'cluster-info/{nspace}'
+                    )
 
             if self.get_option('describe'):
                 # need to drop json formatting for this
-                k_cmd = '%s %s' % (self.kube_cmd, knsp)
+                k_cmd = f'{self.kube_cmd} {knsp}'
                 for res in self.resources:
-                    ret = self.exec_cmd('%s get %s' % (k_cmd, res))
+                    if res == 'events':
+                        continue
+                    ret = self.exec_cmd(f'{k_cmd} get {res}')
                     if ret['status'] == 0:
                         k_list = [k.split()[0] for k in
                                   ret['output'].splitlines()[1:]]
                         for item in k_list:
-                            k_cmd = '%s %s' % (self.kube_cmd, knsp)
+                            k_cmd = f'{self.kube_cmd} {knsp}'
                             self.add_cmd_output(
-                                '%s describe %s %s' % (k_cmd, res, item),
-                                subdir=res
+                                f'{k_cmd} describe {res} {item}',
+                                subdir=f'cluster-info/{nspace}/{res}'
                             )
 
             if self.get_option('podlogs'):
-                k_cmd = '%s %s' % (self.kube_cmd, knsp)
-                ret = self.exec_cmd('%s get pods' % k_cmd)
+                k_cmd = f'{self.kube_cmd} get -o json {knsp}'
+                ret = self.exec_cmd(f'{k_cmd} pods')
                 if ret['status'] == 0:
-                    pods = [p.split()[0] for p in
-                            ret['output'].splitlines()[1:]]
+                    pods = json.loads(ret['output'])
                     # allow shell-style regex
                     reg = (translate(self.get_option('podlogs-filter')) if
                            self.get_option('podlogs-filter') else None)
-                    for pod in pods:
-                        if reg and not re.match(reg, pod):
+                    for pod in pods["items"]:
+                        if reg and not re.match(reg, pod["metadata"]["name"]):
                             continue
-                        self.add_cmd_output('%s logs %s' % (k_cmd, pod),
-                                            subdir='pods')
+                        _subdir = (f'cluster-info/'
+                                   f'{pod["metadata"]["namespace"]}/podlogs/'
+                                   f'{pod["metadata"]["name"]}')
+                        if "containers" in pod["spec"]:
+                            for cont in pod["spec"]["containers"]:
+                                pod_name = pod["metadata"]["name"]
+                                cont_name = cont["name"]
+                                self.add_cmd_output(
+                                    f'{self.kube_cmd} {knsp} logs '
+                                    f'{pod_name} -c {cont_name}',
+                                    subdir=_subdir
+                                )
+                        if "initContainers" in pod["spec"]:
+                            for cont in pod["spec"]["initContainers"]:
+                                pod_name = pod["metadata"]["name"]
+                                cont_name = cont["name"]
+                                self.add_cmd_output(
+                                    f'{self.kube_cmd} {knsp} logs '
+                                    f'{pod_name} -c {cont_name}',
+                                    subdir=_subdir
+                                )
 
     def collect_all_resources(self):
         """ Collect details about all resources """
         if not self.get_option('all'):
-            k_cmd = '%s get --all-namespaces=true' % self.kube_cmd
+            k_cmd = f'{self.kube_cmd} get --all-namespaces=true'
             for res in self.resources:
-                self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res)
+                self.add_cmd_output(
+                    f'{k_cmd} {res}',
+                    subdir='cluster-info'
+                )
 
     def postproc(self):
         # First, clear sensitive data from the json output collected.
@@ -209,12 +267,23 @@ class RedHatKubernetes(Kubernetes, RedHatPlugin):
         # other changes the `oc` binary may implement
         if self.path_exists('/etc/origin/master/admin.kubeconfig'):
             self.kube_cmd = 'oc'
+        self.resources.extend([
+            'limitranges',
+            'policies',
+            'resourcequotas',
+            'routes'
+        ])
+        self.global_resources.extend([
+            'projects',
+            'pvs'
+        ])
         super().setup()
 
 
 class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin):
 
     packages = ('kubernetes',)
+
     files = (
         '/root/cdk/cdk_addons_kubectl_config',
         '/etc/kubernetes/admin.conf',
@@ -229,7 +298,7 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin):
     def setup(self):
         for _kconf in self.files:
             if self.path_exists(_kconf):
-                self.kube_cmd += " --kubeconfig=%s" % _kconf
+                self.kube_cmd += f" --kubeconfig={_kconf}"
                 break
 
         for svc in self.services:
@@ -238,6 +307,10 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin):
         if self.is_installed('microk8s'):
             self.kube_cmd = 'microk8s kubectl'
 
+        self.config_files.extend([
+            '/root/cdk/kubelet/config.yaml',
+            '/root/cdk/audit/audit-policy.yaml'
+        ])
         super().setup()