From 36b2b7d1c87fccde205c0ad85e53877e8043dbcc Mon Sep 17 00:00:00 2001 From: Shunde Zhang Date: Sat, 2 Mar 2024 21:34:15 +1100 Subject: [kubernetes] k8s improvements for Charmed Kubernetes Add more resources to output when kubernetes.all option is true. Remove openshift specific resources from base class. Output json format if kubernetes.all option is true. Export pod logs to an individual file for each container, so the plugin can export logs if there are multiple containers in a pod. Signed-off-by: Shunde Zhang --- sos/report/plugins/kubernetes.py | 175 +++++++++++++++++++++++++++------------ 1 file changed, 124 insertions(+), 51 deletions(-) diff --git a/sos/report/plugins/kubernetes.py b/sos/report/plugins/kubernetes.py index 12b8a776..41c7ddeb 100644 --- a/sos/report/plugins/kubernetes.py +++ b/sos/report/plugins/kubernetes.py @@ -13,6 +13,7 @@ from fnmatch import translate import re from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, PluginOpt) +import json class Kubernetes(Plugin): @@ -21,6 +22,39 @@ class Kubernetes(Plugin): plugin_name = "kubernetes" profiles = ('container',) + plugin_timeout = 1200 + + config_files = [ + "/etc/kubernetes", + "/run/flannel", + ] + resources = [ + 'events', + 'deployments', + 'ingresses', + 'pods', + 'pvc', + 'services', + 'daemonsets', + 'replicasets', + 'endpoints', + 'statefulsets', + 'configmaps', + 'serviceaccounts', + 'secrets', + 'jobs', + 'cronjobs', + 'clusterroles', + 'clusterrolebindings' + ] + + # these are not namespaced, must pull separately. + global_resources = [ + 'sc', + 'pv', + 'roles', + 'rolebindings' + ] option_list = [ PluginOpt('all', default=False, @@ -34,26 +68,13 @@ class Kubernetes(Plugin): ] kube_cmd = "kubectl" - resources = [ - 'deployments', - 'ingresses', - 'limitranges', - 'pods', - 'policies', - 'pvc', - 'rc', - 'resourcequotas', - 'routes', - 'services' - ] def check_is_master(self): """ Check if this is the master node """ return any(self.path_exists(f) for f in self.files) def setup(self): - self.add_copy_spec("/etc/kubernetes") - self.add_copy_spec("/run/flannel") + self.add_copy_spec(self.config_files) self.add_env_var([ 'KUBECONFIG', @@ -83,21 +104,26 @@ class Kubernetes(Plugin): return for subcmd in ['version', 'config view']: - self.add_cmd_output('%s %s' % (self.kube_cmd, subcmd)) - - # these are not namespaced, must pull separately. - global_resources = [ - 'namespaces', - 'projects', - 'pvs' - ] - self.add_cmd_output([ - "%s get %s" % (self.kube_cmd, res) for res in global_resources - ]) + self.add_cmd_output( + f'{self.kube_cmd} {subcmd}', + subdir='cluster-info' + ) + + if self.get_option('all'): + self.add_cmd_output([ + f"{self.kube_cmd} get -o json {res}" + for res in self.global_resources + ], subdir='cluster-info') + else: + self.add_cmd_output([ + f"{self.kube_cmd} get {res}" + for res in self.global_resources + ], subdir='cluster-info') # Get detailed node information - nodes = self.collect_cmd_output("%s get nodes" % self.kube_cmd) - if nodes['status'] == 0: + nodes = self.collect_cmd_output(f"{self.kube_cmd} get nodes", + subdir='cluster-info') + if nodes['status'] == 0 and self.get_option('describe'): for line in nodes['output'].splitlines()[1:]: # find first word in the line and ignore empty+blank lines words = line.split() @@ -105,12 +131,19 @@ class Kubernetes(Plugin): continue node = words[0] self.add_cmd_output( - "%s describe node %s" % (self.kube_cmd, node), - subdir='nodes' + f"{self.kube_cmd} describe node {node}", + subdir='cluster-info' ) + self.add_cmd_output([ + f"{self.kube_cmd} get -o json nodes", + ], subdir='cluster-info') + # Also collect master metrics - self.add_cmd_output("%s get --raw /metrics" % self.kube_cmd) + self.add_cmd_output( + f"{self.kube_cmd} get --raw /metrics", + subdir='cluster-info' + ) # CNV is not part of the base installation, but can be added if self.is_installed('kubevirt-virtctl'): @@ -123,7 +156,8 @@ class Kubernetes(Plugin): def collect_per_resource_details(self): """ Collect details about each resource in all namespaces """ # get all namespaces in use - kns = self.collect_cmd_output('%s get namespaces' % self.kube_cmd) + kns = self.collect_cmd_output(f'{self.kube_cmd} get namespaces', + subdir='cluster-info') # namespace is the 1st word on line, until the line has spaces only kn_output = kns['output'].splitlines()[1:] knsps = [n.split()[0] for n in kn_output if n and len(n.split())] @@ -131,49 +165,73 @@ class Kubernetes(Plugin): for nspace in knsps: knsp = '--namespace=%s' % nspace if self.get_option('all'): - k_cmd = '%s %s %s' % (self.kube_cmd, "get -o json", knsp) - - self.add_cmd_output('%s events' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir=f'cluster-info/{nspace}' + ) if self.get_option('describe'): # need to drop json formatting for this - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' for res in self.resources: - ret = self.exec_cmd('%s get %s' % (k_cmd, res)) + if res == 'events': + continue + ret = self.exec_cmd(f'{k_cmd} get {res}') if ret['status'] == 0: k_list = [k.split()[0] for k in ret['output'].splitlines()[1:]] for item in k_list: - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' self.add_cmd_output( - '%s describe %s %s' % (k_cmd, res, item), - subdir=res + f'{k_cmd} describe {res} {item}', + subdir=f'cluster-info/{nspace}/{res}' ) if self.get_option('podlogs'): - k_cmd = '%s %s' % (self.kube_cmd, knsp) - ret = self.exec_cmd('%s get pods' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' + ret = self.exec_cmd(f'{k_cmd} pods') if ret['status'] == 0: - pods = [p.split()[0] for p in - ret['output'].splitlines()[1:]] + pods = json.loads(ret['output']) # allow shell-style regex reg = (translate(self.get_option('podlogs-filter')) if self.get_option('podlogs-filter') else None) - for pod in pods: - if reg and not re.match(reg, pod): + for pod in pods["items"]: + if reg and not re.match(reg, pod["metadata"]["name"]): continue - self.add_cmd_output('%s logs %s' % (k_cmd, pod), - subdir='pods') + _subdir = (f'cluster-info/' + f'{pod["metadata"]["namespace"]}/podlogs/' + f'{pod["metadata"]["name"]}') + if "containers" in pod["spec"]: + for cont in pod["spec"]["containers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) + if "initContainers" in pod["spec"]: + for cont in pod["spec"]["initContainers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) def collect_all_resources(self): """ Collect details about all resources """ if not self.get_option('all'): - k_cmd = '%s get --all-namespaces=true' % self.kube_cmd + k_cmd = f'{self.kube_cmd} get --all-namespaces=true' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir='cluster-info' + ) def postproc(self): # First, clear sensitive data from the json output collected. @@ -209,12 +267,23 @@ class RedHatKubernetes(Kubernetes, RedHatPlugin): # other changes the `oc` binary may implement if self.path_exists('/etc/origin/master/admin.kubeconfig'): self.kube_cmd = 'oc' + self.resources.extend([ + 'limitranges', + 'policies', + 'resourcequotas', + 'routes' + ]) + self.global_resources.extend([ + 'projects', + 'pvs' + ]) super().setup() class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): packages = ('kubernetes',) + files = ( '/root/cdk/cdk_addons_kubectl_config', '/etc/kubernetes/admin.conf', @@ -229,7 +298,7 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): def setup(self): for _kconf in self.files: if self.path_exists(_kconf): - self.kube_cmd += " --kubeconfig=%s" % _kconf + self.kube_cmd += f" --kubeconfig={_kconf}" break for svc in self.services: @@ -238,6 +307,10 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): if self.is_installed('microk8s'): self.kube_cmd = 'microk8s kubectl' + self.config_files.extend([ + '/root/cdk/kubelet/config.yaml', + '/root/cdk/audit/audit-policy.yaml' + ]) super().setup() -- cgit