diff options
author | Shunde Zhang <shunde.zhang@canonical.com> | 2024-03-02 21:34:15 +1100 |
---|---|---|
committer | Jake Hunsaker <jacob.r.hunsaker@gmail.com> | 2024-04-03 15:53:14 -0400 |
commit | 36b2b7d1c87fccde205c0ad85e53877e8043dbcc (patch) | |
tree | 9ffc7c739debd3692b45915afd79b9a9edc16d0e | |
parent | 701406ce4544860169fd7972af4f63b2e10c651b (diff) | |
download | sos-36b2b7d1c87fccde205c0ad85e53877e8043dbcc.tar.gz |
[kubernetes] k8s improvements for Charmed Kubernetes
Add more resources to output when kubernetes.all option is true.
Remove openshift specific resources from base class.
Output json format if kubernetes.all option is true.
Export pod logs to an individual file for each container, so
the plugin can export logs if there are multiple containers in
a pod.
Signed-off-by: Shunde Zhang <shunde.zhang@canonical.com>
-rw-r--r-- | sos/report/plugins/kubernetes.py | 175 |
1 files changed, 124 insertions, 51 deletions
diff --git a/sos/report/plugins/kubernetes.py b/sos/report/plugins/kubernetes.py index 12b8a776..41c7ddeb 100644 --- a/sos/report/plugins/kubernetes.py +++ b/sos/report/plugins/kubernetes.py @@ -13,6 +13,7 @@ from fnmatch import translate import re from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, PluginOpt) +import json class Kubernetes(Plugin): @@ -21,6 +22,39 @@ class Kubernetes(Plugin): plugin_name = "kubernetes" profiles = ('container',) + plugin_timeout = 1200 + + config_files = [ + "/etc/kubernetes", + "/run/flannel", + ] + resources = [ + 'events', + 'deployments', + 'ingresses', + 'pods', + 'pvc', + 'services', + 'daemonsets', + 'replicasets', + 'endpoints', + 'statefulsets', + 'configmaps', + 'serviceaccounts', + 'secrets', + 'jobs', + 'cronjobs', + 'clusterroles', + 'clusterrolebindings' + ] + + # these are not namespaced, must pull separately. + global_resources = [ + 'sc', + 'pv', + 'roles', + 'rolebindings' + ] option_list = [ PluginOpt('all', default=False, @@ -34,26 +68,13 @@ class Kubernetes(Plugin): ] kube_cmd = "kubectl" - resources = [ - 'deployments', - 'ingresses', - 'limitranges', - 'pods', - 'policies', - 'pvc', - 'rc', - 'resourcequotas', - 'routes', - 'services' - ] def check_is_master(self): """ Check if this is the master node """ return any(self.path_exists(f) for f in self.files) def setup(self): - self.add_copy_spec("/etc/kubernetes") - self.add_copy_spec("/run/flannel") + self.add_copy_spec(self.config_files) self.add_env_var([ 'KUBECONFIG', @@ -83,21 +104,26 @@ class Kubernetes(Plugin): return for subcmd in ['version', 'config view']: - self.add_cmd_output('%s %s' % (self.kube_cmd, subcmd)) - - # these are not namespaced, must pull separately. - global_resources = [ - 'namespaces', - 'projects', - 'pvs' - ] - self.add_cmd_output([ - "%s get %s" % (self.kube_cmd, res) for res in global_resources - ]) + self.add_cmd_output( + f'{self.kube_cmd} {subcmd}', + subdir='cluster-info' + ) + + if self.get_option('all'): + self.add_cmd_output([ + f"{self.kube_cmd} get -o json {res}" + for res in self.global_resources + ], subdir='cluster-info') + else: + self.add_cmd_output([ + f"{self.kube_cmd} get {res}" + for res in self.global_resources + ], subdir='cluster-info') # Get detailed node information - nodes = self.collect_cmd_output("%s get nodes" % self.kube_cmd) - if nodes['status'] == 0: + nodes = self.collect_cmd_output(f"{self.kube_cmd} get nodes", + subdir='cluster-info') + if nodes['status'] == 0 and self.get_option('describe'): for line in nodes['output'].splitlines()[1:]: # find first word in the line and ignore empty+blank lines words = line.split() @@ -105,12 +131,19 @@ class Kubernetes(Plugin): continue node = words[0] self.add_cmd_output( - "%s describe node %s" % (self.kube_cmd, node), - subdir='nodes' + f"{self.kube_cmd} describe node {node}", + subdir='cluster-info' ) + self.add_cmd_output([ + f"{self.kube_cmd} get -o json nodes", + ], subdir='cluster-info') + # Also collect master metrics - self.add_cmd_output("%s get --raw /metrics" % self.kube_cmd) + self.add_cmd_output( + f"{self.kube_cmd} get --raw /metrics", + subdir='cluster-info' + ) # CNV is not part of the base installation, but can be added if self.is_installed('kubevirt-virtctl'): @@ -123,7 +156,8 @@ class Kubernetes(Plugin): def collect_per_resource_details(self): """ Collect details about each resource in all namespaces """ # get all namespaces in use - kns = self.collect_cmd_output('%s get namespaces' % self.kube_cmd) + kns = self.collect_cmd_output(f'{self.kube_cmd} get namespaces', + subdir='cluster-info') # namespace is the 1st word on line, until the line has spaces only kn_output = kns['output'].splitlines()[1:] knsps = [n.split()[0] for n in kn_output if n and len(n.split())] @@ -131,49 +165,73 @@ class Kubernetes(Plugin): for nspace in knsps: knsp = '--namespace=%s' % nspace if self.get_option('all'): - k_cmd = '%s %s %s' % (self.kube_cmd, "get -o json", knsp) - - self.add_cmd_output('%s events' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir=f'cluster-info/{nspace}' + ) if self.get_option('describe'): # need to drop json formatting for this - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' for res in self.resources: - ret = self.exec_cmd('%s get %s' % (k_cmd, res)) + if res == 'events': + continue + ret = self.exec_cmd(f'{k_cmd} get {res}') if ret['status'] == 0: k_list = [k.split()[0] for k in ret['output'].splitlines()[1:]] for item in k_list: - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' self.add_cmd_output( - '%s describe %s %s' % (k_cmd, res, item), - subdir=res + f'{k_cmd} describe {res} {item}', + subdir=f'cluster-info/{nspace}/{res}' ) if self.get_option('podlogs'): - k_cmd = '%s %s' % (self.kube_cmd, knsp) - ret = self.exec_cmd('%s get pods' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' + ret = self.exec_cmd(f'{k_cmd} pods') if ret['status'] == 0: - pods = [p.split()[0] for p in - ret['output'].splitlines()[1:]] + pods = json.loads(ret['output']) # allow shell-style regex reg = (translate(self.get_option('podlogs-filter')) if self.get_option('podlogs-filter') else None) - for pod in pods: - if reg and not re.match(reg, pod): + for pod in pods["items"]: + if reg and not re.match(reg, pod["metadata"]["name"]): continue - self.add_cmd_output('%s logs %s' % (k_cmd, pod), - subdir='pods') + _subdir = (f'cluster-info/' + f'{pod["metadata"]["namespace"]}/podlogs/' + f'{pod["metadata"]["name"]}') + if "containers" in pod["spec"]: + for cont in pod["spec"]["containers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) + if "initContainers" in pod["spec"]: + for cont in pod["spec"]["initContainers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) def collect_all_resources(self): """ Collect details about all resources """ if not self.get_option('all'): - k_cmd = '%s get --all-namespaces=true' % self.kube_cmd + k_cmd = f'{self.kube_cmd} get --all-namespaces=true' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir='cluster-info' + ) def postproc(self): # First, clear sensitive data from the json output collected. @@ -209,12 +267,23 @@ class RedHatKubernetes(Kubernetes, RedHatPlugin): # other changes the `oc` binary may implement if self.path_exists('/etc/origin/master/admin.kubeconfig'): self.kube_cmd = 'oc' + self.resources.extend([ + 'limitranges', + 'policies', + 'resourcequotas', + 'routes' + ]) + self.global_resources.extend([ + 'projects', + 'pvs' + ]) super().setup() class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): packages = ('kubernetes',) + files = ( '/root/cdk/cdk_addons_kubectl_config', '/etc/kubernetes/admin.conf', @@ -229,7 +298,7 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): def setup(self): for _kconf in self.files: if self.path_exists(_kconf): - self.kube_cmd += " --kubeconfig=%s" % _kconf + self.kube_cmd += f" --kubeconfig={_kconf}" break for svc in self.services: @@ -238,6 +307,10 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): if self.is_installed('microk8s'): self.kube_cmd = 'microk8s kubectl' + self.config_files.extend([ + '/root/cdk/kubelet/config.yaml', + '/root/cdk/audit/audit-policy.yaml' + ]) super().setup() |