diff options
| -rw-r--r-- | sos/report/plugins/kubernetes.py | 175 |
1 files changed, 124 insertions, 51 deletions
diff --git a/sos/report/plugins/kubernetes.py b/sos/report/plugins/kubernetes.py index 12b8a776..41c7ddeb 100644 --- a/sos/report/plugins/kubernetes.py +++ b/sos/report/plugins/kubernetes.py @@ -13,6 +13,7 @@ from fnmatch import translate import re from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin, PluginOpt) +import json class Kubernetes(Plugin): @@ -21,6 +22,39 @@ class Kubernetes(Plugin): plugin_name = "kubernetes" profiles = ('container',) + plugin_timeout = 1200 + + config_files = [ + "/etc/kubernetes", + "/run/flannel", + ] + resources = [ + 'events', + 'deployments', + 'ingresses', + 'pods', + 'pvc', + 'services', + 'daemonsets', + 'replicasets', + 'endpoints', + 'statefulsets', + 'configmaps', + 'serviceaccounts', + 'secrets', + 'jobs', + 'cronjobs', + 'clusterroles', + 'clusterrolebindings' + ] + + # these are not namespaced, must pull separately. + global_resources = [ + 'sc', + 'pv', + 'roles', + 'rolebindings' + ] option_list = [ PluginOpt('all', default=False, @@ -34,26 +68,13 @@ class Kubernetes(Plugin): ] kube_cmd = "kubectl" - resources = [ - 'deployments', - 'ingresses', - 'limitranges', - 'pods', - 'policies', - 'pvc', - 'rc', - 'resourcequotas', - 'routes', - 'services' - ] def check_is_master(self): """ Check if this is the master node """ return any(self.path_exists(f) for f in self.files) def setup(self): - self.add_copy_spec("/etc/kubernetes") - self.add_copy_spec("/run/flannel") + self.add_copy_spec(self.config_files) self.add_env_var([ 'KUBECONFIG', @@ -83,21 +104,26 @@ class Kubernetes(Plugin): return for subcmd in ['version', 'config view']: - self.add_cmd_output('%s %s' % (self.kube_cmd, subcmd)) - - # these are not namespaced, must pull separately. - global_resources = [ - 'namespaces', - 'projects', - 'pvs' - ] - self.add_cmd_output([ - "%s get %s" % (self.kube_cmd, res) for res in global_resources - ]) + self.add_cmd_output( + f'{self.kube_cmd} {subcmd}', + subdir='cluster-info' + ) + + if self.get_option('all'): + self.add_cmd_output([ + f"{self.kube_cmd} get -o json {res}" + for res in self.global_resources + ], subdir='cluster-info') + else: + self.add_cmd_output([ + f"{self.kube_cmd} get {res}" + for res in self.global_resources + ], subdir='cluster-info') # Get detailed node information - nodes = self.collect_cmd_output("%s get nodes" % self.kube_cmd) - if nodes['status'] == 0: + nodes = self.collect_cmd_output(f"{self.kube_cmd} get nodes", + subdir='cluster-info') + if nodes['status'] == 0 and self.get_option('describe'): for line in nodes['output'].splitlines()[1:]: # find first word in the line and ignore empty+blank lines words = line.split() @@ -105,12 +131,19 @@ class Kubernetes(Plugin): continue node = words[0] self.add_cmd_output( - "%s describe node %s" % (self.kube_cmd, node), - subdir='nodes' + f"{self.kube_cmd} describe node {node}", + subdir='cluster-info' ) + self.add_cmd_output([ + f"{self.kube_cmd} get -o json nodes", + ], subdir='cluster-info') + # Also collect master metrics - self.add_cmd_output("%s get --raw /metrics" % self.kube_cmd) + self.add_cmd_output( + f"{self.kube_cmd} get --raw /metrics", + subdir='cluster-info' + ) # CNV is not part of the base installation, but can be added if self.is_installed('kubevirt-virtctl'): @@ -123,7 +156,8 @@ class Kubernetes(Plugin): def collect_per_resource_details(self): """ Collect details about each resource in all namespaces """ # get all namespaces in use - kns = self.collect_cmd_output('%s get namespaces' % self.kube_cmd) + kns = self.collect_cmd_output(f'{self.kube_cmd} get namespaces', + subdir='cluster-info') # namespace is the 1st word on line, until the line has spaces only kn_output = kns['output'].splitlines()[1:] knsps = [n.split()[0] for n in kn_output if n and len(n.split())] @@ -131,49 +165,73 @@ class Kubernetes(Plugin): for nspace in knsps: knsp = '--namespace=%s' % nspace if self.get_option('all'): - k_cmd = '%s %s %s' % (self.kube_cmd, "get -o json", knsp) - - self.add_cmd_output('%s events' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir=f'cluster-info/{nspace}' + ) if self.get_option('describe'): # need to drop json formatting for this - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' for res in self.resources: - ret = self.exec_cmd('%s get %s' % (k_cmd, res)) + if res == 'events': + continue + ret = self.exec_cmd(f'{k_cmd} get {res}') if ret['status'] == 0: k_list = [k.split()[0] for k in ret['output'].splitlines()[1:]] for item in k_list: - k_cmd = '%s %s' % (self.kube_cmd, knsp) + k_cmd = f'{self.kube_cmd} {knsp}' self.add_cmd_output( - '%s describe %s %s' % (k_cmd, res, item), - subdir=res + f'{k_cmd} describe {res} {item}', + subdir=f'cluster-info/{nspace}/{res}' ) if self.get_option('podlogs'): - k_cmd = '%s %s' % (self.kube_cmd, knsp) - ret = self.exec_cmd('%s get pods' % k_cmd) + k_cmd = f'{self.kube_cmd} get -o json {knsp}' + ret = self.exec_cmd(f'{k_cmd} pods') if ret['status'] == 0: - pods = [p.split()[0] for p in - ret['output'].splitlines()[1:]] + pods = json.loads(ret['output']) # allow shell-style regex reg = (translate(self.get_option('podlogs-filter')) if self.get_option('podlogs-filter') else None) - for pod in pods: - if reg and not re.match(reg, pod): + for pod in pods["items"]: + if reg and not re.match(reg, pod["metadata"]["name"]): continue - self.add_cmd_output('%s logs %s' % (k_cmd, pod), - subdir='pods') + _subdir = (f'cluster-info/' + f'{pod["metadata"]["namespace"]}/podlogs/' + f'{pod["metadata"]["name"]}') + if "containers" in pod["spec"]: + for cont in pod["spec"]["containers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) + if "initContainers" in pod["spec"]: + for cont in pod["spec"]["initContainers"]: + pod_name = pod["metadata"]["name"] + cont_name = cont["name"] + self.add_cmd_output( + f'{self.kube_cmd} {knsp} logs ' + f'{pod_name} -c {cont_name}', + subdir=_subdir + ) def collect_all_resources(self): """ Collect details about all resources """ if not self.get_option('all'): - k_cmd = '%s get --all-namespaces=true' % self.kube_cmd + k_cmd = f'{self.kube_cmd} get --all-namespaces=true' for res in self.resources: - self.add_cmd_output('%s %s' % (k_cmd, res), subdir=res) + self.add_cmd_output( + f'{k_cmd} {res}', + subdir='cluster-info' + ) def postproc(self): # First, clear sensitive data from the json output collected. @@ -209,12 +267,23 @@ class RedHatKubernetes(Kubernetes, RedHatPlugin): # other changes the `oc` binary may implement if self.path_exists('/etc/origin/master/admin.kubeconfig'): self.kube_cmd = 'oc' + self.resources.extend([ + 'limitranges', + 'policies', + 'resourcequotas', + 'routes' + ]) + self.global_resources.extend([ + 'projects', + 'pvs' + ]) super().setup() class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): packages = ('kubernetes',) + files = ( '/root/cdk/cdk_addons_kubectl_config', '/etc/kubernetes/admin.conf', @@ -229,7 +298,7 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): def setup(self): for _kconf in self.files: if self.path_exists(_kconf): - self.kube_cmd += " --kubeconfig=%s" % _kconf + self.kube_cmd += f" --kubeconfig={_kconf}" break for svc in self.services: @@ -238,6 +307,10 @@ class UbuntuKubernetes(Kubernetes, UbuntuPlugin, DebianPlugin): if self.is_installed('microk8s'): self.kube_cmd = 'microk8s kubectl' + self.config_files.extend([ + '/root/cdk/kubelet/config.yaml', + '/root/cdk/audit/audit-policy.yaml' + ]) super().setup() |