From 0e4bfc519813157f196ec2b8d1076bd21d188271 Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Mon, 8 Jul 2013 09:26:31 +0200 Subject: Add juju status and get-constraints outputs Signed-off-by: Ante Karamatic --- sos/plugins/juju.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sos/plugins/juju.py b/sos/plugins/juju.py index 64b4f1db..c7bbdf17 100644 --- a/sos/plugins/juju.py +++ b/sos/plugins/juju.py @@ -25,3 +25,6 @@ class Juju(Plugin, UbuntuPlugin): def setup(self): self.add_copy_specs(["/var/log/juju", "/var/lib/juju"]) + + self.add_cmd_output("juju -v status") + self.add_cmd_output("juju -v get-constraints") -- cgit From 38163948406013e63dd6d1408c0fc17da78d0749 Mon Sep 17 00:00:00 2001 From: Ante Karamatic Date: Mon, 8 Jul 2013 09:28:30 +0200 Subject: Rearrange openstack plugin so that distribution OpenStack class contains all non-distribution specific details. Also dropped all logrotate parts since those are collected by logrotate plugin. Signed-off-by: Ante Karamatic --- sos/plugins/openstack.py | 145 ++++++++++++++++++++++------------------------- 1 file changed, 69 insertions(+), 76 deletions(-) diff --git a/sos/plugins/openstack.py b/sos/plugins/openstack.py index 8c521365..a35a9e85 100644 --- a/sos/plugins/openstack.py +++ b/sos/plugins/openstack.py @@ -27,6 +27,68 @@ class OpenStack(Plugin): option_list = [("log", "gathers all openstack logs", "slow", False)] + def setup(self): + # Nova + if os.path.exists("nova-manage"): + self.add_cmd_output( + "nova-manage config list 2>/dev/null | sort", + suggest_filename="nova_config_list") + self.add_cmd_output( + "nova-manage service list 2>/dev/null", + suggest_filename="nova_service_list") + self.add_cmd_output( + "nova-manage db version 2>/dev/null", + suggest_filename="nova_db_version") + self.add_cmd_output( + "nova-manage fixed list 2>/dev/null", + suggest_filename="nova_fixed_ip_list") + self.add_cmd_output( + "nova-manage floating list 2>/dev/null", + suggest_filename="nova_floating_ip_list") + self.add_cmd_output( + "nova-manage flavor list 2>/dev/null", + suggest_filename="nova_flavor_list") + self.add_cmd_output( + "nova-manage network list 2>/dev/null", + suggest_filename="nova_network_list") + self.add_cmd_output( + "nova-manage vm list 2>/dev/null", + suggest_filename="nova_vm_list") + self.add_copy_specs(["/etc/nova/", + "/var/log/nova/"]) + + # Glance + if os.path.exists("glance-manage"): + self.add_cmd_output( + "glance-manage db_version", + suggest_filename="glance_db_version") + self.add_copy_specs(["/etc/glance/", + "/var/log/glance/"]) + + # Cinder + if os.path.exists("cinder-manage"): + self.addCmdOutput( + "cinder-manage db version", + suggest_filename="cinder_db_version") + self.add_copy_specs(["/etc/cinder/", + "/var/log/cinder/"]) + + # Keystone + self.add_copy_specs(["/etc/keystone/", + "/var/log/keystone/"]) + + # Quantum + self.add_copy_specs(["/etc/quantum/", + "/var/log/quantum/"]) + + # Swift + self.add_copy_specs("/etc/swift/") + + def postproc(self): + self.do_file_sub('/etc/keystone/keystone.conf', + r"(admin_password\s*=\s*)(.*)", + r"\1******") + class DebianOpenStack(OpenStack, DebianPlugin, UbuntuPlugin): """OpenStack related information for Debian based distributions @@ -95,61 +157,13 @@ class DebianOpenStack(OpenStack, DebianPlugin, UbuntuPlugin): def setup(self): # Nova - if os.path.exists("nova-manage"): - self.add_cmd_output( - "nova-manage config list 2>/dev/null | sort", - suggest_filename="nova_config_list") - self.add_cmd_output( - "nova-manage service list 2>/dev/null", - suggest_filename="nova_service_list") - self.add_cmd_output( - "nova-manage db version 2>/dev/null", - suggest_filename="nova_db_version") - self.add_cmd_output( - "nova-manage fixed list 2>/dev/null", - suggest_filename="nova_fixed_ip_list") - self.add_cmd_output( - "nova-manage floating list 2>/dev/null", - suggest_filename="nova_floating_ip_list") - self.add_cmd_output( - "nova-manage flavor list 2>/dev/null", - suggest_filename="nova_flavor_list") - self.add_cmd_output( - "nova-manage network list 2>/dev/null", - suggest_filename="nova_network_list") - self.add_cmd_output( - "nova-manage vm list 2>/dev/null", - suggest_filename="nova_vm_list") - self.add_copy_specs(["/etc/nova/", - "/var/log/nova/", - "/etc/default/nova-volume", - "/etc/sudoers.d/nova_sudoers", - "/etc/logrotate.d/nova-*"]) + self.add_copy_specs("/etc/sudoers.d/nova_sudoers") + # Cinder - if os.path.exists("/usr/bin/cinder-manage"): - self.addCmdOutput( - "/usr/bin/cinder-manage db version", - suggest_filename="cinder_db_version") - self.addCopySpecs(["/etc/cinder/", - "/var/log/cinder/", - "/etc/logrotate.d/cinder-*"]) - # Glance - if os.path.exists("glance-manage"): - self.add_cmd_output( - "glance-manage db_version", - suggest_filename="glance_db_version") - self.add_copy_specs(["/etc/glance/", - "/var/log/glance/", - "/etc/logrotate.d/glance-*"]) - # Keystone - self.add_copy_specs(["/etc/keystone/", - "/var/log/keystone/", - "/etc/logrotate.d/keystone"]) - # Swift - self.add_copy_specs(["/etc/swift/"]) + self.add_copy_specs("/etc/sudoers.d/cinder_sudoers") + # Quantum - self.add_copy_specs(["/etc/quantum/", - "/var/log/quantum/"]) + self.add_copy_specs("/etc/sudoers.d/quantum_sudoers") class RedHatOpenStack(OpenStack, RedHatPlugin): @@ -181,27 +195,6 @@ class RedHatOpenStack(OpenStack, RedHatPlugin): self.add_cmd_output("openstack-status") # Nova - self.add_copy_specs(["/etc/nova/", - "/var/log/nova/", - "/var/lib/nova/", + self.add_copy_specs(["/var/lib/nova/", "/etc/polkit-1/localauthority/50-local.d/50-nova.pkla", - "/etc/sudoers.d/nova", - "/etc/logrotate.d/openstack-nova"]) - - # Glance - self.add_copy_specs(["/etc/glance/", - "/var/log/glance/", - "/etc/logrotate.d/openstack-glance"]) - - # Keystone - self.add_copy_specs(["/etc/keystone/", - "/var/log/keystone/"]) - - # Quantum - self.add_copy_specs(["/etc/quantum/", - "/var/log/quantum/"]) - - def postproc(self): - self.do_file_sub('/etc/keystone/keystone.conf', - r"(admin_password\s*=\s*)(.*)", - r"\1******") + "/etc/sudoers.d/nova"]) -- cgit From b67718e053b823f9088af3479e7de73b3f13c5c9 Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Tue, 9 Jul 2013 16:02:44 +0100 Subject: Add Openshift plug-in Add an sos plug-in for openshift to gathers several directories and command output sepcific to Openshift systems. Passwords and secrets are scrubbed for privacy reasons. Signed-off-by: Nick Harvey Signed-off-by: Bryn M. Reeves --- sos/plugins/openshift.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 sos/plugins/openshift.py diff --git a/sos/plugins/openshift.py b/sos/plugins/openshift.py new file mode 100644 index 00000000..459e75e1 --- /dev/null +++ b/sos/plugins/openshift.py @@ -0,0 +1,72 @@ +### This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 2 of the License, or +## (at your option) any later version. + +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +from sos.plugins import Plugin, RedHatPlugin + +class Openshift(Plugin, RedHatPlugin): + '''Openshift related information''' + + plugin_name = "Openshift" + + option_list = [("broker", "Gathers broker specific files", "slow", False), + ("node", "Gathers node specific files", "slow", False)] + + def setup(self): + self.add_copy_specs(["/etc/openshift-enterprise-version", + "/etc/openshift/", + "/etc/dhcp/dhclient-*"]) + + if self.option_enabled("broker"): + self.add_copy_specs(["/var/log/activemq", + "/var/log/mongodb", + "/var/log/openshift", + "/var/www/openshift/broker/log", + "/var/www/openshift/broker/httpd/logs/", + "/var/www/openshift/console/log", + "/var/www/openshift/console/httpd/logs", + "/var/log/openshift/user_action.log"]) + + self.add_cmd_output("oo-accpet-broker -v") + self.add_cmd_output("oo-admin-chk -v") + self.add_cmd_output("mco ping") + self.add_cmd_output("gem list --local") + self.add_cmd_output("cd /var/www/openshift/broker/ && bundle --local") + + if self.option_enabled("node"): + self.add_copy_specs(["/var/log/openshift/node", + "/cgroup/all/openshift", + "/var/log/mcollective.log", + "/var/log/openshift-gears-async-start.log", + "/var/log/httpd/error_log"]) + + self.add_cmd_output("oo-accept-node -v") + self.add_cmd_output("oo-admin-ctl-gears list") + self.add_cmd_output("ls -l /var/lib/openshift") + + def postproc(self): + self.do_file_sub('/etc/openshift/broker.conf', + r"(MONGO_PASSWORD=)(.*)", + r"\1*******") + + self.do_file_sub('/etc/openshift/broker.conf', + r"(SESSION_SECRET=)(.*)", + r"\1*******") + + self.do_file_sub('/etc/openshift/console.conf', + r"(SESSION_SECRET=)(.*)", + r"\1*******") + + self.do_file_sub('/etc/openshift/htpasswd', + r"(.*:)(.*)", + r"\1********") -- cgit From 35d529d359643525755c2c50b2c70e490ba7609a Mon Sep 17 00:00:00 2001 From: Jan Pokorný Date: Thu, 18 Jul 2013 23:32:05 +0200 Subject: cluster plugin: add obfuscation of luci secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Pokorný --- sos/plugins/cluster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sos/plugins/cluster.py b/sos/plugins/cluster.py index 8a3432ea..68bfa3f7 100644 --- a/sos/plugins/cluster.py +++ b/sos/plugins/cluster.py @@ -144,6 +144,8 @@ class Cluster(Plugin, RedHatPlugin): self.do_file_sub(cluster_conf, r"(\s*\ Date: Sat, 20 Jul 2013 09:16:13 +0200 Subject: cluster plugin: restrict data collected for luci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Notably, avoid server cert being added. On the other hand, allow collection of rotated log files for luci. Signed-off-by: Jan Pokorný --- sos/plugins/cluster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sos/plugins/cluster.py b/sos/plugins/cluster.py index 68bfa3f7..12a00622 100644 --- a/sos/plugins/cluster.py +++ b/sos/plugins/cluster.py @@ -55,9 +55,10 @@ class Cluster(Plugin, RedHatPlugin): self.add_copy_spec("/etc/sysconfig/cman") self.add_copy_spec("/etc/fence_virt.conf") self.add_copy_spec("/var/lib/ricci") - self.add_copy_spec("/var/lib/luci") + self.add_copy_spec("/var/lib/luci/data/luci.db") + self.add_copy_spec("/var/lib/luci/etc") self.add_copy_spec("/var/log/cluster") - self.add_copy_spec("/var/log/luci/luci.log") + self.add_copy_spec("/var/log/luci") self.add_copy_spec("/etc/fence_virt.conf") if self.get_option('gfslockdump'): -- cgit From d65075a050097087a822381546a8b9bd1ab35220 Mon Sep 17 00:00:00 2001 From: Pierguido Lambri Date: Sat, 6 Jul 2013 21:21:42 +0100 Subject: Added XFS plugin Signed-off-by: Pierguido Lambri --- sos/plugins/xfs.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 sos/plugins/xfs.py diff --git a/sos/plugins/xfs.py b/sos/plugins/xfs.py new file mode 100644 index 00000000..fe84b033 --- /dev/null +++ b/sos/plugins/xfs.py @@ -0,0 +1,40 @@ +### This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 2 of the License, or +## (at your option) any later version. + +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software +## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +from sos.plugins import Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin +import os +import re +from itertools import * + +class Xfs(Plugin, RedHatPlugin, DebianPlugin, UbuntuPlugin): + """information on the XFS filesystem + """ + + plugin_name = 'xfs' + + option_list = [("logprint", 'gathers the log information', 'slow', False)] + + def setup(self): + mounts = '/proc/mounts' + ext_fs_regex = r"^(/dev/.+).+xfs\s+" + for dev in izip(self.do_regex_find_all(ext_fs_regex, mounts)): + for e in dev: + parts = e.split(' ') + self.add_cmd_output("xfs_info %s" % (parts[1])) + + if self.get_option('logprint'): + for dev in izip(self.do_regex_find_all(ext_fs_regex, mounts)): + for e in dev: + parts = e.split(' ') + self.add_cmd_output("xfs_logprint -c %s" % (parts[0])) -- cgit From 49f1529a7a7da196368daf952824fc502859f9af Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Thu, 25 Jul 2013 18:56:58 +0100 Subject: Fix API usage in openstack plug-in One call in openstack.py used the old addCopySpecs() method. This will fail with current sos-3.x code. Update it to use the new add_copy_specs() method instead. Signed-off-by: Bryn M. Reeves --- sos/plugins/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/plugins/openstack.py b/sos/plugins/openstack.py index a35a9e85..69a830bc 100644 --- a/sos/plugins/openstack.py +++ b/sos/plugins/openstack.py @@ -67,7 +67,7 @@ class OpenStack(Plugin): # Cinder if os.path.exists("cinder-manage"): - self.addCmdOutput( + self.add_cmd_output( "cinder-manage db version", suggest_filename="cinder_db_version") self.add_copy_specs(["/etc/cinder/", -- cgit From 8b10cb00ab44239e9f11d8e9ab766908f9f92e16 Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Thu, 25 Jul 2013 19:00:51 +0100 Subject: Fix broken binary checks in openstack.py The OpenStack plug-in attempts to check for the presence of binaries before running them. To do this it passes a relative path to os.path.exists(); clearly this will never succeed. Ideally plug-ins should never need to do these checks. If the binary exists we'll run it and if not we won't (with no harmful side-effects). This is how sos is currently designed and if there is a need for plug-ins to ever do this we should add a method to the base plug-in class e.g. Plugin.find_command() to do this properly (i.e. evaluating the policy-supplied PATH). For now, just make all of this unconditional. Signed-off-by: Bryn M. Reeves --- sos/plugins/openstack.py | 63 +++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/sos/plugins/openstack.py b/sos/plugins/openstack.py index 69a830bc..9c209e63 100644 --- a/sos/plugins/openstack.py +++ b/sos/plugins/openstack.py @@ -29,47 +29,44 @@ class OpenStack(Plugin): def setup(self): # Nova - if os.path.exists("nova-manage"): - self.add_cmd_output( - "nova-manage config list 2>/dev/null | sort", - suggest_filename="nova_config_list") - self.add_cmd_output( - "nova-manage service list 2>/dev/null", - suggest_filename="nova_service_list") - self.add_cmd_output( - "nova-manage db version 2>/dev/null", - suggest_filename="nova_db_version") - self.add_cmd_output( - "nova-manage fixed list 2>/dev/null", - suggest_filename="nova_fixed_ip_list") - self.add_cmd_output( - "nova-manage floating list 2>/dev/null", - suggest_filename="nova_floating_ip_list") - self.add_cmd_output( - "nova-manage flavor list 2>/dev/null", - suggest_filename="nova_flavor_list") - self.add_cmd_output( - "nova-manage network list 2>/dev/null", - suggest_filename="nova_network_list") - self.add_cmd_output( - "nova-manage vm list 2>/dev/null", - suggest_filename="nova_vm_list") + self.add_cmd_output( + "nova-manage config list 2>/dev/null | sort", + suggest_filename="nova_config_list") + self.add_cmd_output( + "nova-manage service list 2>/dev/null", + suggest_filename="nova_service_list") + self.add_cmd_output( + "nova-manage db version 2>/dev/null", + suggest_filename="nova_db_version") + self.add_cmd_output( + "nova-manage fixed list 2>/dev/null", + suggest_filename="nova_fixed_ip_list") + self.add_cmd_output( + "nova-manage floating list 2>/dev/null", + suggest_filename="nova_floating_ip_list") + self.add_cmd_output( + "nova-manage flavor list 2>/dev/null", + suggest_filename="nova_flavor_list") + self.add_cmd_output( + "nova-manage network list 2>/dev/null", + suggest_filename="nova_network_list") + self.add_cmd_output( + "nova-manage vm list 2>/dev/null", + suggest_filename="nova_vm_list") self.add_copy_specs(["/etc/nova/", "/var/log/nova/"]) # Glance - if os.path.exists("glance-manage"): - self.add_cmd_output( - "glance-manage db_version", - suggest_filename="glance_db_version") + self.add_cmd_output( + "glance-manage db_version", + suggest_filename="glance_db_version") self.add_copy_specs(["/etc/glance/", "/var/log/glance/"]) # Cinder - if os.path.exists("cinder-manage"): - self.add_cmd_output( - "cinder-manage db version", - suggest_filename="cinder_db_version") + self.add_cmd_output( + "cinder-manage db version", + suggest_filename="cinder_db_version") self.add_copy_specs(["/etc/cinder/", "/var/log/cinder/"]) -- cgit From 8b2af7f9123b22aa139b8d1e0221f5c82a99bc90 Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Thu, 25 Jul 2013 19:05:59 +0100 Subject: Delete bogus dist checks from RedHatOpenStack We already know we are running on RHEL or Fedora if we are executing a plug-in class tagged with RedHatPlugin. Do not check for /etc/*-release. Signed-off-by: Bryn M. Reeves --- sos/plugins/openstack.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sos/plugins/openstack.py b/sos/plugins/openstack.py index 9c209e63..10ed59f3 100644 --- a/sos/plugins/openstack.py +++ b/sos/plugins/openstack.py @@ -186,11 +186,6 @@ class RedHatOpenStack(OpenStack, RedHatPlugin): 'python-quantumclient') def setup(self): - # If RHEL or Fedora then invoke script for openstack-status - if (os.path.isfile('/etc/redhat-release') - or os.path.isfile('/etc/fedora-release')): - self.add_cmd_output("openstack-status") - # Nova self.add_copy_specs(["/var/lib/nova/", "/etc/polkit-1/localauthority/50-local.d/50-nova.pkla", -- cgit From 854642ca16dc92d050107e20edbf60c323b819c7 Mon Sep 17 00:00:00 2001 From: "Bryn M. Reeves" Date: Thu, 25 Jul 2013 19:16:32 +0100 Subject: Fix add_copy_specs(string) bugs in openstack.py OpenStack was full of the add_copy_specs(string) anti-pattern. While we should address this properly these instances all need fixing to avoid collecting the whole file system. They should also have been tested before being put forward as a pull request. Signed-off-by: Bryn M. Reeves --- sos/plugins/openstack.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sos/plugins/openstack.py b/sos/plugins/openstack.py index 10ed59f3..07d17fbb 100644 --- a/sos/plugins/openstack.py +++ b/sos/plugins/openstack.py @@ -79,7 +79,7 @@ class OpenStack(Plugin): "/var/log/quantum/"]) # Swift - self.add_copy_specs("/etc/swift/") + self.add_copy_spec("/etc/swift/") def postproc(self): self.do_file_sub('/etc/keystone/keystone.conf', @@ -154,13 +154,13 @@ class DebianOpenStack(OpenStack, DebianPlugin, UbuntuPlugin): def setup(self): # Nova - self.add_copy_specs("/etc/sudoers.d/nova_sudoers") + self.add_copy_spec("/etc/sudoers.d/nova_sudoers") # Cinder - self.add_copy_specs("/etc/sudoers.d/cinder_sudoers") + self.add_copy_spec("/etc/sudoers.d/cinder_sudoers") # Quantum - self.add_copy_specs("/etc/sudoers.d/quantum_sudoers") + self.add_copy_spec("/etc/sudoers.d/quantum_sudoers") class RedHatOpenStack(OpenStack, RedHatPlugin): @@ -187,6 +187,8 @@ class RedHatOpenStack(OpenStack, RedHatPlugin): def setup(self): # Nova - self.add_copy_specs(["/var/lib/nova/", - "/etc/polkit-1/localauthority/50-local.d/50-nova.pkla", - "/etc/sudoers.d/nova"]) + self.add_copy_specs([ + "/var/lib/nova/", + "/etc/polkit-1/localauthority/50-local.d/50-nova.pkla", + "/etc/sudoers.d/nova" + ]) -- cgit From ea872349f800b6e1e609d074da45394ada642df2 Mon Sep 17 00:00:00 2001 From: Adam Stokes Date: Thu, 25 Jul 2013 16:47:22 -0400 Subject: Correct archive unittests and pep8 conformity in archive class Signed-off-by: Adam Stokes --- .travis.yml | 1 + sos/archive.py | 70 ++++++++++++++++++++++++-------------------------- tests/archive_tests.py | 62 +++++++++++++++++++++++++------------------- 3 files changed, 69 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d1406ec..5b87103a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ matrix: allow_failures: - python: 3.2 - python: 3.3 + - python: "pypy" notifications: email: false install: diff --git a/sos/archive.py b/sos/archive.py index 42916606..080f2593 100644 --- a/sos/archive.py +++ b/sos/archive.py @@ -1,4 +1,4 @@ -## Copyright (C) 2012 Red Hat, Inc., +## Copyright (C) 2012 Red Hat, Inc., ## Jesse Jaggars ## Bryn M. Reeves ## @@ -17,26 +17,20 @@ ## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. import os import time -import tempfile import tarfile import zipfile import shutil import logging -import shutil import shlex import re # required for compression callout (FIXME: move to policy?) -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE try: import selinux except ImportError: pass -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO class Archive(object): @@ -71,25 +65,26 @@ class Archive(object): def finalize(self, method): """Finalize an archive object via method. This may involve creating - An archive that is subsequently compressed or simply closing an + An archive that is subsequently compressed or simply closing an archive that supports in-line handling. If method is automatic then the following technologies are tried in order: xz, bz2 and gzip""" self.close() + class FileCacheArchive(Archive): _tmp_dir = "" _archive_root = "" _archive_path = "" - + def __init__(self, name, tmpdir): self._name = name self._tmp_dir = tmpdir self._archive_root = os.path.join(tmpdir, name) os.makedirs(self._archive_root, 0700) - self.log.debug("initialised empty FileCacheArchive at %s" - % self._archive_root) + self.log.debug("initialised empty FileCacheArchive at %s" % + (self._archive_root,)) def dest_path(self, name): if os.path.isabs(name): @@ -113,10 +108,10 @@ class FileCacheArchive(Archive): shutil.copystat(src, dest) stat = os.stat(src) os.chown(dest, stat.st_uid, stat.st_gid) - except IOError as e: + except IOError: self.log.info("caught IO error copying %s" % src) - self.log.debug("added %s to FileCacheArchive %s" - % (src, self._archive_root)) + self.log.debug("added %s to FileCacheArchive %s" % + (src, self._archive_root)) def add_string(self, content, dest): src = dest @@ -127,7 +122,7 @@ class FileCacheArchive(Archive): if os.path.exists(src): shutil.copystat(src, dest) self.log.debug("added string at %s to FileCacheArchive %s" - % (src, self._archive_root)) + % (src, self._archive_root)) def add_link(self, source, link_name): dest = self.dest_path(link_name) @@ -135,21 +130,21 @@ class FileCacheArchive(Archive): if not os.path.exists(dest): os.symlink(source, dest) self.log.debug("added symlink at %s to %s in FileCacheArchive %s" - % (dest, source, self._archive_root)) + % (dest, source, self._archive_root)) def add_dir(self, path): self.makedirs(path) def _makedirs(self, path, mode=0700): os.makedirs(path, mode) - + def get_tmp_dir(self): return self._archive_root def makedirs(self, path, mode=0700): self._makedirs(self.dest_path(path)) self.log.debug("created directory at %s in FileCacheArchive %s" - % (path, self._archive_root)) + % (path, self._archive_root)) def open_file(self, path): path = self.dest_path(path) @@ -157,15 +152,16 @@ class FileCacheArchive(Archive): def cleanup(self): shutil.rmtree(self._archive_root) - + def finalize(self, method): self.log.debug("finalizing archive %s" % self._archive_root) self._build_archive() self.cleanup() self.log.debug("built archive at %s (size=%d)" % (self._archive_path, - os.stat(self._archive_path).st_size)) + os.stat(self._archive_path).st_size)) return self._compress() + class TarFileArchive(FileCacheArchive): method = None @@ -186,7 +182,7 @@ class TarFileArchive(FileCacheArchive): tar_info.mode = fstat.st_mode tar_info.uid = fstat.st_uid tar_info.gid = fstat.st_gid - + # this can be used to set permissions if using the # tarfile.add() interface to add directory trees. def copy_permissions_filter(self, tarinfo): @@ -201,7 +197,7 @@ class TarFileArchive(FileCacheArchive): context = self.get_selinux_context(orig_path) if(context): tarinfo.pax_headers['RHT.security.selinux'] = context - self.set_tarinfo_from_stat(tarinfo,fstat) + self.set_tarinfo_from_stat(tarinfo, fstat) return tarinfo def get_selinux_context(self, path): @@ -219,11 +215,12 @@ class TarFileArchive(FileCacheArchive): old_umask = os.umask(0077) os.chdir(self._tmp_dir) tar = tarfile.open(self._archive_path, mode="w") - tar.add(os.path.split(self._name)[1], filter=self.copy_permissions_filter) + tar.add(os.path.split(self._name)[1], + filter=self.copy_permissions_filter) tar.close() os.umask(old_umask) os.chdir(old_pwd) - + def _compress(self): methods = ['xz', 'bzip2', 'gzip'] if self.method in methods: @@ -238,7 +235,7 @@ class TarFileArchive(FileCacheArchive): if cmd != "gzip": cmd = "%s -1" % cmd try: - command = shlex.split("%s %s" % (cmd,self.name())) + command = shlex.split("%s %s" % (cmd, self.name())) p = Popen(command, stdout=PIPE, stderr=PIPE, bufsize=-1) stdout, stderr = p.communicate() if stdout: @@ -252,6 +249,7 @@ class TarFileArchive(FileCacheArchive): else: raise last_error + class ZipFileArchive(Archive): def __init__(self, name): @@ -262,7 +260,8 @@ class ZipFileArchive(Archive): except: self.compression = zipfile.ZIP_STORED - self.zipfile = zipfile.ZipFile(self.name(), mode="w", compression=self.compression) + self.zipfile = zipfile.ZipFile(self.name(), mode="w", + compression=self.compression) def name(self): return "%s.zip" % self._name @@ -284,19 +283,19 @@ class ZipFileArchive(Archive): for filename in filenames: filename = "/".join((path, filename)) if dest: - self.zipfile.write(filename, - self.prepend(re.sub(regex, dest, filename))) + self.zipfile.write(filename, re.sub(regex, dest, + filename)) else: - self.zipfile.write(filename, self.prepend(filename)) + self.zipfile.write(filename) else: if dest: - self.zipfile.write(src, self.prepend(dest)) + self.zipfile.write(src, dest) else: - self.zipfile.write(src, self.prepend(src)) + self.zipfile.write(src) def add_string(self, content, dest): - info = zipfile.ZipInfo(self.prepend(dest), - date_time=time.localtime(time.time())) + info = zipfile.ZipInfo(dest, + date_time=time.localtime(time.time())) info.compress_type = self.compression info.external_attr = 0400 << 16L self.zipfile.writestr(info, content) @@ -305,7 +304,6 @@ class ZipFileArchive(Archive): try: self.zipfile.close() self.zipfile = zipfile.ZipFile(self.name(), mode="r") - name = self.prepend(name) file_obj = self.zipfile.open(name) return file_obj finally: @@ -314,5 +312,3 @@ class ZipFileArchive(Archive): def close(self): self.zipfile.close() - - diff --git a/tests/archive_tests.py b/tests/archive_tests.py index 7847c7a4..abdce994 100644 --- a/tests/archive_tests.py +++ b/tests/archive_tests.py @@ -4,9 +4,12 @@ import unittest import os import tarfile import zipfile +import tempfile +import shutil from sos.archive import TarFileArchive, ZipFileArchive + class ZipFileArchiveTest(unittest.TestCase): def setUp(self): @@ -27,37 +30,37 @@ class ZipFileArchiveTest(unittest.TestCase): self.zf.add_file('tests/ziptest') self.zf.close() - self.check_for_file('test/tests/ziptest') + self.check_for_file('tests/ziptest') def test_add_unicode_file(self): self.zf.add_file(u'tests/') self.zf.close() - self.check_for_file('test/tests/ziptest') + self.check_for_file('tests/ziptest') def test_add_dir(self): self.zf.add_file('tests/') self.zf.close() - self.check_for_file('test/tests/ziptest') + self.check_for_file('tests/ziptest') def test_add_renamed(self): self.zf.add_file('tests/ziptest', dest='tests/ziptest_renamed') self.zf.close() - self.check_for_file('test/tests/ziptest_renamed') + self.check_for_file('tests/ziptest_renamed') def test_add_renamed_dir(self): self.zf.add_file('tests/', 'tests_renamed/') self.zf.close() - self.check_for_file('test/tests_renamed/ziptest') + self.check_for_file('tests_renamed/ziptest') def test_add_string(self): self.zf.add_string('this is content', 'tests/string_test.txt') self.zf.close() - self.check_for_file('test/tests/string_test.txt') + self.check_for_file('tests/string_test.txt') def test_get_file(self): self.zf.add_string('this is my content', 'tests/string_test.txt') @@ -72,40 +75,45 @@ class ZipFileArchiveTest(unittest.TestCase): afp = self.zf.open_file('tests/string_test.txt') self.assertEquals('this is my new content', afp.read()) - def test_make_link(self): - self.zf.add_file('tests/ziptest') - self.zf.add_link('tests/ziptest', 'link_name') +# Disabled as new api doesnt provide a add_link routine +# def test_make_link(self): +# self.zf.add_file('tests/ziptest') +# self.zf.add_link('tests/ziptest', 'link_name') +# +# self.zf.close() +# try: +# self.check_for_file('test/link_name') +# self.fail("link should not exist") +# except KeyError: +# pass - self.zf.close() - try: - self.check_for_file('test/link_name') - self.fail("link should not exist") - except KeyError: - pass +# Disabled as new api doesnt provide a compress routine +# def test_compress(self): +# self.assertEquals(self.zf.compress("zip"), self.zf.name()) - def test_compress(self): - self.assertEquals(self.zf.compress("zip"), self.zf.name()) class TarFileArchiveTest(unittest.TestCase): def setUp(self): - self.tf = TarFileArchive('test') + self.tmpdir = tempfile.mkdtemp() + self.tf = TarFileArchive('test', self.tmpdir) def tearDown(self): - os.unlink(self.tf.name()) + shutil.rmtree(self.tmpdir) def check_for_file(self, filename): - rtf = tarfile.open('test.tar') + rtf = tarfile.open(os.path.join(self.tmpdir, 'test.tar')) rtf.getmember(filename) rtf.close() def test_create(self): - self.tf.close() - self.assertTrue(os.path.exists('test.tar')) + self.tf.finalize('auto') + self.assertTrue(os.path.exists(os.path.join(self.tmpdir, + 'test.tar'))) def test_add_file(self): self.tf.add_file('tests/ziptest') - self.tf.close() + self.tf.finalize('auto') self.check_for_file('test/tests/ziptest') @@ -119,7 +127,7 @@ class TarFileArchiveTest(unittest.TestCase): def test_add_renamed(self): self.tf.add_file('tests/ziptest', dest='tests/ziptest_renamed') - self.tf.close() + self.tf.finalize('auto') self.check_for_file('test/tests/ziptest_renamed') @@ -133,7 +141,7 @@ class TarFileArchiveTest(unittest.TestCase): def test_add_string(self): self.tf.add_string('this is content', 'tests/string_test.txt') - self.tf.close() + self.tf.finalize('auto') self.check_for_file('test/tests/string_test.txt') @@ -154,11 +162,11 @@ class TarFileArchiveTest(unittest.TestCase): self.tf.add_file('tests/ziptest') self.tf.add_link('tests/ziptest', 'link_name') - self.tf.close() + self.tf.finalize('auto') self.check_for_file('test/link_name') def test_compress(self): - name = self.tf.compress("gzip") + self.tf.finalize("auto") if __name__ == "__main__": unittest.main() -- cgit