#!/usr/bin/env python """ Gather information about a system and report it using plugins supplied for application-specific information """ ## sosreport.py ## gather information about a system and report it ## Copyright (C) 2006 Steve Conklin ### This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation; either version 2 of the License, or ## (at your option) any later version. ## This program is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## You should have received a copy of the GNU General Public License ## along with this program; if not, write to the Free Software ## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # pylint: disable-msg = W0611 # pylint: disable-msg = W0702 import sys import os from optparse import OptionParser, Option import sos.policyredhat from sos.helpers import * from snack import * from threading import Thread, activeCount import signal import logging from stat import * from time import strftime, localtime, time from pwd import getpwuid import gettext from threading import Semaphore __version__ = 1.7 __breakHits__ = 0 # Use this to track how many times we enter the exit routine ## Set up routines to be linked to signals for termination handling def exittermhandler(signum, frame): doExitCode() def doExitCode(): from threading import enumerate global __breakHits__ __breakHits__ += 1 if ( ( activeCount() > 1 ) and ( __breakHits__ == 1 ) ): print "SIGTERM received, multiple threads detected, waiting for all threads to exit" for thread in enumerate(): if thread.getName() == "MainThread": continue # until we find a way to kill threads in case of > 1 CTRL+C, ignore KeyboardInterrupt while thread.isAlive(): try: thread.join() except KeyboardInterrupt: pass else: print "All threads ended, cleaning up." if ( ( activeCount() > 1 ) and ( __breakHits__ > 1 ) ): print "Multiple SIGTERMs, multiple threads, attempting to signal threads to die immediately" ## FIXME: Add thread-kill code (see FIXME below) # os.kill(os.getpid(), signal.SIGKILL) print "Threads dead, cleaning up." if ( ( activeCount() == 1 ) and ( __breakHits__ > 2 ) ): print "Multiple SIGTERMs, single thread, exiting without cleaning up." sys.exit(3) # FIXME: Add code here to clean up /tmp sys.exit("Abnormal exit") # Handle any sort of exit signal cleanly # Currently, we intercept only sig 15 (TERM) signal.signal(signal.SIGTERM, exittermhandler) ## FIXME: Need to figure out how to IPC with child threads in case of ## multiple SIGTERMs. # for debugging __raisePlugins__ = 0 class OptionParser_extended(OptionParser): def print_help(self): OptionParser.print_help(self) print print "Some examples:" print print " enable cluster plugin only and collect dlm lockdumps:" print " # sosreport -o cluster -k cluster.lockdump" print print " disable memory and samba plugins, turn off rpm -Va collection:" print " # sosreport -n memory,samba -k rpm.rpmva=off" print class SosOption (Option): """Allow to specify comma delimited list of plugins""" ACTIONS = Option.ACTIONS + ("extend",) STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) def take_action(self, action, dest, opt, value, values, parser): if action == "extend": try: lvalue = value.split(",") except: pass else: values.ensure_value(dest, []).extend(lvalue) else: Option.take_action(self, action, dest, opt, value, values, parser) __cmdParser__ = OptionParser_extended(option_class=SosOption) __cmdParser__.add_option("-l", "--list-plugins", action="store_true", \ dest="listPlugins", default=False, \ help="list plugins and available plugin options") __cmdParser__.add_option("-n", "--skip-plugins", action="extend", \ dest="noplugins", type="string", \ help="skip these plugins", default = []) __cmdParser__.add_option("-e", "--enable-plugins", action="extend", \ dest="enableplugins", type="string", \ help="enable these plugins", default = []) __cmdParser__.add_option("-o", "--only-plugins", action="extend", \ dest="onlyplugins", type="string", \ help="enable these plugins only", default = []) __cmdParser__.add_option("-k", action="extend", \ dest="plugopts", type="string", \ help="plugin options in plugname.option=value format (see -l)") __cmdParser__.add_option("-a", "--alloptions", action="store_true", \ dest="usealloptions", default=False, \ help="enable all options for loaded plugins") __cmdParser__.add_option("-v", "--verbose", action="count", \ dest="verbosity", \ help="increase verbosity") __cmdParser__.add_option("--no-progressbar", action="store_false", \ dest="progressbar", default=True, \ help="do not display a progress bar.") __cmdParser__.add_option("--no-multithread", action="store_true", \ dest="nomultithread", \ help="disable multi-threaded gathering mode (slower)", default=False) (__cmdLineOpts__, __cmdLineArgs__)=__cmdParser__.parse_args() def textcolor(text, fg, raw=0): colors = { "black":"30", "red":"31", "green":"32", "brown":"33", "blue":"34", "purple":"35", "cyan":"36", "lgray":"37", "gray":"1;30", "lred":"1;31", "lgreen":"1;32", "yellow":"1;33", "lblue":"1;34", "pink":"1;35", "lcyan":"1;36", "white":"1;37" } opencol = "\033[" closecol = "m" clear = opencol + "0" + closecol f = opencol + colors[fg] + closecol return "%s%s%s" % (f, text, clear) class progressBar: def __init__(self, minValue = 0, maxValue = 10, totalWidth=40): self.progBar = "[]" # This holds the progress bar string self.min = minValue self.max = maxValue self.width = totalWidth self.amount = 0 # When amount == max, we are 100% done self.time_start = time() self.eta = 0 self.last_amount_update = time() self.update() def updateAmount(self, newAmount = 0): if newAmount < self.min: newAmount = self.min if newAmount > self.max: newAmount = self.max if self.amount != newAmount: self.last_amount_update = time() self.amount = newAmount last_update_relative = round(self.last_amount_update - self.time_start) self.eta = round(last_update_relative * self.max / self.amount) # generate ETA timeElapsed = round(time() - self.time_start) last_update_relative = round(self.last_amount_update - self.time_start) if timeElapsed >= 10 and self.amount > 0: try: percentDone = round(timeElapsed * 100 / self.eta) except: percentDone = 0 if percentDone > 100: percentDone = 100 ETA = timeElapsed elif self.eta < timeElapsed: ETA = timeElapsed else: ETA = self.eta ETA = "[%02d:%02d/%02d:%02d]" % (int(timeElapsed/60), timeElapsed % 60, int(ETA/60), ETA % 60) else: ETA = "[%02d:%02d/--:--]" % (int(timeElapsed/60), timeElapsed % 60) if self.amount < self.max: percentDone = 0 else: percentDone = 100 # Figure out how many hash bars the percentage should be allFull = self.width - 2 numHashes = (percentDone / 100.0) * allFull numHashes = int(round(numHashes)) self.progBar = "" for inc in range(0,allFull): if inc == int(allFull / 2): self.progBar = self.progBar + textcolor("%d%%" % percentDone, "green") elif inc < numHashes: self.progBar = self.progBar + textcolor('#', "gray") else: self.progBar = self.progBar + ' ' self.progBar = " Progress [" + self.progBar + "]" + ETA def incAmount(self, toInc = 1): self.updateAmount(self.amount+toInc) def finished(self): self.updateAmount(self.max) sys.stdout.write(self.progBar + '\n') sys.stdout.flush() def update(self): self.updateAmount(self.amount) sys.stdout.write(self.progBar + '\r') sys.stdout.flush() class XmlReport: def __init__(self): try: import libxml2 except: self.enabled = False return else: self.enabled = True self.doc = libxml2.newDoc("1.0") self.root = self.doc.newChild(None, "sos", None) self.commands = self.root.newChild(None, "commands", None) self.files = self.root.newChild(None, "files", None) def add_command(self, cmdline, exitcode, stdout = None, stderr = None, f_stdout=None, f_stderr=None, runtime=None): if not self.enabled: return cmd = self.commands.newChild(None, "cmd", None) cmd.setNsProp(None, "cmdline", cmdline) cmdchild = cmd.newChild(None, "exitcode", str(exitcode)) if runtime: cmd.newChild(None, "runtime", str(runtime)) if stdout or f_stdout: cmdchild = cmd.newChild(None, "stdout", stdout) if f_stdout: cmdchild.setNsProp(None, "file", f_stdout) if stderr or f_stderr: cmdchild = cmd.newChild(None, "stderr", stderr) if f_stderr: cmdchild.setNsProp(None, "file", f_stderr) def add_file(self,fname,stats): if not self.enabled: return cfile = self.files.newChild(None,"file",None) cfile.setNsProp(None, "fname", fname) cchild = cfile.newChild(None, "uid", str(stats[ST_UID])) cchild = cfile.newChild(None, "gid", str(stats[ST_GID])) cfile.newChild(None, "mode", str(oct(S_IMODE(stats[ST_MODE])))) cchild = cfile.newChild(None, "ctime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_CTIME]))) cchild.setNsProp(None,"tstamp", str(stats[ST_CTIME])) cchild = cfile.newChild(None, "atime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_ATIME]))) cchild.setNsProp(None,"tstamp", str(stats[ST_ATIME])) cchild = cfile.newChild(None, "mtime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_MTIME]))) cchild.setNsProp(None,"tstamp", str(stats[ST_MTIME])) def serialize(self): if not self.enabled: return print self.doc.serialize(None, 1) def serialize_to_file(self,fname): if not self.enabled: return outfn = open(fname,"w") outfn.write(self.doc.serialize(None,1)) outfn.close() def sosreport(): # pylint: disable-msg = R0912 # pylint: disable-msg = R0914 # pylint: disable-msg = R0915 """ This is the top-level function that gathers and processes all sosreport information """ loadedplugins = [] skippedplugins = [] alloptions = [] # perhaps we should automatically locate the policy module?? policy = sos.policyredhat.SosPolicy() # find the plugins path paths = sys.path for path in paths: if path.strip()[-len("site-packages"):] == "site-packages": pluginpath = path + "/sos/plugins" # Set up common info and create destinations dstroot = sosFindTmpDir() cmddir = os.path.join(dstroot, "sos_commands") logdir = os.path.join(dstroot, "sos_logs") rptdir = os.path.join(dstroot, "sos_reports") os.mkdir(cmddir, 0755) os.mkdir(logdir, 0755) os.mkdir(rptdir, 0755) # initialize i18n language localization gettext.install('sos', '/usr/share/locale', unicode=False) # initialize logging soslog = logging.getLogger('sos') soslog.setLevel(logging.DEBUG) logging.VERBOSE = logging.INFO - 1 logging.VERBOSE2 = logging.INFO - 2 logging.VERBOSE3 = logging.INFO - 3 logging.addLevelName(logging.VERBOSE, "verbose") logging.addLevelName(logging.VERBOSE2,"verbose2") logging.addLevelName(logging.VERBOSE3,"verbose3") # FIXME: strip colours if terminal doesn't allow it # log to a file flog = logging.FileHandler(logdir + "/sos.log") flog.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) flog.setLevel(logging.VERBOSE3) soslog.addHandler(flog) # define a Handler which writes INFO messages or higher to the sys.stderr console = logging.StreamHandler(sys.stderr) if __cmdLineOpts__.verbosity > 0: console.setLevel(20 - __cmdLineOpts__.verbosity) __cmdLineOpts__.progressbar = False else: console.setLevel(logging.INFO) console.setFormatter(logging.Formatter('%(message)s')) soslog.addHandler(console) xmlrep = XmlReport() # set up dict so everyone can share the following commons = {'dstroot': dstroot, 'cmddir': cmddir, 'logdir': logdir, 'rptdir': rptdir, 'soslog': soslog, 'policy': policy, 'verbosity' : __cmdLineOpts__.verbosity, 'xmlreport' : xmlrep } # Make policy aware of the commons policy.setCommons(commons) print soslog.info ( _("sosreport (version %s)") % __version__) print # generate list of available plugins plugins = os.listdir(pluginpath) plugins.sort() # FIXME: should at least print a warning if the user references a plugin which does not exist # validate and load plugins for plug in plugins: plugbase = plug[:-3] if not plug[-3:] == '.py' or plugbase == "__init__": continue try: #print "importing plugin: %s" % plugbase try: if policy.validatePlugin(pluginpath + plug): pluginClass = importPlugin("sos.plugins." + plugbase, plugbase) else: soslog.warning(_("plugin %s does not validate, skipping") % plug) skippedplugins.append((plugbase, pluginClass(plugbase, commons))) continue if plugbase in __cmdLineOpts__.noplugins: soslog.log(logging.VERBOSE, _("plugin %s skipped (--skip-plugins)") % plugbase) skippedplugins.append((plugbase, pluginClass(plugbase, commons))) continue if not pluginClass(plugbase, commons).checkenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins: soslog.log(logging.VERBOSE, _("plugin %s is inactive (use -e or -o to enable).") % plug) skippedplugins.append((plugbase, pluginClass(plugbase, commons))) continue if not pluginClass(plugbase, commons).defaultenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins: soslog.log(logging.VERBOSE, "plugin %s not loaded by default (use -e or -o to enable)." % plug) skippedplugins.append((plugbase, pluginClass(plugbase, commons))) continue if __cmdLineOpts__.onlyplugins and not plugbase in __cmdLineOpts__.onlyplugins: soslog.log(logging.VERBOSE, _("plugin %s not specified in --only-plugins list") % plug) skippedplugins.append((plugbase, pluginClass(plugbase, commons))) continue loadedplugins.append((plugbase, pluginClass(plugbase, commons))) except: soslog.warning(_("plugin %s does not install, skipping") % plug) raise except: soslog.warning(_("could not load plugin %s") % plug) if __raisePlugins__: raise # First, gather and process options # using the options specified in the command line (if any) if __cmdLineOpts__.usealloptions: for plugname, plug in loadedplugins: for name, parms in zip(plug.optNames, plug.optParms): if type(parms["enabled"])==bool: parms["enabled"] = True if __cmdLineOpts__.plugopts: opts = {} for opt in __cmdLineOpts__.plugopts: # split up "general.syslogsize=5" try: opt, val = opt.split("=") except: val=True else: if val == "off" or val == "disable" or val == "disabled": val = False else: # try to convert string "val" to int() try: val = int(val) except: pass # split up "general.syslogsize" plug, opt = opt.split(".") try: opts[plug] except KeyError: opts[plug] = [] opts[plug].append( (opt,val) ) for plugname, plug in loadedplugins: if opts.has_key(plugname): for opt,val in opts[plugname]: soslog.log(logging.VERBOSE, "setting option %s for plugin %s to %s" % (plugname,opt,val)) plug.setOption(opt,val) del opt,opts,val for plugname, plug in loadedplugins: soslog.log(logging.VERBOSE3, _("processing options from plugin: %s") % plugname) names, parms = plug.getAllOptions() for optname, optparm in zip(names, parms): alloptions.append((plug, plugname, optname, optparm)) # when --listplugins is specified we do a dry-run # which tells the user which plugins are going to be enabled # and with what options. if __cmdLineOpts__.listPlugins: if not len(loadedplugins) and not len(skippedplugins): soslog.error(_("no valid plugins found")) sys.exit(1) # FIXME: make -l output more concise if len(loadedplugins): print _("The following plugins are currently enabled:") print for (plugname,plug) in loadedplugins: print " %-25s %s" % (textcolor(plugname,"lblue"),plug.get_description()) else: print _("No plugin enabled.") print if len(skippedplugins): print _("The following plugins are currently disabled:") print for (plugname,plugclass) in skippedplugins: print " %-25s %s" % (textcolor(plugname,"cyan"),plugclass.get_description()) print if len(alloptions): print _("The following plugin options are available:") print for (plug, plugname, optname, optparm) in alloptions: # format and colorize option value based on its type (int or bool) if type(optparm["enabled"])==bool: if optparm["enabled"]==True: tmpopt = textcolor("on","lred") else: tmpopt = textcolor("off","red") elif type(optparm["enabled"])==int: if optparm["enabled"] > 0: tmpopt = textcolor(optparm["enabled"],"lred") else: tmpopt = textcolor(optparm["enabled"],"red") else: tmpopt = optparm["enabled"] print " %-21s %-5s %s" % (plugname + "." + optname, tmpopt, optparm["desc"]) del tmpopt else: print _("No plugin options available.") print sys.exit() # to go anywhere further than listing the plugins we will need root permissions. # if os.getuid() != 0: print _('sosreport requires root permissions to run.') sys.exit(1) # we don't need to keep in memory plugins we are not going to use del skippedplugins if not len(loadedplugins): soslog.error(_("no valid plugins were enabled")) sys.exit(1) try: raw_input(_("""This utility will collect some detailed information about the hardware and setup of your Red Hat Enterprise Linux system. The information is collected and an archive is packaged under /tmp, which you can send to a support rappresentative. Red Hat will use this information for diagnostic purposes ONLY and it will be considered confidential information. This process may take a while to complete. No changes will be made to your system. Press ENTER to continue, or CTRL-C to quit. """)) except KeyboardInterrupt: print sys.exit(0) # Call the diagnose() method for each plugin tmpcount = 0 for plugname, plug in loadedplugins: soslog.log(logging.VERBOSE2, "Performing sanity check for plugin %s" % plugname) try: plug.diagnose() except: if __raisePlugins__: raise tmpcount += len(plug.diagnose_msgs) if tmpcount > 0: print _("One or more plugins have detected a problem in your configuration.") print _("Please review the following messages:") print fp = open(rptdir + "/diagnose.txt", "w") for plugname, plug in loadedplugins: for tmpcount2 in range(0,len(plug.diagnose_msgs)): if tmpcount2 == 0: soslog.warning( textcolor("%s:" % plugname, "red") ) soslog.warning(" * %s" % plug.diagnose_msgs[tmpcount2]) fp.write("%s: %s\n" % (plugname, plug.diagnose_msgs[tmpcount2]) ) fp.close() print try: while True: yorno = raw_input( _("Are you sure you would like to continue (y/n) ? ") ) if yorno == _("y") or yorno == _("Y"): print break elif yorno == _("n") or yorno == _("N"): sys.exit(0) del yorno except KeyboardInterrupt: print sys.exit(0) policy.preWork() # Call the setup() method for each plugin for plugname, plug in loadedplugins: soslog.log(logging.VERBOSE2, "Preloading files and commands to be gathered by plugin %s" % plugname) try: plug.setup() except: if __raisePlugins__: raise # Setup the progress bar if __cmdLineOpts__.progressbar: # gather information useful for generating ETA eta_weight = len(loadedplugins) for plugname, plug in loadedplugins: eta_weight += plug.eta_weight pbar = progressBar(minValue = 0, maxValue = eta_weight) # pbar.max = number_of_plugins + weight (default 1 per plugin) if __cmdLineOpts__.nomultithread: soslog.log(logging.VERBOSE, "using single-threading") else: soslog.log(logging.VERBOSE, "using multi-threading") # Call the collect method for each plugin plugrunning = Semaphore(2) for plugname, plug in loadedplugins: soslog.log(logging.VERBOSE, "executing plugin %s" % plugname) try: if not __cmdLineOpts__.nomultithread: plug.copyStuff(threaded = True, semaphore = plugrunning) else: plug.copyStuff() if __cmdLineOpts__.progressbar: pbar.incAmount(plug.eta_weight) pbar.update() except: if __raisePlugins__: raise del plugrunning # Wait for all the collection threads to exit if not __cmdLineOpts__.nomultithread: finishedplugins = [] while len(loadedplugins) > 0: plugname, plug = loadedplugins.pop(0) if not plug.wait(0.5): finishedplugins.append((plugname,plug)) soslog.log(logging.DEBUG, "plugin %s has returned" % plugname) if __cmdLineOpts__.progressbar: pbar.incAmount(plug.eta_weight) else: soslog.log(logging.DEBUG, "plugin %s still hasn't returned" % plugname) loadedplugins.append((plugname,plug)) if __cmdLineOpts__.progressbar: pbar.update() loadedplugins = finishedplugins del finishedplugins for plugname, plug in loadedplugins: for oneFile in plug.copiedFiles: try: xmlrep.add_file(oneFile["srcpath"], os.stat(oneFile["srcpath"])) except: pass xmlrep.serialize_to_file(rptdir + "/sosreport.xml") # Call the analyze method for each plugin for plugname, plug in loadedplugins: soslog.log(logging.VERBOSE2, "Analyzing results of plugin %s" % plugname,) try: plug.analyze() except: # catch exceptions in analyse() and keep working pass if __cmdLineOpts__.progressbar: pbar.incAmount() pbar.update() if __cmdLineOpts__.progressbar: pbar.finished() sys.stdout.write("\n") # Generate the header for the html output file rfd = open(rptdir + "/" + "sosreport.html", "w") rfd.write(""" Sos System Report """) # Make a pass to gather Alerts and a list of module names allAlerts = [] plugNames = [] for plugname, plug in loadedplugins: for alert in plug.alerts: allAlerts.append('%s: %s' % (plugname, plugname, alert)) plugNames.append(plugname) # Create a table of links to the module info rfd.write("

Loaded Plugins:

") rfd.write("\n") rr = 0 for i in range(len(plugNames)): rfd.write('\n' % (plugNames[i], plugNames[i])) rr = divmod(i, 4)[1] if (rr == 3): rfd.write('') if not (rr == 3): rfd.write('') rfd.write('
%s
\n') rfd.write('

Alerts:

') rfd.write('') # Call the report method for each plugin for plugname, plug in loadedplugins: try: html = plug.report() except: if __raisePlugins__: raise else: rfd.write(html) rfd.write("") rfd.close() # Collect any needed user information (name, etc) # Call the postproc method for each plugin for plugname, plug in loadedplugins: try: plug.postproc() except: if __raisePlugins__: raise # package up the results for the support organization policy.packageResults() # delete gathered files os.system("/bin/rm -rf %s" % dstroot) # automated submission will go here # Close all log files and perform any cleanup logging.shutdown() if __name__ == '__main__': try: sosreport() except KeyboardInterrupt: doExitCode()