#!/usr/bin/env python
"""
Gather information about a system and report it using plugins
supplied for application-specific information
"""
## sosreport.py
## gather information about a system and report it
## Copyright (C) 2006 Steve Conklin <sconklin@redhat.com>
### This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# pylint: disable-msg = W0611
# pylint: disable-msg = W0702
import sys
import os
from optparse import OptionParser, Option
import sos.policyredhat
from sos.helpers import *
from snack import *
from threading import Thread, activeCount
import signal
import logging
from stat import *
from time import strftime, localtime, time
from pwd import getpwuid
import gettext
from threading import Semaphore
__version__ = 1.7
__breakHits__ = 0 # Use this to track how many times we enter the exit routine
## Set up routines to be linked to signals for termination handling
def exittermhandler(signum, frame):
doExitCode()
def doExitCode():
from threading import enumerate
global __breakHits__
__breakHits__ += 1
if ( ( activeCount() > 1 ) and ( __breakHits__ == 1 ) ):
print "SIGTERM received, multiple threads detected, waiting for all threads to exit"
for thread in enumerate():
if thread.getName() == "MainThread":
continue
# until we find a way to kill threads in case of > 1 CTRL+C, ignore KeyboardInterrupt
while thread.isAlive():
try:
thread.join()
except KeyboardInterrupt:
pass
else:
print "All threads ended, cleaning up."
if ( ( activeCount() > 1 ) and ( __breakHits__ > 1 ) ):
print "Multiple SIGTERMs, multiple threads, attempting to signal threads to die immediately"
## FIXME: Add thread-kill code (see FIXME below)
# os.kill(os.getpid(), signal.SIGKILL)
print "Threads dead, cleaning up."
if ( ( activeCount() == 1 ) and ( __breakHits__ > 2 ) ):
print "Multiple SIGTERMs, single thread, exiting without cleaning up."
sys.exit(3)
# FIXME: Add code here to clean up /tmp
sys.exit("Abnormal exit")
# Handle any sort of exit signal cleanly
# Currently, we intercept only sig 15 (TERM)
signal.signal(signal.SIGTERM, exittermhandler)
## FIXME: Need to figure out how to IPC with child threads in case of
## multiple SIGTERMs.
# for debugging
__raisePlugins__ = 0
class OptionParser_extended(OptionParser):
def print_help(self):
OptionParser.print_help(self)
print
print "Some examples:"
print
print " enable cluster plugin only and collect dlm lockdumps:"
print " # sosreport -o cluster -k cluster.lockdump"
print
print " disable memory and samba plugins, turn off rpm -Va collection:"
print " # sosreport -n memory,samba -k rpm.rpmva=off"
print
class SosOption (Option):
"""Allow to specify comma delimited list of plugins"""
ACTIONS = Option.ACTIONS + ("extend",)
STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)
def take_action(self, action, dest, opt, value, values, parser):
if action == "extend":
try: lvalue = value.split(",")
except: pass
else: values.ensure_value(dest, []).extend(lvalue)
else:
Option.take_action(self, action, dest, opt, value, values, parser)
__cmdParser__ = OptionParser_extended(option_class=SosOption)
__cmdParser__.add_option("-l", "--list-plugins", action="store_true", \
dest="listPlugins", default=False, \
help="list plugins and available plugin options")
__cmdParser__.add_option("-n", "--skip-plugins", action="extend", \
dest="noplugins", type="string", \
help="skip these plugins", default = [])
__cmdParser__.add_option("-e", "--enable-plugins", action="extend", \
dest="enableplugins", type="string", \
help="enable these plugins", default = [])
__cmdParser__.add_option("-o", "--only-plugins", action="extend", \
dest="onlyplugins", type="string", \
help="enable these plugins only", default = [])
__cmdParser__.add_option("-k", action="extend", \
dest="plugopts", type="string", \
help="plugin options in plugname.option=value format (see -l)")
__cmdParser__.add_option("-a", "--alloptions", action="store_true", \
dest="usealloptions", default=False, \
help="enable all options for loaded plugins")
__cmdParser__.add_option("-v", "--verbose", action="count", \
dest="verbosity", \
help="increase verbosity")
__cmdParser__.add_option("--no-progressbar", action="store_false", \
dest="progressbar", default=True, \
help="do not display a progress bar.")
__cmdParser__.add_option("--no-multithread", action="store_true", \
dest="nomultithread", \
help="disable multi-threaded gathering mode (slower)", default=False)
(__cmdLineOpts__, __cmdLineArgs__)=__cmdParser__.parse_args()
def textcolor(text, fg, raw=0):
colors = { "black":"30", "red":"31", "green":"32", "brown":"33", "blue":"34",
"purple":"35", "cyan":"36", "lgray":"37", "gray":"1;30", "lred":"1;31",
"lgreen":"1;32", "yellow":"1;33", "lblue":"1;34", "pink":"1;35",
"lcyan":"1;36", "white":"1;37" }
opencol = "\033["
closecol = "m"
clear = opencol + "0" + closecol
f = opencol + colors[fg] + closecol
return "%s%s%s" % (f, text, clear)
class progressBar:
def __init__(self, minValue = 0, maxValue = 10, totalWidth=40):
self.progBar = "[]" # This holds the progress bar string
self.min = minValue
self.max = maxValue
self.width = totalWidth
self.amount = 0 # When amount == max, we are 100% done
self.time_start = time()
self.eta = 0
self.last_amount_update = time()
self.update()
def updateAmount(self, newAmount = 0):
if newAmount < self.min:
newAmount = self.min
if newAmount > self.max:
newAmount = self.max
if self.amount != newAmount:
self.last_amount_update = time()
self.amount = newAmount
last_update_relative = round(self.last_amount_update - self.time_start)
self.eta = round(last_update_relative * self.max / self.amount)
# generate ETA
timeElapsed = round(time() - self.time_start)
last_update_relative = round(self.last_amount_update - self.time_start)
if timeElapsed >= 10 and self.amount > 0:
try:
percentDone = round(timeElapsed * 100 / self.eta)
except:
percentDone = 0
if percentDone > 100:
percentDone = 100
ETA = timeElapsed
elif self.eta < timeElapsed:
ETA = timeElapsed
else:
ETA = self.eta
ETA = "[%02d:%02d/%02d:%02d]" % (int(timeElapsed/60), timeElapsed % 60, int(ETA/60), ETA % 60)
else:
ETA = "[%02d:%02d/--:--]" % (int(timeElapsed/60), timeElapsed % 60)
if self.amount < self.max:
percentDone = 0
else:
percentDone = 100
# Figure out how many hash bars the percentage should be
allFull = self.width - 2
numHashes = (percentDone / 100.0) * allFull
numHashes = int(round(numHashes))
self.progBar = ""
for inc in range(0,allFull):
if inc == int(allFull / 2):
self.progBar = self.progBar + textcolor("%d%%" % percentDone, "green")
elif inc < numHashes:
self.progBar = self.progBar + textcolor('#', "gray")
else:
self.progBar = self.progBar + ' '
self.progBar = " Progress [" + self.progBar + "]" + ETA
def incAmount(self, toInc = 1):
self.updateAmount(self.amount+toInc)
def finished(self):
self.updateAmount(self.max)
sys.stdout.write(self.progBar + '\n')
sys.stdout.flush()
def update(self):
self.updateAmount(self.amount)
sys.stdout.write(self.progBar + '\r')
sys.stdout.flush()
class XmlReport:
def __init__(self):
try:
import libxml2
except:
self.enabled = False
return
else:
self.enabled = True
self.doc = libxml2.newDoc("1.0")
self.root = self.doc.newChild(None, "sos", None)
self.commands = self.root.newChild(None, "commands", None)
self.files = self.root.newChild(None, "files", None)
def add_command(self, cmdline, exitcode, stdout = None, stderr = None, f_stdout=None, f_stderr=None, runtime=None):
if not self.enabled: return
cmd = self.commands.newChild(None, "cmd", None)
cmd.setNsProp(None, "cmdline", cmdline)
cmdchild = cmd.newChild(None, "exitcode", str(exitcode))
if runtime:
cmd.newChild(None, "runtime", str(runtime))
if stdout or f_stdout:
cmdchild = cmd.newChild(None, "stdout", stdout)
if f_stdout:
cmdchild.setNsProp(None, "file", f_stdout)
if stderr or f_stderr:
cmdchild = cmd.newChild(None, "stderr", stderr)
if f_stderr:
cmdchild.setNsProp(None, "file", f_stderr)
def add_file(self,fname,stats):
if not self.enabled: return
cfile = self.files.newChild(None,"file",None)
cfile.setNsProp(None, "fname", fname)
cchild = cfile.newChild(None, "uid", str(stats[ST_UID]))
cchild = cfile.newChild(None, "gid", str(stats[ST_GID]))
cfile.newChild(None, "mode", str(oct(S_IMODE(stats[ST_MODE]))))
cchild = cfile.newChild(None, "ctime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_CTIME])))
cchild.setNsProp(None,"tstamp", str(stats[ST_CTIME]))
cchild = cfile.newChild(None, "atime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_ATIME])))
cchild.setNsProp(None,"tstamp", str(stats[ST_ATIME]))
cchild = cfile.newChild(None, "mtime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_MTIME])))
cchild.setNsProp(None,"tstamp", str(stats[ST_MTIME]))
def serialize(self):
if not self.enabled: return
print self.doc.serialize(None, 1)
def serialize_to_file(self,fname):
if not self.enabled: return
outfn = open(fname,"w")
outfn.write(self.doc.serialize(None,1))
outfn.close()
def sosreport():
# pylint: disable-msg = R0912
# pylint: disable-msg = R0914
# pylint: disable-msg = R0915
"""
This is the top-level function that gathers and processes all sosreport information
"""
loadedplugins = []
skippedplugins = []
alloptions = []
# perhaps we should automatically locate the policy module??
policy = sos.policyredhat.SosPolicy()
# find the plugins path
paths = sys.path
for path in paths:
if path.strip()[-len("site-packages"):] == "site-packages":
pluginpath = path + "/sos/plugins"
# Set up common info and create destinations
dstroot = sosFindTmpDir()
cmddir = os.path.join(dstroot, "sos_commands")
logdir = os.path.join(dstroot, "sos_logs")
rptdir = os.path.join(dstroot, "sos_reports")
os.mkdir(cmddir, 0755)
os.mkdir(logdir, 0755)
os.mkdir(rptdir, 0755)
# initialize i18n language localization
gettext.install('sos', '/usr/share/locale', unicode=False)
# initialize logging
soslog = logging.getLogger('sos')
soslog.setLevel(logging.DEBUG)
logging.VERBOSE = logging.INFO - 1
logging.VERBOSE2 = logging.INFO - 2
logging.VERBOSE3 = logging.INFO - 3
logging.addLevelName(logging.VERBOSE, "verbose")
logging.addLevelName(logging.VERBOSE2,"verbose2")
logging.addLevelName(logging.VERBOSE3,"verbose3")
# FIXME: strip colours if terminal doesn't allow it
# log to a file
flog = logging.FileHandler(logdir + "/sos.log")
flog.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
flog.setLevel(logging.VERBOSE3)
soslog.addHandler(flog)
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler(sys.stderr)
if __cmdLineOpts__.verbosity > 0:
console.setLevel(20 - __cmdLineOpts__.verbosity)
__cmdLineOpts__.progressbar = False
else:
console.setLevel(logging.INFO)
console.setFormatter(logging.Formatter('%(message)s'))
soslog.addHandler(console)
xmlrep = XmlReport()
# set up dict so everyone can share the following
commons = {'dstroot': dstroot, 'cmddir': cmddir, 'logdir': logdir, 'rptdir': rptdir,
'soslog': soslog, 'policy': policy, 'verbosity' : __cmdLineOpts__.verbosity,
'xmlreport' : xmlrep }
# Make policy aware of the commons
policy.setCommons(commons)
print
soslog.info ( _("sosreport (version %s)") % __version__)
print
# generate list of available plugins
plugins = os.listdir(pluginpath)
plugins.sort()
# FIXME: should at least print a warning if the user references a plugin which does not exist
# validate and load plugins
for plug in plugins:
plugbase = plug[:-3]
if not plug[-3:] == '.py' or plugbase == "__init__":
continue
try:
#print "importing plugin: %s" % plugbase
try:
if policy.validatePlugin(pluginpath + plug):
pluginClass = importPlugin("sos.plugins." + plugbase, plugbase)
else:
soslog.warning(_("plugin %s does not validate, skipping") % plug)
skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
continue
if plugbase in __cmdLineOpts__.noplugins:
soslog.log(logging.VERBOSE, _("plugin %s skipped (--skip-plugins)") % plugbase)
skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
continue
if not pluginClass(plugbase, commons).checkenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins:
soslog.log(logging.VERBOSE, _("plugin %s is inactive (use -e or -o to enable).") % plug)
skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
continue
if not pluginClass(plugbase, commons).defaultenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins:
soslog.log(logging.VERBOSE, "plugin %s not loaded by default (use -e or -o to enable)." % plug)
skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
continue
if __cmdLineOpts__.onlyplugins and not plugbase in __cmdLineOpts__.onlyplugins:
soslog.log(logging.VERBOSE, _("plugin %s not specified in --only-plugins list") % plug)
skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
continue
loadedplugins.append((plugbase, pluginClass(plugbase, commons)))
except:
soslog.warning(_("plugin %s does not install, skipping") % plug)
raise
except:
soslog.warning(_("could not load plugin %s") % plug)
if __raisePlugins__:
raise
# First, gather and process options
# using the options specified in the command line (if any)
if __cmdLineOpts__.usealloptions:
for plugname, plug in loadedplugins:
for name, parms in zip(plug.optNames, plug.optParms):
if type(parms["enabled"])==bool:
parms["enabled"] = True
if __cmdLineOpts__.plugopts:
opts = {}
for opt in __cmdLineOpts__.plugopts:
# split up "general.syslogsize=5"
try:
opt, val = opt.split("=")
except:
val=True
else:
if val == "off" or val == "disable" or val == "disabled":
val = False
else:
# try to convert string "val" to int()
try: val = int(val)
except: pass
# split up "general.syslogsize"
plug, opt = opt.split(".")
try: opts[plug]
except KeyError: opts[plug] = []
opts[plug].append( (opt,val) )
for plugname, plug in loadedplugins:
if opts.has_key(plugname):
for opt,val in opts[plugname]:
soslog.log(logging.VERBOSE, "setting option %s for plugin %s to %s" % (plugname,opt,val))
plug.setOption(opt,val)
del opt,opts,val
for plugname, plug in loadedplugins:
soslog.log(logging.VERBOSE3, _("processing options from plugin: %s") % plugname)
names, parms = plug.getAllOptions()
for optname, optparm in zip(names, parms):
alloptions.append((plug, plugname, optname, optparm))
# when --listplugins is specified we do a dry-run
# which tells the user which plugins are going to be enabled
# and with what options.
if __cmdLineOpts__.listPlugins:
if not len(loadedplugins) and not len(skippedplugins):
soslog.error(_("no valid plugins found"))
sys.exit(1)
# FIXME: make -l output more concise
if len(loadedplugins):
print _("The following plugins are currently enabled:")
print
for (plugname,plug) in loadedplugins:
print " %-25s %s" % (textcolor(plugname,"lblue"),plug.get_description())
else:
print _("No plugin enabled.")
print
if len(skippedplugins):
print _("The following plugins are currently disabled:")
print
for (plugname,plugclass) in skippedplugins:
print " %-25s %s" % (textcolor(plugname,"cyan"),plugclass.get_description())
print
if len(alloptions):
print _("The following plugin options are available:")
print
for (plug, plugname, optname, optparm) in alloptions:
# format and colorize option value based on its type (int or bool)
if type(optparm["enabled"])==bool:
if optparm["enabled"]==True:
tmpopt = textcolor("on","lred")
else:
tmpopt = textcolor("off","red")
elif type(optparm["enabled"])==int:
if optparm["enabled"] > 0:
tmpopt = textcolor(optparm["enabled"],"lred")
else:
tmpopt = textcolor(optparm["enabled"],"red")
else:
tmpopt = optparm["enabled"]
print " %-21s %-5s %s" % (plugname + "." + optname, tmpopt, optparm["desc"])
del tmpopt
else:
print _("No plugin options available.")
print
sys.exit()
# to go anywhere further than listing the plugins we will need root permissions.
#
if os.getuid() != 0:
print _('sosreport requires root permissions to run.')
sys.exit(1)
# we don't need to keep in memory plugins we are not going to use
del skippedplugins
if not len(loadedplugins):
soslog.error(_("no valid plugins were enabled"))
sys.exit(1)
try:
raw_input(_("""This utility will collect some detailed information about the
hardware and setup of your Red Hat Enterprise Linux system.
The information is collected and an archive is packaged under
/tmp, which you can send to a support rappresentative.
Red Hat will use this information for diagnostic purposes ONLY
and it will be considered confidential information.
This process may take a while to complete.
No changes will be made to your system.
Press ENTER to continue, or CTRL-C to quit.
"""))
except KeyboardInterrupt:
print
sys.exit(0)
# Call the diagnose() method for each plugin
tmpcount = 0
for plugname, plug in loadedplugins:
soslog.log(logging.VERBOSE2, "Performing sanity check for plugin %s" % plugname)
try:
plug.diagnose()
except:
if __raisePlugins__:
raise
tmpcount += len(plug.diagnose_msgs)
if tmpcount > 0:
print _("One or more plugins have detected a problem in your configuration.")
print _("Please review the following messages:")
print
fp = open(rptdir + "/diagnose.txt", "w")
for plugname, plug in loadedplugins:
for tmpcount2 in range(0,len(plug.diagnose_msgs)):
if tmpcount2 == 0:
soslog.warning( textcolor("%s:" % plugname, "red") )
soslog.warning(" * %s" % plug.diagnose_msgs[tmpcount2])
fp.write("%s: %s\n" % (plugname, plug.diagnose_msgs[tmpcount2]) )
fp.close()
print
try:
while True:
yorno = raw_input( _("Are you sure you would like to continue (y/n) ? ") )
if yorno == _("y") or yorno == _("Y"):
print
break
elif yorno == _("n") or yorno == _("N"):
sys.exit(0)
del yorno
except KeyboardInterrupt:
print
sys.exit(0)
policy.preWork()
# Call the setup() method for each plugin
for plugname, plug in loadedplugins:
soslog.log(logging.VERBOSE2, "Preloading files and commands to be gathered by plugin %s" % plugname)
try:
plug.setup()
except:
if __raisePlugins__:
raise
# Setup the progress bar
if __cmdLineOpts__.progressbar:
# gather information useful for generating ETA
eta_weight = len(loadedplugins)
for plugname, plug in loadedplugins:
eta_weight += plug.eta_weight
pbar = progressBar(minValue = 0, maxValue = eta_weight)
# pbar.max = number_of_plugins + weight (default 1 per plugin)
if __cmdLineOpts__.nomultithread:
soslog.log(logging.VERBOSE, "using single-threading")
else:
soslog.log(logging.VERBOSE, "using multi-threading")
# Call the collect method for each plugin
plugrunning = Semaphore(2)
for plugname, plug in loadedplugins:
soslog.log(logging.VERBOSE, "executing plugin %s" % plugname)
try:
if not __cmdLineOpts__.nomultithread:
plug.copyStuff(threaded = True, semaphore = plugrunning)
else:
plug.copyStuff()
if __cmdLineOpts__.progressbar:
pbar.incAmount(plug.eta_weight)
pbar.update()
except:
if __raisePlugins__:
raise
del plugrunning
# Wait for all the collection threads to exit
if not __cmdLineOpts__.nomultithread:
finishedplugins = []
while len(loadedplugins) > 0:
plugname, plug = loadedplugins.pop(0)
if not plug.wait(0.5):
finishedplugins.append((plugname,plug))
soslog.log(logging.DEBUG, "plugin %s has returned" % plugname)
if __cmdLineOpts__.progressbar:
pbar.incAmount(plug.eta_weight)
else:
soslog.log(logging.DEBUG, "plugin %s still hasn't returned" % plugname)
loadedplugins.append((plugname,plug))
if __cmdLineOpts__.progressbar:
pbar.update()
loadedplugins = finishedplugins
del finishedplugins
for plugname, plug in loadedplugins:
for oneFile in plug.copiedFiles:
try:
xmlrep.add_file(oneFile["srcpath"], os.stat(oneFile["srcpath"]))
except:
pass
xmlrep.serialize_to_file(rptdir + "/sosreport.xml")
# Call the analyze method for each plugin
for plugname, plug in loadedplugins:
soslog.log(logging.VERBOSE2, "Analyzing results of plugin %s" % plugname,)
try:
plug.analyze()
except:
# catch exceptions in analyse() and keep working
pass
if __cmdLineOpts__.progressbar:
pbar.incAmount()
pbar.update()
if __cmdLineOpts__.progressbar:
pbar.finished()
sys.stdout.write("\n")
# Generate the header for the html output file
rfd = open(rptdir + "/" + "sosreport.html", "w")
rfd.write("""
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<link rel="stylesheet" type="text/css" media="screen" href="donot.css" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Sos System Report</title>
</head>
<body>
""")
# Make a pass to gather Alerts and a list of module names
allAlerts = []
plugNames = []
for plugname, plug in loadedplugins:
for alert in plug.alerts:
allAlerts.append('<a href="#%s">%s</a>: %s' % (plugname, plugname, alert))
plugNames.append(plugname)
# Create a table of links to the module info
rfd.write("<hr/><h3>Loaded Plugins:</h3>")
rfd.write("<table><tr>\n")
rr = 0
for i in range(len(plugNames)):
rfd.write('<td><a href="#%s">%s</a></td>\n' % (plugNames[i], plugNames[i]))
rr = divmod(i, 4)[1]
if (rr == 3):
rfd.write('</tr>')
if not (rr == 3):
rfd.write('</tr>')
rfd.write('</table>\n')
rfd.write('<hr/><h3>Alerts:</h3>')
rfd.write('<ul>')
for alert in allAlerts:
rfd.write('<li>%s</li>' % alert)
rfd.write('</ul>')
# Call the report method for each plugin
for plugname, plug in loadedplugins:
try:
html = plug.report()
except:
if __raisePlugins__:
raise
else:
rfd.write(html)
rfd.write("</body></html>")
rfd.close()
# Collect any needed user information (name, etc)
# Call the postproc method for each plugin
for plugname, plug in loadedplugins:
try:
plug.postproc()
except:
if __raisePlugins__:
raise
# package up the results for the support organization
policy.packageResults()
# delete gathered files
os.system("/bin/rm -rf %s" % dstroot)
# automated submission will go here
# Close all log files and perform any cleanup
logging.shutdown()
if __name__ == '__main__':
try:
sosreport()
except KeyboardInterrupt:
doExitCode()
