path: root/src/sosreport

#!/usr/bin/env python
"""
Gather information about a system and report it using plugins
supplied for application-specific information
"""
## sosreport.py
## gather information about a system and report it

## Copyright (C) 2006 Steve Conklin <sconklin@redhat.com>

### This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.

## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# pylint: disable-msg = W0611
# pylint: disable-msg = W0702

import sys
import os
from optparse import OptionParser, Option
import sos.policyredhat
from sos.helpers import *
from snack import *
from threading import Thread, activeCount
import signal
import logging
from stat import *
from time import strftime, localtime, time
from pwd import getpwuid
import gettext
from threading import Semaphore

__version__ = 1.7

__breakHits__ = 0  # Use this to track how many times we enter the exit routine

## Set up routines to be linked to signals for termination handling
def exittermhandler(signum, frame):
    doExitCode()

def doExitCode():
    from threading import enumerate
    global __breakHits__, loadedplugins, dstroot

    __breakHits__ += 1
    if ( ( activeCount() > 1 ) and ( __breakHits__ == 1 ) ):
        print "SIGTERM received, multiple threads detected, waiting for all threads to exit"

        for plugname, plug in loadedplugins:
            plug.exit_please()

        for thread in enumerate():
            if thread.getName() == "MainThread":
                continue
            # until we find a way to kill threads in case of > 1 CTRL+C, ignore KeyboardInterrupt
            while thread.isAlive():
                try:
                    thread.join()
                except KeyboardInterrupt:
                    doExitCode()
        else:
            print "All threads ended, cleaning up."
            doExit(1)

    if  ( ( activeCount() > 1 ) and ( __breakHits__ > 1 ) ):
        print "Multiple SIGTERMs, multiple threads, attempting to signal threads to die immediately"
        ## FIXME: Add thread-kill code (see FIXME below)
        return
    elif ( ( activeCount() > 1 ) and ( __breakHits__ > 2 ) ):
        print "Multiple SIGTERMs, multiple threads, process suicides."
        os.kill(os.getpid(), signal.SIGKILL)
    elif ( ( activeCount() == 1 ) and ( __breakHits__ > 2 ) ):
        print "Multiple SIGTERMs, single thread, exiting without cleaning up."
        doExit(3)

    # FIXME: Add code here to clean up /tmp
    doExit("Abnormal exit")

def doExit(error=0):
    global policy
    policy.cleanDstroot()
    sys.exit(error)

def doException(type, value, tb):
   if hasattr(sys, 'ps1') or not sys.stderr.isatty():
      # we are in interactive mode or we don't have a tty-like
      # device, so we call the default hook
      sys.__excepthook__(type, value, tb)
   else:
      import traceback, pdb
      # we are NOT in interactive mode, print the exception...
      traceback.print_exception(type, value, tb)
      print
      # ...then start the debugger in post-mortem mode.
      pdb.pm()

# Handle any sort of exit signal cleanly
# Currently, we intercept only sig 15 (TERM)
signal.signal(signal.SIGTERM, exittermhandler)

## FIXME: Need to figure out how to IPC with child threads in case of
## multiple SIGTERMs.

class OptionParser_extended(OptionParser):
    def print_help(self):
        OptionParser.print_help(self)
        print
        print "Some examples:"
        print
        print " enable cluster plugin only and collect dlm lockdumps:"
        print "   # sosreport -o cluster -k cluster.lockdump"
        print 
        print " disable memory and samba plugins, turn off rpm -Va collection:"
        print "   # sosreport -n memory,samba -k rpm.rpmva=off"
        print 

class SosOption (Option):
    """Allow to specify comma delimited list of plugins"""
    ACTIONS = Option.ACTIONS + ("extend",)
    STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
    TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)

    def take_action(self, action, dest, opt, value, values, parser):
        if action == "extend":
            try: lvalue = value.split(",")
            except: pass
            else: values.ensure_value(dest, []).extend(lvalue)
        else:
            Option.take_action(self, action, dest, opt, value, values, parser)

__cmdParser__ = OptionParser_extended(option_class=SosOption)
__cmdParser__.add_option("-l", "--list-plugins", action="store_true", \
                     dest="listPlugins", default=False, \
                     help="list plugins and available plugin options")
__cmdParser__.add_option("-n", "--skip-plugins", action="extend", \
                     dest="noplugins", type="string", \
                     help="skip these plugins", default = [])
__cmdParser__.add_option("-e", "--enable-plugins", action="extend", \
                     dest="enableplugins", type="string", \
                     help="enable these plugins", default = [])
__cmdParser__.add_option("-o", "--only-plugins", action="extend", \
                     dest="onlyplugins", type="string", \
                     help="enable these plugins only", default = [])
__cmdParser__.add_option("-k", action="extend", \
                     dest="plugopts", type="string", \
                     help="plugin options in plugname.option=value format (see -l)")
__cmdParser__.add_option("-a", "--alloptions", action="store_true", \
                     dest="usealloptions", default=False, \
                     help="enable all options for loaded plugins")
__cmdParser__.add_option("-u", "--upload", action="store_true", \
                     dest="upload", default=False, \
                     help="upload the report to Red Hat support")
__cmdParser__.add_option("--encrypt", action="store_true", \
                     dest="encrypt", default=False, \
                     help="encrypt with GPG using Red Hat support's public key")
__cmdParser__.add_option("-v", "--verbose", action="count", \
                     dest="verbosity", \
                     help="increase verbosity")
__cmdParser__.add_option("--debug", action="count", \
                     dest="debug", \
                     help="enabling debugging")
__cmdParser__.add_option("--no-progressbar", action="store_false", \
                     dest="progressbar", default=True, \
                     help="do not display a progress bar.")
__cmdParser__.add_option("--no-multithread", action="store_true", \
                     dest="nomultithread", \
                     help="disable multi-threaded gathering mode (slower)", default=False)
(__cmdLineOpts__, __cmdLineArgs__)=__cmdParser__.parse_args()

def textcolor(text, fg, raw=0):
    colors = {  "black":"30", "red":"31", "green":"32", "brown":"33", "blue":"34",
		"purple":"35", "cyan":"36", "lgray":"37", "gray":"1;30", "lred":"1;31",
		"lgreen":"1;32", "yellow":"1;33", "lblue":"1;34", "pink":"1;35",
		"lcyan":"1;36", "white":"1;37" }
    opencol = "\033["
    closecol = "m"
    clear = opencol + "0" + closecol
    f = opencol + colors[fg] + closecol
    return "%s%s%s" % (f, text, clear)

class progressBar:
    def __init__(self, minValue = 0, maxValue = 10, totalWidth=40):
        self.progBar = "[]"   # This holds the progress bar string
        self.min = minValue
        self.max = maxValue
        self.width = totalWidth
        self.amount = 0       # When amount == max, we are 100% done
        self.time_start = time()
        self.eta = 0
        self.last_amount_update = time()
        self.update()

    def updateAmount(self, newAmount = 0):
        if newAmount < self.min:
            newAmount = self.min
        if newAmount > self.max:
            newAmount = self.max
        if self.amount != newAmount:
            self.last_amount_update = time()
            self.amount = newAmount
            last_update_relative = round(self.last_amount_update - self.time_start)
            self.eta = round(last_update_relative * self.max / self.amount)

        # generate ETA
        timeElapsed = round(time() - self.time_start)
        last_update_relative = round(self.last_amount_update - self.time_start)
        if timeElapsed >= 10 and self.amount > 0:
            try:
                percentDone = round(timeElapsed * 100 / self.eta)
            except:
                percentDone = 0
            if percentDone > 100:
                percentDone = 100
                ETA = timeElapsed
            elif self.eta < timeElapsed:
                ETA = timeElapsed
            else:
                ETA = self.eta
            ETA = "[%02d:%02d/%02d:%02d]" % (int(timeElapsed/60), timeElapsed % 60, int(ETA/60), ETA % 60)
        else:
            ETA = "[%02d:%02d/--:--]" % (int(timeElapsed/60), timeElapsed % 60)
            if self.amount < self.max:
                percentDone = 0
            else:
                percentDone = 100

        # Figure out how many hash bars the percentage should be
        allFull = self.width - 2
        numHashes = (percentDone / 100.0) * allFull
        numHashes = int(round(numHashes))

        self.progBar = ""
        for inc in range(0,allFull):
            if inc == int(allFull / 2):
                self.progBar = self.progBar + textcolor("%d%%" % percentDone, "green")
            elif inc < numHashes:
                self.progBar = self.progBar + textcolor('#', "gray")
            else:
                self.progBar = self.progBar + ' '
        self.progBar = " Progress [" + self.progBar + "]" + ETA

    def incAmount(self, toInc = 1):
        self.updateAmount(self.amount+toInc)

    def finished(self):
        self.updateAmount(self.max)
        sys.stdout.write(self.progBar + '\n')
        sys.stdout.flush()

    def update(self):
        self.updateAmount(self.amount)
        sys.stdout.write(self.progBar + '\r')
        sys.stdout.flush()

class XmlReport:
    def __init__(self):
        try:
            import libxml2
        except:
            self.enabled = False
            return
        else:
            self.enabled = True
        self.doc = libxml2.newDoc("1.0")
        self.root = self.doc.newChild(None, "sos", None)
        self.commands = self.root.newChild(None, "commands", None)
        self.files = self.root.newChild(None, "files", None)

    def add_command(self, cmdline, exitcode, stdout = None, stderr = None, f_stdout=None, f_stderr=None, runtime=None):
        if not self.enabled: return

        cmd = self.commands.newChild(None, "cmd", None)

        cmd.setNsProp(None, "cmdline", cmdline)

        cmdchild = cmd.newChild(None, "exitcode", str(exitcode))

        if runtime:
            cmd.newChild(None, "runtime", str(runtime))

        if stdout or f_stdout:
            cmdchild = cmd.newChild(None, "stdout", stdout)
            if f_stdout:
                cmdchild.setNsProp(None, "file", f_stdout)

        if stderr or f_stderr:
            cmdchild = cmd.newChild(None, "stderr", stderr)
            if f_stderr:
                cmdchild.setNsProp(None, "file", f_stderr)

    def add_file(self,fname,stats):
        if not self.enabled: return

        cfile = self.files.newChild(None,"file",None)

        cfile.setNsProp(None, "fname", fname)

        cchild = cfile.newChild(None, "uid", str(stats[ST_UID]))
        cchild = cfile.newChild(None, "gid", str(stats[ST_GID]))
        cfile.newChild(None, "mode", str(oct(S_IMODE(stats[ST_MODE]))))
        cchild = cfile.newChild(None, "ctime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_CTIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_CTIME]))
        cchild = cfile.newChild(None, "atime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_ATIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_ATIME]))
        cchild = cfile.newChild(None, "mtime", strftime('%a %b %d %H:%M:%S %Y', localtime(stats[ST_MTIME])))
        cchild.setNsProp(None,"tstamp", str(stats[ST_MTIME]))

    def serialize(self):
        if not self.enabled: return

        print self.doc.serialize(None,  1)

    def serialize_to_file(self,fname):
        if not self.enabled: return

        outfn = open(fname,"w")
        outfn.write(self.doc.serialize(None,1))
        outfn.close()

# if debugging is enabled, allow plugins to raise exceptions

if __cmdLineOpts__.debug:
    sys.excepthook = doException
    __raisePlugins__ = 1
else:
    __raisePlugins__ = 0

def sosreport():
    # pylint: disable-msg = R0912
    # pylint: disable-msg = R0914
    # pylint: disable-msg = R0915
    """
    This is the top-level function that gathers and processes all sosreport information
    """

    global loadedplugins, dstroot, policy

    loadedplugins = []
    skippedplugins = []
    alloptions = []

    # perhaps we should automatically locate the policy module??
    policy = sos.policyredhat.SosPolicy()

    # find the plugins path
    paths = sys.path
    for path in paths:
        if path.strip()[-len("site-packages"):] == "site-packages":
            pluginpath = path + "/sos/plugins"

    # Set up common info and create destinations
    
    dstroot = policy.getDstroot()
    if not dstroot:
        print _("Could not create temporary directory.")
        doExit()

    cmddir = os.path.join(dstroot, "sos_commands")
    logdir = os.path.join(dstroot, "sos_logs")
    rptdir = os.path.join(dstroot, "sos_reports")
    os.mkdir(cmddir, 0755)
    os.mkdir(logdir, 0755)
    os.mkdir(rptdir, 0755)

    # initialize i18n language localization
    gettext.install('sos', '/usr/share/locale', unicode=False) 

    # initialize logging
    soslog = logging.getLogger('sos')
    soslog.setLevel(logging.DEBUG)

    logging.VERBOSE  = logging.INFO - 1
    logging.VERBOSE2 = logging.INFO - 2
    logging.VERBOSE3 = logging.INFO - 3 
    logging.addLevelName(logging.VERBOSE, "verbose")
    logging.addLevelName(logging.VERBOSE2,"verbose2")
    logging.addLevelName(logging.VERBOSE3,"verbose3")

    # FIXME: strip colours if terminal doesn't allow it

    # log to a file
    flog = logging.FileHandler(logdir + "/sos.log")
    flog.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
    flog.setLevel(logging.VERBOSE3)
    soslog.addHandler(flog)

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler(sys.stderr)
    if __cmdLineOpts__.verbosity > 0:
        console.setLevel(20 - __cmdLineOpts__.verbosity)
        __cmdLineOpts__.progressbar = False
    else:
        console.setLevel(logging.INFO)
    console.setFormatter(logging.Formatter('%(message)s'))
    soslog.addHandler(console)

    xmlrep = XmlReport()

    # set up dict so everyone can share the following
    commons = {'dstroot': dstroot, 'cmddir': cmddir, 'logdir': logdir, 'rptdir': rptdir,
               'soslog': soslog, 'policy': policy, 'verbosity' : __cmdLineOpts__.verbosity,
               'xmlreport' : xmlrep }

    # Make policy aware of the commons
    policy.setCommons(commons)

    print
    soslog.info ( _("sosreport (version %s)") % __version__)
    print
    
    # generate list of available plugins
    plugins = os.listdir(pluginpath)
    plugins.sort()
    plugin_names = []

    # validate and load plugins
    for plug in plugins:
        plugbase =  plug[:-3]
        if not plug[-3:] == '.py' or plugbase == "__init__":
            continue
        try:
            if policy.validatePlugin(pluginpath + plug):
                pluginClass = importPlugin("sos.plugins." + plugbase, plugbase)
            else:
                soslog.warning(_("plugin %s does not validate, skipping") % plug)
                skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                continue
            # plug-in is valid, let's decide whether run it or not
            plugin_names.append(plugbase)
            if plugbase in __cmdLineOpts__.noplugins:
                soslog.log(logging.VERBOSE, _("plugin %s skipped (--skip-plugins)") % plugbase)
                skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                continue
            if not pluginClass(plugbase, commons).checkenabled() and not plugbase in __cmdLineOpts__.enableplugins  and not plugbase in __cmdLineOpts__.onlyplugins:
                soslog.log(logging.VERBOSE, _("plugin %s is inactive (use -e or -o to enable).") % plug)
                skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                continue
            if not pluginClass(plugbase, commons).defaultenabled() and not plugbase in __cmdLineOpts__.enableplugins and not plugbase in __cmdLineOpts__.onlyplugins:
                soslog.log(logging.VERBOSE, "plugin %s not loaded by default (use -e or -o to enable)." % plug)
                skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                continue
            if __cmdLineOpts__.onlyplugins and not plugbase in __cmdLineOpts__.onlyplugins:
                soslog.log(logging.VERBOSE, _("plugin %s not specified in --only-plugins list") % plug)
                skippedplugins.append((plugbase, pluginClass(plugbase, commons)))
                continue
            loadedplugins.append((plugbase, pluginClass(plugbase, commons)))
        except:
            soslog.warning(_("plugin %s does not install, skipping") % plug)
            if __raisePlugins__:
                raise

    # First, gather and process options
    # using the options specified in the command line (if any)
    if __cmdLineOpts__.usealloptions:
        for plugname, plug in loadedplugins:
            for name, parms in zip(plug.optNames, plug.optParms):
                if type(parms["enabled"])==bool:
                    parms["enabled"] = True

    if __cmdLineOpts__.plugopts:
        opts = {}
        for opt in __cmdLineOpts__.plugopts:
            # split up "general.syslogsize=5"
            try:
                opt, val = opt.split("=")
            except:
                val=True
            else:
                if val == "off" or val == "disable" or val == "disabled":
                    val = False
                else:
                    # try to convert string "val" to int()
                    try:    val = int(val)
                    except: pass

            # split up "general.syslogsize"
            try:
                plug, opt = opt.split(".")
            except:
                plug = opt
                opt = True

            try: opts[plug]
            except KeyError: opts[plug] = []
            opts[plug].append( (opt,val) )

        for plugname, plug in loadedplugins:
           if opts.has_key(plugname):
               for opt,val in opts[plugname]:
                   soslog.log(logging.VERBOSE, 'setting option "%s" for plugin (%s) to "%s"' % (plugname,opt,val))
                   if not plug.setOption(opt,val):
                       soslog.error('no such option "%s" for plugin (%s)' % (opt,plugname))
                       doExit(1)
               del opts[plugname]
        for plugname in opts.keys():
            soslog.error('unable to set option for disabled or non-existing plugin (%s)' % (plugname))
            doExit(1)
        del opt,opts,val

    # error if the user references a plugin which does not exist
    unk_plugs =  [plugname.split(".")[0] for plugname in __cmdLineOpts__.onlyplugins   if not plugname.split(".")[0] in plugin_names]
    unk_plugs += [plugname.split(".")[0] for plugname in __cmdLineOpts__.noplugins     if not plugname.split(".")[0] in plugin_names]
    unk_plugs += [plugname.split(".")[0] for plugname in __cmdLineOpts__.enableplugins if not plugname.split(".")[0] in plugin_names]
    if len(unk_plugs):
        for plugname in unk_plugs:
            soslog.error('a non-existing plugin (%s) was specified in the command line' % (plugname))
        doExit(1)
    del unk_plugs

    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE3, _("processing options from plugin: %s") % plugname)
        names, parms = plug.getAllOptions()
        for optname, optparm  in zip(names, parms):
            alloptions.append((plug, plugname, optname, optparm))

    # when --listplugins is specified we do a dry-run
    # which tells the user which plugins are going to be enabled
    # and with what options.

    if __cmdLineOpts__.listPlugins:
        if not len(loadedplugins) and not len(skippedplugins):
           soslog.error(_("no valid plugins found"))
           doExit(1)

        # FIXME: make -l output more concise
        if len(loadedplugins):
            print _("The following plugins are currently enabled:")
            print
            for (plugname,plug) in loadedplugins:
               print " %-25s  %s" % (textcolor(plugname,"lblue"),plug.get_description())
        else:
            print _("No plugin enabled.")
        print

        if len(skippedplugins):
           print _("The following plugins are currently disabled:")
           print
           for (plugname,plugclass) in skippedplugins:
              print " %-25s  %s" % (textcolor(plugname,"cyan"),plugclass.get_description())
        print

        if len(alloptions):
           print _("The following plugin options are available:")
           print
           for (plug, plugname, optname, optparm)  in alloptions:
              # format and colorize option value based on its type (int or bool)
              if type(optparm["enabled"])==bool:
                 if optparm["enabled"]==True:
                    tmpopt = textcolor("on","lred")
                 else:
                    tmpopt = textcolor("off","red")
              elif type(optparm["enabled"])==int:
                 if optparm["enabled"] > 0:
                    tmpopt = textcolor(optparm["enabled"],"lred")
                 else:
                    tmpopt = textcolor(optparm["enabled"],"red")
              else:
                 tmpopt = optparm["enabled"]

              print " %-21s %-5s %s" % (plugname + "." + optname, tmpopt, optparm["desc"])
              del tmpopt
        else:
           print _("No plugin options available.")

        print
        doExit()

    # to go anywhere further than listing the plugins we will need root permissions.
    #
    if os.getuid() != 0:
        print _('sosreport requires root permissions to run.')
        doExit(1)

    # we don't need to keep in memory plugins we are not going to use
    del skippedplugins

    if not len(loadedplugins):
        soslog.error(_("no valid plugins were enabled"))
        doExit(1)

    try:
        raw_input(_("""This utility will collect some detailed  information about the
hardware and  setup of your  Red Hat Enterprise Linux  system.
The information is collected and an archive is  packaged under
/tmp, which you can send to a support representative.
Red Hat will use this information for diagnostic purposes ONLY
and it will be considered confidential information.

This process may take a while to complete.
No changes will be made to your system.

Press ENTER to continue, or CTRL-C to quit.
"""))
    except:
        print
        doExit()

    # Call the diagnose() method for each plugin
    tmpcount = 0
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Performing sanity check for plugin %s" % plugname)
        try:
           plug.diagnose()
        except:
           if __raisePlugins__:
              raise
        tmpcount += len(plug.diagnose_msgs)
    if tmpcount > 0:
        print _("One or more plugins have detected a problem in your configuration.")
        print _("Please review the following messages:")
        print

        fp = open(rptdir + "/diagnose.txt", "w")
        for plugname, plug in loadedplugins:
            for tmpcount2 in range(0,len(plug.diagnose_msgs)):
                if tmpcount2 == 0:
                    soslog.warning( textcolor("%s:" % plugname, "red") )
                soslog.warning("    * %s" % plug.diagnose_msgs[tmpcount2])
                fp.write("%s: %s\n" % (plugname, plug.diagnose_msgs[tmpcount2]) )
        fp.close()

        print
        try:
           while True:
              yorno = raw_input( _("Are you sure you would like to continue (y/n) ? ") )
              if yorno == _("y") or yorno == _("Y"):
                 print
                 break
              elif yorno == _("n") or yorno == _("N"):
                 doExit(0)
           del yorno
        except KeyboardInterrupt:
           print
           doExit(0)

    policy.preWork()

    # Call the setup() method for each plugin
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Preloading files and commands to be gathered by plugin %s" % plugname)
        try:
            plug.setup()
        except KeyboardInterrupt:
            raise
        except:
            if __raisePlugins__:
               raise

    # Setup the progress bar
    if __cmdLineOpts__.progressbar:
        # gather information useful for generating ETA
        eta_weight = len(loadedplugins)
        for plugname, plug in loadedplugins:
            eta_weight += plug.eta_weight
        pbar = progressBar(minValue = 0, maxValue = eta_weight)
        # pbar.max = number_of_plugins + weight (default 1 per plugin)

    if __cmdLineOpts__.nomultithread:
        soslog.log(logging.VERBOSE, "using single-threading")
    else:
        soslog.log(logging.VERBOSE, "using multi-threading")

    # Call the collect method for each plugin
    plugrunning = Semaphore(2)
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE, "requesting plugin %s" % plugname)
        try:
            if not __cmdLineOpts__.nomultithread:
                plug.copyStuff(threaded = True, semaphore = plugrunning)
            else:
                plug.copyStuff()    
                if __cmdLineOpts__.progressbar:
                    pbar.incAmount(plug.eta_weight)
                    pbar.update()
        except KeyboardInterrupt:
            raise
        except:
            if __raisePlugins__:
                raise
    del plugrunning

    # Wait for all the collection threads to exit
    if not __cmdLineOpts__.nomultithread:
        finishedplugins = []
        while len(loadedplugins) > 0:
            plugname, plug = loadedplugins.pop(0)
            if not plug.wait(0.5):
                finishedplugins.append((plugname,plug))
                soslog.log(logging.DEBUG, "plugin %s has returned" % plugname)
                if __cmdLineOpts__.progressbar:
                   pbar.incAmount(plug.eta_weight)
            else:
                soslog.log(logging.DEBUG, "plugin %s still hasn't returned" % plugname)
                loadedplugins.append((plugname,plug))
            if __cmdLineOpts__.progressbar:
                pbar.update()
        loadedplugins = finishedplugins
        del finishedplugins

    for plugname, plug in loadedplugins:
        for oneFile in plug.copiedFiles:
            try:
                xmlrep.add_file(oneFile["srcpath"], os.stat(oneFile["srcpath"]))
            except:
                pass

    xmlrep.serialize_to_file(rptdir + "/sosreport.xml")

    # Call the analyze method for each plugin
    for plugname, plug in loadedplugins:
        soslog.log(logging.VERBOSE2, "Analyzing results of plugin %s" % plugname,)
        try:
            plug.analyze()
        except:
            # catch exceptions in analyse() and keep working
            pass
        if __cmdLineOpts__.progressbar:
            pbar.incAmount()
            pbar.update()

    if __cmdLineOpts__.progressbar:
        pbar.finished()
        sys.stdout.write("\n")

    # Generate the header for the html output file
    rfd = open(rptdir + "/" + "sosreport.html", "w")
    rfd.write("""
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
    <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
        <head>
    <link rel="stylesheet" type="text/css" media="screen" href="donot.css" />
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>Sos System Report</title>
        </head>

        <body>
    """)


    # Make a pass to gather Alerts and a list of module names
    allAlerts = []
    plugNames = []
    for plugname, plug in loadedplugins:
        for alert in plug.alerts:
            allAlerts.append('<a href="#%s">%s</a>: %s' % (plugname, plugname, alert))
        plugNames.append(plugname)



    # Create a table of links to the module info
    rfd.write("<hr/><h3>Loaded Plugins:</h3>")
    rfd.write("<table><tr>\n")
    rr = 0
    for i in range(len(plugNames)):
        rfd.write('<td><a href="#%s">%s</a></td>\n' % (plugNames[i], plugNames[i]))
        rr = divmod(i, 4)[1]
        if (rr == 3):
            rfd.write('</tr>')
    if not (rr == 3):
        rfd.write('</tr>')
    rfd.write('</table>\n')

    rfd.write('<hr/><h3>Alerts:</h3>')
    rfd.write('<ul>')
    for alert in allAlerts:
        rfd.write('<li>%s</li>' % alert)
    rfd.write('</ul>')


    # Call the report method for each plugin
    for plugname, plug in loadedplugins:
        try:
            html = plug.report()
        except:
            if __raisePlugins__:
                raise
        else:
            rfd.write(html)

    rfd.write("</body></html>")

    rfd.close()

    # Call the postproc method for each plugin
    for plugname, plug in loadedplugins:
        try:
           plug.postproc()
        except:
           if __raisePlugins__:
                raise
  
    # package up the results for the support organization
    policy.packageResults()

    # delete gathered files
    policy.cleanDstroot()

    # let's encrypt the tar-ball
    if __cmdLineOpts__.encrypt:
        policy.encryptResults()

    # automated submission will go here
    if not __cmdLineOpts__.upload:
        policy.displayResults()
    else:
        policy.uploadResults()

    # Close all log files and perform any cleanup
    logging.shutdown()
 
    
if __name__ == '__main__':
    try:
        sosreport()
    except KeyboardInterrupt:
        doExitCode()