merged navid-devel branch r248

git-svn-id: svn+ssh://svn.fedorahosted.org/svn/sos/trunk@255 ef72aa8b-4018-0410-8976-d6e080ef94d8

13 files changed, 641 insertions, 215 deletions

diff --git a/src/Makefile b/src/Makefile
index 99d2ed5d..9e052778 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -70,7 +70,7 @@ clean:
 
 rpm:	mo
 	@test -f sos.spec
-	@mkdir -p $(TOPDIR)/SOURCES $(TOPDIR)/SRPMS $(TOPDIR)/RPMS $(TOPDIR)/BUILD
+	@mkdir -p $(TOPDIR)/SOURCES $(TOPDIR)/SRPMS $(TOPDIR)/RPMS $(TOPDIR)/BUILD $(SRCDIR)/dist
 
 #	this builds an RPM from the current working copy
 	@cd $(TOPDIR)/BUILD ; \
diff --git a/src/extras/htmlog b/src/extras/htmlog
new file mode 100755
index 00000000..7765d09f
--- /dev/null
+++ b/src/extras/htmlog
@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+
+from optparse import OptionParser, Option
+import time, sys
+
+__cmdParser__ = OptionParser()
+__cmdParser__.add_option("-i", "--logfile", action="append", \
+                     dest="logfiles", type="string", \
+                     help="system log to parse")
+__cmdParser__.add_option("-v", "--verbose", action="count", \
+                     dest="verbosity", \
+                     help="How obnoxious we're being about telling the user what we're doing.")
+(__cmdLineOpts__, __cmdLineArgs__)=__cmdParser__.parse_args()
+
+class logfile_class:
+
+   def __init__(self,fname):
+      self.events = []
+      self.curline = ""
+      self.prevline = ""
+      self.eof = False
+      self.hostname = ""
+
+      self.fname = fname
+      self.fp = open(fname)
+      first_line = self.fp.readline().strip()
+      multip = 1
+      readblock = 64
+      while True:
+         self.fp.seek(-readblock * multip,2)
+         newlnpos = self.fp.read(readblock).find("\n")
+         if newlnpos > 0 and newlnpos < readblock - 1:
+            self.fp.seek(-readblock * multip + newlnpos +1, 2)
+            break
+         multip+=1
+      last_line = self.fp.readline().strip()
+      self.fp.seek(0)
+
+      # let's convert the first and last timestamp to something useful
+      # Jul 22 04:48:05
+      self.time_begin = time.strptime(first_line[0:15], "%b %d %H:%M:%S")
+      self.time_end   = time.strptime( last_line[0:15], "%b %d %H:%M:%S")
+
+      # FIXME: check that first_line < last_line
+
+   def readline(self):
+      self.curline_pos = self.fp.tell()
+      self.prevline = self.curline
+      self.curline = self.fp.readline().strip()
+      if len(self.curline) == 0:
+         self.eof = True
+      return self.curline
+
+   def seek(self,pos):
+      self.fp.seek(pos)
+      self.eof = False
+      self.curline = ""
+
+   def parse(self):
+      self.seek(0)
+      while not self.eof:
+         self.readline()
+         self.parse_line()
+      self.seek(0)
+
+   def curmessage(self):
+      return self.curline[17 + self.curline[16:].find(" "):]
+
+   def parse_line(self):
+
+      # is valid log line ?
+      if not self.time_current():
+         return
+
+      # store hostname, if we don't already have it
+      if len(self.hostname) == 0:
+         self.curline.split()[3]
+
+      # system is booting
+      if self.curmessage().startswith("Linux version"):
+         self.add_event("system boot")
+
+      # hostname has changed
+      if len(self.hostname) and self.hostname != self.curline.split()[3]:
+         self.add_event("hostname changed")
+         self.hostname = self.curline.split()[3]
+
+      # the clock is wrong wrong
+      if self.prevline and time.strptime(self.prevline[0:15], "%b %d %H:%M:%S") > self.time_current():
+         self.add_event("clock is fucked up")
+
+   def add_event(self, message):
+      self.events.append( (self.curline_pos,len(self.curline),message) )
+
+   def time_current(self):
+      if len(self.curline) == 0: return None
+      try:
+         return time.strptime(self.curline[0:15], "%b %d %H:%M:%S")
+      except ValueError:
+         print "could not parse time", self.curline
+         return False
+
+logs = []
+
+for logname in __cmdLineOpts__.logfiles:
+   log = logfile_class(logname)
+   log.parse()
+   logs.append(log)
+
+print """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<HTML>
+<HEAD>
+<TITLE>HTMLogs output</TITLE>
+<style type="text/css">
+body {
+	background: #FFF;
+	color: #000;
+	font: normal normal 12px Verdana, Geneva, Arial, Helvetica, sans-serif;
+	margin: 10px;
+	padding: 0
+}
+
+table, td, a {
+	color: #000;
+	font: normal normal 12px Verdana, Geneva, Arial, Helvetica, sans-serif
+}
+
+/* make the TH elements pretty */
+thead.fixedHeader th {
+	background: #C96;
+	border-left: 1px solid #EB8;
+	border-right: 1px solid #B74;
+	border-top: 1px solid #EB8;
+	font-weight: normal;
+	padding: 4px 3px;
+	text-align: left
+}
+
+/* make the A elements pretty. makes for nice clickable headers                */
+thead.fixedHeader a, thead.fixedHeader a:link, thead.fixedHeader a:visited {
+	color: #FFF;
+	display: block;
+	text-decoration: none;
+	width: 100%
+}
+
+/* make the A elements pretty. makes for nice clickable headers                */
+/* WARNING: swapping the background on hover may cause problems in WinIE 6.x   */
+thead.fixedHeader a:hover {
+	color: #FFF;
+	display: block;
+	text-decoration: underline;
+	width: 100%
+}
+
+ul#toc {list-style:none;width:320px;}
+#toc li {background:url(dot.gif) repeat-x 0 0.85em;}
+#toc li a {float:left;background:#FFF;padding: 0 4px 0 0;}
+#toc li span {float:right;background:#FFF; padding 0 0 0 4px;}
+#toc li br {clear:both;}
+
+-->
+</style>
+</HEAD>
+<BODY>
+"""
+
+print '<ul id="toc">'
+for log in logs:
+#   print logs[idx].fname, logs[idx].events
+   for line, msglen, event in log.events:
+      print ' <li><span>%s</span> <a href="#">Link</a><br /></li>' % event
+print '</ul>'
+
+print "<TABLE>"
+while True:
+   # who is next ?
+   lowest_date = None
+   for log in logs:
+      if log.eof:
+         continue
+
+      if not len(log.curline):
+         log.readline()
+
+      if lowest_date == None or log.time_current() < lowest_date:
+         lowest_date = log.time_current()
+
+   if lowest_date == None:
+      # all logs are EOF
+      break
+
+   print " <TR>"
+   # FIXME: if this tick has an event, add <a name="...">
+   print '  <TD STYLE="white-space:nowrap">' + time.strftime("%b %d %H:%M:%S", lowest_date) + "</TD>"
+   for log in logs:
+      if log.time_current() == lowest_date:
+         print "  <TD>" + log.curmessage() + "</TD>"
+         log.curline = ""
+      else:
+         print "  <TD></TD>"
+#      print log.curline_pos, time.strftime("%b %d %H:%M:%S", log.time_current()), log.curmessage()
+   print " </TR>"
+print "<TABLE>"
+   
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/lib/sos/helpers.py b/src/lib/sos/helpers.py
index 54a86dba..dcff1405 100755
--- a/src/lib/sos/helpers.py
+++ b/src/lib/sos/helpers.py
@@ -113,7 +113,11 @@ def sosRelPath(path1, path2, sep=os.path.sep, pardir=os.path.pardir):
         In particular: the empty string, if path1 == path2;
                        path2, if path1 and path2 have no common prefix.
     '''
-    common, (u1, u2) = commonPrefix(path1.split(sep), path2.split(sep))
+    try:
+        common, (u1, u2) = commonPrefix(path1.split(sep), path2.split(sep))
+    except AttributeError:
+        return path2
+
     if not common:
         return path2      # leave path absolute if nothing at all in common
     return sep.join( [pardir]*len(u1) + u2 )
diff --git a/src/lib/sos/plugins/cluster.py b/src/lib/sos/plugins/cluster.py
index fbd3b29d..02fede4f 100644
--- a/src/lib/sos/plugins/cluster.py
+++ b/src/lib/sos/plugins/cluster.py
@@ -14,10 +14,16 @@
 
 import sos.plugintools
 import commands, os
+import time, libxml2
 
 class cluster(sos.plugintools.PluginBase):
     """cluster suite and GFS related information
     """
+
+    optionList = [("gfslockdump", 'gather output of gfs lockdumps', 'slow', False),
+                  ('lockdump', 'gather dlm lockdumps', 'slow', False),
+                  ('taskdump', 'trigger 3 SysRq+t dumps every 5 seconds', 'slow', False)]
+
     def checkenabled(self):
        # enable if any related package is installed
        for pkg in [ "ccs", "cman", "cman-kernel", "magma", "magma-plugins", 
@@ -35,29 +41,121 @@ class cluster(sos.plugintools.PluginBase):
        # no data related to RHCS/GFS exists
        return False
 
+    def has_gfs(self):
+        fp = open("/proc/mounts","r")
+        for line in fp.readlines():
+           mntline = line.split(" ")
+           if mntline[2] == "gfs":
+              return True
+        fp.close()
+        return False
+
     def diagnose(self):
         try: rhelver = self.cInfo["policy"].pkgDictByName("redhat-release")[0]
         except: rhelver = None
-        if rhelver == "4" or not rhelver:
+
+        if rhelver == "4" or True:
+           # check that kernel module packages are installed for
+           # running kernel version
+           pkgs_check = [ "dlm-kernel" , "cman-kernel" ]
+           if self.has_gfs(): pkgs_check.append("GFS-kernel")
+
+           for pkgname in pkgs_check:
+               found = 0
+               if self.cInfo["policy"].isKernelSMP() and self.cInfo["policy"].pkgByName(pkgname):
+                   found = 1 # -one- means package found (but not for same version as kernel)
+                   pkgname = pkgname + "-smp"
+
+               for pkg in self.cInfo["policy"].allPkgsByName(pkgname):
+                   found = 1
+                   for reqline in self.cInfo["policy"].pkgRequires("%s-%s-%s" % (pkg[0],pkg[1],pkg[2]) ):
+                       if reqline[0] == 'kernel-smp' and reqline[1] == '=':
+                           reqline[2] = reqline[2] + "smp"
+
+                       if ( (not self.cInfo["policy"].isKernelSMP() and reqline[0] == 'kernel') or (self.cInfo["policy"].isKernelSMP() and reqline[0] == 'kernel-smp') ) and reqline[1] == '=' and reqline[2] == self.cInfo["policy"].kernelVersion():
+                           found = 2
+                           break
+
+               if found == 0:
+                   self.addDiagnose("required package is missing: %s" % pkgname)
+               elif found == 1:
+                   self.addDiagnose("required package is not installed for current kernel: %s" % pkgname)
+
            # check if the minimum set of packages is installed
            # for RHEL4 RHCS(ccs, cman, cman-kernel, magma, magma-plugins, (dlm, dlm-kernel) || gulm, perl-Net-Telnet, rgmanager, fence)
            # RHEL4 GFS (GFS, GFS-kernel, ccs, lvm2-cluster, fence)
-           for pkg in [ "ccs", "cman", "cman-kernel", "magma", "magma-plugins", "perl-Net-Telnet", "rgmanager", "fence" ]:
-              if self.cInfo["policy"].pkgByName(pkg) == None:
-                 self.addDiagnose("required package is missing: %s" % pkg)
+
+           for pkg in [ "ccs", "cman", "magma", "magma-plugins", "perl-Net-Telnet", "rgmanager", "fence" ]:
+               if self.cInfo["policy"].pkgByName(pkg) == None:
+                  self.addDiagnose("required package is missing: %s" % pkg)
+
+           # (dlm, dlm-kernel) || gulm
+           if not ((self.cInfo["policy"].pkgByName("dlm") and self.cInfo["policy"].pkgByName("dlm-kernel")) or self.cInfo["policy"].pkgByName("gulm")):
+               self.addDiagnose("required packages are missing: (dlm, dlm-kernel) || gulm")
 
            # check if all the needed daemons are active at sosreport time
            # check if they are started at boot time in RHEL4 RHCS (cman, ccsd, rgmanager, fenced)
            # and GFS (gfs, ccsd, clvmd, fenced)
-           for service in [ "cman", "ccsd", "rgmanager", "fence" ]:
-              if commands.getstatus("/sbin/service %s status" % service):
-                 self.addDiagnose("service %s is not running" % service)
+           checkserv = [ "cman", "ccsd", "rgmanager", "fenced" ]
+           if self.has_gfs(): checkserv.extend( ["gfs", "clvmd"] )
+           for service in checkserv:
+               status, output = commands.getstatusoutput("/sbin/service %s status" % service)
+               if status:
+                   self.addDiagnose("service %s is not running" % service)
+
+               if not self.cInfo["policy"].runlevelDefault() in self.cInfo["policy"].runlevelByService(service):
+                   self.addDiagnose("service %s is not started in default runlevel" % service)
+
+           # is cluster quorate
+           if not self.is_cluster_quorate():
+               self.addDiagnose("cluster node is not quorate")
+
+           # if there is no cluster.conf, diagnose() finishes here.
+           try:    os.stat("/etc/cluster/cluster.conf")
+           except: return
+
+           # setup XML xpath context
+           xml = libxml2.parseFile("/etc/cluster/cluster.conf")
+           xpathContext = xml.xpathNewContext()
+
+           # check fencing (warn on empty or manual)
+           if len(xpathContext.xpathEval("/cluster/clusternodes/clusternode[not(fence/method/device)]")):
+               self.addDiagnose("one or more nodes have no fencing agent configured")
+
+           if len(xpathContext.xpathEval("/cluster/clusternodes/clusternode[/cluster/fencedevices/fencedevice[@agent='fence_manual']/@name=fence/method/device/@name]")):
+               self.addDiagnose("one or more nodes have manual fencing agent configured (data integrity is not guaranteed)")
+
+           # check for fs exported via nfs without nfsid attribute
+           if len(xpathContext.xpathEval("/cluster/rm/service//fs[not(@fsid)]/nfsexport")):
+               self.addDiagnose("one or more nfs file-system doesn't have a fsid attribute set.")
 
-              if not self.cInfo["policy"].runlevelDefault() in self.cInfo["policy"].runlevelByService(service):
-                 self.addDiagnose("service %s is not started in default runlevel" % service)
+           # cluster.conf file version and the in-memory cluster configuration version matches
+           status, cluster_version = commands.getstatusoutput("cman_tool status | grep 'Config version'")
+           if not status: cluster_version = cluster_version[16:]
+           else: cluster_version = None
+           conf_version = xpathContext.xpathEval("/cluster/@config_version")[0].content
 
-           # FIXME: what locking are we using ? check if packages exist
-#           if self.cInfo["policy"].pkgByName(pkg) and self.cInfo["policy"].pkgByName(pkg) and not self.cInfo["policy"].pkgByName(pkg)
+           if status == 0 and conf_version != cluster_version:
+               self.addDiagnose("cluster.conf and in-memory configuration version differ (%s != %s)" % (conf_version, cluster_version) )
+
+           # make sure the first part of the lock table matches the cluster name
+           # and that the locking protocol is sane
+           cluster_name = xpathContext.xpathEval("/cluster/@name")[0].content
+
+           fp = open("/etc/fstab","r")
+           for fs in fp.readline().split():
+#               fs = fs.split()
+               if not fs or fs[0] == "#" or len(fs) < 6 or fs[2]!="gfs": continue
+               lockproto = get_gfs_sb_field(fs[1], "sb_lockproto")
+               if lockproto != "lock_" + get_locking_proto:
+                   self.addDiagnose("gfs mountpoint (%s) is using the wrong locking protocol (%s)" % (fs[0], lockproto) )
+
+               locktable = get_gfs_sb_field(fs[1], "sb_locktable")
+               if not locktable: continue
+               try: locktable = locktable.split(":")[0]
+               except: continue
+               if locktable != cluster_name:
+                   self.addDiagnose("gfs mountpoint (%s) is using the wrong locking table" % fs[0])
 
     def setup(self):
         self.collectExtOutput("/sbin/fdisk -l")
@@ -72,8 +170,88 @@ class cluster(sos.plugintools.PluginBase):
         self.collectExtOutput("/usr/bin/ccs_tool lsnode")
         self.collectExtOutput("/usr/bin/openais-cfgtool -s")
         self.collectExtOutput("/usr/bin/clustat")
+
+        if self.isOptionEnabled('gfslockdump'): self.do_gfslockdump()
+        if self.isOptionEnabled('lockdump'): self.do_lockdump()
+        if self.isOptionEnabled('taskdump'): self.do_taskdump()
+
         return
 
+    def do_taskdump(self):
+        commands.getstatusoutput("echo t > /proc/sysrq-trigger")
+        time.sleep(3)
+        commands.getstatusoutput("echo t > /proc/sysrq-trigger")
+        time.sleep(3)
+        commands.getstatusoutput("echo t > /proc/sysrq-trigger")
+        self.addCopySpec("/var/log/messages")
+
+    def do_lockdump(self):
+        fp = open("/proc/cluster/services","r")
+        for line in fp.readlines():
+           if line[0:14] == "DLM Lock Space":
+              try:
+                 lockspace = line.split('"')[1]
+              except:
+                 pass
+              else:
+                 commands.getstatusoutput("echo %s > /proc/cluster/dlm_locks" % lockspace)
+                 self.collectOutputNow("cat /proc/cluster/dlm_locks", root_symlink = "dlm_locks_%s" % lockspace)
+        fp.close()
+
+    def get_locking_proto(self):
+        # FIXME: what's the best way to find out ?
+        return "dlm"
+        return "gulm"
+
+    def do_gfslockdump(self):
+        fp = open("/proc/mounts","r")
+        for line in fp.readlines():
+           mntline = line.split(" ")
+           if mntline[2] == "gfs":
+              self.collectExtOutput("/sbin/gfs_tool lockdump %s" % mntline[1])
+        fp.close()
+
     def postproc(self):
         self.doRegexSub("/etc/cluster/cluster.conf", r"(\s*\<fencedevice\s*.*\s*passwd\s*=\s*)\S+(\")", r"\1***")
         return
+
+    def is_cluster_quorate(self):
+        output = commands.getoutput("/bin/cat /proc/cluster/status | grep '^Membership state: '")
+        try:
+            if output[18:] == "Cluster-Member":
+                return True
+            else:
+                return False
+        except:
+            pass
+        return None
+
+    def get_gfs_sb_field(self, mntpoint, field):
+        for line in commands.getoutput("/sbin/gfs_tool sb %s all" % fs[1]):
+            tostrip = "  " + field + " = "
+            if line.startwith(tostrip):
+                return line[len(tostrip):]
+        return None
+
+    def xpath_query_count(self, fname, query):
+        return len(self.xpath_query(fname, query))
+
+    def xpath_query(self, fname, query):
+        xml = libxml2.parseFile(fname)
+        xpathContext = xml.xpathNewContext()
+        return xpathContext.xpathEval(query)
+
+        # FIXME: use python libxml internals
+        tmpout = commands.getoutput("/bin/echo xpath %s | /usr/bin/xmllint --shell /etc/cluster/cluster.conf")
+        for tmpline in tmpout:
+            if tmpline.startswith("Set contains "):
+                tmpvalue = tmpline[14:].split(" ")[0]
+                if tmpvalue == "NULL": return 0
+                try: tmpvalue = int(tmpvalue)
+                except: return False
+                return tmpvalue
+        return False
+
+
+
+
diff --git a/src/lib/sos/plugins/kernel.py b/src/lib/sos/plugins/kernel.py
index 09381b38..07cc6ac7 100644
--- a/src/lib/sos/plugins/kernel.py
+++ b/src/lib/sos/plugins/kernel.py
@@ -18,8 +18,8 @@ import commands, os, re
 class kernel(sos.plugintools.PluginBase):
     """kernel related information
     """
-    optionList = [("modinfo", 'Gathers module information on all modules', 'fast', 1),
-                  ('sysrq', 'Trigger SysRq dumps', 'fast', 0)]
+    optionList = [("modinfo", 'Gathers module information on all modules', 'fast', True),
+                  ('sysrq', 'Trigger SysRq dumps', 'fast', False)]
     moduleFile = ""
     taintList = [
         {'regex':'mvfs*', 'description':'Clearcase module'},
@@ -57,6 +57,7 @@ class kernel(sos.plugintools.PluginBase):
           if len(runcmd):
             self.collectExtOutput("/sbin/modinfo " + runcmd)
         self.collectExtOutput("/sbin/ksyms")
+        self.addCopySpec("/sys/module")
         self.addCopySpec("/proc/filesystems")
         self.addCopySpec("/proc/ksyms")
         self.addCopySpec("/proc/slabinfo")
diff --git a/src/lib/sos/plugins/networking.py b/src/lib/sos/plugins/networking.py
index 54723663..6d11146b 100644
--- a/src/lib/sos/plugins/networking.py
+++ b/src/lib/sos/plugins/networking.py
@@ -18,7 +18,7 @@ import os,re,commands
 class networking(sos.plugintools.PluginBase):
     """network related information
     """
-    optionList = [("traceroute", "collects a traceroute to rhn.redhat.com", "slow", 0)]
+    optionList = [("traceroute", "collects a traceroute to rhn.redhat.com", "slow", False)]
 
     def get_interface_name(self,ifconfigFile):
         """Return a dictionary for which key are interface name according to the
diff --git a/src/lib/sos/plugins/rhn.py b/src/lib/sos/plugins/rhn.py
index 049e51bc..46ff2fbf 100644
--- a/src/lib/sos/plugins/rhn.py
+++ b/src/lib/sos/plugins/rhn.py
@@ -12,80 +12,83 @@
 ## along with this program; if not, write to the Free Software
 ## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
-from sos.plugintools import PluginBase
+import sos.plugintools
 
-class rhn(PluginBase):
-    """RHN server related information
+class rhn(sos.plugintools.PluginBase):
+    """RHN Satellite related information
     """
+    satellite = False
+    proxy = False
+
+    def defaultenabled(self):
+        return False
+
     def checkenabled(self):
-        # XXX check for the presence of requisite packages
-        satellite = self.cInfo["policy"].pkgByName("rhns-satellite-tools")
-        proxy = self.cInfo["policy"].pkgByName("rhns-proxy-tools")
-        if not satellite and not proxy:
-            return False
-        return True
+        # enable if any related package is installed
+
+        self.satellite = self.cInfo["policy"].pkgByName("rhns-satellite-tools")
+        self.proxy = self.cInfo["policy"].pkgByName("rhns-proxy-tools")
+
+        if self.satellite or self.proxy:
+            return True
+
+        return False
 
     def setup(self):
-        #
-        # First, grab things needed from both Satellite and Proxy systems
-        #
-        # TODO: add chain load so we can use existing modules for httpd, &c.
-        #
-
-        # basic RHN logs and configs
-        self.addCopySpec("/var/log/rhn*")
-        self.addCopySpec("/etc/rhn")
-        self.collectExtOutput("/usr/share/rhn/up2date_client/hardware.py")
+        # made from:
+        # http://svn.rhndev.redhat.com/viewcvs/branches/eng/RELEASE-5.0.5-dev/backend/satellite_tools/satellite-debug?rev=114478&view=markup
+        # http://cvs.devel.redhat.com/cgi-bin/cvsweb.cgi/rhn/proxy/proxy/tools/rhn-proxy-debug?rev=1.3;content-type=text%2Fplain;cvsroot=RHN
+        # FIXME: symlinks and directories for copySpec (same as root_symlink for commands)
 
-        # httpd
         self.addCopySpec("/etc/httpd/conf")
-        self.addCopySpec("/var/log/httpd")
+        self.addCopySpec("/etc/rhn")
+        self.addCopySpec("/etc/sysconfig/rhn")
+        self.addCopySpec("/var/log/httpd")	# httpd-logs
+        self.addCopySpec("/var/log/rhn*")	# rhn-logs
+        self.addCopySpec("/var/log/rhn/rhn-database-installation.log")
 
-        # RPM manifests
-        self.collectExtOutput("/bin/rpm -qa --last | sort")
+	# all these used to go in $DIR/mon-logs/
+        self.addCopySpec("/opt/notification/var/*.log*")
+        self.addCopySpec("/var/tmp/ack_handler.log*")
+        self.addCopySpec("/var/tmp/enqueue.log*")
 
         # monitoring scout logs
         self.addCopySpec("/home/nocpulse/var/*.log*")
         self.addCopySpec("/home/nocpulse/var/commands/*.log*")
-
-        satellite = self.cInfo["policy"].pkgByName("rhns-satellite-tools")
-        proxy = self.cInfo["policy"].pkgByName("rhns-proxy-tools")
-
-        #
-        # Now, go for product-specific data
-        #
-        if satellite:
-            self.setupSatellite(satellite)
-
-        if proxy:
-            self.setupProxy(proxy)
-
-    def setupSatellite(self, satellite):
-        self.collectExtOutput("/usr/bin/rhn-schema-version")
-        self.collectExtOutput("/usr/bin/rhn-charsets")
-
-        # oracle
-        self.addCopySpec("/etc/tnsnames.ora")
-
-        # tomcat (4.x and newer satellites only)
-        if not self.cInfo["policy"].pkgNVRA(satellite)[1].startswith("3."):
-            self.addCopySpec("/etc/tomcat5")
-            self.addCopySpec("/var/log/tomcat5")
-
-        # jabberd
-        #  - logs to /var/log/messages
-        self.addCopySpec("/etc/jabberd")
-
-        # SSL build
-        self.addCopySpec("/root/ssl-build")
-    
-        # monitoring logs
-        self.addCopySpec("/opt/notification/var/*.log*")
         self.addCopySpec("/var/tmp/ack_handler.log*")
         self.addCopySpec("/var/tmp/enqueue.log*")
 
-    def setupProxy(self, proxy):
-        # squid
-        self.addCopySpec("/etc/squid")
-        self.addCopySpec("/var/log/squid")
- 
+        self.addCopySpec("/root/ssl-build")
+        self.collectExtOutput("rpm -qa --last", root_symlink = "rpm-manifest")
+        self.collectExtOutput("/usr/bin/rhn-schema-version", root_symlink = "database-schema-version")
+        self.collectExtOutput("/usr/bin/rhn-charsets", root_symlink = "database-character-sets")
+
+        if self.satellite:
+           self.addCopySpec("/etc/tnsnames.ora")	
+           self.addCopySpec("/etc/jabberd")
+
+           # tomcat (4.x and newer satellites only)
+           if not self.cInfo["policy"].pkgNVRA(satellite)[1].startswith("3."):
+              self.addCopySpec("/etc/tomcat5")
+              self.addCopySpec("/var/log/tomcat5")
+
+           self.addCopySpec("/etc/tomcat5")
+           self.addCopySpec("/var/log/tomcat5")
+
+        if self.proxy:
+           # copying configuration information
+           self.addCopySpec("/etc/httpd/conf")
+           self.addCopySpec("/etc/squid")
+           self.addCopySpec("/etc/rhn")
+           self.addCopySpec("/etc/sysconfig/rhn")
+
+           # copying logs
+           self.addCopySpec("/var/log/httpd")
+           self.addCopySpec("/var/log/squid")
+           self.addCopySpec("/var/log/rhn*")
+
+        return
+
+#    def diagnose(self):
+        # RHN Proxy:
+        # * /etc/sysconfig/rhn/systemid is owned by root.apache with the permissions 0640
diff --git a/src/lib/sos/plugins/rpm.py b/src/lib/sos/plugins/rpm.py
index b6fdb699..9c6f2659 100644
--- a/src/lib/sos/plugins/rpm.py
+++ b/src/lib/sos/plugins/rpm.py
@@ -17,8 +17,8 @@ import sos.plugintools
 class rpm(sos.plugintools.PluginBase):
     """RPM information
     """
-    optionList = [("rpmq", "Queries for package information via rpm -q", "fast", 1),
-                  ("rpmva", "Runs a verify on all packages", "slow", 1)]
+    optionList = [("rpmq", "Queries for package information via rpm -q", "fast", True),
+                  ("rpmva", "Runs a verify on all packages", "slow", True)]
                   
     def setup(self):
         self.addCopySpec("/var/log/rpmpkgs")
diff --git a/src/lib/sos/plugins/satellite.py b/src/lib/sos/plugins/satellite.py
deleted file mode 100644
index 5d0b2b1e..00000000
--- a/src/lib/sos/plugins/satellite.py
+++ /dev/null
@@ -1,62 +0,0 @@
-### This program is free software; you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 2 of the License, or
-## (at your option) any later version.
-
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-
-## You should have received a copy of the GNU General Public License
-## along with this program; if not, write to the Free Software
-## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-import sos.plugintools
-
-class satellite(sos.plugintools.PluginBase):
-    """RHN Satellite related information
-    """
-
-    def defaultenabled(self):
-        # enable if any related package is installed
-        for pkg in [ "rhn-satellite" ]:
-           if self.cInfo["policy"].pkgByName(pkg) != None:
-              return True
-
-        return False
-
-    def setup(self):
-        # made from:
-        # http://svn.rhndev.redhat.com/viewcvs/branches/eng/RELEASE-5.0.5-dev/backend/satellite_tools/satellite-debug?rev=114478&view=markup
-        # FIXME: symlinks and directories for copySpec (same as root_symlink for commands)
-
-        self.addCopySpec("/etc/httpd/conf")
-        self.addCopySpec("/etc/rhn")
-        self.addCopySpec("/etc/sysconfig/rhn")
-        self.addCopySpec("/etc/tnsnames.ora")	
-        self.addCopySpec("/var/log/httpd")	# httpd-logs
-        self.addCopySpec("/var/log/rhn*")	# rhn-logs
-        self.addCopySpec("/var/log/rhn/rhn-database-installation.log")
-        self.addCopySpec("/etc/jabberd")
-
-	# tomcat for satellite 400+
-        self.addCopySpec("/etc/tomcat5")
-        self.addCopySpec("/var/log/tomcat5")
-
-	# all these used to go in $DIR/mon-logs/
-        self.addCopySpec("/opt/notification/var/*.log*")
-        self.addCopySpec("/var/tmp/ack_handler.log*")
-        self.addCopySpec("/var/tmp/enqueue.log*")
-
-        self.addCopySpec("/home/nocpulse/var/*.log*")
-        self.addCopySpec("/home/nocpulse/var/commands/*.log*")
-        self.addCopySpec("/var/tmp/ack_handler.log*")
-        self.addCopySpec("/var/tmp/enqueue.log*")
-
-        self.addCopySpec("/root/ssl-build")
-        self.collectExtOutput("rpm -qa --last", root_symlink = "rpm-manifest")
-        self.collectExtOutput("/usr/bin/rhn-schema-version", root_symlink = "database-schema-version")
-        self.collectExtOutput("/usr/bin/rhn-charsets", root_symlink = "database-character-sets")
-
-        return
diff --git a/src/lib/sos/plugintools.py b/src/lib/sos/plugintools.py
index 238036ce..8f07fb62 100644
--- a/src/lib/sos/plugintools.py
+++ b/src/lib/sos/plugintools.py
@@ -113,11 +113,12 @@ class PluginBase:
 
         if copyProhibited:
             return ''
-                
+
         if os.path.islink(srcpath):
             # This is a symlink - We need to also copy the file that it points to
             # file and dir symlinks ar ehandled the same
             link = os.readlink(srcpath)
+
             if os.path.isabs(link):
                 # the link was an absolute path, and will not point to the new
                 # tree. We must adjust it.
@@ -138,11 +139,22 @@ class PluginBase:
                 # no adjustment, symlink is the relative path
                 dstslname = link
 
-            if os.path.isdir(srcpath):
+            if os.path.isdir(srcpath): # symlink to a directory
+                # FIXME: don't recurse symlinks until vicious loops are detected
+                return
+
+                abslink = os.path.abspath(os.path.dirname(srcpath) + "/" + link)
+                self.soslog.log(logging.VERBOSE2, "DIRLINK %s to %s [%s]" % (srcpath,link,abslink))
+                for tmplink in self.copiedDirs:
+                   if tmplink["srcpath"] == abslink or tmplink["pointsto"] == abslink:
+                      self.soslog.log(logging.VERBOSE2, "already copied [%s]" % srcpath)
+                      return
+
                 for afile in os.listdir(srcpath):
                     if afile == '.' or afile == '..':
                         pass
                     else:
+                        self.soslog.log(logging.VERBOSE2, "copying (file or dir) %s" % srcpath+'/'+afile)
                         try:
                             abspath = self.doCopyFileOrDir(srcpath+'/'+afile)
                         except SystemExit:
@@ -150,13 +162,14 @@ class PluginBase:
                         except KeyboardInterrupt:
                           raise KeyboardInterrupt
                         except Exception, e:
-                            self.soslog.warning(traceback.format_exc())
+                            self.soslog.verbose(traceback.format_exc())
 
                         # if on forbidden list, abspath is null
                         if not abspath == '':
                             dstslname = sosRelPath(self.cInfo['rptdir'], abspath)
-                            self.copiedDirs.append({'srcpath':srcpath, 'dstpath':dstslname, 'symlink':"yes", 'pointsto':link})
+                            self.copiedDirs.append({'srcpath':srcpath, 'dstpath':dstslname, 'symlink':"yes", 'pointsto':os.path.abspath(srcpath+'/'+afile) })
             else:
+                self.soslog.log(logging.VERBOSE2, "copying symlink %s" % srcpath)
                 try:
                     dstslname, abspath = self.__copyFile(srcpath)
                     self.copiedFiles.append({'srcpath':srcpath, 'dstpath':dstslname, 'symlink':"yes", 'pointsto':link})
@@ -170,10 +183,10 @@ class PluginBase:
 
             return abspath
 
-        else:
+        else: # not a symlink
             if not os.path.exists(srcpath):
                 self.soslog.debug("File or directory %s does not exist\n" % srcpath)
-            elif  os.path.isdir(srcpath):
+            elif os.path.isdir(srcpath):
                 for afile in os.listdir(srcpath):
                     if afile == '.' or afile == '..':
                         pass
@@ -309,12 +322,12 @@ class PluginBase:
         outfn = self.cInfo['cmddir'] + "/" + self.piName + "/" + mangledname
 
         # check for collisions
-        inc = 0
         if os.path.exists(outfn):
             inc = 2
             while True:
-               newfn = outfn + "_" + inc
+               newfn = "%s_%d" % (outfn, inc)
                if not os.path.exists(newfn):
+                  outfn = newfn
                   break
                inc +=1
 
@@ -339,18 +352,22 @@ class PluginBase:
         if not os.path.isdir(os.path.dirname(outfn)):
             os.mkdir(os.path.dirname(outfn))
 
-        outfd = open(outfn, "w")
-        if status == 127: outfd.write("# the command returned exit status 127, this normally means that the command was not found.\n\n")
-        if len(shout):    outfd.write(shout+"\n")
-        outfd.close()
+        if not (status == 127 or status == 32512):
+            outfd = open(outfn, "w")
+            if len(shout):    outfd.write(shout+"\n")
+            outfd.close()
 
-        if root_symlink:
-            curdir = os.getcwd()
-            os.chdir(self.cInfo['dstroot'])
-            os.symlink(outfn[len(self.cInfo['dstroot'])+1:], root_symlink.strip("/."))
-            os.chdir(curdir)
+            if root_symlink:
+                curdir = os.getcwd()
+                os.chdir(self.cInfo['dstroot'])
+                os.symlink(outfn[len(self.cInfo['dstroot'])+1:], root_symlink.strip("/."))
+                os.chdir(curdir)
 
-        outfn = outfn[len(self.cInfo['cmddir'])+1:]
+            outfn = outfn[len(self.cInfo['cmddir'])+1:]
+
+        else:
+            self.soslog.log(logging.VERBOSE, "could not run command: %s" % exe)
+            outfn = None
 
         # sosStatus(status)
         # save info for later
diff --git a/src/lib/sos/policyredhat.py b/src/lib/sos/policyredhat.py
index d4a4074d..7382f740 100755
--- a/src/lib/sos/policyredhat.py
+++ b/src/lib/sos/policyredhat.py
@@ -53,11 +53,16 @@ class SosPolicy:
         #print "validating %s" % pluginpath
         return True
 
+    def pkgRequires(self, name):
+        # FIXME: we're relying on rpm to sort the output list
+        cmd = "/bin/rpm -q --requires %s" % (name)
+        return [requires[:-1].split() for requires in os.popen(cmd).readlines()]
+
     def allPkgsByName(self, name):
         # FIXME: we're relying on rpm to sort the output list
-        cmd = "/bin/rpm --qf '%%{N}-%%{V}-%%{R}-%%{ARCH}\n' -q %s" % (name,)
+        cmd = "/bin/rpm --qf '%%{N} %%{V} %%{R} %%{ARCH}\n' -q %s" % (name,)
         pkgs = os.popen(cmd).readlines()
-        return [pkg[:-1] for pkg in pkgs if pkg.startswith(name)]
+        return [pkg[:-1].split() for pkg in pkgs if pkg.startswith(name)]
 
     def pkgByName(self, name):
         # TODO: do a full NEVRA compare and return newest version, best arch
@@ -91,6 +96,13 @@ class SosPolicy:
         # FIXME: get this from /etc/inittab
         return 3
 
+    def kernelVersion(self):
+        return commands.getoutput("/bin/uname -r").strip("\n")
+
+    def isKernelSMP(self):
+        if self.kernelVersion()[-3:]=="smp": return True
+        else: return False
+
     def pkgNVRA(self, pkg):
         fields = pkg.split("-")
         version, release, arch = fields[-3:]
@@ -107,8 +119,9 @@ class SosPolicy:
             ticketNumber = raw_input("Please enter the case number that you are generating this report for: ")
         except KeyboardInterrupt:
             print _("<interrupted>")
-            print _("Temporary files have been stored in ") % self.cInfo['dstroot']
-            return
+            print _("Temporary files have been stored in %s") % self.cInfo['dstroot']
+            print
+            sys.exit(1)
 
         if len(ticketNumber):
             namestr = name + "." + ticketNumber
diff --git a/src/sos.spec b/src/sos.spec
index 34423b15..5a61a5d0 100644
--- a/src/sos.spec
+++ b/src/sos.spec
@@ -21,6 +21,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 BuildArch: noarch
 Url: http://sos.108.redhat.com/
 BuildRequires: python-devel
+Requires: libxml2-python
 
 %description
 Sos is a set of tools that gathers information about system
diff --git a/src/sosreport b/src/sosreport
index 86d66b4d..6607c453 100755
--- a/src/sosreport
+++ b/src/sosreport
@@ -231,9 +231,9 @@ class progressBar:
                ETA = timeElapsed
             else:
                ETA = self.eta
-            ETA = "[%02d:%02d/%02d:%02d]" % (round(timeElapsed/60), timeElapsed % 60, round(ETA/60), ETA % 60)
+            ETA = "[%02d:%02d/%02d:%02d]" % (int(timeElapsed/60), timeElapsed % 60, round(ETA/60), ETA % 60)
         else:
-            ETA = "[%02d:%02d/--:--]" % (round(timeElapsed/60), timeElapsed % 60)
+            ETA = "[%02d:%02d/--:--]" % (int(timeElapsed/60), timeElapsed % 60)
             if self.amount < self.max:
                percentDone = 0
             else:
@@ -453,12 +453,65 @@ def sosreport():
                 raise
 
     # First, gather and process options
+    # using the options specified in the command line (if any)
+    if __cmdLineOpts__.plugopts:
+        opts = {}
+        for opt in __cmdLineOpts__.plugopts:
+           # split up "general.syslogsize=5"
+           try:
+              opt, val = opt.split("=")
+           except:
+              val=True
+           else:
+              if val == "off" or val == "disable" or val == "disabled":
+                 val = False
+              else:
+                 # try to convert string "val" to int()
+                 try:    val = int(val)
+                 except: pass
+
+           # split up "general.syslogsize"
+           plug, opt = opt.split(".")
+
+           try: opts[plug]
+           except KeyError: opts[plug] = []
+           opts[plug].append( (opt,val) )
+
+        for plugname, plug in loadedplugins:
+           if opts.has_key(plugname):
+               for opt,val in opts[plugname]:
+                   soslog.log(logging.VERBOSE, "setting option %s for plugin %s to %s" % (plugname,opt,val))
+                   plug.setOption(opt,val)
+        del opt,opts,val
+    elif not __cmdLineOpts__.fastoptions and not __cmdLineOpts__.usealloptions:
+        if len(alloptions) and __cmdLineOpts__.use_curses:
+            try:
+                get_curse_options(alloptions)
+            except "Cancelled":
+                sys.exit(_("Exiting."))
+    elif __cmdLineOpts__.fastoptions:
+        # FIXME: with int and bool options, fast/slow/all breaks
+        for i in range(len(alloptions)):
+            for plug, plugname, optname, optparm in alloptions:
+                if optparm['speed'] == 'fast':
+                    plug.setOption(optname, 1)
+                else:
+                    plug.setOption(optname, 0)
+    elif __cmdLineOpts__.usealloptions:
+        for i in range(len(alloptions)):
+            for plug, plugname, optname, optparm in alloptions:
+                plug.setOption(optname, 1)
+
     for plugname, plug in loadedplugins:
         soslog.log(logging.VERBOSE3, _("processing options from plugin: %s") % plugname)
         names, parms = plug.getAllOptions()
         for optname, optparm  in zip(names, parms):
             alloptions.append((plug, plugname, optname, optparm))
 
+    # when --listplugins is specified we do a dry-run
+    # which tells the user which plugins are going to be enabled
+    # and with what options.
+
     if __cmdLineOpts__.listPlugins:
         if not len(loadedplugins) and not len(skippedplugins):
            soslog.error(_("no valid plugins found"))
@@ -473,21 +526,36 @@ def sosreport():
            print _("No plugin enabled.")
         print
 
+        if len(skippedplugins):
+           print _("The following plugins are currently disabled:")
+           print
+           for (plugname,plugclass) in skippedplugins:
+              print " %-25s  %s" % (textcolor(plugname,"blue"),plugclass.get_description())
+        print
+
         if len(alloptions):
            print _("The following plugin options are available:")
            print
            for (plug, plugname, optname, optparm)  in alloptions:
-              print " %-25s  %s [%d]" % (plugname + "." + optname, optparm["desc"], optparm["enabled"])
+              # format and colorize option value based on its type (int or bool)
+              if type(optparm["enabled"])==bool:
+                 if optparm["enabled"]==True:
+                    tmpopt = textcolor("on","lred")
+                 else:
+                    tmpopt = textcolor("off","red")
+              elif type(optparm["enabled"])==int:
+                 if optparm["enabled"] > 0:
+                    tmpopt = textcolor(optparm["enabled"],"lred")
+                 else:
+                    tmpopt = textcolor(optparm["enabled"],"red")
+              else:
+                 tmpopt = optparm["enabled"]
+
+              print " %-21s %-5s %s" % (plugname + "." + optname, tmpopt, optparm["desc"])
+              del tmpopt
         else:
            print _("No plugin options available.")
 
-        if len(skippedplugins):
-           print
-           print _("The following plugins are currently disabled:")
-           print
-           for (plugname,plugclass) in skippedplugins:
-              print " %-25s  %s" % (textcolor(plugname,"blue"),plugclass.get_description())
-
         print
         sys.exit()
 
@@ -520,58 +588,35 @@ Press ENTER to continue, or CTRL-C to quit.
         print
         sys.exit(0)
 
-    # setup plugin options
-    if __cmdLineOpts__.plugopts:
-        opts = {}
-        for opt in __cmdLineOpts__.plugopts:
-           try: opt, val = opt.split("=")
-           except: val=1
-           plug, opt = opt.split(".")
-           try: val = int(val) # try to convert string "val" to int()
-           except: pass
-           try: opts[plug]
-           except KeyError: opts[plug] = []
-           opts[plug].append( (opt,val) )
-        for plugname, plug in loadedplugins:
-           if opts.has_key(plugname):
-               for opt,val in opts[plugname]:
-                   soslog.log(logging.VERBOSE, "setting option %s for plugin %s to %s" % (plugname,opt,val))
-                   plug.setOption(opt,val)
-        del opt,opts,val
-    elif not __cmdLineOpts__.fastoptions and not __cmdLineOpts__.usealloptions:
-        if len(alloptions) and __cmdLineOpts__.use_curses:
-            try:
-                get_curse_options(alloptions)
-            except "Cancelled":
-                sys.exit(_("Exiting."))
-    elif __cmdLineOpts__.fastoptions:
-        for i in range(len(alloptions)):
-            for plug, plugname, optname, optparm in alloptions:
-                if optparm['speed'] == 'fast':
-                    plug.setOption(optname, 1)
-                else:
-                    plug.setOption(optname, 0)
-    elif __cmdLineOpts__.usealloptions:
-        for i in range(len(alloptions)):
-            for plug, plugname, optname, optparm in alloptions:
-                plug.setOption(optname, 1)
-
     # Call the diagnose() method for each plugin
     tmpcount = 0
     for plugname, plug in loadedplugins:
         soslog.log(logging.VERBOSE2, "Performing sanity check for plugin %s" % plugname)
-        plug.diagnose()
+        try:
+           plug.diagnose()
+        except:
+           if __raisePlugins__:
+              raise
         tmpcount += len(plug.diagnose_msgs)
     if tmpcount > 0:
         print _("One or more plugin has detected a problem in your configuration.")
         print _("Please review the following messages:")
         print
         for plugname, plug in loadedplugins:
-           for msg in plug.diagnose_msgs:
-               soslog.warning(" * %s: %s", plugname, msg)
+           for tmpcount2 in range(0,len(plug.diagnose_msgs)):
+              if tmpcount2 == 0:
+                 soslog.warning( textcolor("%s:" % plugname, "red") )
+              soslog.warning("    * %s" % plug.diagnose_msgs[tmpcount2])
         print
         try:
-           raw_input( _("Press ENTER to continue, or CTRL-C to quit.\n") )
+           while True:
+              yorno = raw_input( _("Are you sure you would like to continue (Y/n) ? ") )
+              if yorno == "y" or yorno == "Y":
+                 del yorno
+                 print
+                 break
+              else:
+                 sys.exit(0)
         except KeyboardInterrupt:
            print
            sys.exit(0)
@@ -579,7 +624,11 @@ Press ENTER to continue, or CTRL-C to quit.
     # Call the setup() method for each plugin
     for plugname, plug in loadedplugins:
         soslog.log(logging.VERBOSE2, "Preloading files and commands to be gathered by plugin %s" % plugname)
-        plug.setup()
+        try:
+           plug.setup()
+        except:
+           if __raisePlugins__:
+              raise
 
     # Setup the progress bar
     if __cmdLineOpts__.progressbar: