From 09d32a855b52efcef55e33bf11a87e7ddfdb84c9 Mon Sep 17 00:00:00 2001 From: navid Date: Mon, 10 Dec 2007 09:39:46 +0000 Subject: developement moved to http://code.google.com/p/logpox git-svn-id: svn+ssh://svn.fedorahosted.org/svn/sos/trunk@465 ef72aa8b-4018-0410-8976-d6e080ef94d8 --- src/extras/sos-html-logs/lib/__init__.py | 0 src/extras/sos-html-logs/lib/database.py | 26 -- src/extras/sos-html-logs/lib/helpers.py | 41 -- src/extras/sos-html-logs/lib/logs_abstraction.py | 415 --------------------- src/extras/sos-html-logs/lib/parsers/__init__.py | 0 .../sos-html-logs/lib/parsers/crazyclock_parser.py | 24 -- .../sos-html-logs/lib/parsers/dmapper_parser.py | 106 ------ .../sos-html-logs/lib/parsers/simple_parser.py | 29 -- .../sos-html-logs/lib/parsers/simple_parser.rules | 38 -- 9 files changed, 679 deletions(-) delete mode 100644 src/extras/sos-html-logs/lib/__init__.py delete mode 100644 src/extras/sos-html-logs/lib/database.py delete mode 100644 src/extras/sos-html-logs/lib/helpers.py delete mode 100644 src/extras/sos-html-logs/lib/logs_abstraction.py delete mode 100644 src/extras/sos-html-logs/lib/parsers/__init__.py delete mode 100644 src/extras/sos-html-logs/lib/parsers/crazyclock_parser.py delete mode 100644 src/extras/sos-html-logs/lib/parsers/dmapper_parser.py delete mode 100644 src/extras/sos-html-logs/lib/parsers/simple_parser.py delete mode 100644 src/extras/sos-html-logs/lib/parsers/simple_parser.rules (limited to 'src/extras/sos-html-logs/lib') diff --git a/src/extras/sos-html-logs/lib/__init__.py b/src/extras/sos-html-logs/lib/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/extras/sos-html-logs/lib/database.py b/src/extras/sos-html-logs/lib/database.py deleted file mode 100644 index d1082f52..00000000 --- a/src/extras/sos-html-logs/lib/database.py +++ /dev/null @@ -1,26 +0,0 @@ -try: from pysqlite2 import dbapi2 as sqlite -except: print "python-sqlite is missing. Exiting."; sys.exit(1) - -from threading import Lock - -class myDB_class: - def __init__(self): - self.dbcon = sqlite.connect(":memory:", check_same_thread=False) - self.dbcon.row_factory = sqlite.Row -# self.dbcur = dbcon.cursor() - self.dblock = Lock() - - self.execute("create table events(eid INTEGER PRIMARY KEY AUTOINCREMENT, date KEY, host, position, parser, message, css_style, tooltip BLOB)") - - def execute(self, query): - self.dblock.acquire() - toret = self.dbcon.execute(query) - self.dblock.release() - return toret - - def execute_and_fetch(self, query): - self.dblock.acquire() - dbcur = self.dbcon.execute(query) - toret = dbcur.fetchall() - self.dblock.release() - return toret diff --git a/src/extras/sos-html-logs/lib/helpers.py b/src/extras/sos-html-logs/lib/helpers.py deleted file mode 100644 index 995cf16b..00000000 --- a/src/extras/sos-html-logs/lib/helpers.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -def ksort(d, func = None): - keys = d.keys() - keys.sort(func) - return keys - -class Memoize: - """Memoize(fn) - an instance which acts like fn but memoizes its arguments - Will only work on functions with non-mutable arguments - """ - def __init__(self, fn): - self.fn = fn - self.memo = {} - def __call__(self, *args): - if not self.memo.has_key(args): - self.memo[args] = self.fn(*args) - return self.memo[args] - -class SQL: - def __init__(self): - self.dbcon = sqlite.connect(":memory:", check_same_thread=False) - self.dbcur = self.dbcon.cursor() - - self.dbcon.execute("create table events(date, host, position, message, css_style)") - - def execute(self, query): - return self.dbcon.execute(query) - -def color_gradient(src, dst, percent): - csrc = [ col for col in src ] - cdst = [ col for col in dst ] - toret = [] - - for inc in range(0,3): - toret.append(csrc[inc] + ((cdst[inc] - csrc[inc]) * percent / 100)) - - return toret - -def rgb_to_hex(rgb): - return "%X%X%X" % (rgb[0], rgb[1], rgb[2]) diff --git a/src/extras/sos-html-logs/lib/logs_abstraction.py b/src/extras/sos-html-logs/lib/logs_abstraction.py deleted file mode 100644 index 5708c960..00000000 --- a/src/extras/sos-html-logs/lib/logs_abstraction.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env python - -import os, sys, time, re, pdb -from threading import Thread, Lock -from helpers import * -from operator import itemgetter -import traceback - -class cluster_class: - - def __init__(self): - self.hosts = {} - self.index = {} - self.daemon_log_counter = [] - self.parsers = [] - - def host_names(self): - return ksort(self.hosts) - - def register_parser(self, parser_class): - self.parsers.append(parser_class) - - def get_parser(self, parser_name): - for parser in self.parsers: - if parser.__class__.__name__ == parser_name: - return parser - - def get_host(self, host): - return self.hosts[host] - - def tell(self): - toret = {} - for host in self.hosts: - toret[host] = self.hosts[host].tell() - return toret - - def tell_sum(self): - toret = 0 - for host in self.hosts: - toret += self.hosts[host].tell() - return toret - - def size(self): - toret = 0 - for host in self.hosts: - toret += self.hosts[host].size() - return toret - - def seek(self, positions): - # make sure positions in argument are valid - for host in self.hosts: - if host not in positions.keys(): - print "cannot find", positions - raise "Invalid_Positions" - - # seek each host to saved position - for host in positions: - self.hosts[host].seek(positions[host]) - - return True - - def seek_beginning(self): - for host in self.hosts: - self.hosts[host].seek(0) - - return True - - def add_log(self, logname): - log = logfile_class(logname) - hostname = log.hostname() - sys.stderr.write("""adding log "%s" for host %s\n""" % (logname, hostname)) - if not self.hosts.has_key(hostname): - self.hosts[hostname] = host_class() - self.hosts[hostname].add_log(log) - - def get_position_by_date(self, goto_date): - try: - return self.index[goto_date]["position"] - except KeyError: - # can't find position in cache, calculate on the fly - # - for cmp_date in ksort(self.index): - if goto_date <= cmp_date: - return self.index[cmp_date]["position"] - return None - - def parse(self, threaded = False): - - if threaded and (not hasattr(self,"parse_t") or self.parse_t == None): - self.parse_t = Thread(target=self.parse, name='parse-thread', args = [True] ) - self.parse_t.start() - return self.parse_t - - print "parsing begins" - - daemon_log_counter = {} - - self.seek_beginning() - - for date in self: - - self.index[date.date] = { "position":date.position, "log_counter":{} } - - for host in self.hosts: - self.index[date.date]["log_counter"][host]=0 - - try: - for log in date[host]: - self.index[date.date]["log_counter"][host]+=1 - - for parser_class in self.parsers: - parser_class.parse_line(date, log) - - # count how many logs per daemon - try: - daemon_log_counter[log.daemon()]+=1 - except KeyError: - daemon_log_counter[log.daemon()]=1 - - except "Eof": - # no more logs for this host - pass - - self.daemon_log_counter = sorted(daemon_log_counter.items(), key=itemgetter(1), reverse=True) - - print "parsing ends." - - def eof(self): - for host in self.hosts: - if not self.hosts[host].eof(): -# print "All logs are not EOF yet", host - return False - print "All logs are EOF!" - return True - - def __iter__(self): - return self - - def next(self): - if self.eof(): - raise StopIteration - - return log_date_class(cluster = self) - - def instance(self): - toret = cluster_class() - - for host in self.hosts: - toret.hosts[host] = host_class() - - for log in self.hosts[host].logs: - toret.hosts[host].logs.append(logfile_class(log.fname)) - - toret.index = self.index - toret.daemon_log_counter = self.daemon_log_counter - toret.parsers = self.parsers - - return toret - -class log_date_class: - def __init__(self, cluster): - self.cluster = cluster - self.date = None - self.hosts = cluster.hosts.keys() - - self.position = cluster.tell() - - newtime = None - - # 1st run, must find out what is the oldest date for each host - for host in self.hosts: - while True: - try: - newtime = time.strptime("2007 " + cluster.hosts[host].readline()[0:15], "%Y %b %d %H:%M:%S") - except "Eof": - break - except ValueError: - print "parsing error in line", cluster.hosts[host].tell() - else: - break - - if newtime: - if not self.date or newtime < self.date: - self.date = newtime - - if not cluster.hosts[host].eof(): - cluster.hosts[host].backline() - - # this should almost never happen, but just in case. - if not self.date: - raise "Strange_Eof" - - def __str__(self): - return time.strftime("%b %d %H:%M:%S", self.date) - - def __getitem__(self, host): - return log_date_host(self.cluster, self.cluster.hosts[host], self.date) - - def __iter__(self): - return self - -class log_date_host: - def __init__(self, cluster, host, date): - self.cluster = cluster - self.host = host - self.date = date - - self.parent_date = date - - def __iter__(self): - return self - - def next(self): - position = self.host.tell() - - self.host.readline() - - try: - if time.strptime("2007 " + self.host.cur_line[0:15], "%Y %b %d %H:%M:%S") <= self.date: - return log_line_class(self.parent_date, self.host, position, self.host.cur_line) - except: - return log_line_class(self.parent_date, self.host, position, self.host.cur_line) - - self.host.backline() - - raise StopIteration - -class log_line_class: - def __init__(self, date, host, position, line): - self.host = host - self.position = position - self.line = line - self.parse = Memoize(self.parse_uncached) - - self.parent_date = date - self.parent_host = host - - def parse_uncached(self): - try: - return re.findall(r"""^(... .. ..:..:..) %s ([-_0-9a-zA-Z \.\/\(\)]+)(\[[0-9]+\])?(:)? (.*)$""" % self.hostname(), self.line)[0] - except: - return [ None, None, None, None, None ] - - def __str__(self): - return self.line - - def date(self): - try: - return time.strptime("2007 " + self.line[0:15], "%Y %b %d %H:%M:%S") - except: - return False - - def hostname(self): - return self.line[16:].split(" ", 1)[0] - - def daemon(self): - return self.parse()[1] - - def message(self): - return self.parse()[4] - -class host_class: - - def __init__(self): - self.logs = [] - - self.log_idx = 0 # first log - self.log_ptr = 0 # first char - - self.cur_line = None - - def __str__(self): - return self.hostname() - - def add_log(self, logfile): - - for inc in range(0,len(self.logs)): - if logfile.time_end() < self.logs[inc].time_begin(): - self.logs.insert(inc, logfile) - break - else: - self.logs.append(logfile) - - def hostname(self): - return self.logs[0].hostname() -# try: return self.logs[0].hostname() -# except: return None - - def tell(self): - sumsize = 0 - if self.log_idx > 0: - for inc in range(0, self.log_idx): - sumsize += self.logs[inc].size() - try: - sumsize += self.fp().tell() - except TypeError: - pass - return sumsize - - def size(self): - sumsize = 0 - for inc in range(0, len(self.logs)): - sumsize += self.logs[inc].size() - return sumsize - - def eof(self): - if self.tell() >= self.size(): - return True - return False - - def seek(self, offset, whence = 0): - if whence == 1: offset = self.tell() + offset - elif whence == 2: offset = self.size() + offset - - sumsize = 0 - for inc in range(0, len(self.logs)): - if offset <= sumsize + self.logs[inc].size(): - offset -= sumsize - self.log_idx = inc - self.log_ptr = offset - self.logs[inc].seek(offset) - return True - sumsize += self.logs[inc].size() - raise "Off_Boundaries" - - def seek_and_read(self, offset, whence = 0): - self.seek(offset, whence) - return self.readline() - - def time(self): - return time.strptime("2007 " + self.cur_line[0:15], "%Y %b %d %H:%M:%S") - - def fp(self): - return self.logs[self.log_idx] - - def backline(self): - self.seek(-len(self.cur_line), 1) - - def readline(self): - if self.eof(): - raise "Eof" - - while True: - position = self.fp().tell() - fromfile = self.fp().fname - toret = self.fp().readline() - if len(toret) == 0: - if self.log_idx < len(self.logs): - self.log_idx += 1 - self.fp().seek(0) - continue - else: - return "" - - if len(toret) > 0 or toret == "": - self.cur_line = toret - self.cur_file = fromfile - self.cur_pos = position - return toret - else: - print "invalid line", toret - -class logfile_class: - - def __init__(self,fname): - self.fname = fname - self.fp = open(fname) - - def hostname(self): - pos = self.fp.tell() - self.seek(0) - toret = self.fp.readline()[16:].split(" ")[0] - self.fp.seek(pos) - return toret - - def time_begin(self): - pos = self.fp.tell() - self.fp.seek(0) - toret = time.strptime(self.fp.readline()[0:15], "%b %d %H:%M:%S") - self.fp.seek(pos) - return toret - - def time_end(self): - pos = self.fp.tell() - bs = 1024 - if self.size() < bs: bs = self.size() - self.fp.seek(-bs, 2) - buf = self.fp.read(bs) - bufsplit = buf.split("\n") - bufsplit.reverse() - for line in bufsplit: - if len(line) == 0: continue - try: toret = time.strptime(line[0:15], "%b %d %H:%M:%S") - except ValueError: print "Error in conversion"; continue - else: break - self.fp.seek(pos) - return toret - - def size(self): - return os.path.getsize(self.fname) - - def eof(self): - return self.fp.tell() > self.size() - - def readline(self): - return self.fp.readline() - - def seek(self,pos): -# if cmdline["verbose"]: -# print "seeking to position %d for file %s" % (pos, self.fname) -# traceback.print_stack() - self.fp.seek(pos) - - def tell(self): - return self.fp.tell() diff --git a/src/extras/sos-html-logs/lib/parsers/__init__.py b/src/extras/sos-html-logs/lib/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/extras/sos-html-logs/lib/parsers/crazyclock_parser.py b/src/extras/sos-html-logs/lib/parsers/crazyclock_parser.py deleted file mode 100644 index 3e794a6f..00000000 --- a/src/extras/sos-html-logs/lib/parsers/crazyclock_parser.py +++ /dev/null @@ -1,24 +0,0 @@ -import time -import soshtmllogs.parsers_base as parsers_base - -class crazyclock_Parser(parsers_base.base_Parser_class): - - def initialize(self): - # in this dict, we store the last date for each host - self.last_date = {} - - def parse_line(self, date, log): - - if date.date != log.date(): - self.add_event(log, "Time skew (%d seconds in the past)" % int(time.mktime(date.date) - time.mktime(log.date())), "color:purple; background-color:yellow") - - self.last_date[log.parent_host] = log.date() - - return - - def analyse_line(self, log): - - yield """The following line matched the rule:
""" - yield """
%s
""" % log.line - - yield "The logged time for this message is before the one for the previous message appearing in the log." diff --git a/src/extras/sos-html-logs/lib/parsers/dmapper_parser.py b/src/extras/sos-html-logs/lib/parsers/dmapper_parser.py deleted file mode 100644 index 279df50e..00000000 --- a/src/extras/sos-html-logs/lib/parsers/dmapper_parser.py +++ /dev/null @@ -1,106 +0,0 @@ -import re -import soshtmllogs.parsers_base as parsers_base - -class dmapper_Parser(parsers_base.base_Parser_class): - default_css = "color:black; background-color:orange; font-size:larger" - - def initialize(self): - self.db.execute("create table dmapper(disk PRIMARY KEY, host INTEGER, channel INTEGER, id INTEGER, lun INTEGER)") - - def parse_line(self, date, logline): - - # kernel: sd 1:0:0:49152: Attached scsi disk sdt - found = re.findall(r"""^sd (.+):(.+):(.+):(.+): Attached scsi disk (.+)$""", logline.message()) - if found: - # we can learn a little about the layout - found = found[0] - self.db.execute("""replace into dmapper(disk,host,channel,id,lun) values ("%s",%d,%d,%d,%d)""" - % (found[4], int(found[0]), int(found[1]), int(found[2]), int(found[3])) - ) - - found = re.findall(r"""^Attached scsi disk (.+) at scsi(.+), channel (.+), id (.+), lun (.+)$""", logline.message()) - if found: - # we can learn a little about the layout - found = found[0] - self.db.execute("""replace into dmapper(disk,host,channel,id,lun) values ("%s",%d,%d,%d,%d)""" - % (found[0], int(found[1]), int(found[2]), int(found[3]), int(found[4])) - ) - - #Nov 7 12:55:38 itrac415 kernel: SCSI error : <2 0 3 0> return code = 0x20000 - found = re.findall(r"""^SCSI error : <(.+)> return code = (.+)$""", logline.message()) - if found: - found = found[0] - scsi_address_split = [ int(sid) for sid in found[0].split(" ") ] - results = self.db.execute_and_fetch("select disk from dmapper where host = %d AND channel = %d AND id = %d AND lun = %d" % - (scsi_address_split[0], scsi_address_split[1], scsi_address_split[2], scsi_address_split[3]) - ) - for row in results: found[0] = row["disk"] - self.add_event(logline, "SCSI error on %s - %s" % (found[0], self._get_did_error(found[1])), self.default_css) - return - - found = re.findall(r"""^end_request: I/O error, dev (.*), sector .*$""", logline.message()) - if found: - self.add_event(logline, "I/O error on %s" % (found[0]), self.default_css) - return - - if logline.daemon() != "multipathd": - return - - found = re.findall(r"""(.*): mark as failed$""", logline.message()) - if found: - disk = self._get_disk_from_majmin(found[0]) - self.add_event(logline, "Multipath path %s (%s) failed" % (found[0], disk), self.default_css) - return - - found = re.findall(r"""(.*): reinstated$""", logline.message()) - if found: - disk = self._get_disk_from_majmin(found[0]) - self.add_event(logline, "Multipath path %s (%s) reinstated" % (found[0], disk), self.default_css) - return - - return - - def _get_disk_from_majmin(self, majmin): - major, minor = majmin.split(":") - major, minor = int(major), int(minor) - - block_majors = [8, 65, 66, 67, 68, 69, 70, 71, 128, 129, 130, 131, 132, 133, 134, 135] - - disk = (block_majors.index(major) * 16) + int(minor / 16) - partition = minor % 16 - - # 97 = ord('a') - # 25 = ord('z') - ord('a') - - rchar = chr(97 + (disk % 25)) - - if disk > 25: - lchar = chr(97 - 1 + int(disk / 25)) - rchar = chr(ord(rchar)-1) - else: - lchar = "" - - return "sd" + lchar + rchar - - def _get_did_error(self, hexerr): - # hexherr = 0x20000 - - if not hexerr.startswith("0x"): - return "Unknown error code (%s)" % hexerr - - err = hexerr[2] - - if err == "0": return "DID_OK (NO error)" - if err == "1": return "DID_NO_CONNECT (Couldn\\'t connect before timeout period)" - if err == "2": return "DID_BUS_BUSY (BUS stayed busy through time out period)" - if err == "3": return "DID_TIME_OUT (TIMED OUT for other reason)" - if err == "4": return "DID_BAD_TARGET (BAD target)" - if err == "5": return "DID_ABORT (Told to abort for some other reason)" - if err == "6": return "DID_PARITY (Parity error)" - if err == "7": return "DID_ERROR (Internal error)" - if err == "8": return "DID_RESET (Reset by somebody)" - if err == "9": return "DID_BAD_INTR (Got an interrupt we weren't expecting)" - if err == "a": return "DID_PASSTHROUGH (Force command past mid-layer)" - if err == "b": return "DID_SOFT_ERROR (The low level driver just wish a retry)" - if err == "c": return "DID_IMM_RETRY (Retry without decrementing retry count)" - if err == "d": return "DID_REQUEUE (Requeue command (no immediate retry) also without decrementing the retry count)" diff --git a/src/extras/sos-html-logs/lib/parsers/simple_parser.py b/src/extras/sos-html-logs/lib/parsers/simple_parser.py deleted file mode 100644 index 9beb10f8..00000000 --- a/src/extras/sos-html-logs/lib/parsers/simple_parser.py +++ /dev/null @@ -1,29 +0,0 @@ -import ConfigParser, re -import soshtmllogs.parsers_base as parsers_base - -class simple_Parser(parsers_base.base_Parser_class): - - def initialize(self): - self.config = ConfigParser.ConfigParser() - self.config.readfp(open('/usr/lib/python2.4/site-packages/soshtmllogs/parsers/simple_parser.rules')) - - def parse_line(self, date, log): - - for section in self.config.sections(): - match = False - - if self.config.has_option(section, "find"): - if log.line.find(self.config.get(section, "find")) >= 0: - match = True - elif self.config.has_option(section, "regex"): - if re.match(self.config.get(section, "regex"), log.line): - match = True - - if not match: - continue - - self.add_event(log, section, "color:green; background-color:yellow; font-size:larger") - - return - - return None diff --git a/src/extras/sos-html-logs/lib/parsers/simple_parser.rules b/src/extras/sos-html-logs/lib/parsers/simple_parser.rules deleted file mode 100644 index 724fbfa8..00000000 --- a/src/extras/sos-html-logs/lib/parsers/simple_parser.rules +++ /dev/null @@ -1,38 +0,0 @@ -[Syslog restart (possible system restart)] -regex = .*syslogd .*: restart.$ - -[System reboot] -regex = ^kernel: Linux version - -[Fencing node] -regex = ^.*fencing node - -[Fencing node succeeded] -regex = .*fence ".*" success - -[Fencing node failed] -regex = .*fence ".* - -[Quorum lost] -find = quorum lost, blocking activity - -[Quorum regained] -find = quorum regained, resuming activity - -[Segmentation fault] -find = segfault at - -[Inittab reloaded] -find = Re-reading inittab - -[Init line respawning too fast] -find = respawning too fast: disabled for - -[Ext3 file-system error] -find = EXT3-fs error - -[File-system remounted read-only] -find = Remounting filesystem read-only - -[Shutting down] -find = shutdown: shutting down -- cgit