diff options
Diffstat (limited to 'libbe')
38 files changed, 2326 insertions, 1245 deletions
diff --git a/libbe/bug.py b/libbe/bug.py index 3b9be59..8b81842 100644 --- a/libbe/bug.py +++ b/libbe/bug.py @@ -317,6 +317,97 @@ class Bug (settings_object.SavedSettingsObject): return output def xml(self, indent=0, show_comments=False): + """ + >>> bugA = Bug(uuid='0123', summary='Need to test Bug.xml()') + >>> bugA.uuid = 'bugA' + >>> bugA.time_string = 'Thu, 01 Jan 1970 00:00:00 +0000' + >>> bugA.creator = u'Frank' + >>> bugA.extra_strings += ['TAG: very helpful'] + >>> commA = bugA.comment_root.new_reply(body='comment A') + >>> commA.uuid = 'commA' + >>> commA.date = 'Thu, 01 Jan 1970 00:01:00 +0000' + >>> commB = commA.new_reply(body='comment B') + >>> commB.uuid = 'commB' + >>> commB.date = 'Thu, 01 Jan 1970 00:02:00 +0000' + >>> commC = commB.new_reply(body='comment C') + >>> commC.uuid = 'commC' + >>> commC.date = 'Thu, 01 Jan 1970 00:03:00 +0000' + >>> print(bugA.xml(show_comments=True)) # doctest: +REPORT_UDIFF + <bug> + <uuid>bugA</uuid> + <short-name>/bug</short-name> + <severity>minor</severity> + <status>open</status> + <creator>Frank</creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Need to test Bug.xml()</summary> + <extra-string>TAG: very helpful</extra-string> + <comment> + <uuid>commA</uuid> + <short-name>/bug/commA</short-name> + <author></author> + <date>Thu, 01 Jan 1970 00:01:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment A</body> + </comment> + <comment> + <uuid>commB</uuid> + <short-name>/bug/commB</short-name> + <in-reply-to>commA</in-reply-to> + <author></author> + <date>Thu, 01 Jan 1970 00:02:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment B</body> + </comment> + <comment> + <uuid>commC</uuid> + <short-name>/bug/commC</short-name> + <in-reply-to>commB</in-reply-to> + <author></author> + <date>Thu, 01 Jan 1970 00:03:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment C</body> + </comment> + </bug> + >>> print(bugA.xml(show_comments=True, indent=2)) + ... # doctest: +REPORT_UDIFF + <bug> + <uuid>bugA</uuid> + <short-name>/bug</short-name> + <severity>minor</severity> + <status>open</status> + <creator>Frank</creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Need to test Bug.xml()</summary> + <extra-string>TAG: very helpful</extra-string> + <comment> + <uuid>commA</uuid> + <short-name>/bug/commA</short-name> + <author></author> + <date>Thu, 01 Jan 1970 00:01:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment A</body> + </comment> + <comment> + <uuid>commB</uuid> + <short-name>/bug/commB</short-name> + <in-reply-to>commA</in-reply-to> + <author></author> + <date>Thu, 01 Jan 1970 00:02:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment B</body> + </comment> + <comment> + <uuid>commC</uuid> + <short-name>/bug/commC</short-name> + <in-reply-to>commB</in-reply-to> + <author></author> + <date>Thu, 01 Jan 1970 00:03:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment C</body> + </comment> + </bug> + """ if self.time == None: timestring = "" else: @@ -339,7 +430,8 @@ class Bug (settings_object.SavedSettingsObject): lines.append(' <extra-string>%s</extra-string>' % estr) if show_comments == True: comout = self.comment_root.xml_thread(indent=indent+2) - if len(comout) > 0: + if comout: + comout = comout[indent:] # strip leading indent spaces lines.append(comout) lines.append('</bug>') istring = ' '*indent @@ -384,7 +476,7 @@ class Bug (settings_object.SavedSettingsObject): bug = ElementTree.XML(xml_string) if bug.tag != 'bug': raise utility.InvalidXML( \ - 'bug', bug, 'root element must be <comment>') + 'bug', bug, 'root element must be <bug>') tags=['uuid','short-name','severity','status','assigned', 'reporter', 'creator','created','summary','extra-string'] self.explicit_attrs = [] @@ -405,13 +497,16 @@ class Bug (settings_object.SavedSettingsObject): text = settings_object.EMPTY else: text = xml.sax.saxutils.unescape(child.text) - text = text.decode('unicode_escape').strip() + if not isinstance(text, unicode): + text = text.decode('unicode_escape') + text = text.strip() if child.tag == 'uuid' and not preserve_uuids: uuid = text continue # don't set the bug's uuid tag. elif child.tag == 'created': - self.time = utility.str_to_time(text) - self.explicit_attrs.append('time') + if text is not settings_object.EMPTY: + self.time = utility.str_to_time(text) + self.explicit_attrs.append('time') continue elif child.tag == 'extra-string': estrs.append(text) @@ -605,20 +700,21 @@ class Bug (settings_object.SavedSettingsObject): </comment> </bug> """ - for attr in other.explicit_attrs: - old = getattr(self, attr) - new = getattr(other, attr) - if old != new: - if accept_changes == True: - setattr(self, attr, new) - elif change_exception == True: - raise ValueError, \ - 'Merge would change %s "%s"->"%s" for bug %s' \ - % (attr, old, new, self.uuid) + if hasattr(other, 'explicit_attrs'): + for attr in other.explicit_attrs: + old = getattr(self, attr) + new = getattr(other, attr) + if old != new: + if accept_changes: + setattr(self, attr, new) + elif change_exception: + raise ValueError( + ('Merge would change {} "{}"->"{}" for bug {}' + ).format(attr, old, new, self.uuid)) for estr in other.extra_strings: if not estr in self.extra_strings: if accept_extra_strings == True: - self.extra_strings.append(estr) + self.extra_strings += [estr] elif change_exception == True: raise ValueError, \ 'Merge would add extra string "%s" for bug %s' \ diff --git a/libbe/bugdir.py b/libbe/bugdir.py index fdf786b..8f11075 100644 --- a/libbe/bugdir.py +++ b/libbe/bugdir.py @@ -28,6 +28,12 @@ import errno import os import os.path import time +import types +try: # import core module, Python >= 2.5 + from xml.etree import ElementTree +except ImportError: # look for non-core module + from elementtree import ElementTree +import xml.sax.saxutils import libbe import libbe.storage as storage @@ -214,6 +220,8 @@ class BugDir (list, settings_object.SavedSettingsObject): directory=False) self.save_settings() for bug in self: + bug.bugdir = self + bug.storage = self.storage bug.save() # methods for managing bugs @@ -249,12 +257,19 @@ class BugDir (list, settings_object.SavedSettingsObject): def new_bug(self, summary=None, _uuid=None): bg = bug.Bug(bugdir=self, uuid=_uuid, summary=summary, from_storage=False) - self.append(bg) - self._bug_map_gen() - if hasattr(self, '_uuids_cache') and not bg.uuid in self._uuids_cache: - self._uuids_cache.add(bg.uuid) + self.append(bg, update=True) return bg + def append(self, bug, update=False): + super(BugDir, self).append(bug) + if update: + bug.bugdir = self + bug.storage = self.storage + self._bug_map_gen() + if (hasattr(self, '_uuids_cache') and + not bug.uuid in self._uuids_cache): + self._uuids_cache.add(bug.uuid) + def remove_bug(self, bug): if hasattr(self, '_uuids_cache') and bug.uuid in self._uuids_cache: self._uuids_cache.remove(bug.uuid) @@ -278,6 +293,379 @@ class BugDir (list, settings_object.SavedSettingsObject): return False return True + def xml(self, indent=0, show_bugs=False, show_comments=False): + """ + >>> bug.load_severities(bug.severity_def) + >>> bug.load_status( + ... active_status_def=bug.active_status_def, + ... inactive_status_def=bug.inactive_status_def) + >>> bugdirA = SimpleBugDir(memory=True) + >>> bugdirA.severities + >>> bugdirA.severities = (('minor', 'The standard bug level.'),) + >>> bugdirA.inactive_status = ( + ... ('closed', 'The bug is no longer relevant.'),) + >>> bugA = bugdirA.bug_from_uuid('a') + >>> commA = bugA.comment_root.new_reply(body='comment A') + >>> commA.uuid = 'commA' + >>> commA.date = 'Thu, 01 Jan 1970 00:03:00 +0000' + >>> print(bugdirA.xml(show_bugs=True, show_comments=True)) + ... # doctest: +REPORT_UDIFF + <bugdir> + <uuid>abc123</uuid> + <short-name>abc</short-name> + <severities> + <entry> + <key>minor</key> + <value>The standard bug level.</value> + </entry> + </severities> + <inactive-status> + <entry> + <key>closed</key> + <value>The bug is no longer relevant.</value> + </entry> + </inactive-status> + <bug> + <uuid>a</uuid> + <short-name>abc/a</short-name> + <severity>minor</severity> + <status>open</status> + <creator>John Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug A</summary> + <comment> + <uuid>commA</uuid> + <short-name>abc/a/com</short-name> + <author></author> + <date>Thu, 01 Jan 1970 00:03:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment A</body> + </comment> + </bug> + <bug> + <uuid>b</uuid> + <short-name>abc/b</short-name> + <severity>minor</severity> + <status>closed</status> + <creator>Jane Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug B</summary> + </bug> + </bugdir> + >>> bug.load_severities(bug.severity_def) + >>> bug.load_status( + ... active_status_def=bug.active_status_def, + ... inactive_status_def=bug.inactive_status_def) + >>> bugdirA.cleanup() + """ + info = [('uuid', self.uuid), + ('short-name', self.id.user()), + ('target', self.target), + ('severities', self.severities), + ('active-status', self.active_status), + ('inactive-status', self.inactive_status), + ] + lines = ['<bugdir>'] + for (k,v) in info: + if v is not None: + if k in ['severities', 'active-status', 'inactive-status']: + lines.append(' <{}>'.format(k)) + for vk,vv in v: + lines.extend([ + ' <entry>', + ' <key>{}</key>'.format( + xml.sax.saxutils.escape(vk)), + ' <value>{}</value>'.format( + xml.sax.saxutils.escape(vv)), + ' </entry>', + ]) + lines.append(' </{}>'.format(k)) + else: + v = xml.sax.saxutils.escape(v) + lines.append(' <{0}>{1}</{0}>'.format(k, v)) + for estr in self.extra_strings: + lines.append(' <extra-string>{}</extra-string>'.format(estr)) + if show_bugs: + for bug in self: + bug_xml = bug.xml(indent=indent+2, show_comments=show_comments) + if bug_xml: + bug_xml = bug_xml[indent:] # strip leading indent spaces + lines.append(bug_xml) + lines.append('</bugdir>') + istring = ' '*indent + sep = '\n' + istring + return istring + sep.join(lines).rstrip('\n') + + def from_xml(self, xml_string, preserve_uuids=False, verbose=True): + """ + Note: If a bugdir uuid is given, set .alt_id to it's value. + >>> bug.load_severities(bug.severity_def) + >>> bug.load_status( + ... active_status_def=bug.active_status_def, + ... inactive_status_def=bug.inactive_status_def) + >>> bugdirA = SimpleBugDir(memory=True) + >>> bugdirA.severities = (('minor', 'The standard bug level.'),) + >>> bugdirA.inactive_status = ( + ... ('closed', 'The bug is no longer relevant.'),) + >>> bugA = bugdirA.bug_from_uuid('a') + >>> commA = bugA.comment_root.new_reply(body='comment A') + >>> commA.uuid = 'commA' + >>> xml = bugdirA.xml(show_bugs=True, show_comments=True) + >>> bugdirB = BugDir(storage=None) + >>> bugdirB.from_xml(xml) + >>> bugdirB.xml(show_bugs=True, show_comments=True) == xml + False + >>> bugdirB.uuid = bugdirB.alt_id + >>> for bug_ in bugdirB: + ... bug_.uuid = bug_.alt_id + ... bug_.alt_id = None + ... for comm in bug_.comments(): + ... comm.uuid = comm.alt_id + ... comm.alt_id = None + >>> bugdirB.xml(show_bugs=True, show_comments=True) == xml + True + >>> bugdirB.explicit_attrs # doctest: +NORMALIZE_WHITESPACE + ['severities', 'inactive_status'] + >>> bugdirC = BugDir(storage=None) + >>> bugdirC.from_xml(xml, preserve_uuids=True) + >>> bugdirC.uuid == bugdirA.uuid + True + >>> bugdirC.xml(show_bugs=True, show_comments=True) == xml + True + >>> bug.load_severities(bug.severity_def) + >>> bug.load_status( + ... active_status_def=bug.active_status_def, + ... inactive_status_def=bug.inactive_status_def) + >>> bugdirA.cleanup() + """ + if type(xml_string) == types.UnicodeType: + xml_string = xml_string.strip().encode('unicode_escape') + if hasattr(xml_string, 'getchildren'): # already an ElementTree Element + bugdir = xml_string + else: + bugdir = ElementTree.XML(xml_string) + if bugdir.tag != 'bugdir': + raise utility.InvalidXML( + 'bugdir', bugdir, 'root element must be <bugdir>') + tags = ['uuid', 'short-name', 'target', 'severities', 'active-status', + 'inactive-status', 'extra-string'] + self.explicit_attrs = [] + uuid = None + estrs = [] + for child in bugdir.getchildren(): + if child.tag == 'short-name': + pass + elif child.tag == 'bug': + bg = bug.Bug(bugdir=self) + bg.from_xml( + child, preserve_uuids=preserve_uuids, verbose=verbose) + self.append(bg, update=True) + continue + elif child.tag in tags: + if child.text == None or len(child.text) == 0: + text = settings_object.EMPTY + elif child.tag in ['severities', 'active-status', + 'inactive-status']: + entries = [] + for entry in child.getchildren(): + if entry.tag != 'entry': + raise utility.InvalidXML( + '{} child element {} must be <entry>'.format( + child.tag, entry)) + key = value = None + for kv in entry.getchildren(): + if kv.tag == 'key': + if key is not None: + raise utility.InvalidXML( + ('duplicate keys ({} and {}) in {}' + ).format(key, kv.text, child.tag)) + key = xml.sax.saxutils.unescape(kv.text) + elif kv.tag == 'value': + if value is not None: + raise utility.InvalidXML( + ('duplicate values ({} and {}) in {}' + ).format( + value, kv.text, child.tag)) + value = xml.sax.saxutils.unescape(kv.text) + else: + raise utility.InvalidXML( + ('{} child element {} must be <key> or ' + '<value>').format(child.tag, kv)) + if key is None: + raise utility.InvalidXML( + 'no key for {}'.format(child.tag)) + if value is None: + raise utility.InvalidXML( + 'no key for {}'.format(child.tag)) + entries.append((key, value)) + text = entries + else: + text = xml.sax.saxutils.unescape(child.text) + if not isinstance(text, unicode): + text = text.decode('unicode_escape') + text = text.strip() + if child.tag == 'uuid' and not preserve_uuids: + uuid = text + continue # don't set the bug's uuid tag. + elif child.tag == 'extra-string': + estrs.append(text) + continue # don't set the bug's extra_string yet. + attr_name = child.tag.replace('-','_') + self.explicit_attrs.append(attr_name) + setattr(self, attr_name, text) + elif verbose == True: + sys.stderr.write('Ignoring unknown tag {} in {}\n'.format( + child.tag, bugdir.tag)) + if uuid != self.uuid: + if not hasattr(self, 'alt_id') or self.alt_id == None: + self.alt_id = uuid + self.extra_strings = estrs + + def merge(self, other, accept_changes=True, + accept_extra_strings=True, accept_bugs=True, + accept_comments=True, change_exception=False): + """Merge info from other into this bugdir. + + Overrides any attributes in self that are listed in + other.explicit_attrs. + + >>> bugdirA = SimpleBugDir() + >>> bugdirA.extra_strings += ['TAG: favorite'] + >>> bugdirB = SimpleBugDir() + >>> bugdirB.explicit_attrs = ['target'] + >>> bugdirB.target = '1234' + >>> bugdirB.extra_strings += ['TAG: very helpful'] + >>> bugdirB.extra_strings += ['TAG: useful'] + >>> bugA = bugdirB.bug_from_uuid('a') + >>> commA = bugA.comment_root.new_reply(body='comment A') + >>> commA.uuid = 'uuid-commA' + >>> commA.date = 'Thu, 01 Jan 1970 00:01:00 +0000' + >>> bugC = bugdirB.new_bug(summary='bug C', _uuid='c') + >>> bugC.alt_id = 'alt-c' + >>> bugC.time_string = 'Thu, 01 Jan 1970 00:02:00 +0000' + >>> bugdirA.merge( + ... bugdirB, accept_changes=False, accept_extra_strings=False, + ... accept_bugs=False, change_exception=False) + >>> print(bugdirA.target) + None + >>> bugdirA.merge( + ... bugdirB, accept_changes=False, accept_extra_strings=False, + ... accept_bugs=False, change_exception=True) + Traceback (most recent call last): + ... + ValueError: Merge would change target "None"->"1234" for bugdir abc123 + >>> print(bugdirA.target) + None + >>> bugdirA.merge( + ... bugdirB, accept_changes=True, accept_extra_strings=False, + ... accept_bugs=False, change_exception=True) + Traceback (most recent call last): + ... + ValueError: Merge would add extra string "TAG: useful" for bugdir abc123 + >>> print(bugdirA.target) + 1234 + >>> print(bugdirA.extra_strings) + ['TAG: favorite'] + >>> bugdirA.merge( + ... bugdirB, accept_changes=True, accept_extra_strings=True, + ... accept_bugs=False, change_exception=True) + Traceback (most recent call last): + ... + ValueError: Merge would add bug c (alt: alt-c) to bugdir abc123 + >>> print(bugdirA.extra_strings) + ['TAG: favorite', 'TAG: useful', 'TAG: very helpful'] + >>> bugdirA.merge( + ... bugdirB, accept_changes=True, accept_extra_strings=True, + ... accept_bugs=True, change_exception=True) + >>> print(bugdirA.xml(show_bugs=True, show_comments=True)) + ... # doctest: +ELLIPSIS, +REPORT_UDIFF + <bugdir> + <uuid>abc123</uuid> + <short-name>abc</short-name> + <target>1234</target> + <extra-string>TAG: favorite</extra-string> + <extra-string>TAG: useful</extra-string> + <extra-string>TAG: very helpful</extra-string> + <bug> + <uuid>a</uuid> + <short-name>abc/a</short-name> + <severity>minor</severity> + <status>open</status> + <creator>John Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug A</summary> + <comment> + <uuid>uuid-commA</uuid> + <short-name>abc/a/uui</short-name> + <author></author> + <date>Thu, 01 Jan 1970 00:01:00 +0000</date> + <content-type>text/plain</content-type> + <body>comment A</body> + </comment> + </bug> + <bug> + <uuid>b</uuid> + <short-name>abc/b</short-name> + <severity>minor</severity> + <status>closed</status> + <creator>Jane Doe <jdoe@example.com></creator> + <created>Thu, 01 Jan 1970 00:00:00 +0000</created> + <summary>Bug B</summary> + </bug> + <bug> + <uuid>c</uuid> + <short-name>abc/c</short-name> + <severity>minor</severity> + <status>open</status> + <created>Thu, 01 Jan 1970 00:02:00 +0000</created> + <summary>bug C</summary> + </bug> + </bugdir> + >>> bugdirA.cleanup() + >>> bugdirB.cleanup() + """ + if hasattr(other, 'explicit_attrs'): + for attr in other.explicit_attrs: + old = getattr(self, attr) + new = getattr(other, attr) + if old != new: + if accept_changes: + setattr(self, attr, new) + elif change_exception: + raise ValueError( + ('Merge would change {} "{}"->"{}" for bugdir {}' + ).format(attr, old, new, self.uuid)) + for estr in other.extra_strings: + if not estr in self.extra_strings: + if accept_extra_strings: + self.extra_strings += [estr] + elif change_exception: + raise ValueError( + ('Merge would add extra string "{}" for bugdir {}' + ).format(estr, self.uuid)) + for o_bug in other: + try: + s_bug = self.bug_from_uuid(o_bug.uuid) + except KeyError as e: + try: + s_bug = self.bug_from_uuid(o_bug.alt_id) + except KeyError as e: + s_bug = None + if s_bug is None: + if accept_bugs: + o_bug_copy = copy.copy(o_bug) + o_bug_copy.bugdir = self + o_bug_copy.id = libbe.util.id.ID(o_bug_copy, 'bug') + self.append(o_bug_copy) + elif change_exception: + raise ValueError( + ('Merge would add bug {} (alt: {}) to bugdir {}' + ).format(o_bug.uuid, o_bug.alt_id, self.uuid)) + else: + s_bug.merge(o_bug, accept_changes=accept_changes, + accept_extra_strings=accept_extra_strings, + change_exception=change_exception) + # methods for id generation def sibling_uuids(self): diff --git a/libbe/command/assign.py b/libbe/command/assign.py index c710662..f9658e5 100644 --- a/libbe/command/assign.py +++ b/libbe/command/assign.py @@ -76,10 +76,11 @@ class Assign (libbe.command.Command): def _run(self, **params): assigned = parse_assigned(self, params['assigned']) - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() for bug_id in params['bug-id']: - bug,dummy_comment = \ - libbe.command.util.bug_comment_from_user_id(bugdir, bug_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, bug_id)) if bug.assigned != assigned: bug.assigned = assigned if bug.status == 'open': diff --git a/libbe/command/base.py b/libbe/command/base.py index 4e1e41f..b9038a0 100644 --- a/libbe/command/base.py +++ b/libbe/command/base.py @@ -23,11 +23,14 @@ import optparse import os.path import StringIO import sys +import urlparse +import yaml import libbe import libbe.storage import libbe.ui.util.user import libbe.util.encoding +import libbe.util.http import libbe.util.plugin @@ -261,11 +264,13 @@ class Command (object): <BLANKLINE> A detailed help message. """ + user_agent = 'BE-HTTP-Command' name = 'command' - def __init__(self, ui=None): + def __init__(self, ui=None, server=None): self.ui = ui # calling user-interface + self.server = server # location of eventual execution self.status = None self.result = None self.restrict_file_access = True @@ -291,7 +296,10 @@ class Command (object): else: params.pop('complete') - self.status = self._run(**params) + if self.server: + self.status = self._run_remote(**params) + else: + self.status = self._run(**params) return self.status def _parse_options_args(self, options=None, args=None): @@ -339,6 +347,17 @@ class Command (object): def _run(self, **kwargs): raise NotImplementedError + def _run_remote(self, **kwargs): + data = yaml.safe_dump({ + 'command': self.name, + 'parameters': kwargs, + }) + url = urlparse.urljoin(self.server, 'run') + page,final_url,info = libbe.util.http.get_post_url( + url=url, get=False, data=data, agent=self.user_agent) + self.stdout.write(page) + return 0 + def help(self, *args): return '\n\n'.join([self.usage(), self._option_help(), @@ -483,8 +502,7 @@ class StringInputOutput (InputOutput): def get_stdout(self): ret = self.stdout.getvalue() - self.stdout = StringIO.StringIO() # clear stdout for next read - self.stdin.encoding = 'utf-8' + self.stdout.truncate(size=0) return ret class UnconnectedStorageGetter (object): @@ -504,7 +522,7 @@ class StorageCallbacks (object): def setup_command(self, command): command._get_unconnected_storage = self.get_unconnected_storage command._get_storage = self.get_storage - command._get_bugdir = self.get_bugdir + command._get_bugdirs = self.get_bugdirs def get_unconnected_storage(self): """ @@ -537,15 +555,20 @@ class StorageCallbacks (object): def set_storage(self, storage): self._storage = storage - def get_bugdir(self): + def get_bugdirs(self): """Callback for use by commands that need it.""" - if not hasattr(self, '_bugdir'): - self._bugdir = libbe.bugdir.BugDir(self.get_storage(), - from_storage=True) - return self._bugdir - - def set_bugdir(self, bugdir): - self._bugdir = bugdir + if not hasattr(self, '_bugdirs'): + storage = self.get_storage() + self._bugdirs = dict( + (uuid, libbe.bugdir.BugDir( + storage=storage, + uuid=uuid, + from_storage=True)) + for uuid in storage.children()) + return self._bugdirs + + def set_bugdirs(self, bugdirs): + self._bugdirs = bugdirs def cleanup(self): if hasattr(self, '_storage'): @@ -567,11 +590,11 @@ class UserInterface (object): return command.run(options, args) def setup_command(self, command): - if command.ui == None: + if command.ui is None: command.ui = self - if self.io != None: + if self.io is not None: self.io.setup_command(command) - if self.storage_callbacks != None: + if self.storage_callbacks is not None: self.storage_callbacks.setup_command(command) command.restrict_file_access = self.restrict_file_access command._get_user_id = self._get_user_id @@ -584,5 +607,6 @@ class UserInterface (object): return self._user_id def cleanup(self): - self.storage_callbacks.cleanup() + if self.storage_callbacks is not None: + self.storage_callbacks.cleanup() self.io.cleanup() diff --git a/libbe/command/comment.py b/libbe/command/comment.py index 9695ff6..399d8a7 100644 --- a/libbe/command/comment.py +++ b/libbe/command/comment.py @@ -107,6 +107,8 @@ class Comment (libbe.command.Command): help='Set comment content-type (e.g. text/plain)', arg=libbe.command.Argument(name='content-type', metavar='MIME')), + libbe.command.Option(name='full-uuid', short_name='f', + help='Print the full UUID for the new bug') ]) self.args.extend([ libbe.command.Argument( @@ -119,9 +121,10 @@ class Comment (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() - bug,parent = \ - libbe.command.util.bug_comment_from_user_id(bugdir, params['id']) + bugdirs = self._get_bugdirs() + bugdir,bug,parent = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['id'])) if params['comment'] == None: # try to launch an editor for comment-body entry try: @@ -159,7 +162,11 @@ class Comment (libbe.command.Command): for key in ['alt-id', 'author']: if params[key] != None: setattr(new, new._setting_name_to_attr_name(key), params[key]) - print >> self.stdout, 'Created comment with ID %s' % new.id.user() + if params['full-uuid']: + comment_id = new.id.long_user() + else: + comment_id = new.id.user() + self.stdout.write('Created comment with ID %s\n' % (comment_id)) return 0 def _long_help(self): diff --git a/libbe/command/depend.py b/libbe/command/depend.py index 395409f..e3765d0 100644 --- a/libbe/command/depend.py +++ b/libbe/command/depend.py @@ -19,10 +19,12 @@ # Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. import copy +import itertools import os import libbe import libbe.bug +import libbe.bugdir import libbe.command import libbe.command.util import libbe.util.tree @@ -40,7 +42,7 @@ class Filter (object): self.target = target self.extra_strings_regexps = extra_strings_regexps - def __call__(self, bugdir, bug): + def __call__(self, bugdirs, bug): if self.status != 'all' and not bug.status in self.status: return False if self.severity != 'all' and not bug.severity in self.severity: @@ -50,7 +52,7 @@ class Filter (object): if self.target == 'all': pass else: - target_bug = libbe.command.target.bug_target(bugdir, bug) + target_bug = libbe.command.target.bug_target(bugdirs, bug) if self.target in ['none', None]: if target_bug.summary != None: return False @@ -113,7 +115,6 @@ class Depend (libbe.command.Command): """Add/remove bug dependencies >>> import sys - >>> import libbe.bugdir >>> bd = libbe.bugdir.SimpleBugDir(memory=False) >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout @@ -204,9 +205,10 @@ class Depend (libbe.command.Command): and params['blocking-bug-id'] != None: raise libbe.command.UserError( 'Only one bug id used in tree mode.') - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() if params['repair'] == True: - good,fixed,broken = check_dependencies(bugdir, repair_broken_links=True) + good,fixed,broken = check_dependencies( + bugdirs, repair_broken_links=True) assert len(broken) == 0, broken if len(fixed) > 0: print >> self.stdout, 'Fixed the following links:' @@ -218,11 +220,12 @@ class Depend (libbe.command.Command): severity = parse_severity(params['severity']) filter = Filter(status, severity) - bugA, dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['bug-id']) + bugdir,bugA,dummy_comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['bug-id'])) if params['tree-depth'] != None: - dtree = DependencyTree(bugdir, bugA, params['tree-depth'], filter) + dtree = DependencyTree(bugdirs, bugA, params['tree-depth'], filter) if len(dtree.blocked_by_tree()) > 0: print >> self.stdout, '%s blocked by:' % bugA.id.user() for depth,node in dtree.blocked_by_tree().thread(): @@ -240,21 +243,22 @@ class Depend (libbe.command.Command): return 0 if params['blocking-bug-id'] != None: - bugB,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['blocking-bug-id']) + bugdirB,bugB,dummy_comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['blocking-bug-id'])) if params['remove'] == True: remove_block(bugA, bugB) else: # add the dependency add_block(bugA, bugB) - blocked_by = get_blocked_by(bugdir, bugA) + blocked_by = get_blocked_by(bugdirs, bugA) if len(blocked_by) > 0: print >> self.stdout, '%s blocked by:' % bugA.id.user() print >> self.stdout, \ '\n'.join([self.bug_string(_bug, params) for _bug in blocked_by]) - blocks = get_blocks(bugdir, bugA) + blocks = get_blocks(bugdirs, bugA) if len(blocks) > 0: print >> self.stdout, '%s blocks:' % bugA.id.user() print >> self.stdout, \ @@ -355,35 +359,37 @@ def remove_block(blocked_bug, blocking_bug): blocks_string = _generate_blocks_string(blocked_bug) _add_remove_extra_string(blocking_bug, blocks_string, add=False) -def get_blocks(bugdir, bug): +def get_blocks(bugdirs, bug): """ Return a list of bugs that the given bug blocks. """ blocks = [] for uuid in _get_blocks(bug): - blocks.append(bugdir.bug_from_uuid(uuid)) + blocks.append(libbe.command.util.bug_from_uuid(bugdirs, uuid)) return blocks -def get_blocked_by(bugdir, bug): +def get_blocked_by(bugdirs, bug): """ Return a list of bugs blocking the given bug. """ blocked_by = [] for uuid in _get_blocked_by(bug): - blocked_by.append(bugdir.bug_from_uuid(uuid)) + blocked_by.append(libbe.command.util.bug_from_uuid(bugdirs, uuid)) return blocked_by -def check_dependencies(bugdir, repair_broken_links=False): +def check_dependencies(bugdirs, repair_broken_links=False): """ Check that links are bi-directional for all bugs in bugdir. >>> import libbe.bugdir - >>> bd = libbe.bugdir.SimpleBugDir() - >>> a = bd.bug_from_uuid("a") - >>> b = bd.bug_from_uuid("b") + >>> bugdir = libbe.bugdir.SimpleBugDir() + >>> bugdirs = {bugdir.uuid: bugdir} + >>> a = bugdir.bug_from_uuid('a') + >>> b = bugdir.bug_from_uuid('b') >>> blocked_by_string = _generate_blocked_by_string(b) >>> _add_remove_extra_string(a, blocked_by_string, add=True) - >>> good,repaired,broken = check_dependencies(bd, repair_broken_links=False) + >>> good,repaired,broken = check_dependencies( + ... bugdirs, repair_broken_links=False) >>> good [] >>> repaired @@ -392,7 +398,8 @@ def check_dependencies(bugdir, repair_broken_links=False): [(Bug(uuid='a'), Bug(uuid='b'))] >>> _get_blocks(b) [] - >>> good,repaired,broken = check_dependencies(bd, repair_broken_links=True) + >>> good,repaired,broken = check_dependencies( + ... bugdirs, repair_broken_links=True) >>> _get_blocks(b) ['a'] >>> good @@ -401,45 +408,48 @@ def check_dependencies(bugdir, repair_broken_links=False): [(Bug(uuid='a'), Bug(uuid='b'))] >>> broken [] + >>> bugdir.cleanup() """ - if bugdir.storage != None: - bugdir.load_all_bugs() + for bugdir in bugdirs.values(): + if bugdir.storage is not None: + bugdir.load_all_bugs() good_links = [] fixed_links = [] broken_links = [] - for bug in bugdir: - for blocker in get_blocked_by(bugdir, bug): - blocks = get_blocks(bugdir, blocker) - if (bug, blocks) in good_links+fixed_links+broken_links: - continue # already checked that link - if bug not in blocks: - if repair_broken_links == True: - _repair_one_way_link(bug, blocker, blocks=True) - fixed_links.append((bug, blocker)) + for bugdir in bugdirs.values(): + for bug in bugdir: + for blocker in get_blocked_by(bugdirs, bug): + blocks = get_blocks(bugdirs, blocker) + if (bug, blocks) in good_links+fixed_links+broken_links: + continue # already checked that link + if bug not in blocks: + if repair_broken_links == True: + _repair_one_way_link(bug, blocker, blocks=True) + fixed_links.append((bug, blocker)) + else: + broken_links.append((bug, blocker)) else: - broken_links.append((bug, blocker)) - else: - good_links.append((bug, blocker)) - for blockee in get_blocks(bugdir, bug): - blocked_by = get_blocked_by(bugdir, blockee) - if (blockee, bug) in good_links+fixed_links+broken_links: - continue # already checked that link - if bug not in blocked_by: - if repair_broken_links == True: - _repair_one_way_link(blockee, bug, blocks=False) - fixed_links.append((blockee, bug)) + good_links.append((bug, blocker)) + for blockee in get_blocks(bugdirs, bug): + blocked_by = get_blocked_by(bugdirs, blockee) + if (blockee, bug) in good_links+fixed_links+broken_links: + continue # already checked that link + if bug not in blocked_by: + if repair_broken_links == True: + _repair_one_way_link(blockee, bug, blocks=False) + fixed_links.append((blockee, bug)) + else: + broken_links.append((blockee, bug)) else: - broken_links.append((blockee, bug)) - else: - good_links.append((blockee, bug)) + good_links.append((blockee, bug)) return (good_links, fixed_links, broken_links) class DependencyTree (object): """ Note: should probably be DependencyDiGraph. """ - def __init__(self, bugdir, root_bug, depth_limit=0, filter=None): - self.bugdir = bugdir + def __init__(self, bugdirs, root_bug, depth_limit=0, filter=None): + self.bugdirs = bugdirs self.root_bug = root_bug self.depth_limit = depth_limit self.filter = filter @@ -453,8 +463,8 @@ class DependencyTree (object): node = stack.pop() if self.depth_limit > 0 and node.depth == self.depth_limit: continue - for bug in child_fn(self.bugdir, node.bug): - if not self.filter(self.bugdir, bug): + for bug in child_fn(self.bugdirs, node.bug): + if not self.filter(self.bugdirs, bug): continue child = libbe.util.tree.Tree() child.bug = bug diff --git a/libbe/command/diff.py b/libbe/command/diff.py index 991a206..0d03ebf 100644 --- a/libbe/command/diff.py +++ b/libbe/command/diff.py @@ -91,11 +91,16 @@ class Diff (libbe.command.Command): params['subscribe']) except ValueError, e: raise libbe.command.UserError(e.msg) - bugdir = self._get_bugdir() - if bugdir.storage.versioned == False: - raise libbe.command.UserError( - 'This repository is not revision-controlled.') + bugdirs = self._get_bugdirs() + for uuid,bugdir in sorted(bugdirs.items()): + self.diff(bugdir, subscriptions, params=params) + + + def diff(self, bugdir, subscriptions, params): if params['repo'] == None: + if bugdir.storage.versioned == False: + raise libbe.command.UserError( + 'This repository is not revision-controlled.') if params['revision'] == None: # get the most recent revision params['revision'] = bugdir.storage.revision_id(-1) old_bd = libbe.bugdir.RevisionedBugDir(bugdir, params['revision']) @@ -108,8 +113,8 @@ class Diff (libbe.command.Command): else: if old_bd_current.storage.versioned == False: raise libbe.command.UserError( - '%s is not revision-controlled.' - % storage.repo) + '{} is not revision-controlled.'.format( + bugdir.storage.repo)) old_bd = libbe.bugdir.RevisionedBugDir(old_bd_current,revision) d = libbe.diff.Diff(old_bd, bugdir) tree = d.report_tree(subscriptions) diff --git a/libbe/command/due.py b/libbe/command/due.py index b026ed7..00ad742 100644 --- a/libbe/command/due.py +++ b/libbe/command/due.py @@ -61,9 +61,10 @@ class Due (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() - bug,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['bug-id']) + bugdirs = self._get_bugdirs() + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['bug-id'])) if params['due'] == None: due_time = get_due(bug) if due_time is None: diff --git a/libbe/command/html.py b/libbe/command/html.py index 4ab7e62..36ceeec 100644 --- a/libbe/command/html.py +++ b/libbe/command/html.py @@ -20,6 +20,7 @@ import codecs import htmlentitydefs +import itertools import os import os.path import re @@ -43,28 +44,34 @@ class HTML (libbe.command.Command): >>> import sys >>> import libbe.bugdir - >>> bd = libbe.bugdir.SimpleBugDir(memory=False) + >>> bugdir = libbe.bugdir.SimpleBugDir(memory=False) >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_storage(bd.storage) + >>> ui.storage_callbacks.set_storage(bugdir.storage) >>> cmd = HTML(ui=ui) - >>> ret = ui.run(cmd, {'output':os.path.join(bd.storage.repo, 'html_export')}) - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export')) + >>> ret = ui.run(cmd, { + ... 'output':os.path.join(bugdir.storage.repo, 'html_export')}) + >>> os.path.exists(os.path.join(bugdir.storage.repo, 'html_export')) True - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export', 'index.html')) + >>> os.path.exists(os.path.join( + ... bugdir.storage.repo, 'html_export', 'index.html')) True - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export', 'index_inactive.html')) + >>> os.path.exists(os.path.join( + ... bugdir.storage.repo, 'html_export', 'index_inactive.html')) True - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export', 'bugs')) + >>> os.path.exists(os.path.join( + ... bugdir.storage.repo, 'html_export', 'bugs')) True - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export', 'bugs', 'a', 'index.html')) + >>> os.path.exists(os.path.join( + ... bugdir.storage.repo, 'html_export', 'bugs', 'a', 'index.html')) True - >>> os.path.exists(os.path.join(bd.storage.repo, 'html_export', 'bugs', 'b', 'index.html')) + >>> os.path.exists(os.path.join( + ... bugdir.storage.repo, 'html_export', 'bugs', 'b', 'index.html')) True >>> ui.cleanup() - >>> bd.cleanup() + >>> bugdir.cleanup() """ name = 'html' @@ -110,11 +117,12 @@ class HTML (libbe.command.Command): def _run(self, **params): if params['export-template'] == True: - bugdir = None + bugdirs = None else: - bugdir = self._get_bugdir() - bugdir.load_all_bugs() - html_gen = HTMLGen(bugdir, + bugdirs = self._get_bugdirs() + for bugdir in bugdirs.values(): + bugdir.load_all_bugs() + html_gen = HTMLGen(bugdirs, template_dir=params['template-dir'], title=params['title'], header=params['index-header'], @@ -135,13 +143,13 @@ directory. Html = HTML # alias for libbe.command.base.get_command_class() class HTMLGen (object): - def __init__(self, bd, template_dir=None, + def __init__(self, bugdirs, template_dir=None, title="Site Title", header="Header", min_id_length=-1, verbose=False, encoding=None, stdout=None, ): self.generation_time = time.ctime() - self.bd = bd + self.bugdirs = bugdirs self.title = title self.header = header self.verbose = verbose @@ -162,7 +170,9 @@ class HTMLGen (object): bugs_active = [] bugs_inactive = [] bugs_target = [] - bugs = [b for b in self.bd] + bugs = list(itertools.chain(*list( + [bug for bug in bugdir] + for bugdir in self.bugdirs.values()))) bugs.sort() for b in bugs: @@ -294,7 +304,7 @@ class HTMLGen (object): template = self.template.get_template('target_index.html') template_info['targets'] = [ (target, sorted(libbe.command.depend.get_blocked_by( - self.bd, target))) + target.bugdir, target))) for target in bugs] else: template = self.template.get_template('standard_index.html') @@ -304,14 +314,14 @@ class HTMLGen (object): def _long_to_linked_user(self, text): """ >>> import libbe.bugdir - >>> bd = libbe.bugdir.SimpleBugDir(memory=False) - >>> h = HTMLGen(bd) + >>> bugdir = libbe.bugdir.SimpleBugDir(memory=False) + >>> h = HTMLGen({bugdir.uuid: bugdir}) >>> h._long_to_linked_user('A link #abc123/a#, and a non-link #x#y#.') 'A link <a href="./a/">abc/a</a>, and a non-link #x#y#.' - >>> bd.cleanup() + >>> bugdir.cleanup() """ replacer = libbe.util.id.IDreplacer( - [self.bd], self._long_to_linked_user_replacer, wrap=False) + self.bugdirs, self._long_to_linked_user_replacer, wrap=False) return re.sub( libbe.util.id.REGEXP, replacer, text) @@ -319,29 +329,30 @@ class HTMLGen (object): """ >>> import libbe.bugdir >>> import libbe.util.id - >>> bd = libbe.bugdir.SimpleBugDir(memory=False) - >>> a = bd.bug_from_uuid('a') + >>> bugdir = libbe.bugdir.SimpleBugDir(memory=False) + >>> bugdirs = {bugdir.uuid: bugdir} + >>> a = bugdir.bug_from_uuid('a') >>> uuid_gen = libbe.util.id.uuid_gen >>> libbe.util.id.uuid_gen = lambda : '0123' >>> c = a.new_comment('comment for link testing') >>> libbe.util.id.uuid_gen = uuid_gen >>> c.uuid '0123' - >>> h = HTMLGen(bd) - >>> h._long_to_linked_user_replacer([bd], 'abc123') + >>> h = HTMLGen(bugdirs) + >>> h._long_to_linked_user_replacer(bugdirs, 'abc123') '#abc123#' - >>> h._long_to_linked_user_replacer([bd], 'abc123/a') + >>> h._long_to_linked_user_replacer(bugdirs, 'abc123/a') '<a href="./a/">abc/a</a>' - >>> h._long_to_linked_user_replacer([bd], 'abc123/a/0123') + >>> h._long_to_linked_user_replacer(bugdirs, 'abc123/a/0123') '<a href="./a/#0123">abc/a/012</a>' - >>> h._long_to_linked_user_replacer([bd], 'x') + >>> h._long_to_linked_user_replacer(bugdirs, 'x') '#x#' - >>> h._long_to_linked_user_replacer([bd], '') + >>> h._long_to_linked_user_replacer(bugdirs, '') '##' - >>> bd.cleanup() + >>> bugdir.cleanup() """ try: - p = libbe.util.id.parse_user(bugdirs[0], long_id) + p = libbe.util.id.parse_user(bugdirs, long_id) except (libbe.util.id.MultipleIDMatches, libbe.util.id.NoIDMatches, libbe.util.id.InvalidIDStructure), e: @@ -349,13 +360,15 @@ class HTMLGen (object): if p['type'] == 'bugdir': return '#%s#' % long_id elif p['type'] == 'bug': - bug,comment = libbe.command.util.bug_comment_from_user_id( - bugdirs[0], long_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, long_id)) return '<a href="./%s/">%s</a>' \ % (self._truncated_bug_id(bug), bug.id.user()) elif p['type'] == 'comment': - bug,comment = libbe.command.util.bug_comment_from_user_id( - bugdirs[0], long_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, long_id)) return '<a href="./%s/#%s">%s</a>' \ % (self._truncated_bug_id(bug), self._truncated_comment_id(comment), diff --git a/libbe/command/import_xml.py b/libbe/command/import_xml.py index c3b1f42..298ad8a 100644 --- a/libbe/command/import_xml.py +++ b/libbe/command/import_xml.py @@ -27,10 +27,12 @@ except ImportError: # look for non-core module import libbe import libbe.bug +import libbe.bugdir import libbe.command import libbe.command.util import libbe.comment import libbe.util.encoding +import libbe.util.id import libbe.util.utility if libbe.TESTING == True: @@ -40,6 +42,7 @@ if libbe.TESTING == True: import libbe.bugdir + class Import_XML (libbe.command.Command): """Import comments and bugs from XML @@ -54,7 +57,7 @@ class Import_XML (libbe.command.Command): >>> cmd = Import_XML(ui=ui) >>> ui.io.set_stdin('<be-xml><comment><uuid>c</uuid><body>This is a comment about a</body></comment></be-xml>') - >>> ret = ui.run(cmd, {'comment-root':'/a'}, ['-']) + >>> ret = ui.run(cmd, {'root':'/a'}, ['-']) >>> bd.flush_reload() >>> bug = bd.bug_from_uuid('a') >>> bug.load_comments(load_full=False) @@ -80,10 +83,13 @@ class Import_XML (libbe.command.Command): help='If any bug or comment listed in the XML file already exists in the bug repository, do not alter the repository version.'), libbe.command.Option(name='preserve-uuids', short_name='p', help='Preserve UUIDs for trusted input (potential name collisions).'), - libbe.command.Option(name='comment-root', short_name='c', - help='Supply a bug or comment ID as the root of any <comment> elements that are direct children of the <be-xml> element. If any such <comment> elements exist, you are required to set this option.', + libbe.command.Option(name='root', short_name='r', + help='Supply a bugdir, bug, or comment ID as the root of ' + 'any non-bugdir elements that are direct children of the ' + '<be-xml> element. If any such elements exist, you are ' + 'required to set this option.', arg=libbe.command.Argument( - name='comment-root', metavar='ID', + name='root', metavar='ID', completion_callback=libbe.command.util.complete_bug_comment_id)), ]) self.args.extend([ @@ -92,53 +98,102 @@ class Import_XML (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() - writeable = bugdir.storage.writeable - bugdir.storage.writeable = False - if params['comment-root'] != None: - croot_bug,croot_comment = \ - libbe.command.util.bug_comment_from_user_id( - bugdir, params['comment-root']) - croot_bug.load_comments(load_full=True) - if croot_comment.uuid == libbe.comment.INVALID_UUID: - croot_comment = croot_bug.comment_root - else: - croot_comment = croot_bug.comment_from_uuid(croot_comment.uuid) - new_croot_bug = libbe.bug.Bug(bugdir=bugdir, uuid=croot_bug.uuid) - new_croot_bug.explicit_attrs = [] - new_croot_bug.comment_root = copy.deepcopy(croot_bug.comment_root) - if croot_comment.uuid == libbe.comment.INVALID_UUID: - new_croot_comment = new_croot_bug.comment_root - else: - new_croot_comment = \ - new_croot_bug.comment_from_uuid(croot_comment.uuid) - for new in new_croot_bug.comments(): - new.explicit_attrs = [] + storage = self._get_storage() + bugdirs = self._get_bugdirs() + writeable = storage.writeable + storage.writeable = False + if params['root'] != None: + root_bugdir,root_bug,root_comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['root'])) else: - croot_bug,croot_comment = (None, None) + root_bugdir,root_bug,root_comment = (None, None, None) + xml = self._read_xml(storage, params) + version,root_bugdirs,root_bugs,root_comments = self._parse_xml( + xml, params) + + if params['add-only']: + accept_changes = False + accept_extra_strings = False + else: + accept_changes = True + accept_extra_strings = True + + dirty_items = list(self._merge_comments( + bugdirs, root_bug, root_comment, root_comments, + params, accept_changes, accept_extra_strings)) + dirty_items.extend(self._merge_bugs( + bugdirs, root_bugdir, root_bugs, + params, accept_changes, accept_extra_strings)) + dirty_items.extend(self._merge_bugdirs( + bugdirs, root_bugdirs, + params, accept_changes, accept_extra_strings)) + + # protect against programmer error causing data loss: + if root_bug is not None: + # check for each of the new comments + comms = [] + for c in root_bug.comments(): + comms.append(c.uuid) + if c.alt_id != None: + comms.append(c.alt_id) + if root_comment.uuid == libbe.comment.INVALID_UUID: + root_text = root_bug.id.user() + else: + root_text = root_comment.id.user() + for new in root_comments: + assert new.uuid in comms or new.alt_id in comms, \ + "comment %s (alt: %s) wasn't added to %s" \ + % (new.uuid, new.alt_id, root_text) + for new in root_bugs: + # check for each of the new bugs + try: + libbe.command.util.bug_from_uuid(bugdirs, new.uuid) + except libbe.bugdir.NoBugMatches: + try: + libbe.command.util.bug_from_uuid(bugdirs, new.alt_id) + except libbe.bugdir.NoBugMatches: + raise AssertionError( + "bug {} (alt: {}) wasn't added to {}".format( + new.uuid, new.alt_id, root_bugdir.id.user())) + for new in root_bugdirs: + assert new.uuid in bugdirs or new.alt_id in bugdirs, ( + "bugdir {} wasn't added to {}".format( + new.uuid, sorted(bugdirs.keys()))) + + # save new information + storage.writeable = writeable + for item in dirty_items: + item.save() + + def _read_xml(self, storage, params): if params['xml-file'] == '-': - xml = self.stdin.read().encode(self.stdin.encoding) + return self.stdin.read().encode(self.stdin.encoding) else: - self._check_restricted_access(bugdir.storage, params['xml-file']) - xml = libbe.util.encoding.get_file_contents( - params['xml-file']) + self._check_restricted_access(storage, params['xml-file']) + return libbe.util.encoding.get_file_contents(params['xml-file']) - # parse the xml + def _parse_xml(self, xml, params): + version = {} + root_bugdirs = [] root_bugs = [] root_comments = [] - version = {} be_xml = ElementTree.XML(xml) if be_xml.tag != 'be-xml': raise libbe.util.utility.InvalidXML( 'import-xml', be_xml, 'root element must be <be-xml>') for child in be_xml.getchildren(): - if child.tag == 'bug': - new = libbe.bug.Bug(bugdir=bugdir) + if child.tag == 'bugdir': + new = libbe.bugdir.BugDir(storage=None) + new.from_xml(child, preserve_uuids=params['preserve-uuids']) + root_bugdirs.append(new) + elif child.tag == 'bug': + new = libbe.bug.Bug() new.from_xml(child, preserve_uuids=params['preserve-uuids']) root_bugs.append(new) elif child.tag == 'comment': - new = libbe.comment.Comment(croot_bug) + new = libbe.comment.Comment() new.from_xml(child, preserve_uuids=params['preserve-uuids']) root_comments.append(new) elif child.tag == 'version': @@ -148,84 +203,82 @@ class Import_XML (libbe.command.Command): text = text.decode('unicode_escape').strip() version[child.tag] = text else: - print >> sys.stderr, 'ignoring unknown tag %s in %s' \ - % (gchild.tag, child.tag) + sys.stderr.write( + 'ignoring unknown tag {} in {}\n'.format( + gchild.tag, child.tag)) else: - print >> sys.stderr, 'ignoring unknown tag %s in %s' \ - % (child.tag, comment_list.tag) + sys.stderr.write('ignoring unknown tag {} in {}\n'.format( + child.tag, be_xml.tag)) + return (version, root_bugdirs, root_bugs, root_comments) - # merge the new root_comments - if params['add-only'] == True: - accept_changes = False - accept_extra_strings = False + def _merge_comments(self, bugdirs, bug, root_comment, comments, + params, accept_changes, accept_extra_strings, + accept_comments=True): + if len(comments) == 0: + return + if bug is None: + raise libbe.command.UserError( + 'No root bug for merging comments:\n{}'.format( + '\n\n'.join([c.string() for c in comments]))) + bug.load_comments(load_full=True) + if root_comment.uuid == libbe.comment.INVALID_UUID: + root_comment = bug.comment_root else: - accept_changes = True - accept_extra_strings = True - accept_comments = True - if len(root_comments) > 0: - if croot_bug == None: - raise libbe.command.UserError( - '--comment-root option is required for your root comments:\n%s' - % '\n\n'.join([c.string() for c in root_comments])) - try: - # link new comments - new_croot_bug.add_comments(root_comments, - default_parent=new_croot_comment, - ignore_missing_references= \ - params['ignore-missing-references']) - except libbe.comment.MissingReference, e: - raise libbe.command.UserError(e) - croot_bug.merge(new_croot_bug, accept_changes=accept_changes, - accept_extra_strings=accept_extra_strings, - accept_comments=accept_comments) - - # merge the new croot_bugs - merged_bugs = [] - old_bugs = [] - for new in root_bugs: + root_comment = bug.comment_from_uuid(root_comment.uuid) + new_bug = libbe.bug.Bug(bugdir=bug.bugdir, uuid=bug.uuid) + new_bug.explicit_attrs = [] + new_bug.comment_root = copy.deepcopy(bug.comment_root) + if root_comment.uuid == libbe.comment.INVALID_UUID: + new_root_comment = new_bug.comment_root + else: + new_root_comment = new_bug.comment_from_uuid( + root_comment.uuid) + for new in new_bug.comments(): + new.explicit_attrs = [] + try: + new_bug.add_comments( + comments, + default_parent=root_comment, + ignore_missing_references=params['ignore-missing-references']) + except libbe.comment.MissingReference as e: + raise libbe.command.UserError(e) + bug.merge(new_bug, accept_changes=accept_changes, + accept_extra_strings=accept_extra_strings, + accept_comments=accept_comments) + yield bug + + def _merge_bugs(self, bugdirs, bugdir, bugs, + params, accept_changes, accept_extra_strings, + accept_comments=True): + for new in bugs: try: old = bugdir.bug_from_uuid(new.alt_id) except KeyError: - old = None - if old == None: - bugdir.append(new) + bugdir.append(new, update=True) + yield new else: old.load_comments(load_full=True) old.merge(new, accept_changes=accept_changes, accept_extra_strings=accept_extra_strings, accept_comments=accept_comments) - merged_bugs.append(new) - old_bugs.append(old) + yield old - # protect against programmer error causing data loss: - if croot_bug != None: - comms = [] - for c in croot_comment.traverse(): - comms.append(c.uuid) - if c.alt_id != None: - comms.append(c.alt_id) - if croot_comment.uuid == libbe.comment.INVALID_UUID: - root_text = croot_bug.id.user() + def _merge_bugdirs(self, bugdirs, new_bugdirs, + params, accept_changes, accept_extra_strings, + accept_comments=True): + for new in new_bugdirs: + if new.alt_id in bugdirs: + old = bugdirs[new.alt_id] + old.load_all_bugs() + old.merge(new, accept_changes=accept_changes, + accept_extra_strings=accept_extra_strings, + accept_bugs=True, + accept_comments=accept_comments) + yield old else: - root_text = croot_comment.id.user() - for new in root_comments: - assert new.uuid in comms or new.alt_id in comms, \ - "comment %s (alt: %s) wasn't added to %s" \ - % (new.uuid, new.alt_id, root_text) - for new in root_bugs: - if not new in merged_bugs: - assert bugdir.has_bug(new.uuid), \ - "bug %s wasn't added" % (new.uuid) - - # save new information - bugdir.storage.writeable = writeable - if croot_bug != None: - croot_bug.save() - for new in root_bugs: - if not new in merged_bugs: - new.save() - for old in old_bugs: - old.save() + bugdirs[new.uuid] = new + new.storage = self._get_storage() + yield new def _long_help(self): return """ @@ -238,7 +291,8 @@ VCSs are compatible, it's better to use their builtin merge/push/pull to share this information, as that will preserve a more detailed history. -The XML file should be formatted similarly to +The XML file should be formatted similarly to: + <be-xml> <version> <tag>1.0.0</tag> @@ -246,29 +300,34 @@ The XML file should be formatted similarly to <revno>446</revno> <revision-id>a@b.com-20091119214553-iqyw2cpqluww3zna</revision-id> <version> - <bug> - ... - <comment>...</comment> - <comment>...</comment> - </bug> + <bugdir> + <bug> + ... + <comment>...</comment> + <comment>...</comment> + </bug> + <bug>...</bug> + </bugdir> + <bug>...</bug> <bug>...</bug> <comment>...</comment> <comment>...</comment> </be-xml> -where the ellipses mark output commpatible with Bug.xml() and -Comment.xml(). Take a look at the output of `be show --xml` for some -explicit examples. Unrecognized tags are ignored. Missing tags are -left at the default value. The version tag is not required, but is -strongly recommended. - -The bug and comment UUIDs are always auto-generated, so if you set a -<uuid> field, but no <alt-id> field, your <uuid> will be used as the -comment's <alt-id>. An exception is raised if <alt-id> conflicts with -an existing comment. Bugs do not have a permantent alt-id, so they -the <uuid>s you specify are not saved. The <uuid>s _are_ used to -match agains prexisting bug and comment uuids, and comment alt-ids, -and fields explicitly given in the XML file will replace old versions -unless the --add-only flag. + +where the ellipses mark output commpatible with BugDir.xml(), +Bug.xml(), and Comment.xml(). Take a look at the output of `be show +--xml` for some explicit examples. Unrecognized tags are ignored. +Missing tags are left at the default value. The version tag is not +required, but is strongly recommended. + +The bugdir, bug, and comment UUIDs are always auto-generated, so if +you set a <uuid> field, but no <alt-id> field, your <uuid> will be +used as the object's <alt-id>. An exception is raised if <alt-id> +conflicts with an existing object. Bugdirs and bugs do not have a +permantent alt-id, so they the <uuid>s you specify are not saved. The +<uuid>s _are_ used to match agains prexisting bug and comment uuids, +and comment alt-ids, and fields explicitly given in the XML file will +replace old versions unless the --add-only flag. *.extra_strings recieves special treatment, and if --add-only is not set, the resulting list concatenates both source lists and removes @@ -276,53 +335,65 @@ repeats. Here's an example of import activity: Repository - bug (uuid=B, creator=John, status=open) - estr (don't forget your towel) - estr (helps with space travel) - com (uuid=C1, author=Jane, body=Hello) - com (uuid=C2, author=Jess, body=World) + bugdir (uuid=abc123) + bug (uuid=B, creator=John, status=open) + estr (don't forget your towel) + estr (helps with space travel) + com (uuid=C1, author=Jane, body=Hello) + com (uuid=C2, author=Jess, body=World) XML - bug (uuid=B, status=fixed) - estr (don't forget your towel) - estr (watch out for flying dolphins) - com (uuid=C1, body=So long) - com (uuid=C3, author=Jed, body=And thanks) + bugdir (uuid=abc123) + bug (uuid=B, status=fixed) + estr (don't forget your towel) + estr (watch out for flying dolphins) + com (uuid=C1, body=So long) + com (uuid=C3, author=Jed, body=And thanks) Result - bug (uuid=B, creator=John, status=fixed) - estr (don't forget your towel) - estr (helps with space travel) - estr (watch out for flying dolphins) - com (uuid=C1, author=Jane, body=So long) - com (uuid=C2, author=Jess, body=World) - com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) + bugdir (uuid=abc123) + bug (uuid=B, creator=John, status=fixed) + estr (don't forget your towel) + estr (helps with space travel) + estr (watch out for flying dolphins) + com (uuid=C1, author=Jane, body=So long) + com (uuid=C2, author=Jess, body=World) + com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) Result, with --add-only - bug (uuid=B, creator=John, status=open) - estr (don't forget your towel) - estr (helps with space travel) - com (uuid=C1, author=Jane, body=Hello) - com (uuid=C2, author=Jess, body=World) - com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) + bugdir (uuid=abc123) + bug (uuid=B, creator=John, status=open) + estr (don't forget your towel) + estr (helps with space travel) + com (uuid=C1, author=Jane, body=Hello) + com (uuid=C2, author=Jess, body=World) + com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) Examples: -Import comments (e.g. emails from an mbox) and append to bug XYZ - $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ - -Or you can append those emails underneath the prexisting comment XYZ-3 - $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ-3 - +Import comments (e.g. emails from an mbox) and append to bug /XYZ: + + $ be-mbox-to-xml mail.mbox | be import-xml --r /XYZ - + +Or you can append those emails underneath the prexisting comment /XYZ/3: + + $ be-mbox-to-xml mail.mbox | be import-xml --r /XYZ/3 - + +User creates a new bug: -User creates a new bug user$ be new "The demuxulizer is broken" Created bug with ID 48f user$ be comment 48f <Describe bug> ... -User exports bug as xml and emails it to the developers + +User exports bug as xml and emails it to the developers: + user$ be show --xml 48f > 48f.xml user$ cat 48f.xml | mail -s "Demuxulizer bug xml" devs@b.com or equivalently (with a slightly fancier be-handle-mail compatible email): user$ be email-bugs 48f -Devs recieve email, and save it's contents as demux-bug.xml + +Devs recieve email, and save it's contents as demux-bug.xml: + dev$ cat demux-bug.xml | be import-xml - """ @@ -360,22 +431,25 @@ if libbe.TESTING == True: bugB.save() self.xml = """ <be-xml> - <bug> - <uuid>b</uuid> - <status>fixed</status> - <summary>a test bug</summary> - <extra-string>don't forget your towel</extra-string> - <extra-string>watch out for flying dolphins</extra-string> - <comment> - <uuid>c1</uuid> - <body>So long</body> - </comment> - <comment> - <uuid>c3</uuid> - <author>Jed</author> - <body>And thanks</body> - </comment> - </bug> + <bugdir> + <uuid>abc123</uuid> + <bug> + <uuid>b</uuid> + <status>fixed</status> + <summary>a test bug</summary> + <extra-string>don't forget your towel</extra-string> + <extra-string>watch out for flying dolphins</extra-string> + <comment> + <uuid>c1</uuid> + <body>So long</body> + </comment> + <comment> + <uuid>c3</uuid> + <author>Jed</author> + <body>And thanks</body> + </comment> + </bug> + </bugdir> </be-xml> """ self.root_comment_xml = """ @@ -473,7 +547,7 @@ if libbe.TESTING == True: def testRootCommentsNotAddOnly(self): bugB = self.bugdir.bug_from_uuid('b') initial_bugB_summary = bugB.summary - self._execute(self.root_comment_xml, {'comment-root':'/b'}, ['-']) + self._execute(self.root_comment_xml, {'root':'/b'}, ['-']) uuids = list(self.bugdir.uuids()) uuids = list(self.bugdir.uuids()) self.failUnless(uuids == ['b'], uuids) @@ -510,7 +584,7 @@ if libbe.TESTING == True: bugB = self.bugdir.bug_from_uuid('b') initial_bugB_summary = bugB.summary self._execute(self.root_comment_xml, - {'comment-root':'/b', 'add-only':True}, ['-']) + {'root':'/b', 'add-only':True}, ['-']) uuids = list(self.bugdir.uuids()) self.failUnless(uuids == ['b'], uuids) bugB = self.bugdir.bug_from_uuid('b') diff --git a/libbe/command/init.py b/libbe/command/init.py index 21d2303..421ca0d 100644 --- a/libbe/command/init.py +++ b/libbe/command/init.py @@ -97,7 +97,7 @@ class Init (libbe.command.Command): storage.connect() self.ui.storage_callbacks.set_storage(storage) bd = libbe.bugdir.BugDir(storage, from_storage=False) - self.ui.storage_callbacks.set_bugdir(bd) + self.ui.storage_callbacks.set_bugdirs({bd.uuid: bd}) if bd.storage.name is not 'None': print >> self.stdout, \ 'Using %s for revision control.' % storage.name diff --git a/libbe/command/list.py b/libbe/command/list.py index c310b11..e47a4ce 100644 --- a/libbe/command/list.py +++ b/libbe/command/list.py @@ -20,6 +20,7 @@ # You should have received a copy of the GNU General Public License along with # Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. +import itertools import os import re @@ -125,15 +126,18 @@ class List (libbe.command.Command): # ]) def _run(self, **params): - bugdir = self._get_bugdir() - writeable = bugdir.storage.writeable - bugdir.storage.writeable = False + storage = self._get_storage() + bugdirs = self._get_bugdirs() + writeable = storage.writeable + storage.writeable = False cmp_list, status, severity, assigned, extra_strings_regexps = \ - self._parse_params(bugdir, params) + self._parse_params(bugdirs, params) filter = Filter(status, severity, assigned, extra_strings_regexps=extra_strings_regexps) - bugs = [bugdir.bug_from_uuid(uuid) for uuid in bugdir.uuids()] - bugs = [b for b in bugs if filter(bugdir, b) == True] + bugs = list(itertools.chain(*list( + [bugdir.bug_from_uuid(uuid) for uuid in bugdir.uuids()] + for bugdir in bugdirs.values()))) + bugs = [b for b in bugs if filter(bugdirs, b) == True] self.result = bugs if len(bugs) == 0 and params['xml'] == False: print >> self.stdout, 'No matching bugs found' @@ -147,10 +151,10 @@ class List (libbe.command.Command): print >> self.stdout, bug.id.user() else: self._list_bugs(bugs, show_tags=params['tags'], xml=params['xml']) - bugdir.storage.writeable = writeable + storage.writeable = writeable return 0 - def _parse_params(self, bugdir, params): + def _parse_params(self, bugdirs, params): cmp_list = [] if params['sort'] != None: for cmp in params['sort'].split(','): @@ -170,7 +174,7 @@ class List (libbe.command.Command): assigned = 'all' else: assigned = libbe.command.util.select_values( - params['assigned'], libbe.command.util.assignees(bugdir)) + params['assigned'], libbe.command.util.assignees(bugdirs)) for i in range(len(assigned)): if assigned[i] == '-': assigned[i] = params['user-id'] diff --git a/libbe/command/merge.py b/libbe/command/merge.py index e2c8951..5d74c7e 100644 --- a/libbe/command/merge.py +++ b/libbe/command/merge.py @@ -35,7 +35,7 @@ class Merge (libbe.command.Command): >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_storage(bd.storage) >>> cmd = Merge(ui=ui) >>> a = bd.bug_from_uuid('a') @@ -61,7 +61,8 @@ class Merge (libbe.command.Command): ... cmp=libbe.comment.cmp_time) >>> mergeA = a_comments[0] >>> mergeA.time = 3 - >>> print a.string(show_comments=True) # doctest: +ELLIPSIS + >>> print a.string(show_comments=True) + ... # doctest: +ELLIPSIS, +REPORT_UDIFF ID : a Short name : abc/a Severity : minor @@ -107,7 +108,8 @@ class Merge (libbe.command.Command): ... libbe.comment.cmp_time) >>> mergeB = b_comments[0] >>> mergeB.time = 3 - >>> print b.string(show_comments=True) # doctest: +ELLIPSIS + >>> print b.string(show_comments=True) + ... # doctest: +ELLIPSIS, +REPORT_UDIFF ID : b Short name : abc/b Severity : minor @@ -154,14 +156,15 @@ class Merge (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() - bugA,dummy_comment = \ - libbe.command.util.bug_comment_from_user_id( - bugdir, params['bug-id']) + storage = self._get_storage() + bugdirs = self._get_bugdirs() + bugdirA,bugA,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['bug-id'])) bugA.load_comments() - bugB,dummy_comment = \ - libbe.command.util.bug_comment_from_user_id( - bugdir, params['bug-id-to-merge']) + bugdirB,bugB,dummy_comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['bug-id-to-merge'])) bugB.load_comments() mergeA = bugA.new_comment('Merged from bug #%s#' % bugB.id.long_user()) newCommTree = copy.deepcopy(bugB.comment_root) @@ -171,7 +174,7 @@ class Merge (libbe.command.Command): if comment.alt_id == None: comment.storage = None comment.alt_id = comment.uuid - comment.storage = bugdir.storage + comment.storage = storage comment.uuid = libbe.util.id.uuid_gen() comment.save() # force onto disk under bugA diff --git a/libbe/command/new.py b/libbe/command/new.py index e34d05c..5404271 100644 --- a/libbe/command/new.py +++ b/libbe/command/new.py @@ -95,6 +95,15 @@ class New (libbe.command.Command): arg=libbe.command.Argument( name='severity', metavar='SEVERITY', completion_callback=libbe.command.util.complete_severity)), + libbe.command.Option(name='bugdir', short_name='b', + help='Short bugdir UUID for the new bug. You ' + 'only need to set this if you have multiple bugdirs in ' + 'your repository.', + arg=libbe.command.Argument( + name='bugdir', metavar='ID', default=None, + completion_callback=libbe.command.util.complete_bugdir_id)), + libbe.command.Option(name='full-uuid', short_name='f', + help='Print the full UUID for the new bug') ]) self.args.extend([ libbe.command.Argument(name='summary', metavar='SUMMARY') @@ -105,8 +114,16 @@ class New (libbe.command.Command): summary = self.stdin.readline() else: summary = params['summary'] - bugdir = self._get_bugdir() - bugdir.storage.writeable = False + storage = self._get_storage() + bugdirs = self._get_bugdirs() + if params['bugdir']: + bugdir = bugdirs[bugdir] + elif len(bugdirs) == 1: + bugdir = bugdirs.values()[0] + else: + raise libbe.command.UserError( + 'Ambiguous bugdir {}'.format(sorted(bugdirs.values()))) + storage.writeable = False bug = bugdir.new_bug(summary=summary.strip()) if params['creator'] != None: bug.creator = params['creator'] @@ -122,9 +139,13 @@ class New (libbe.command.Command): bug.status = params['status'] if params['severity'] != None: bug.severity = params['severity'] - bugdir.storage.writeable = True + storage.writeable = True bug.save() - print >> self.stdout, 'Created bug with ID %s' % bug.id.user() + if params['full-uuid']: + bug_id = bug.id.long_user() + else: + bug_id = bug.id.user() + self.stdout.write('Created bug with ID %s\n' % (bug_id)) return 0 def _long_help(self): diff --git a/libbe/command/remove.py b/libbe/command/remove.py index dcca3d1..19b5e6f 100644 --- a/libbe/command/remove.py +++ b/libbe/command/remove.py @@ -62,11 +62,12 @@ class Remove (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() user_ids = [] for bug_id in params['bug-id']: - bug,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, bug_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, bug_id)) user_ids.append(bug.id.user()) bugdir.remove_bug(bug) if len(user_ids) == 1: diff --git a/libbe/command/serve_commands.py b/libbe/command/serve_commands.py new file mode 100644 index 0000000..d8598e2 --- /dev/null +++ b/libbe/command/serve_commands.py @@ -0,0 +1,215 @@ +# Copyright (C) 2010-2012 Chris Ball <cjb@laptop.org> +# W. Trevor King <wking@drexel.edu> +# +# This file is part of Bugs Everywhere. +# +# Bugs Everywhere is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 2 of the License, or (at your option) any +# later version. +# +# Bugs Everywhere is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. + +"""Define the :class:`ServeCommands` serving BE Commands over HTTP. + +See Also +-------- +:py:meth:`be-libbe.command.base.Command._run_remote` : the associated client +""" + +import logging +import os.path +import posixpath +import re +import urllib +import wsgiref.simple_server + +import yaml + +import libbe +import libbe.command +import libbe.command.base +import libbe.util.wsgi +import libbe.version + +if libbe.TESTING: + import copy + import doctest + import StringIO + import sys + import unittest + import wsgiref.validate + try: + import cherrypy.test.webtest + cherrypy_test_webtest = True + except ImportError: + cherrypy_test_webtest = None + + import libbe.bugdir + import libbe.command.list + + +class ServerApp (libbe.util.wsgi.WSGI_AppObject, + libbe.util.wsgi.WSGI_DataObject): + """WSGI server for a BE Command invocation over HTTP. + + RESTful_ WSGI request handler for serving the + libbe.command.base.Command._run_remote backend with GET, POST, and + HEAD commands. + + This serves all commands from a single, persistant storage + instance, usually a VCS-based repository located on the local + machine. + """ + server_version = "BE-command-server/" + libbe.version.version() + + def __init__(self, storage=None, notify=False, **kwargs): + super(ServerApp, self).__init__( + urls=[ + (r'^run/?$', self.run), + ], + **kwargs) + self.storage = storage + self.ui = libbe.command.base.UserInterface() + self.notify = notify + self.http_user_error = 418 + + # handlers + def run(self, environ, start_response): + self.check_login(environ) + data = self.post_data(environ) + source = 'post' + name = data['command'] + parameters = data['parameters'] + try: + Class = libbe.command.get_command_class(command_name=name) + except libbe.command.UnknownCommand, e: + raise libbe.util.wsgi.HandlerError( + self.http_user_error, 'UnknownCommand {}'.format(e)) + command = Class(ui=self.ui) + self.ui.setup_command(command) + command.status = command._run(**parameters) # already parsed params + assert command.status == 0, command.status + stdout = self.ui.io.get_stdout() + if self.notify: # TODO, check what notify does + self._notify(environ, 'run', command) + return self.ok_response(environ, start_response, stdout) + + # handler utility functions + def _parse_post(self, post): + return yaml.safe_load(post) + + def check_login(self, environ): + user = environ.get('be-auth.user', None) + if user is not None: # we're running under AuthenticationApp + if environ['REQUEST_METHOD'] == 'POST': + # TODO: better detection of commands requiring writes + if user == 'guest' or self.storage.is_writeable() == False: + raise _Unauthorized() # only non-guests allowed to write + # allow read-only commands for all users + + def _notify(self, environ, command, id, params): + message = self._format_notification(environ, command, id, params) + self._submit_notification(message) + + def _format_notification(self, environ, command, id, params): + key_length = len('command') + for key,value in params: + if len(key) > key_length and '\n' not in str(value): + key_length = len(key) + key_length += 1 + lines = [] + multi_line_params = [] + for key,value in [('address', environ.get('REMOTE_ADDR', '-')), + ('command', command), ('id', id)]+params: + v = str(value) + if '\n' in v: + multi_line_params.append((key,v)) + continue + lines.append('%*.*s %s' % (key_length, key_length, key+':', v)) + lines.append('') + for key,value in multi_line_params: + lines.extend(['=== START %s ===' % key, v, + '=== STOP %s ===' % key, '']) + lines.append('') + return '\n'.join(lines) + + def _submit_notification(self, message): + libbe.util.subproc.invoke(self.notify, stdin=message, shell=True) + + +class ServeCommands (libbe.util.wsgi.ServerCommand): + """Serve commands over HTTP. + + This allows you to run local `be` commands interfacing with remote + data, transmitting command requests over the network. + + :class:`~libbe.command.base.Command` wrapper around + :class:`ServerApp`. + """ + + name = 'serve-commands' + + def _get_app(self, logger, storage, **kwargs): + return ServerApp( + logger=logger, storage=storage, notify=kwargs.get('notify', False)) + + def _long_help(self): + return """ +Example usage:: + + $ be serve-commands + +And in another terminal (or after backgrounding the server):: + + $ be --server http://localhost:8000/ list + +If you bind your server to a public interface, take a look at the +``--read-only`` option or the combined ``--ssl --auth FILE`` +options so other people can't mess with your repository. If you do use +authentication, you'll need to send in your username and password with, +for example:: + + $ be --repo http://username:password@localhost:8000/ list +""" + + +# alias for libbe.command.base.get_command_class() +Serve_commands = ServeCommands + + +if libbe.TESTING: + class ServerAppTestCase (libbe.util.wsgi.WSGITestCase): + def setUp(self): + libbe.util.wsgi.WSGITestCase.setUp(self) + self.bd = libbe.bugdir.SimpleBugDir(memory=False) + self.app = ServerApp(self.bd.storage, logger=self.logger) + + def tearDown(self): + self.bd.cleanup() + libbe.util.wsgi.WSGITestCase.tearDown(self) + + def test_run_list(self): + list = libbe.command.list.List() + params = list._parse_options_args() + data = yaml.safe_dump({ + 'command': 'list', + 'parameters': params, + }) + self.getURL(self.app, '/run', method='POST', data=data) + self.failUnless(self.status.startswith('200 '), self.status) + self.failUnless( + ('Content-Type', 'application/octet-stream' + ) in self.response_headers, + self.response_headers) + self.failUnless(self.exc_info == None, self.exc_info) + # TODO: integration tests on ServeCommands? + + unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) + suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) diff --git a/libbe/command/serve_storage.py b/libbe/command/serve_storage.py new file mode 100644 index 0000000..966c932 --- /dev/null +++ b/libbe/command/serve_storage.py @@ -0,0 +1,353 @@ +# Copyright (C) 2010-2012 Chris Ball <cjb@laptop.org> +# W. Trevor King <wking@drexel.edu> +# +# This file is part of Bugs Everywhere. +# +# Bugs Everywhere is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 2 of the License, or (at your option) any +# later version. +# +# Bugs Everywhere is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. + +"""Define the :class:`Serve` serving BE Storage over HTTP. + +See Also +-------- +:mod:`libbe.storage.http` : the associated client +""" + +import logging +import os.path + +import libbe +import libbe.command +import libbe.command.util +import libbe.util.subproc +import libbe.util.wsgi +import libbe.version + +if libbe.TESTING: + import copy + import doctest + import StringIO + import sys + import unittest + import wsgiref.validate + try: + import cherrypy.test.webtest + cherrypy_test_webtest = True + except ImportError: + cherrypy_test_webtest = None + + import libbe.bugdir + import libbe.util.wsgi + + +class ServerApp (libbe.util.wsgi.WSGI_AppObject, + libbe.util.wsgi.WSGI_DataObject): + """WSGI server for a BE Storage instance over HTTP. + + RESTful_ WSGI request handler for serving the + libbe.storage.http.HTTP backend with GET, POST, and HEAD commands. + For more information on authentication and REST, see John + Calcote's `Open Sourcery article`_ + + .. _RESTful: http://www.ics.uci.edu/~fielding/pubs/dissertation/rest_arch_style.htm + .. _Open Sourcery article: http://jcalcote.wordpress.com/2009/08/10/restful-authentication/ + + This serves files from a connected storage instance, usually + a VCS-based repository located on the local machine. + + Notes + ----- + + The GET and HEAD requests are identical except that the HEAD + request omits the actual content of the file. + """ + server_version = "BE-server/" + libbe.version.version() + + def __init__(self, storage=None, notify=False, **kwargs): + super(ServerApp, self).__init__( + urls=[ + (r'^add/?', self.add), + (r'^exists/?', self.exists), + (r'^remove/?', self.remove), + (r'^ancestors/?', self.ancestors), + (r'^children/?', self.children), + (r'^get/(.+)', self.get), + (r'^set/(.+)', self.set), + (r'^commit/?', self.commit), + (r'^revision-id/?', self.revision_id), + (r'^changed/?', self.changed), + (r'^version/?', self.version), + ], + **kwargs) + self.storage = storage + self.notify = notify + self.http_user_error = 418 + + # handlers + def add(self, environ, start_response): + self.check_login(environ) + data = self.post_data(environ) + source = 'post' + id = self.data_get_id(data, source=source) + parent = self.data_get_string( + data, 'parent', default=None, source=source) + directory = self.data_get_boolean( + data, 'directory', default=False, source=source) + self.storage.add(id, parent=parent, directory=directory) + if self.notify: + self._notify(environ, 'add', id, + [('parent', parent), ('directory', directory)]) + return self.ok_response(environ, start_response, None) + + def exists(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + id = self.data_get_id(data, source=source) + revision = self.data_get_string( + data, 'revision', default=None, source=source) + content = str(self.storage.exists(id, revision)) + return self.ok_response(environ, start_response, content) + + def remove(self, environ, start_response): + self.check_login(environ) + data = self.post_data(environ) + source = 'post' + id = self.data_get_id(data, source=source) + recursive = self.data_get_boolean( + data, 'recursive', default=False, source=source) + if recursive == True: + self.storage.recursive_remove(id) + else: + self.storage.remove(id) + if self.notify: + self._notify(environ, 'remove', id, [('recursive', recursive)]) + return self.ok_response(environ, start_response, None) + + def ancestors(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + id = self.data_get_id(data, source=source) + revision = self.data_get_string( + data, 'revision', default=None, source=source) + content = '\n'.join(self.storage.ancestors(id, revision))+'\n' + return self.ok_response(environ, start_response, content) + + def children(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + id = self.data_get_id(data, default=None, source=source) + revision = self.data_get_string( + data, 'revision', default=None, source=source) + content = '\n'.join(self.storage.children(id, revision)) + return self.ok_response(environ, start_response, content) + + def get(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + try: + id = environ['be-server.url_args'][0] + except: + raise libbe.util.wsgi.HandlerError(404, 'Not Found') + revision = self.data_get_string( + data, 'revision', default=None, source=source) + content = self.storage.get(id, revision=revision) + be_version = self.storage.storage_version(revision) + return self.ok_response(environ, start_response, content, + headers=[('X-BE-Version', be_version)]) + + def set(self, environ, start_response): + self.check_login(environ) + data = self.post_data(environ) + try: + id = environ['be-server.url_args'][0] + except: + raise libbe.util.wsgi.HandlerError(404, 'Not Found') + if not 'value' in data: + raise libbe.util.wsgi.HandlerError(406, 'Missing query key value') + value = data['value'] + self.storage.set(id, value) + if self.notify: + self._notify(environ, 'set', id, [('value', value)]) + return self.ok_response(environ, start_response, None) + + def commit(self, environ, start_response): + self.check_login(environ) + data = self.post_data(environ) + if not 'summary' in data: + raise libbe.util.wsgi.HandlerError( + 406, 'Missing query key summary') + summary = data['summary'] + if not 'body' in data or data['body'] == 'None': + data['body'] = None + body = data['body'] + if not 'allow_empty' in data \ + or data['allow_empty'] == 'True': + allow_empty = True + else: + allow_empty = False + try: + revision = self.storage.commit(summary, body, allow_empty) + except libbe.storage.EmptyCommit, e: + raise libbe.util.wsgi.HandlerError( + self.http_user_error, 'EmptyCommit') + if self.notify: + self._notify(environ, 'commit', id, + [('allow_empty', allow_empty), ('summary', summary), + ('body', body)]) + return self.ok_response(environ, start_response, revision) + + def revision_id(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + index = int(self.data_get_string( + data, 'index', default=libbe.util.wsgi.HandlerError, + source=source)) + content = self.storage.revision_id(index) + return self.ok_response(environ, start_response, content) + + def changed(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + revision = self.data_get_string( + data, 'revision', default=None, source=source) + add,mod,rem = self.storage.changed(revision) + content = '\n\n'.join(['\n'.join(p) for p in (add,mod,rem)]) + return self.ok_response(environ, start_response, content) + + def version(self, environ, start_response): + self.check_login(environ) + data = self.query_data(environ) + source = 'query' + revision = self.data_get_string( + data, 'revision', default=None, source=source) + content = self.storage.storage_version(revision) + return self.ok_response(environ, start_response, content) + + # handler utility functions + def check_login(self, environ): + user = environ.get('be-auth.user', None) + if user is not None: # we're running under AuthenticationApp + if environ['REQUEST_METHOD'] == 'POST': + if user == 'guest' or self.storage.is_writeable() == False: + raise _Unauthorized() # only non-guests allowed to write + # allow read-only commands for all users + + def _notify(self, environ, command, id, params): + message = self._format_notification(environ, command, id, params) + self._submit_notification(message) + + def _format_notification(self, environ, command, id, params): + key_length = len('command') + for key,value in params: + if len(key) > key_length and '\n' not in str(value): + key_length = len(key) + key_length += 1 + lines = [] + multi_line_params = [] + for key,value in [('address', environ.get('REMOTE_ADDR', '-')), + ('command', command), ('id', id)]+params: + v = str(value) + if '\n' in v: + multi_line_params.append((key,v)) + continue + lines.append('%*.*s %s' % (key_length, key_length, key+':', v)) + lines.append('') + for key,value in multi_line_params: + lines.extend(['=== START %s ===' % key, v, + '=== STOP %s ===' % key, '']) + lines.append('') + return '\n'.join(lines) + + def _submit_notification(self, message): + libbe.util.subproc.invoke(self.notify, stdin=message, shell=True) + + +class ServeStorage (libbe.util.wsgi.ServerCommand): + """Serve bug directory storage over HTTP. + + This allows you to run local `be` commands interfacing with remote + data, transmitting file reads/writes/etc. over the network. + + :class:`~libbe.command.base.Command` wrapper around + :class:`ServerApp`. + """ + + name = 'serve-storage' + + def _get_app(self, logger, storage, **kwargs): + return ServerApp( + logger=logger, storage=storage, notify=kwargs.get('notify', False)) + + def _long_help(self): + return """ +Example usage:: + + $ be serve-storage + +And in another terminal (or after backgrounding the server):: + + $ be --repo http://localhost:8000/ list + +If you bind your server to a public interface, take a look at the +``--read-only`` option or the combined ``--ssl --auth FILE`` +options so other people can't mess with your repository. If you do use +authentication, you'll need to send in your username and password with, +for example:: + + $ be --repo http://username:password@localhost:8000/ list +""" + + +# alias for libbe.command.base.get_command_class() +Serve_storage = ServeStorage + + +if libbe.TESTING: + class ServerAppTestCase (libbe.util.wsgi.WSGITestCase): + def setUp(self): + super(ServerAppTestCase, self).setUp() + self.bd = libbe.bugdir.SimpleBugDir(memory=False) + self.app = ServerApp(self.bd.storage, logger=self.logger) + + def tearDown(self): + self.bd.cleanup() + super(ServerAppTestCase, self).tearDown() + + def test_add_get(self): + try: + self.getURL(self.app, '/add/', method='GET') + except libbe.util.wsgi.HandlerError as e: + self.failUnless(e.code == 404, e) + else: + self.fail('GET /add/ did not raise 404') + + def test_add_post(self): + self.getURL(self.app, '/add/', method='POST', + data_dict={'id':'123456', 'parent':'abc123', + 'directory':'True'}) + self.failUnless(self.status == '200 OK', self.status) + self.failUnless(self.response_headers == [], + self.response_headers) + self.failUnless(self.exc_info is None, self.exc_info) + # Note: other methods tested in libbe.storage.http + + # TODO: integration tests on Serve? + + unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) + suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) diff --git a/libbe/command/set.py b/libbe/command/set.py index f635faa..e575b08 100644 --- a/libbe/command/set.py +++ b/libbe/command/set.py @@ -57,6 +57,15 @@ class Set (libbe.command.Command): def __init__(self, *args, **kwargs): libbe.command.Command.__init__(self, *args, **kwargs) + self.options.extend([ + libbe.command.Option(name='bugdir', short_name='b', + help='Short bugdir UUID to act on. You ' + 'only need to set this if you have multiple bugdirs in ' + 'your repository.', + arg=libbe.command.Argument( + name='bugdir', metavar='ID', default=None, + completion_callback=libbe.command.util.complete_bugdir_id)), + ]) self.args.extend([ libbe.command.Argument( name='setting', metavar='SETTING', optional=True, @@ -66,7 +75,14 @@ class Set (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() + if params['bugdir']: + bugdir = bugdirs[bugdir] + elif len(bugdirs) == 1: + bugdir = bugdirs.values()[0] + else: + raise libbe.command.UserError( + 'Ambiguous bugdir {}'.format(sorted(bugdirs.values()))) if params['setting'] == None: keys = bugdir.settings_properties keys.sort() diff --git a/libbe/command/severity.py b/libbe/command/severity.py index 67e4c38..51096a7 100644 --- a/libbe/command/severity.py +++ b/libbe/command/severity.py @@ -36,7 +36,7 @@ class Severity (libbe.command.Command): >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_storage(bd.storage) >>> cmd = Severity(ui=ui) >>> bd.bug_from_uuid('a').severity @@ -66,10 +66,11 @@ class Severity (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() for bug_id in params['bug-id']: - bug,dummy_comment = \ - libbe.command.util.bug_comment_from_user_id(bugdir, bug_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, bug_id)) if bug.severity != params['severity']: try: bug.severity = params['severity'] @@ -82,7 +83,7 @@ class Severity (libbe.command.Command): def _long_help(self): try: # See if there are any per-tree severity configurations - bd = self._get_bugdir() + bugdirs = self._get_bugdirs() except NotImplementedError: pass # No tree, just show the defaults longest_severity_len = max([len(s) for s in libbe.bug.severity_values]) diff --git a/libbe/command/show.py b/libbe/command/show.py index 3175df8..4bf5bd8 100644 --- a/libbe/command/show.py +++ b/libbe/command/show.py @@ -39,7 +39,7 @@ class Show (libbe.command.Command): >>> io.stdout = sys.stdout >>> io.stdout.encoding = 'ascii' >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_storage(bd.storage) >>> cmd = Show(ui=ui) >>> ret = ui.run(cmd, args=['/a',]) # doctest: +ELLIPSIS @@ -98,13 +98,14 @@ class Show (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() if params['only-raw-body'] == True: if len(params['id']) != 1: raise libbe.command.UserError( 'only one ID accepted with --only-raw-body') - bug,comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['id'][0]) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['id'][0])) if comment == bug.comment_root: raise libbe.command.UserError( "--only-raw-body requires a comment ID, not '%s'" @@ -112,7 +113,7 @@ class Show (libbe.command.Command): sys.__stdout__.write(comment.body) return 0 print >> self.stdout, \ - output(bugdir, params['id'], encoding=self.stdout.encoding, + output(bugdirs, params['id'], encoding=self.stdout.encoding, as_xml=params['xml'], with_comments=not params['no-comments']) return 0 @@ -134,11 +135,11 @@ placed at the end of the output, so the ordering may not match the order of the listed IDs. """ -def _sort_ids(bugdir, ids, with_comments=True): +def _sort_ids(bugdirs, ids, with_comments=True): bugs = [] root_comments = {} for id in ids: - p = libbe.util.id.parse_user(bugdir, id) + p = libbe.util.id.parse_user(bugdirs, id) if p['type'] == 'bug': bugs.append(p['bug']) elif with_comments == True: @@ -165,18 +166,20 @@ def _xml_header(encoding): def _xml_footer(): return ['</be-xml>'] -def output(bd, ids, encoding, as_xml=True, with_comments=True): +def output(bugdirs, ids, encoding, as_xml=True, with_comments=True): if ids == None or len(ids) == 0: - bd.load_all_bugs() - ids = [bug.id.user() for bug in bd] - bugs,root_comments = _sort_ids(bd, ids, with_comments) + ids = [] + for bugdir in bugdirs.values(): + bugdir.load_all_bugs() + ids.extend([bug.id.user() for bug in bugdir]) + uuids,root_comments = _sort_ids(bugdirs, ids, with_comments) lines = [] if as_xml: lines.extend(_xml_header(encoding)) else: spaces_left = len(ids) - 1 - for bugname in bugs: - bug = bd.bug_from_uuid(bugname) + for bugname in uuids: + bug = libbe.command.util.bug_from_uuid(bugdirs, bugname) if as_xml: lines.append(bug.xml(indent=2, show_comments=with_comments)) else: @@ -185,7 +188,7 @@ def output(bd, ids, encoding, as_xml=True, with_comments=True): spaces_left -= 1 lines.append('') # add a blank line between bugs/comments for bugname,comments in root_comments.items(): - bug = bd.bug_from_uuid(bugname) + bug = libbe.command.util.bug_from_uuid(bugdirs, bugname) if as_xml: lines.extend([' <bug>', ' <uuid>%s</uuid>' % bug.uuid]) for commname in comments: diff --git a/libbe/command/status.py b/libbe/command/status.py index bdc9159..dd41190 100644 --- a/libbe/command/status.py +++ b/libbe/command/status.py @@ -36,7 +36,7 @@ class Status (libbe.command.Command): >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_storage(bd.storage) >>> cmd = Status(ui=ui) >>> cmd._storage = bd.storage @@ -67,10 +67,11 @@ class Status (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() for bug_id in params['bug-id']: - bug,dummy_comment = \ - libbe.command.util.bug_comment_from_user_id(bugdir, bug_id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, bug_id)) if bug.status != params['status']: try: bug.status = params['status'] @@ -83,7 +84,7 @@ class Status (libbe.command.Command): def _long_help(self): try: # See if there are any per-tree status configurations - bd = self._get_bugdir() + bugdirs = self._get_bugdirs() except NotImplementedError: pass # No tree, just show the defaults longest_status_len = max([len(s) for s in libbe.bug.status_values]) diff --git a/libbe/command/subscribe.py b/libbe/command/subscribe.py index e80c408..5a89c68 100644 --- a/libbe/command/subscribe.py +++ b/libbe/command/subscribe.py @@ -41,7 +41,7 @@ class Subscribe (libbe.command.Command): >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_storage(bd.storage) >>> cmd = Subscribe(ui=ui) >>> a = bd.bug_from_uuid('a') @@ -74,11 +74,15 @@ class Subscribe (libbe.command.Command): Subscriptions for abc/a: John Doe <j@doe.com> all * >>> ret = ui.run(cmd, {'unsubscribe':True, 'subscriber':'John Doe <j@doe.com>'}, ['/a']) - >>> ret = ui.run(cmd, {'subscriber':'Jane Doe <J@doe.com>', 'types':'new'}, ['DIR']) # doctest: +NORMALIZE_WHITESPACE - Subscriptions for bug directory: + >>> ret = ui.run(cmd, + ... {'subscriber':'Jane Doe <J@doe.com>', 'types':'new'}, + ... [bd.uuid[:3]]) # doctest: +NORMALIZE_WHITESPACE + Subscriptions for abc: Jane Doe <J@doe.com> new * - >>> ret = ui.run(cmd, {'subscriber':'Jane Doe <J@doe.com>'}, ['DIR']) # doctest: +NORMALIZE_WHITESPACE - Subscriptions for bug directory: + >>> ret = ui.run(cmd, + ... {'subscriber':'Jane Doe <J@doe.com>'}, + ... [bd.uuid]) # doctest: +NORMALIZE_WHITESPACE + Subscriptions for abc: Jane Doe <J@doe.com> all * >>> ui.cleanup() >>> bd.cleanup() @@ -115,10 +119,11 @@ class Subscribe (libbe.command.Command): ]) def _run(self, **params): - bugdir = self._get_bugdir() + storage = self._get_storage() + bugdirs = self._get_bugdirs() if params['list-all'] == True or params['list'] == True: - writeable = bugdir.storage.writeable - bugdir.storage.writeable = False + writeable = storage.writeable + storage.writeable = False if params['list-all'] == True: assert len(params['id']) == 0, params['id'] subscriber = params['subscriber'] @@ -138,18 +143,19 @@ class Subscribe (libbe.command.Command): types = params['types'].split(',') if len(params['id']) == 0: - params['id'] = [libbe.diff.BUGDIR_ID] + params['id'] = bugdirs.keys() for _id in params['id']: - if _id == libbe.diff.BUGDIR_ID: # directory-wide subscriptions + p = libbe.util.id.parse_user(bugdirs, _id) + if p['type'] == 'bugdir': type_root = libbe.diff.BUGDIR_TYPE_ALL - entity = bugdir - entity_name = 'bug directory' + entity = bugdirs[p['bugdir']] else: # bug-specific subscriptions type_root = libbe.diff.BUG_TYPE_ALL - bug,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, _id) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, _id)) entity = bug - entity_name = bug.id.user() + entity_name = entity.id.user() if params['list-all'] == True: entity_name = 'anything in the bug directory' types = [libbe.diff.type_from_name(name, type_root, default=libbe.diff.INVALID_TYPE, @@ -166,8 +172,11 @@ class Subscribe (libbe.command.Command): entity.extra_strings = estrs # reassign to notice change if params['list-all'] == True: - bugdir.load_all_bugs() - subscriptions = get_bugdir_subscribers(bugdir, servers[0]) + subscriptions = [] + for bugdir in bugdirs.values(): + bugdir.load_all_bugs() + subscriptions.extend( + get_bugdir_subscribers(bugdir, servers[0])) else: subscriptions = [] for estr in entity.extra_strings: @@ -178,13 +187,13 @@ class Subscribe (libbe.command.Command): print >> self.stdout, 'Subscriptions for %s:' % entity_name print >> self.stdout, '\n'.join(subscriptions) if params['list-all'] == True or params['list'] == True: - bugdir.storage.writeable = writeable + storage.writeable = writeable return 0 def _long_help(self): return """ -ID can be either a bug id, or blank/"DIR", in which case it refers to the -whole bug directory. +ID can be either a bug ID, a bugdir ID, or blank, in which case it +refers to all known bugdirs. SERVERS specifies the servers from which you would like to receive notification. Multiple severs may be specified in a comma-separated diff --git a/libbe/command/tag.py b/libbe/command/tag.py index 5607b77..58c04d0 100644 --- a/libbe/command/tag.py +++ b/libbe/command/tag.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License along with # Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. +import itertools + import libbe import libbe.command import libbe.command.util @@ -34,7 +36,7 @@ class Tag (libbe.command.Command): >>> io = libbe.command.StringInputOutput() >>> io.stdout = sys.stdout >>> ui = libbe.command.UserInterface(io=io) - >>> ui.storage_callbacks.set_bugdir(bd) + >>> ui.storage_callbacks.set_bugdirs({bd.uuid: bd}) >>> cmd = Tag(ui=ui) >>> a = bd.bug_from_uuid('a') @@ -107,16 +109,18 @@ class Tag (libbe.command.Command): if params['id'] != None and params['list'] == True: raise libbe.command.UserError( 'Do not specify a bug id with the --list option.') - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() if params['list'] == True: - tags = get_all_tags(bugdir) + tags = list(itertools.chain(* + [get_all_tags(bugdir) for bugdir in bugdirs.values()])) tags.sort() if len(tags) > 0: print >> self.stdout, '\n'.join(tags) return 0 - bug,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['id']) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['id'])) if len(params['tag']) > 0: tags = get_tags(bug) for tag in params['tag']: diff --git a/libbe/command/target.py b/libbe/command/target.py index e18d515..3f14048 100644 --- a/libbe/command/target.py +++ b/libbe/command/target.py @@ -24,6 +24,7 @@ import libbe import libbe.command import libbe.command.util import libbe.command.depend +import libbe.util.id class Target (libbe.command.Command): @@ -70,6 +71,13 @@ class Target (libbe.command.Command): help="Print the UUID for the target bug whose summary " "matches TARGET. If TARGET is not given, print the UUID " "of the current bugdir target."), + libbe.command.Option(name='bugdir', short_name='b', + help='Short bugdir UUID for the target resolution. You ' + 'only need to set this if you have multiple bugdirs in ' + 'your repository.', + arg=libbe.command.Argument( + name='bugdir', metavar='ID', default=None, + completion_callback=libbe.command.util.complete_bugdir_id)), ]) self.args.extend([ libbe.command.Argument( @@ -88,27 +96,35 @@ class Target (libbe.command.Command): if params['target'] != None: raise libbe.command.UserError('Too many arguments') params['target'] = params.pop('id') - bugdir = self._get_bugdir() + bugdirs = self._get_bugdirs() if params['resolve'] == True: - bug = bug_from_target_summary(bugdir, params['target']) + if params['bugdir']: + bugdir = bugdirs[bugdir] + elif len(bugdirs) == 1: + bugdir = bugdirs.values()[0] + else: + raise libbe.command.UserError( + 'Ambiguous bugdir {}'.format(sorted(bugdirs.values()))) + bug = bug_from_target_summary(bugdirs, bugdir, params['target']) if bug == None: print >> self.stdout, 'No target assigned.' else: print >> self.stdout, bug.uuid return 0 - bug,dummy_comment = libbe.command.util.bug_comment_from_user_id( - bugdir, params['id']) + bugdir,bug,comment = ( + libbe.command.util.bugdir_bug_comment_from_user_id( + bugdirs, params['id'])) if params['target'] == None: - target = bug_target(bugdir, bug) + target = bug_target(bugdirs, bug) if target == None: print >> self.stdout, 'No target assigned.' else: print >> self.stdout, target.summary else: if params['target'] == 'none': - target = remove_target(bugdir, bug) + target = remove_target(bugdirs, bug) else: - target = add_target(bugdir, bug, params['target']) + target = add_target(bugdirs, bugdir, bug, params['target']) return 0 def usage(self): @@ -140,7 +156,7 @@ by UUID), try $ be set target $(be target --resolve SUMMARY) """ -def bug_from_target_summary(bugdir, summary=None): +def bug_from_target_summary(bugdirs, bugdir, summary=None): if summary == None: if bugdir.target == None: return None @@ -158,11 +174,11 @@ def bug_from_target_summary(bugdir, summary=None): % '\n '.join([bug.uuid for bug in matched])) return matched[0] -def bug_target(bugdir, bug): +def bug_target(bugdirs, bug): if bug.severity == 'target': return bug matched = [] - for blocked in libbe.command.depend.get_blocks(bugdir, bug): + for blocked in libbe.command.depend.get_blocks(bugdirs, bug): if blocked.severity == 'target': matched.append(blocked) if len(matched) == 0: @@ -173,38 +189,37 @@ def bug_target(bugdir, bug): '\n '.join([b.uuid for b in matched]))) return matched[0] -def remove_target(bugdir, bug): - target = bug_target(bugdir, bug) +def remove_target(bugdirs, bug): + target = bug_target(bugdirs, bug) libbe.command.depend.remove_block(target, bug) return target -def add_target(bugdir, bug, summary): - target = bug_from_target_summary(bugdir, summary) +def add_target(bugdirs, bugdir, bug, summary): + target = bug_from_target_summary(bugdirs, bugdir, summary) if target == None: target = bugdir.new_bug(summary=summary) target.severity = 'target' libbe.command.depend.add_block(target, bug) return target -def targets(bugdir): +def targets(bugdirs): """Generate all possible target bug summaries.""" - bugdir.load_all_bugs() - for bug in bugdir: - if bug.severity == 'target': - yield bug.summary + for bugdir in bugdirs.values(): + bugdir.load_all_bugs() + for bug in bugdir: + if bug.severity == 'target': + yield bug.summary -def target_dict(bugdir): +def target_dict(bugdirs): """ Return a dict with bug UUID keys and bug summary values for all target bugs. """ ret = {} - bugdir.load_all_bugs() - for bug in bugdir: - if bug.severity == 'target': - ret[bug.uuid] = bug.summary + for bug in targets(bugdirs): + ret[bug.uuid] = bug return ret def complete_target(command, argument, fragment=None): """List possible command completions for fragment.""" - return targets(command._get_bugdir()) + return targets(command._get_bugdirs()) diff --git a/libbe/command/util.py b/libbe/command/util.py index 75d301d..4c6756f 100644 --- a/libbe/command/util.py +++ b/libbe/command/util.py @@ -25,7 +25,7 @@ import libbe.command class Completer (object): def __init__(self, options): self.options = options - def __call__(self, bugdir, fragment=None): + def __call__(self, bugdirs, fragment=None): return [fragment] def complete_command(command, argument, fragment=None): @@ -50,28 +50,35 @@ def complete_path(command, argument, fragment=None): return comp_path(fragment) def complete_status(command, argument, fragment=None): - bd = command._get_bugdir() + bd = sorted(command._get_bugdirs().items())[1] import libbe.bug return libbe.bug.status_values def complete_severity(command, argument, fragment=None): - bd = command._get_bugdir() + bd = sorted(command._get_bugdirs().items())[1] import libbe.bug return libbe.bug.severity_values -def assignees(bugdir): - bugdir.load_all_bugs() - return list(set([bug.assigned for bug in bugdir - if bug.assigned != None])) +def assignees(bugdirs): + ret = set() + for bugdir in bugdirs.values(): + bugdir.load_all_bugs() + ret.update(set([bug.assigned for bug in bugdir + if bug.assigned != None])) + return list(ret) def complete_assigned(command, argument, fragment=None): - return assignees(command._get_bugdir()) + return assignees(command._get_bugdirs()) def complete_extra_strings(command, argument, fragment=None): if fragment == None: return [] return [fragment] +def complete_bugdir_id(command, argument, fragment=None): + bugdirs = command._get_bugdirs() + return bugdirs.keys() + def complete_bug_id(command, argument, fragment=None): return complete_bug_comment_id(command, argument, fragment, comments=False) @@ -80,11 +87,11 @@ def complete_bug_comment_id(command, argument, fragment=None, active_only=True, comments=True): import libbe.bugdir import libbe.util.id - bd = command._get_bugdir() + bugdirs = command._get_bugdirs() if fragment == None or len(fragment) == 0: fragment = '/' try: - p = libbe.util.id.parse_user(bd, fragment) + p = libbe.util.id.parse_user(bugdirs, fragment) matches = None root,residual = (fragment, None) if not root.endswith('/'): @@ -100,28 +107,32 @@ def complete_bug_comment_id(command, argument, fragment=None, common = e.common matches = e.matches root,residual = libbe.util.id.residual(common, fragment) - p = libbe.util.id.parse_user(bd, e.common) + p = libbe.util.id.parse_user(bugdirs, e.common) bug = None if matches == None: # fragment was complete, get a list of children uuids if p['type'] == 'bugdir': - matches = bd.uuids() - common = bd.id.user() + bugdir = bugdirs[p['bugdir']] + matches = bugdir.uuids() + common = bugdir.id.user() elif p['type'] == 'bug': if comments == False: return [fragment] - bug = bd.bug_from_uuid(p['bug']) + bugdir = bugdirs[p['bugdir']] + bug = bugdir.bug_from_uuid(p['bug']) matches = bug.uuids() common = bug.id.user() else: assert p['type'] == 'comment', p return [fragment] if p['type'] == 'bugdir': - child_fn = bd.bug_from_uuid + bugdir = bugdirs[p['bugdir']] + child_fn = bugdir.bug_from_uuid elif p['type'] == 'bug': if comments == False: return[fragment] + bugdir = bugdirs[p['bugdir']] if bug == None: - bug = bd.bug_from_uuid(p['bug']) + bug = bugdir.bug_from_uuid(p['bug']) child_fn = bug.comment_from_uuid elif p['type'] == 'comment': assert matches == None, matches @@ -188,18 +199,38 @@ def select_values(string, possible_values, name="unkown"): possible_values = whitelisted_values return possible_values -def bug_comment_from_user_id(bugdir, id): - p = libbe.util.id.parse_user(bugdir, id) - if not p['type'] in ['bug', 'comment']: +def bugdir_bug_comment_from_user_id(bugdirs, id): + p = libbe.util.id.parse_user(bugdirs, id) + if not p['type'] in ['bugdir', 'bug', 'comment']: + raise libbe.command.UserError( + '{} is a {} id, not a bugdir, bug, or comment id'.format( + id, p['type'])) + if p['bugdir'] not in bugdirs: raise libbe.command.UserError( - '%s is a %s id, not a bug or comment id' % (id, p['type'])) + "{} doesn't belong to any bugdirs in {}".format( + id, sorted(bugdirs.keys()))) + bugdir = bugdirs[p['bugdir']] if p['bugdir'] != bugdir.uuid: raise libbe.command.UserError( "%s doesn't belong to this bugdir (%s)" % (id, bugdir.uuid)) - bug = bugdir.bug_from_uuid(p['bug']) - if 'comment' in p: - comment = bug.comment_from_uuid(p['comment']) + if 'bug' in p: + bug = bugdir.bug_from_uuid(p['bug']) + if 'comment' in p: + comment = bug.comment_from_uuid(p['comment']) + else: + comment = bug.comment_root else: - comment = bug.comment_root - return (bug, comment) + bug = comment = None + return (bugdir, bug, comment) + +def bug_from_uuid(bugdirs, uuid): + error = None + for bugdir in bugdirs.values(): + try: + bug = bugdir.bug_from_uuid(uuid) + except libbe.bugdir.NoBugMatches as e: + error = e + else: + return bug + raise error diff --git a/libbe/comment.py b/libbe/comment.py index 082a46f..0eadbb2 100644 --- a/libbe/comment.py +++ b/libbe/comment.py @@ -86,6 +86,8 @@ def load_comments(bug, load_full=False): def save_comments(bug): for comment in bug.comment_root.traverse(): + comment.bug = bug + comment.storage = bug.storage comment.save() @@ -156,7 +158,8 @@ class Comment (Tree, settings_object.SavedSettingsObject): assert self.uuid != INVALID_UUID, self if self.content_type.startswith('text/') \ and self.bug != None and self.bug.bugdir != None: - new = libbe.util.id.short_to_long_text([self.bug.bugdir], new) + new = libbe.util.id.short_to_long_text( + {self.bug.bugdir.uuid: self.bug.bugdir}, new) if (self.storage != None and self.storage.writeable == True) \ or force==True: assert new != None, "Can't save empty comment" @@ -377,7 +380,9 @@ class Comment (Tree, settings_object.SavedSettingsObject): text = settings_object.EMPTY else: text = xml.sax.saxutils.unescape(child.text) - text = text.decode('unicode_escape').strip() + if not isinstance(text, unicode): + text = text.decode('unicode_escape') + text = text.strip() if child.tag == 'uuid' and not preserve_uuids: uuid = text continue # don't set the comment's uuid tag. @@ -456,16 +461,17 @@ class Comment (Tree, settings_object.SavedSettingsObject): <extra-string>TAG: very helpful</extra-string> </comment> """ - for attr in other.explicit_attrs: - old = getattr(self, attr) - new = getattr(other, attr) - if old != new: - if accept_changes == True: - setattr(self, attr, new) - elif change_exception == True: - raise ValueError, \ - 'Merge would change %s "%s"->"%s" for comment %s' \ - % (attr, old, new, self.uuid) + if hasattr(other, 'explicit_attrs'): + for attr in other.explicit_attrs: + old = getattr(self, attr) + new = getattr(other, attr) + if old != new: + if accept_changes: + setattr(self, attr, new) + elif change_exception: + raise ValueError( + ('Merge would change {} "{}"->"{}" for comment {}' + ).format(attr, old, new, self.uuid)) if self.alt_id == self.uuid: self.alt_id = None for estr in other.extra_strings: @@ -501,7 +507,8 @@ class Comment (Tree, settings_object.SavedSettingsObject): if self.content_type.startswith("text/"): body = (self.body or "") if self.bug != None and self.bug.bugdir != None: - body = libbe.util.id.long_to_short_text([self.bug.bugdir], body) + body = libbe.util.id.long_to_short_text( + {self.bug.bugdir.uuid: self.bug.bugdir}, body) lines.extend(body.splitlines()) else: lines.append("Content type %s not printable. Try XML output instead" % self.content_type) diff --git a/libbe/storage/base.py b/libbe/storage/base.py index 14c54bd..0c7ca10 100644 --- a/libbe/storage/base.py +++ b/libbe/storage/base.py @@ -681,6 +681,24 @@ if TESTING == True: s = sorted(self.s.children('parent')) self.failUnless(s == ids, '\n %s\n !=\n %s' % (s, ids)) + def test_grandchildren(self): + """Grandchildren should not be returned as children. + """ + self.s.add('parent', directory=True) + ids = [] + for i in range(5): + child = 'parent/%s' % str(i) + directory = (i % 2 == 0) + ids.append(child) + self.s.add(child, 'parent', directory=directory) + if directory: + for j in range(3): + grandchild = '%s/%s' % (child, str(j)) + directory = (j % 2 == 0) + self.s.add(grandchild, child, directory=directory) + s = sorted(self.s.children('parent')) + self.failUnless(s == ids, '\n %s\n !=\n %s' % (s, ids)) + def test_add_invalid_directory(self): """Should not be able to add children to non-directories. """ @@ -908,7 +926,7 @@ if TESTING == True: self.s.commit('Added initialization files') except EmptyCommit: pass - + def test_revision_id_exception(self): """Invalid revision id should raise InvalidRevision. """ @@ -1000,6 +1018,38 @@ if TESTING == True: % (vars(self.Class)['name'], ret, children[i], revs[i])) + def test_avoid_previous_grandchildren(self): + """Previous grandchildren should not be returned as children. + """ + self.s.add('parent', directory=True) + revs = [] + cur_children = [] + children = [] + for i in range(5): + new_child = 'parent/%s' % str(i) + directory = (i % 2 == 0) + self.s.add(new_child, 'parent', directory=directory) + cur_children.append(new_child) + children.append(list(cur_children)) + if directory: + for j in range(3): + new_grandchild = '%s/%s' % (new_child, str(j)) + directory = (j % 2 == 0) + self.s.add( + new_grandchild, new_child, directory=directory) + if not directory: + self.s.set(new_grandchild, self.val) + else: + self.s.set(new_child, self.val) + revs.append(self.s.commit('%s: %d' % (self.commit_msg, i), + self.commit_body)) + for rev,cur_children in zip(revs, children): + ret = sorted(self.s.children('parent', revision=rev)) + self.failUnless(ret == cur_children, + "%s.children() returned %s not %s for revision %s" + % (vars(self.Class)['name'], ret, + cur_children, rev)) + class VersionedStorage_changed_TestCase (VersionedStorageTestCase): """Test cases for VersionedStorage.changed() method.""" diff --git a/libbe/storage/http.py b/libbe/storage/http.py index 877fc96..b1a213f 100644 --- a/libbe/storage/http.py +++ b/libbe/storage/http.py @@ -16,30 +16,25 @@ # You should have received a copy of the GNU General Public License along with # Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. -# For urllib2 information, see -# urllib2, from urllib2 - The Missing Manual -# http://www.voidspace.org.uk/python/articles/urllib2.shtml -# -# A dictionary of response codes is available in -# httplib.responses - """Define an HTTP-based :class:`~libbe.storage.base.VersionedStorage` implementation. See Also -------- -:mod:`libbe.command.serve` : the associated server - +:mod:`libbe.command.serve_storage` : the associated server """ +from __future__ import absolute_import import sys import urllib -import urllib2 import urlparse import libbe import libbe.version -import base +import libbe.util.http +from libbe.util.http import HTTP_VALID, HTTP_USER_ERROR +from . import base + from libbe import TESTING if TESTING == True: @@ -49,78 +44,9 @@ if TESTING == True: import unittest import libbe.bugdir - import libbe.command.serve - - -USER_AGENT = 'BE-HTTP-Storage' -HTTP_OK = 200 -HTTP_FOUND = 302 -HTTP_TEMP_REDIRECT = 307 -HTTP_USER_ERROR = 418 -"""Status returned to indicate exceptions on the server side. - -A BE-specific extension to the HTTP/1.1 protocol (See `RFC 2616`_). - -.. _RFC 2616: http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 -""" - -HTTP_VALID = [HTTP_OK, HTTP_FOUND, HTTP_TEMP_REDIRECT, HTTP_USER_ERROR] - -class InvalidURL (Exception): - def __init__(self, error=None, url=None, msg=None): - Exception.__init__(self, msg) - self.url = url - self.error = error - self.msg = msg - def __str__(self): - if self.msg == None: - if self.error == None: - return "Unknown URL error: %s" % self.url - return self.error.__str__() - return self.msg - -def get_post_url(url, get=True, data_dict=None, headers=[]): - """Execute a GET or POST transaction. - - Parameters - ---------- - url : str - The base URL (query portion added internally, if necessary). - get : bool - Use GET if True, otherwise use POST. - data_dict : dict - Data to send, either by URL query (if GET) or by POST (if POST). - headers : list - Extra HTTP headers to add to the request. - """ - if data_dict == None: - data_dict = {} - if get == True: - if data_dict != {}: - # encode get parameters in the url - param_string = urllib.urlencode(data_dict) - url = "%s?%s" % (url, param_string) - data = None - else: - data = urllib.urlencode(data_dict) - headers = dict(headers) - headers['User-Agent'] = USER_AGENT - req = urllib2.Request(url, data=data, headers=headers) - try: - response = urllib2.urlopen(req) - except urllib2.HTTPError, e: - if hasattr(e, 'reason'): - msg = 'We failed to reach a server.\nURL: %s\nReason: %s' \ - % (url, e.reason) - elif hasattr(e, 'code'): - msg = "The server couldn't fulfill the request.\nURL: %s\nError code: %s" \ - % (url, e.code) - raise InvalidURL(error=e, url=url, msg=msg) - page = response.read() - final_url = response.geturl() - info = response.info() - response.close() - return (page, final_url, info) + import libbe.command.serve_storage + import libbe.util.http + import libbe.util.wsgi class HTTP (base.VersionedStorage): @@ -130,6 +56,7 @@ class HTTP (base.VersionedStorage): Uses GET to retrieve information and POST to set information. """ name = 'HTTP' + user_agent = 'BE-HTTP-Storage' def __init__(self, repo, *args, **kwargs): repo,self.uname,self.password = self.parse_repo(repo) @@ -165,7 +92,9 @@ class HTTP (base.VersionedStorage): if self.uname != None and self.password != None: headers.append(('Authorization','Basic %s' % \ ('%s:%s' % (self.uname, self.password)).encode('base64'))) - return get_post_url(url, get, data_dict, headers) + return libbe.util.http.get_post_url( + url, get, data_dict=data_dict, headers=headers, + agent=self.user_agent) def storage_version(self, revision=None): """Return the storage format for this backend.""" @@ -234,7 +163,7 @@ class HTTP (base.VersionedStorage): page,final_url,info = self.get_post_url( url, get=True, data_dict={'revision':revision}) - except InvalidURL, e: + except libbe.util.http.HTTPError, e: if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): raise elif default == base.InvalidObject: @@ -252,7 +181,7 @@ class HTTP (base.VersionedStorage): page,final_url,info = self.get_post_url( url, get=False, data_dict={'value':value}) - except InvalidURL, e: + except libbe.util.http.HTTPError, e: if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): raise if e.error.code == HTTP_USER_ERROR \ @@ -268,7 +197,7 @@ class HTTP (base.VersionedStorage): url, get=False, data_dict={'summary':summary, 'body':body, 'allow_empty':allow_empty}) - except InvalidURL, e: + except libbe.util.http.HTTPError, e: if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): raise if e.error.code == HTTP_USER_ERROR: @@ -302,7 +231,7 @@ class HTTP (base.VersionedStorage): page,final_url,info = self.get_post_url( url, get=True, data_dict={'index':index}) - except InvalidURL, e: + except libbe.util.http.HTTPError, e: if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): raise if e.error.code == HTTP_USER_ERROR: @@ -332,31 +261,16 @@ class HTTP (base.VersionedStorage): return page.rstrip('\n') if TESTING == True: - class GetPostUrlTestCase (unittest.TestCase): - """Test cases for get_post_url()""" - def test_get(self): - url = 'http://bugseverywhere.org/' - page,final_url,info = get_post_url(url=url) - self.failUnless(final_url == url, - 'Redirect?\n Expected: "%s"\n Got: "%s"' - % (url, final_url)) - def test_get_redirect(self): - url = 'http://physics.drexel.edu/~wking/code/be/redirect' - expected = 'http://physics.drexel.edu/~wking/' - page,final_url,info = get_post_url(url=url) - self.failUnless(final_url == expected, - 'Redirect?\n Expected: "%s"\n Got: "%s"' - % (expected, final_url)) - class TestingHTTP (HTTP): name = 'TestingHTTP' def __init__(self, repo, *args, **kwargs): self._storage_backend = base.VersionedStorage(repo) - self.app = libbe.command.serve.ServerApp( + app = libbe.command.serve_storage.ServerApp( storage=self._storage_backend) + self.app = libbe.util.wsgi.BEExceptionApp(app=app) HTTP.__init__(self, repo='http://localhost:8000/', *args, **kwargs) self.intitialized = False - # duplicated from libbe.command.serve.WSGITestCase + # duplicated from libbe.util.wsgi.WSGITestCase self.default_environ = { 'REQUEST_METHOD': 'GET', # 'POST', 'HEAD' 'REMOTE_ADDR': '192.168.0.123', @@ -378,7 +292,7 @@ if TESTING == True: } def getURL(self, app, path='/', method='GET', data=None, scheme='http', environ={}): - # duplicated from libbe.command.serve.WSGITestCase + # duplicated from libbe.util.wsgi.WSGITestCase env = copy.copy(self.default_environ) env['PATH_INFO'] = path env['REQUEST_METHOD'] = method @@ -393,7 +307,11 @@ if TESTING == True: env['QUERY_STRING'] = enc_data for key,value in environ.items(): env[key] = value - return ''.join(app(env, self.start_response)) + try: + result = app(env, self.start_response) + except libbe.util.wsgi.HandlerError as e: + raise libbe.util.http.HTTPError(error=e, url=path, msg=str(e)) + return ''.join(result) def start_response(self, status, response_headers, exc_info=None): self.status = status self.response_headers = response_headers @@ -417,7 +335,8 @@ if TESTING == True: def __str__(self): return self.string error = __estr(self.status) - raise InvalidURL(error=error, url=url, msg=output) + raise libbe.util.http.HTTPError( + error=error, url=url, msg=output) info = dict(self.response_headers) return (output, url, info) def _init(self): diff --git a/libbe/storage/vcs/base.py b/libbe/storage/vcs/base.py index 5f13c01..2a5f600 100644 --- a/libbe/storage/vcs/base.py +++ b/libbe/storage/vcs/base.py @@ -219,7 +219,8 @@ class CachedPathID (object): else: self._cache = cache spaced_root = os.path.join(self._root, self._spacer_dirs[0]) - for dirpath, dirnames, filenames in os.walk(spaced_root): + for dirpath, dirnames, filenames in os.walk(spaced_root, + followlinks=True): if dirpath == spaced_root: continue try: diff --git a/libbe/storage/vcs/hg.py b/libbe/storage/vcs/hg.py index 176719c..2f4469d 100644 --- a/libbe/storage/vcs/hg.py +++ b/libbe/storage/vcs/hg.py @@ -152,7 +152,10 @@ class Hg(base.VCS): output = self._u_invoke_client('manifest', '--rev', revision) files = output.splitlines() path = path.rstrip(os.path.sep) + os.path.sep - return [self._u_rel_path(f, path) for f in files if f.startswith(path)] + descendent_files = [self._u_rel_path(f, path) for f in files + if f.startswith(path)] + return sorted(set( + f.split(os.path.sep, 1)[0] for f in descendent_files)) def _vcs_commit(self, commitfile, allow_empty=False): args = ['commit', '--logfile', commitfile] @@ -242,7 +245,7 @@ class Hg(base.VCS): assert file_a.startswith('a/'), \ 'missformed file_a %s' % file_a assert file_b.startswith('b/'), \ - 'missformed file_a %s' % file_b + 'missformed file_b %s' % file_b file = file_a[2:] assert file_b[2:] == file, \ 'diff file missmatch %s != %s' % (file_a, file_b) diff --git a/libbe/ui/command_line.py b/libbe/ui/command_line.py index 59a9560..1cafc36 100644 --- a/libbe/ui/command_line.py +++ b/libbe/ui/command_line.py @@ -28,9 +28,11 @@ import libbe import libbe.bugdir import libbe.command import libbe.command.util +import libbe.storage import libbe.version import libbe.ui.util.pager import libbe.util.encoding +import libbe.util.http if libbe.TESTING == True: @@ -228,6 +230,12 @@ class BE (libbe.command.Command): arg=libbe.command.Argument( name='repo', metavar='REPO', default='.', completion_callback=libbe.command.util.complete_path)), + libbe.command.Option(name='server', short_name='s', + help='Select BE command server (see `be help ' + 'command-server`) rather than executing commands ' + 'locally', + arg=libbe.command.Argument( + name='server', metavar='URL')), libbe.command.Option(name='paginate', help='Pipe all output into less (or if set, $PAGER).'), libbe.command.Option(name='no-pager', @@ -302,9 +310,15 @@ def dispatch(ui, command, args): except libbe.command.UserError, e: print >> ui.io.stdout, 'ERROR:\n', e return 1 + except OSError, e: + print >> ui.io.stdout, 'OSError:\n', e + return 1 except libbe.storage.ConnectionError, e: print >> ui.io.stdout, 'Connection Error:\n', e return 1 + except libbe.util.http.HTTPError, e: + print >> ui.io.stdout, 'HTTP Error:\n', e + return 1 except (libbe.util.id.MultipleIDMatches, libbe.util.id.NoIDMatches, libbe.util.id.InvalidIDStructure), e: print >> ui.io.stdout, 'Invalid id:\n', e @@ -347,10 +361,11 @@ def main(): return 1 ui.storage_callbacks = libbe.command.StorageCallbacks(options['repo']) - command = Class(ui=ui) + command = Class(ui=ui, server=options['server']) ui.setup_command(command) - if command.name in ['comment', 'commit', 'import-xml', 'serve']: + if command.name in [ + 'new', 'comment', 'commit', 'import-xml', 'serve', 'serve-commands']: paginate = 'never' else: paginate = 'auto' @@ -361,7 +376,12 @@ def main(): libbe.ui.util.pager.run_pager(paginate) ret = dispatch(ui, command, args) - ui.cleanup() + try: + ui.cleanup() + except IOError, e: + print >> ui.io.stdout, 'IOError:\n', e + return 1 + return ret if __name__ == '__main__': diff --git a/libbe/ui/util/pager.py b/libbe/ui/util/pager.py index 08b1024..e6e6d1b 100644 --- a/libbe/ui/util/pager.py +++ b/libbe/ui/util/pager.py @@ -50,7 +50,6 @@ def run_pager(paginate='auto'): if os.fork() == 0: # child process os.close(read_fd) - os.close(0) os.dup2(write_fd, 1) os.close(write_fd) if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() == True: diff --git a/libbe/util/encoding.py b/libbe/util/encoding.py index ee51993..a33678f 100644 --- a/libbe/util/encoding.py +++ b/libbe/util/encoding.py @@ -34,6 +34,10 @@ if libbe.TESTING == True: ENCODING = os.environ.get('BE_ENCODING', None) "override get_encoding() output" +INPUT_ENCODING = os.environ.get('BE_INPUT_ENCODING', None) +"override get_input_encoding() output" +OUTPUT_ENCODING = os.environ.get('BE_OUTPUT_ENCODING', None) +"override get_output_encoding() output" def get_encoding(): """ @@ -46,9 +50,13 @@ def get_encoding(): return encoding def get_input_encoding(): + if INPUT_ENCODING != None: + return INPUT_ENCODING return sys.__stdin__.encoding or get_encoding() def get_output_encoding(): + if OUTPUT_ENCODING != None: + return OUTPUT_ENCODING return sys.__stdout__.encoding or get_encoding() def get_text_file_encoding(): diff --git a/libbe/util/http.py b/libbe/util/http.py new file mode 100644 index 0000000..8af97eb --- /dev/null +++ b/libbe/util/http.py @@ -0,0 +1,129 @@ +# Copyright + +# For urllib2 information, see +# urllib2, from urllib2 - The Missing Manual +# http://www.voidspace.org.uk/python/articles/urllib2.shtml +# +# A dictionary of response codes is available in +# httplib.responses +# but it is slow to load. + +import urllib +import urllib2 + +from libbe import TESTING + +if TESTING: + import unittest + + +HTTP_OK = 200 +HTTP_FOUND = 302 +HTTP_TEMP_REDIRECT = 307 +HTTP_USER_ERROR = 418 +"""Status returned to indicate exceptions on the server side. + +A BE-specific extension to the HTTP/1.1 protocol (See `RFC 2616`_). + +.. _RFC 2616: http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 +""" + +HTTP_VALID = [HTTP_OK, HTTP_FOUND, HTTP_TEMP_REDIRECT, HTTP_USER_ERROR] + + +USER_AGENT = 'BE-agent' + + +class HTTPError (Exception): + def __init__(self, error=None, url=None, msg=None): + Exception.__init__(self, msg) + self.url = url + self.error = error + self.msg = msg + + def __str__(self): + if self.msg is None: + if self.error is None: + return 'Unknown HTTP error: {}'.format(self.url) + return str(self.error) + return self.msg + + +def get_post_url(url, get=True, data=None, data_dict=None, headers=[], + agent=None): + """Execute a GET or POST transaction. + + Parameters + ---------- + url : str + The base URL (query portion added internally, if necessary). + get : bool + Use GET if True, otherwise use POST. + data : str + Raw data to send by POST (requires POST). + data_dict : dict + Data to send, either by URL query (if GET) or by POST (if POST). + Cannot be given in combination with `data`. + headers : list + Extra HTTP headers to add to the request. + agent : str + User agent string overriding the BE default. + """ + if agent is None: + agent = USER_AGENT + if data is None: + if data_dict is None: + data_dict = {} + if get is True: + if data_dict != {}: + # encode get parameters in the url + param_string = urllib.urlencode(data_dict) + url = '{}?{}'.format(url, param_string) + else: + data = urllib.urlencode(data_dict) + else: + assert get is False, (data, get) + assert data_dict is None, (data, data_dict) + headers = dict(headers) + headers['User-Agent'] = agent + req = urllib2.Request(url, data=data, headers=headers) + try: + response = urllib2.urlopen(req) + except urllib2.HTTPError, e: + lines = [ + 'We failed to connect to the server (HTTPError).', + 'URL: {}'.format(url), + ] + if hasattr(e, 'reason'): + lines.append('Reason: {}'.format(e.reason)) + lines.append('Error code: {}'.format(e.code)) + msg = '\n'.join(lines) + raise HTTPError(error=e, url=url, msg=msg) + except urllib2.URLError, e: + msg = ('We failed to connect to the server (URLError).\nURL: {}\n' + 'Reason: {}').format(url, e.reason) + raise HTTPError(error=e, url=url, msg=msg) + page = response.read() + final_url = response.geturl() + info = response.info() + response.close() + return (page, final_url, info) + + +if TESTING: + class GetPostUrlTestCase (unittest.TestCase): + """Test cases for get_post_url()""" + def test_get(self): + url = 'http://bugseverywhere.org/' + page,final_url,info = get_post_url(url=url) + self.failUnless(final_url == url, + 'Redirect?\n Expected: "{}"\n Got: "{}"'.format( + url, final_url)) + + def test_get_redirect(self): + url = 'http://physics.drexel.edu/~wking/code/be/redirect' + expected = 'http://physics.drexel.edu/~wking/' + page,final_url,info = get_post_url(url=url) + self.failUnless(final_url == expected, + 'Redirect?\n Expected: "{}"\n Got: "{}"'.format( + expected, final_url)) diff --git a/libbe/util/id.py b/libbe/util/id.py index 8dc6075..e73640b 100644 --- a/libbe/util/id.py +++ b/libbe/util/id.py @@ -414,11 +414,11 @@ def long_to_short_user(bugdirs, id): long_to_short_text : conversion on a block of text """ ids = _split(id, check_length=True) - matching_bugdirs = [bd for bd in bugdirs if bd.uuid == ids[0]] + matching_bugdirs = [bd for bd in bugdirs.values() if bd.uuid == ids[0]] if len(matching_bugdirs) == 0: - raise NoIDMatches(id, [bd.uuid for bd in bugdirs]) + raise NoIDMatches(id, [bd.uuid for bd in bugdirs.values()]) elif len(matching_bugdirs) > 1: - raise MultipleIDMatches(id, '', [bd.uuid for bd in bugdirs]) + raise MultipleIDMatches(id, '', [bd.uuid for bd in bugdirs.values()]) bugdir = matching_bugdirs[0] objects = [bugdir] if len(ids) >= 2: @@ -443,10 +443,10 @@ def short_to_long_user(bugdirs, id): """ ids = _split(id, check_length=True) ids[0] = _expand(ids[0], common=None, - other_ids=[bd.uuid for bd in bugdirs]) + other_ids=[bd.uuid for bd in bugdirs.values()]) if len(ids) == 1: return _assemble(ids) - bugdir = [bd for bd in bugdirs if bd.uuid == ids[0]][0] + bugdir = [bd for bd in bugdirs.values() if bd.uuid == ids[0]][0] ids[1] = _expand(ids[1], common=bugdir.id.user(), other_ids=bugdir.uuids()) if len(ids) == 2: @@ -471,17 +471,24 @@ class IDreplacer (object): -------- short_to_long_text, long_to_short_text """ - def __init__(self, bugdirs, replace_fn, wrap=True): + def __init__(self, bugdirs, replace_fn, wrap=True, strict=True): self.bugdirs = bugdirs self.replace_fn = replace_fn self.wrap = wrap + self.strict = strict + def __call__(self, match): ids = [] for m in match.groups(): if m == None: m = '' ids.append(m) - replacement = self.replace_fn(self.bugdirs, ''.join(ids)) + try: + replacement = self.replace_fn(self.bugdirs, ''.join(ids)) + except (MultipleIDMatches, NoIDMatches, InvalidIDStructure): + if self.strict: + raise + replacement = ''.join(ids) if self.wrap == True: return '#%s#' % replacement return replacement @@ -496,7 +503,8 @@ def short_to_long_text(bugdirs, text): short_to_long_user : conversion on a single ID long_to_short_text : inverse """ - return re.sub(REGEXP, IDreplacer(bugdirs, short_to_long_user), text) + return re.sub( + REGEXP, IDreplacer(bugdirs, short_to_long_user, strict=False), text) def long_to_short_text(bugdirs, text): """Convert long user IDs to short user IDs in text (see :class:`ID`). @@ -576,7 +584,7 @@ def _parse_user(id): ret[type] = arg return ret -def parse_user(bugdir, id): +def parse_user(bugdirs, id): """Parse a user ID (see :class:`ID`), returning a dict of parsed information. @@ -588,7 +596,7 @@ def parse_user(bugdir, id): This function tries to expand IDs before parsing, so it can handle both short and long IDs successfully. """ - long_id = short_to_long_user([bugdir], id) + long_id = short_to_long_user(bugdirs, id) return _parse_user(long_id) if libbe.TESTING == True: @@ -651,6 +659,7 @@ if libbe.TESTING == True: class ShortLongParseTestCase(unittest.TestCase): def setUp(self): self.bugdir = DummyObject('1234abcd') + self.bugdirs = {self.bugdir.uuid: self.bugdir} self.bug = DummyObject('abcdef', self.bugdir, ['a1234', 'ab9876']) self.comment = DummyObject('12345678', self.bug, ['1234abcd', '1234cdef']) self.bd_id = self.bugdir.id @@ -681,20 +690,25 @@ if libbe.TESTING == True: None, '123/abc', ['1234abcd','1234cdef','12345678'])), ] def test_short_to_long_text(self): - self.failUnless(short_to_long_text([self.bugdir], self.short) == self.long, - '\n' + self.short + '\n' + short_to_long_text([self.bugdir], self.short) + '\n' + self.long) + self.failUnless(short_to_long_text( + self.bugdirs, self.short) == self.long, + '\n' + self.short + '\n' + short_to_long_text( + self.bugdirs, self.short) + '\n' + self.long) def test_long_to_short_text(self): - self.failUnless(long_to_short_text([self.bugdir], self.long) == self.short, - '\n' + long_to_short_text([self.bugdir], self.long) + '\n' + self.short) + self.failUnless(long_to_short_text( + self.bugdirs, self.long) == self.short, + '\n' + long_to_short_text( + self.bugdirs, self.long + ) + '\n' + self.short) def test_parse_user(self): for short_id,parsed in self.short_id_parse_pairs: - ret = parse_user(self.bugdir, short_id) + ret = parse_user(self.bugdirs, short_id) self.failUnless(ret == parsed, 'got %s\nexpected %s' % (ret, parsed)) def test_parse_user_exceptions(self): for short_id,exception in self.short_id_exception_pairs: try: - ret = parse_user(self.bugdir, short_id) + ret = parse_user(self.bugdirs, short_id) self.fail('Expected parse_user(bugdir, "%s") to raise %s,' '\n but it returned %s' % (short_id, exception.__class__.__name__, ret)) diff --git a/libbe/util/plugin.py b/libbe/util/plugin.py index 211e608..ca5686d 100644 --- a/libbe/util/plugin.py +++ b/libbe/util/plugin.py @@ -27,6 +27,7 @@ submodules (i.e. "plugins"). import os import os.path import sys +import zipfile _PLUGIN_PATH = os.path.realpath( @@ -52,6 +53,14 @@ def import_by_name(modname): module = getattr(module, comp) return module +def zip_listdir(path, components): + """Lists items in a directory contained in a zip file + """ + dirpath = '/'.join(components) + with zipfile.ZipFile(path, 'r') as f: + return [os.path.relpath(f, dirpath) for f in f.namelist() + if f.startswith(dirpath)] + def modnames(prefix): """ >>> 'list' in [n for n in modnames('libbe.command')] @@ -60,10 +69,17 @@ def modnames(prefix): True """ components = prefix.split('.') - modfiles = os.listdir(os.path.join(_PLUGIN_PATH, *components)) - modfiles.sort() + egg = os.path.isfile(_PLUGIN_PATH) # are we inside a zip archive (egg)? + if egg: + modfiles = zip_listdir(_PLUGIN_PATH, components) + else: + modfiles = os.listdir(os.path.join(_PLUGIN_PATH, *components)) + # normalize .py/.pyc extensions and sort + base_ext = [os.path.splitext(f) for f in modfiles] + modfiles = sorted(set( + base + '.py' for base,ext in base_ext if ext in ['.py', '.pyc'])) for modfile in modfiles: - if modfile.startswith('.'): - continue # the occasional emacs temporary file + if modfile.startswith('.') or not modfile: + continue # the occasional emacs temporary file or .* file if modfile.endswith('.py') and modfile != '__init__.py': yield modfile[:-3] diff --git a/libbe/util/subproc.py b/libbe/util/subproc.py index 3a66f49..0bda520 100644 --- a/libbe/util/subproc.py +++ b/libbe/util/subproc.py @@ -95,144 +95,5 @@ def invoke(args, stdin=None, stdout=PIPE, stderr=PIPE, expect=(0,), raise CommandError(list_args, status, stdout, stderr) return status, stdout, stderr -class Pipe (object): - """Simple interface for executing POSIX-style pipes. - - Based on the `subprocess` module. The only complication is the - adaptation of `subprocess.Popen._communicate` to listen to the - stderrs of all processes involved in the pipe, as well as the - terminal process' stdout. There are two implementations of - `Pipe._communicate`, one for MS Windows, and one for POSIX - systems. The MS Windows implementation is currently untested. - - >>> p = Pipe([['find', '/etc/'], ['grep', '^/etc/ssh$']]) - >>> p.stdout - '/etc/ssh\\n' - >>> p.status - 1 - >>> p.statuses - [1, 0] - >>> p.stderrs # doctest: +ELLIPSIS - [...find: ...: Permission denied..., ''] - """ - def __init__(self, cmds, stdin=None): - # spawn processes - self._procs = [] - for cmd in cmds: - if len(self._procs) != 0: - stdin = self._procs[-1].stdout - self._procs.append(Popen(cmd, stdin=stdin, stdout=PIPE, stderr=PIPE)) - - self.stdout,self.stderrs = self._communicate(input=None) - - # collect process statuses - self.statuses = [] - self.status = 0 - for proc in self._procs: - self.statuses.append(proc.wait()) - if self.statuses[-1] != 0: - self.status = self.statuses[-1] - - # Code excerpted from subprocess.Popen._communicate() - if _MSWINDOWS == True: - def _communicate(self, input=None): - assert input == None, 'stdin != None not yet supported' - # listen to each process' stderr - threads = [] - std_X_arrays = [] - for proc in self._procs: - stderr_array = [] - thread = Thread(target=proc._readerthread, - args=(proc.stderr, stderr_array)) - thread.setDaemon(True) - thread.start() - threads.append(thread) - std_X_arrays.append(stderr_array) - - # also listen to the last processes stdout - stdout_array = [] - thread = Thread(target=proc._readerthread, - args=(proc.stdout, stdout_array)) - thread.setDaemon(True) - thread.start() - threads.append(thread) - std_X_arrays.append(stdout_array) - - # join threads as they die - for thread in threads: - thread.join() - - # read output from reader threads - std_X_strings = [] - for std_X_array in std_X_arrays: - std_X_strings.append(std_X_array[0]) - - stdout = std_X_strings.pop(-1) - stderrs = std_X_strings - return (stdout, stderrs) - else: - assert _POSIX==True, 'invalid platform' - def _communicate(self, input=None): - read_set = [] - write_set = [] - read_arrays = [] - stdout = None # Return - stderr = None # Return - - if self._procs[0].stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self._procs[0].stdin.flush() - if input: - write_set.append(self._procs[0].stdin) - else: - self._procs[0].stdin.close() - for proc in self._procs: - read_set.append(proc.stderr) - read_arrays.append([]) - read_set.append(self._procs[-1].stdout) - read_arrays.append([]) - - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, xlist = select.select(read_set, write_set, []) - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - if self._procs[0].stdin in wlist: - # When select has indicated that the file is writable, - # we can write up to PIPE_BUF bytes without risk - # blocking. POSIX defines PIPE_BUF >= 512 - chunk = input[input_offset : input_offset + 512] - bytes_written = os.write(self.stdin.fileno(), chunk) - input_offset += bytes_written - if input_offset >= len(input): - self._procs[0].stdin.close() - write_set.remove(self._procs[0].stdin) - if self._procs[-1].stdout in rlist: - data = os.read(self._procs[-1].stdout.fileno(), 1024) - if data == '': - self._procs[-1].stdout.close() - read_set.remove(self._procs[-1].stdout) - read_arrays[-1].append(data) - for i,proc in enumerate(self._procs): - if proc.stderr in rlist: - data = os.read(proc.stderr.fileno(), 1024) - if data == '': - proc.stderr.close() - read_set.remove(proc.stderr) - read_arrays[i].append(data) - - # All data exchanged. Translate lists into strings. - read_strings = [] - for read_array in read_arrays: - read_strings.append(''.join(read_array)) - - stdout = read_strings.pop(-1) - stderrs = read_strings - return (stdout, stderrs) - if libbe.TESTING == True: suite = doctest.DocTestSuite() diff --git a/libbe/command/serve.py b/libbe/util/wsgi.py index 09ff0ff..9887830 100644 --- a/libbe/command/serve.py +++ b/libbe/util/wsgi.py @@ -1,45 +1,24 @@ -# Copyright (C) 2010-2012 Chris Ball <cjb@laptop.org> -# W. Trevor King <wking@drexel.edu> -# -# This file is part of Bugs Everywhere. -# -# Bugs Everywhere is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 2 of the License, or (at your option) any -# later version. -# -# Bugs Everywhere is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>. - -"""Define the :class:`Serve` serving BE Storage over HTTP. +# Copyright + +"""Utilities for building WSGI commands. See Also -------- -:mod:`libbe.storage.http` : the associated client +:py:mod:`libbe.command.serve_storage` and +:py:mod:`libbe.command.serve_commands`. """ import hashlib import logging -import os.path -import posixpath import re import sys import time import traceback import types import urllib +import urlparse import wsgiref.simple_server -try: - # Python >= 2.6 - from urlparse import parse_qs -except ImportError: - # Python <= 2.5 - from cgi import parse_qs + try: import cherrypy import cherrypy.wsgiserver @@ -50,17 +29,18 @@ if cherrypy != None: import cherrypy.wsgiserver.ssl_builtin except ImportError: # CherryPy <= 3.1.X cherrypy.wsgiserver.ssl_builtin = None + try: import OpenSSL except ImportError: OpenSSL = None -import libbe -import libbe.command -import libbe.command.util + import libbe.util.encoding -import libbe.util.subproc -import libbe.version +import libbe.command +import libbe.command.base +import libbe.storage + if libbe.TESTING == True: import copy @@ -74,71 +54,86 @@ if libbe.TESTING == True: except ImportError: cherrypy_test_webtest = None - import libbe.bugdir - -class _HandlerError (Exception): + +class HandlerError (Exception): def __init__(self, code, msg, headers=[]): - Exception.__init__(self, '%d %s' % (code, msg)) + super(HandlerError, self).__init__('{} {}'.format(code, msg)) self.code = code self.msg = msg self.headers = headers -class _Unauthenticated (_HandlerError): + +class Unauthenticated (HandlerError): def __init__(self, realm, msg='User Not Authenticated', headers=[]): - _HandlerError.__init__(self, 401, msg, headers+[ - ('WWW-Authenticate','Basic realm="%s"' % realm)]) + super(Unauthenticated, self).__init__(401, msg, headers+[ + ('WWW-Authenticate','Basic realm="{}"'.format(realm))]) + -class _Unauthorized (_HandlerError): +class Unauthorized (HandlerError): def __init__(self, msg='User Not Authorized', headers=[]): - _HandlerError.__init__(self, 403, msg, headers) + super(Unauthorized, self).__init__(403, msg, headers) + class User (object): def __init__(self, uname=None, name=None, passhash=None, password=None): self.uname = uname self.name = name self.passhash = passhash - if passhash == None: - if password != None: + if passhash is None: + if password is not None: self.passhash = self.hash(password) else: - assert password == None, \ - 'Redundant password %s with passhash %s' % (password, passhash) + assert password is None, ( + 'Redundant password {} with passhash {}'.format( + password, passhash)) self.users = None + def from_string(self, string): string = string.strip() fields = string.split(':') if len(fields) != 3: - raise ValueError, '%d!=3 fields in "%s"' % (len(fields), string) + raise ValueError, '{}!=3 fields in "{}"'.format( + len(fields), string) self.uname,self.name,self.passhash = fields + def __str__(self): return ':'.join([self.uname, self.name, self.passhash]) + def __cmp__(self, other): return cmp(self.uname, other.uname) + def hash(self, password): return hashlib.sha1(password).hexdigest() + def valid_login(self, password): if self.hash(password) == self.passhash: return True return False + def set_name(self, name): self._set_property('name', name) + def set_password(self, password): self._set_property('passhash', self.hash(password)) + def _set_property(self, property, value): if self.uname == 'guest': - raise _Unauthorized('guest user not allowed to change %s' % property) - if getattr(self, property) != value \ - and self.users != None: + raise Unauthorized( + 'guest user not allowed to change {}'.format(property)) + if (getattr(self, property) != value and + self.users is not None): self.users.changed = True setattr(self, property, value) + class Users (dict): def __init__(self, filename=None): - dict.__init__(self) + super(Users, self).__init__() self.filename = filename self.changed = False + def load(self): - if self.filename == None: + if self.filename is None: return user_file = libbe.util.encoding.get_file_contents( self.filename, decode=True) @@ -147,22 +142,24 @@ class Users (dict): user = User() user.from_string(line) self.add_user(user) + def save(self): - if self.filename != None and self.changed == True: + if self.filename is not None and self.changed: lines = [] for user in sorted(self.users): lines.append(str(user)) libbe.util.encoding.set_file_contents(self.filename) self.changed = False + def add_user(self, user): - assert user.users == None, user.users + assert user.users is None, user.users user.users = self self[user.uname] = user + def valid_login(self, uname, password): - if uname in self and \ - self[uname].valid_login(password) == True: - return True - return False + return (uname in self and + self[uname].valid_login(password)) + class WSGI_Object (object): """Utility class for WGSI clients and middleware. @@ -174,15 +171,25 @@ class WSGI_Object (object): def __init__(self, logger=None, log_level=logging.INFO, log_format=None): self.logger = logger self.log_level = log_level - if log_format == None: + if log_format is None: self.log_format = ( - '%(REMOTE_ADDR)s - %(REMOTE_USER)s [%(time)s] ' - '"%(REQUEST_METHOD)s %(REQUEST_URI)s %(HTTP_VERSION)s" ' - '%(status)s %(bytes)s "%(HTTP_REFERER)s" "%(HTTP_USER_AGENT)s"') + '{REMOTE_ADDR} - {REMOTE_USER} [{time}] ' + '"{REQUEST_METHOD} {REQUEST_URI} {HTTP_VERSION}" ' + '{status} {bytes} "{HTTP_REFERER}" "{HTTP_USER_AGENT}"') else: self.log_format = log_format def __call__(self, environ, start_response): + if self.logger is not None: + self.logger.log( + logging.DEBUG, 'entering {}'.format(self.__class__.__name__)) + ret = self._call(environ, start_response) + if self.logger is not None: + self.logger.log( + logging.DEBUG, 'leaving {}'.format(self.__class__.__name__)) + return ret + + def _call(self, environ, start_response): """The main WSGI entry point.""" raise NotImplementedError # start_response() is a callback for setting response headers @@ -196,31 +203,31 @@ class WSGI_Object (object): def error(self, environ, start_response, error, message, headers=[]): """Make it easy to call start_response for errors.""" - response = '%d %s' % (error, message) + response = '{} {}'.format(error, message) self.log_request(environ, status=response, bytes=len(message)) start_response(response, [('Content-Type', 'text/plain')]+headers) return [message] def log_request(self, environ, status='-1 OK', bytes=-1): - if self.logger == None: + if self.logger is None or self.logger.level > self.log_level: return req_uri = urllib.quote(environ.get('SCRIPT_NAME', '') + environ.get('PATH_INFO', '')) if environ.get('QUERY_STRING'): - req_uri += '?'+environ['QUERY_STRING'] + req_uri += '?' + environ['QUERY_STRING'] start = time.localtime() if time.daylight: offset = time.altzone / 60 / 60 * -100 else: offset = time.timezone / 60 / 60 * -100 if offset >= 0: - offset = "+%0.4d" % (offset) + offset = '+{:04d}'.format(offset) elif offset < 0: - offset = "%0.4d" % (offset) + offset = '{:04d}'.format(offset) d = { - 'REMOTE_ADDR': environ.get('REMOTE_ADDR') or '-', - 'REMOTE_USER': environ.get('REMOTE_USER') or '-', + 'REMOTE_ADDR': environ.get('REMOTE_ADDR', '-'), + 'REMOTE_USER': environ.get('REMOTE_USER', '-'), 'REQUEST_METHOD': environ['REQUEST_METHOD'], 'REQUEST_URI': req_uri, 'HTTP_VERSION': environ.get('SERVER_PROTOCOL'), @@ -230,20 +237,26 @@ class WSGI_Object (object): 'HTTP_REFERER': environ.get('HTTP_REFERER', '-'), 'HTTP_USER_AGENT': environ.get('HTTP_USER_AGENT', '-'), } - self.logger.log(self.log_level, self.log_format % d) + self.logger.log(self.log_level, self.log_format.format(**d)) -class ExceptionApp (WSGI_Object): - """Some servers (e.g. cherrypy) eat app-raised exceptions. - Work around that by logging tracebacks by hand. +class WSGI_Middleware (WSGI_Object): + """Utility class for WGSI middleware. """ def __init__(self, app, *args, **kwargs): - WSGI_Object.__init__(self, *args, **kwargs) + super(WSGI_Middleware, self).__init__(*args, **kwargs) self.app = app - def __call__(self, environ, start_response): - if self.logger != None: - self.logger.log(logging.DEBUG, 'ExceptionApp') + def _call(self, environ, start_response): + return self.app(environ, start_response) + + +class ExceptionApp (WSGI_Middleware): + """Some servers (e.g. cherrypy) eat app-raised exceptions. + + Work around that by logging tracebacks by hand. + """ + def _call(self, environ, start_response): try: return self.app(environ, start_response) except Exception, e: @@ -253,7 +266,27 @@ class ExceptionApp (WSGI_Object): self.logger.log(self.log_level, trace) raise -class UppercaseHeaderApp (WSGI_Object): + +class BEExceptionApp (WSGI_Middleware): + """Translate BE-specific exceptions + """ + def __init__(self, *args, **kwargs): + super(BEExceptionApp, self).__init__(*args, **kwargs) + self.http_user_error = 418 + + def _call(self, environ, start_response): + try: + return self.app(environ, start_response) + except libbe.storage.NotReadable as e: + raise libbe.util.wsgi.HandlerError(403, 'Read permission denied') + except libbe.storage.NotWriteable as e: + raise libbe.util.wsgi.HandlerError(403, 'Write permission denied') + except libbe.storage.InvalidID as e: + raise libbe.util.wsgi.HandlerError( + self.http_user_error, 'InvalidID {}'.format(e)) + + +class UppercaseHeaderApp (WSGI_Middleware): """WSGI middleware that uppercases incoming HTTP headers. From PEP 333, `The start_response() Callable`_ : @@ -266,13 +299,7 @@ class UppercaseHeaderApp (WSGI_Object): .. _The start_response() Callable: http://www.python.org/dev/peps/pep-0333/#id20 """ - def __init__(self, app, *args, **kwargs): - WSGI_Object.__init__(self, *args, **kwargs) - self.app = app - - def __call__(self, environ, start_response): - if self.logger != None: - self.logger.log(logging.DEBUG, 'UppercaseHeaderApp') + def _call(self, environ, start_response): for key,value in environ.items(): if key.startswith('HTTP_'): uppercase = key.upper() @@ -280,33 +307,30 @@ class UppercaseHeaderApp (WSGI_Object): environ[uppercase] = environ.pop(key) return self.app(environ, start_response) -class AuthenticationApp (WSGI_Object): + +class AuthenticationApp (WSGI_Middleware): """WSGI middleware for handling user authentication. """ - def __init__(self, app, realm, setting='be-auth', users=None, *args, **kwargs): - WSGI_Object.__init__(self, *args, **kwargs) - self.app = app + def __init__(self, realm, setting='be-auth', users=None, *args, **kwargs): + super(AuthenticationApp, self).__init__(*args, **kwargs) self.realm = realm self.setting = setting self.users = users - def __call__(self, environ, start_response): - if self.logger != None: - self.logger.log(logging.DEBUG, 'AuthenticationApp') - environ['%s.realm' % self.setting] = self.realm + def _call(self, environ, start_response): + environ['{}.realm'.format(self.setting)] = self.realm try: username = self.authenticate(environ) - environ['%s.user' % self.setting] = username - environ['%s.user.name' % self.setting] = \ - self.users[username].name + environ['{}.user'.format(self.setting)] = username + environ['{}.user.name'.format(self.setting)] = self.users[username].name return self.app(environ, start_response) - except _Unauthorized, e: + except Unauthorized, e: return self.error(environ, start_response, e.code, e.msg, e.headers) def authenticate(self, environ): """Handle user-authentication sent in the "Authorization" header. - + This function implements ``Basic`` authentication as described in HTTP/1.0 specification [1]_ . Do not use this module unless you are using SSL, as it transmits unencrypted passwords. @@ -332,38 +356,39 @@ class AuthenticationApp (WSGI_Object): http://www.opensource.org/licenses/mit-license.php """ authorization = environ.get('HTTP_AUTHORIZATION', None) - if authorization == None: - raise _Unauthorized('Authorization required') + if authorization is None: + raise Unauthorized('Authorization required') try: - authmeth,auth = authorization.split(' ',1) + authmeth,auth = authorization.split(' ', 1) except ValueError: return None if 'basic' != authmeth.lower(): - return None # non-basic HTTP authorization not implemented + return None # non-basic HTTP authorization not implemented auth = auth.strip().decode('base64') try: - username,password = auth.split(':',1) + username,password = auth.split(':', 1) except ValueError: return None - if self.authfunc(environ, username, password) == True: + if self.authfunc(environ, username, password): return username def authfunc(self, environ, username, password): if not username in self.users: return False - if self.users[username].valid_login(password) == True: - if self.logger != None: + if self.users[username].valid_login(password): + if self.logger is not None: self.logger.log(self.log_level, - 'Authenticated %s' % self.users[username].name) + 'Authenticated {}'.format(self.users[username].name)) return True return False -class WSGI_AppObject (WSGI_Object): + +class WSGI_DataObject (WSGI_Object): """Useful WSGI utilities for handling data (POST, QUERY) and returning responses. """ def __init__(self, *args, **kwargs): - WSGI_Object.__init__(self, *args, **kwargs) + super(WSGI_DataObject, self).__init__(*args, **kwargs) # Maximum input we will accept when REQUEST_METHOD is POST # 0 ==> unlimited input @@ -372,10 +397,10 @@ class WSGI_AppObject (WSGI_Object): def ok_response(self, environ, start_response, content, content_type='application/octet-stream', headers=[]): - if content == None: + if content is None: start_response('200 OK', []) return [] - if type(content) == types.UnicodeType: + if type(content) is types.UnicodeType: content = content.encode('utf-8') for i,header in enumerate(headers): header_name,header_value = header @@ -384,23 +409,23 @@ class WSGI_AppObject (WSGI_Object): response = '200 OK' content_length = len(content) self.log_request(environ, status=response, bytes=content_length) - start_response('200 OK', [ + start_response(response, [ ('Content-Type', content_type), ('Content-Length', str(content_length)), ]+headers) - if self.is_head(environ) == True: + if self.is_head(environ): return [] return [content] def query_data(self, environ): if not environ['REQUEST_METHOD'] in ['GET', 'HEAD']: - raise _HandlerError(404, 'Not Found') + raise HandlerError(404, 'Not Found') return self._parse_query(environ.get('QUERY_STRING', '')) def _parse_query(self, query): if len(query) == 0: return {} - data = parse_qs( + data = urlparse.parse_qs( query, keep_blank_values=True, strict_parsing=True) for k,v in data.items(): if len(v) == 1: @@ -409,7 +434,7 @@ class WSGI_AppObject (WSGI_Object): def post_data(self, environ): if environ['REQUEST_METHOD'] != 'POST': - raise _HandlerError(404, 'Not Found') + raise HandlerError(404, 'Not Found') post_data = self._read_post_data(environ) return self._parse_post(post_data) @@ -429,12 +454,13 @@ class WSGI_AppObject (WSGI_Object): def data_get_string(self, data, key, default=None, source='query'): if not key in data or data[key] in [None, 'None']: - if default == _HandlerError: - raise _HandlerError(406, 'Missing %s key %s' % (source, key)) + if default == HandlerError: + raise HandlerError( + 406, 'Missing {} key {}'.format(source, key)) return default return data[key] - def data_get_id(self, data, key='id', default=_HandlerError, + def data_get_id(self, data, key='id', default=HandlerError, source='query'): return self.data_get_string(data, key, default, source) @@ -450,314 +476,76 @@ class WSGI_AppObject (WSGI_Object): return environ['REQUEST_METHOD'] == 'HEAD' -class AdminApp (WSGI_AppObject): - """WSGI middleware for managing users (changing passwords, - usernames, etc.). +class WSGI_AppObject (WSGI_Object): + """Useful WSGI utilities for handling URL delegation. """ - def __init__(self, app, users=None, url=r'^admin/?', *args, **kwargs): - WSGI_AppObject.__init__(self, *args, **kwargs) - self.app = app - self.users = users - self.url = url + def __init__(self, urls=tuple(), default_handler=None, setting='be-server', + *args, **kwargs): + super(WSGI_AppObject, self).__init__(*args, **kwargs) + self.urls = [(re.compile(regexp),callback) for regexp,callback in urls] + self.default_callback = default_handler + self.setting = setting - def __call__(self, environ, start_response): - if self.logger != None: - self.logger.log(logging.DEBUG, 'AdminApp') + def _call(self, environ, start_response): path = environ.get('PATH_INFO', '').lstrip('/') - match = re.search(self.url, path) - if match is not None: - return self.admin(environ, start_response) - return self.app(environ, start_response) + for regexp,callback in self.urls: + match = regexp.match(path) + if match is not None: + setting = '{}.url_args'.format(self.setting) + environ[setting] = match.groups() + return callback(environ, start_response) + if self.default_handler is None: + raise HandlerError(404, 'Not Found') + return self.default_handler(environ, start_response) + + +class AdminApp (WSGI_AppObject, WSGI_DataObject, WSGI_Middleware): + """WSGI middleware for managing users + + Changing passwords, usernames, etc. + """ + def __init__(self, users=None, setting='be-auth', *args, **kwargs): + handler = ('^admin/?', self.admin) + if 'urls' not in kwargs: + kwargs['urls'] = [handler] + else: + kwargs.urls.append(handler) + super(AdminApp, self).__init__(*args, **kwargs) + self.users = users + self.setting = setting def admin(self, environ, start_response): - if not 'be-auth.user' in environ: - raise _Unauthenticated(realm=envirion.get('be-auth.realm')) - uname = environ.get('be-auth.user') + if not '{}.user'.format(self.setting) in environ: + realm = envirion.get('{}.realm'.format(self.setting)) + raise Unauthenticated(realm=realm) + uname = environ.get('{}.user'.format(self.setting)) user = self.users[uname] data = self.post_data(environ) source = 'post' name = self.data_get_string( data, 'name', default=None, source=source) - if name != None: + if name is not None: self.users[uname].set_name(name) password = self.data_get_string( data, 'password', default=None, source=source) - if password != None: + if password is not None: self.users[uname].set_password(password) self.users.save() return self.ok_response(environ, start_response, None) -class ServerApp (WSGI_AppObject): - """WSGI server for a BE Storage instance over HTTP. - RESTful_ WSGI request handler for serving the - libbe.storage.http.HTTP backend with GET, POST, and HEAD commands. - For more information on authentication and REST, see John - Calcote's `Open Sourcery article`_ +class SilentRequestHandler (wsgiref.simple_server.WSGIRequestHandler): + def log_message(self, format, *args): + pass - .. _RESTful: http://www.ics.uci.edu/~fielding/pubs/dissertation/rest_arch_style.htm - .. _Open Sourcery article: http://jcalcote.wordpress.com/2009/08/10/restful-authentication/ - This serves files from a connected storage instance, usually - a VCS-based repository located on the local machine. +class ServerCommand (libbe.command.base.Command): + """Serve something over HTTP. - Notes - ----- - - The GET and HEAD requests are identical except that the HEAD - request omits the actual content of the file. + Use this as a base class to build commands that serve a web interface. """ - server_version = "BE-server/" + libbe.version.version() - - def __init__(self, storage, notify=False, **kwargs): - WSGI_AppObject.__init__(self, **kwargs) - self.storage = storage - self.notify = notify - self.http_user_error = 418 - - self.urls = [ - (r'^add/?', self.add), - (r'^exists/?', self.exists), - (r'^remove/?', self.remove), - (r'^ancestors/?', self.ancestors), - (r'^children/?', self.children), - (r'^get/(.+)', self.get), - (r'^set/(.+)', self.set), - (r'^commit/?', self.commit), - (r'^revision-id/?', self.revision_id), - (r'^changed/?', self.changed), - (r'^version/?', self.version), - ] - - def __call__(self, environ, start_response): - """The main WSGI application. - - Dispatch the current request to the functions from above and - store the regular expression captures in the WSGI environment - as `be-server.url_args` so that the functions from above can - access the url placeholders. - - URL dispatcher from Armin Ronacher's "Getting Started with WSGI" - http://lucumr.pocoo.org/2007/5/21/getting-started-with-wsgi - """ - if self.logger != None: - self.logger.log(logging.DEBUG, 'ServerApp') - path = environ.get('PATH_INFO', '').lstrip('/') - try: - for regex, callback in self.urls: - match = re.search(regex, path) - if match is not None: - environ['be-server.url_args'] = match.groups() - try: - return callback(environ, start_response) - except libbe.storage.NotReadable, e: - raise _HandlerError(403, 'Read permission denied') - except libbe.storage.NotWriteable, e: - raise _HandlerError(403, 'Write permission denied') - except libbe.storage.InvalidID, e: - raise _HandlerError( - self.http_user_error, 'InvalidID %s' % e) - raise _HandlerError(404, 'Not Found') - except _HandlerError, e: - return self.error(environ, start_response, - e.code, e.msg, e.headers) - - # handlers - def add(self, environ, start_response): - self.check_login(environ) - data = self.post_data(environ) - source = 'post' - id = self.data_get_id(data, source=source) - parent = self.data_get_string( - data, 'parent', default=None, source=source) - directory = self.data_get_boolean( - data, 'directory', default=False, source=source) - self.storage.add(id, parent=parent, directory=directory) - if self.notify: - self._notify(environ, 'add', id, - [('parent', parent), ('directory', directory)]) - return self.ok_response(environ, start_response, None) - - def exists(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - id = self.data_get_id(data, source=source) - revision = self.data_get_string( - data, 'revision', default=None, source=source) - content = str(self.storage.exists(id, revision)) - return self.ok_response(environ, start_response, content) - - def remove(self, environ, start_response): - self.check_login(environ) - data = self.post_data(environ) - source = 'post' - id = self.data_get_id(data, source=source) - recursive = self.data_get_boolean( - data, 'recursive', default=False, source=source) - if recursive == True: - self.storage.recursive_remove(id) - else: - self.storage.remove(id) - if self.notify: - self._notify(environ, 'remove', id, [('recursive', recursive)]) - return self.ok_response(environ, start_response, None) - - def ancestors(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - id = self.data_get_id(data, source=source) - revision = self.data_get_string( - data, 'revision', default=None, source=source) - content = '\n'.join(self.storage.ancestors(id, revision))+'\n' - return self.ok_response(environ, start_response, content) - - def children(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - id = self.data_get_id(data, default=None, source=source) - revision = self.data_get_string( - data, 'revision', default=None, source=source) - content = '\n'.join(self.storage.children(id, revision)) - return self.ok_response(environ, start_response, content) - - def get(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - try: - id = environ['be-server.url_args'][0] - except: - raise _HandlerError(404, 'Not Found') - revision = self.data_get_string( - data, 'revision', default=None, source=source) - content = self.storage.get(id, revision=revision) - be_version = self.storage.storage_version(revision) - return self.ok_response(environ, start_response, content, - headers=[('X-BE-Version', be_version)]) - - def set(self, environ, start_response): - self.check_login(environ) - data = self.post_data(environ) - try: - id = environ['be-server.url_args'][0] - except: - raise _HandlerError(404, 'Not Found') - if not 'value' in data: - raise _HandlerError(406, 'Missing query key value') - value = data['value'] - self.storage.set(id, value) - if self.notify: - self._notify(environ, 'set', id, [('value', value)]) - return self.ok_response(environ, start_response, None) - - def commit(self, environ, start_response): - self.check_login(environ) - data = self.post_data(environ) - if not 'summary' in data: - raise _HandlerError(406, 'Missing query key summary') - summary = data['summary'] - if not 'body' in data or data['body'] == 'None': - data['body'] = None - body = data['body'] - if not 'allow_empty' in data \ - or data['allow_empty'] == 'True': - allow_empty = True - else: - allow_empty = False - try: - revision = self.storage.commit(summary, body, allow_empty) - except libbe.storage.EmptyCommit, e: - raise _HandlerError(self.http_user_error, 'EmptyCommit') - if self.notify: - self._notify(environ, 'commit', id, - [('allow_empty', allow_empty), ('summary', summary), - ('body', body)]) - return self.ok_response(environ, start_response, revision) - - def revision_id(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - index = int(self.data_get_string( - data, 'index', default=_HandlerError, source=source)) - content = self.storage.revision_id(index) - return self.ok_response(environ, start_response, content) - - def changed(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - revision = self.data_get_string( - data, 'revision', default=None, source=source) - add,mod,rem = self.storage.changed(revision) - content = '\n\n'.join(['\n'.join(p) for p in (add,mod,rem)]) - return self.ok_response(environ, start_response, content) - - def version(self, environ, start_response): - self.check_login(environ) - data = self.query_data(environ) - source = 'query' - revision = self.data_get_string( - data, 'revision', default=None, source=source) - content = self.storage.storage_version(revision) - return self.ok_response(environ, start_response, content) - - # handler utility functions - def check_login(self, environ): - user = environ.get('be-auth.user', None) - if user != None: # we're running under AuthenticationApp - if environ['REQUEST_METHOD'] == 'POST': - if user == 'guest' or self.storage.is_writeable() == False: - raise _Unauthorized() # only non-guests allowed to write - # allow read-only commands for all users - - def _notify(self, environ, command, id, params): - message = self._format_notification(environ, command, id, params) - self._submit_notification(message) - - def _format_notification(self, environ, command, id, params): - key_length = len('command') - for key,value in params: - if len(key) > key_length and '\n' not in str(value): - key_length = len(key) - key_length += 1 - lines = [] - multi_line_params = [] - for key,value in [('address', environ.get('REMOTE_ADDR', '-')), - ('command', command), ('id', id)]+params: - v = str(value) - if '\n' in v: - multi_line_params.append((key,v)) - continue - lines.append('%*.*s %s' % (key_length, key_length, key+':', v)) - lines.append('') - for key,value in multi_line_params: - lines.extend(['=== START %s ===' % key, v, - '=== STOP %s ===' % key, '']) - lines.append('') - return '\n'.join(lines) - - def _submit_notification(self, message): - libbe.util.subproc.invoke(self.notify, stdin=message, shell=True) - - -class Serve (libbe.command.Command): - """Serve bug directory storage over HTTP. - - This allows you to run local `be` commands interfacing with remote - data, transmitting file reads/writes/etc. over the network. - - :class:`~libbe.command.base.Command` wrapper around - :class:`ServerApp`. - """ - - name = 'serve' - def __init__(self, *args, **kwargs): - libbe.command.Command.__init__(self, *args, **kwargs) + super(ServerCommand, self).__init__(*args, **kwargs) self.options.extend([ libbe.command.Option(name='port', help='Bind server to port (%default)', @@ -766,7 +554,7 @@ class Serve (libbe.command.Command): libbe.command.Option(name='host', help='Set host string (blank for localhost, %default)', arg=libbe.command.Argument( - name='host', metavar='HOST', default='')), + name='host', metavar='HOST', default='localhost')), libbe.command.Option(name='read-only', short_name='r', help='Dissable operations that require writing'), libbe.command.Option(name='notify', short_name='n', @@ -776,7 +564,11 @@ class Serve (libbe.command.Command): libbe.command.Option(name='ssl', short_name='s', help='Use CherryPy to serve HTTPS (HTTP over SSL/TLS)'), libbe.command.Option(name='auth', short_name='a', - help='Require authentication. FILE should be a file containing colon-separated UNAME:USER:sha1(PASSWORD) lines, for example: "jdoe:John Doe <jdoe@example.com>:read:d99f8e5a4b02dc25f49da2ea67c0034f61779e72"', + help=('Require authentication. FILE should be a file ' + 'containing colon-separated ' + 'UNAME:USER:sha1(PASSWORD) lines, for example: ' + '"jdoe:John Doe <jdoe@example.com>:' + 'd99f8e5a4b02dc25f49da2ea67c0034f61779e72"'), arg=libbe.command.Argument( name='auth', metavar='FILE', default=None, completion_callback=libbe.command.util.complete_path)), @@ -785,18 +577,15 @@ class Serve (libbe.command.Command): def _run(self, **params): self._setup_logging() storage = self._get_storage() - if params['read-only'] == True: + if params['read-only']: writeable = storage.writeable storage.writeable = False - if params['host'] == '': - params['host'] = 'localhost' - if params['auth'] != None: + if params['auth']: self._check_restricted_access(storage, params['auth']) users = Users(params['auth']) users.load() - app = ServerApp( - storage=storage, notify=params['notify'], logger=self.logger) - if params['auth'] != None: + app = self._get_app(logger=self.logger, storage=storage, **params) + if params['auth']: app = AdminApp(app, users=users, logger=self.logger) app = AuthenticationApp(app, realm=storage.repo, users=users, logger=self.logger) @@ -808,11 +597,14 @@ class Serve (libbe.command.Command): except KeyboardInterrupt: pass self._stop_server(params, server) - if params['read-only'] == True: + if params['read-only']: storage.writeable = writeable + def _get_app(self, logger, storage, **kwargs): + raise NotImplementedError() + def _setup_logging(self, log_level=logging.INFO): - self.logger = logging.getLogger('be-serve') + self.logger = logging.getLogger('be-{}'.format(self.name)) self.log_level = logging.INFO console = logging.StreamHandler(self.stdout) console.setFormatter(logging.Formatter('%(message)s')) @@ -823,42 +615,43 @@ class Serve (libbe.command.Command): self.logger.setLevel(log_level) def _get_server(self, params, app): - details = {'port':params['port']} + details = { + 'socket-name':params['host'], + 'port':params['port'], + } + app = BEExceptionApp(app, logger=self.logger) + app = ExceptionApp(app, logger=self.logger) if params['ssl'] == True: details['protocol'] = 'HTTPS' if cherrypy == None: - raise libbe.command.UserError, \ - '--ssl requires the cherrypy module' - app = ExceptionApp(app, logger=self.logger) + raise libbe.command.UserError( + '--ssl requires the cherrypy module') server = cherrypy.wsgiserver.CherryPyWSGIServer( (params['host'], params['port']), app) #server.throw_errors = True #server.show_tracebacks = True - private_key,certificate = get_cert_filenames( + private_key,certificate = _get_cert_filenames( 'be-server', logger=self.logger) if cherrypy.wsgiserver.ssl_builtin == None: server.ssl_module = 'builtin' server.ssl_private_key = private_key server.ssl_certificate = certificate else: - server.ssl_adapter = \ + server.ssl_adapter = ( cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter( - certificate=certificate, private_key=private_key) - details['socket-name'] = params['host'] + certificate=certificate, private_key=private_key)) else: details['protocol'] = 'HTTP' server = wsgiref.simple_server.make_server( - params['host'], params['port'], app) - details['socket-name'] = server.socket.getsockname()[0] + params['host'], params['port'], app, + handler_class=SilentRequestHandler) return (server, details) def _start_server(self, params, server, details): self.logger.log(self.log_level, - 'Serving %(protocol)s on %(socket-name)s port %(port)s ...' \ - % details) - self.logger.log(self.log_level, - 'BE repository %(repo)s' % details) - if params['ssl'] == True: + ('Serving {protocol} on {socket-name} port {port} ...\n' + 'BE repository {repo}').format(**details)) + if params['ssl']: server.start() else: server.serve_forever() @@ -871,40 +664,14 @@ class Serve (libbe.command.Command): server.server_close() def _long_help(self): - return """ -Example usage:: - - $ be serve - -And in another terminal (or after backgrounding the server):: - - $ be --repo http://localhost:8000/ list - -If you bind your server to a public interface, take a look at the -``--read-only`` option or the combined ``--ssl --auth FILE`` -options so other people can't mess with your repository. If you do use -authentication, you'll need to send in your username and password with, -for example:: - - $ be --repo http://username:password@localhost:8000/ list -""" + raise NotImplementedError() -def random_string(length=256): - if os.path.exists(os.path.join('dev', 'urandom')): - return open("/dev/urandom").read(length) - else: - import array - from random import randint - d = array.array('B') - for i in xrange(1000000): - d.append(randint(0,255)) - return d.tostring() -if libbe.TESTING == True: +if libbe.TESTING: class WSGITestCase (unittest.TestCase): def setUp(self): self.logstream = StringIO.StringIO() - self.logger = logging.getLogger('be-serve-test') + self.logger = logging.getLogger('be-wsgi-test') console = logging.StreamHandler(self.logstream) console.setFormatter(logging.Formatter('%(message)s')) self.logger.addHandler(console) @@ -930,32 +697,40 @@ if libbe.TESTING == True: 'wsgi.multiprocess':False, 'wsgi.run_once':False, } + def getURL(self, app, path='/', method='GET', data=None, - scheme='http', environ={}): + data_dict=None, scheme='http', environ={}): env = copy.copy(self.default_environ) env['PATH_INFO'] = path env['REQUEST_METHOD'] = method env['scheme'] = scheme - if data != None: - enc_data = urllib.urlencode(data) + if data_dict is not None: + assert data is None, (data, data_dict) + data = urllib.urlencode(data_dict) + if data is not None: + if data_dict is None: + assert method == 'POST', (method, data) if method == 'POST': - env['CONTENT_LENGTH'] = len(enc_data) - env['wsgi.input'] = StringIO.StringIO(enc_data) + env['CONTENT_LENGTH'] = len(data) + env['wsgi.input'] = StringIO.StringIO(data) else: assert method in ['GET', 'HEAD'], method - env['QUERY_STRING'] = enc_data + env['QUERY_STRING'] = data for key,value in environ.items(): env[key] = value return ''.join(app(env, self.start_response)) + def start_response(self, status, response_headers, exc_info=None): self.status = status self.response_headers = response_headers self.exc_info = exc_info + class WSGI_ObjectTestCase (WSGITestCase): def setUp(self): WSGITestCase.setUp(self) self.app = WSGI_Object(self.logger) + def test_error(self): contents = self.app.error( environ=self.default_environ, @@ -970,18 +745,21 @@ if libbe.TESTING == True: ('X-Dummy-Header','Dummy Value')], self.response_headers) self.failUnless(self.exc_info == None, self.exc_info) + def test_log_request(self): self.app.log_request( environ=self.default_environ, status='-1 OK', bytes=123) log = self.logstream.getvalue() self.failUnless(log.startswith('192.168.0.123 -'), log) + class ExceptionAppTestCase (WSGITestCase): def setUp(self): WSGITestCase.setUp(self) def child_app(environ, start_response): raise ValueError('Dummy Error') self.app = ExceptionApp(child_app, self.logger) + def test_traceback(self): try: self.getURL(self.app) @@ -992,6 +770,7 @@ if libbe.TESTING == True: self.failUnless('child_app' in log, log) self.failUnless('ValueError: Dummy Error' in log, log) + class AdminAppTestCase (WSGITestCase): def setUp(self): WSGITestCase.setUp(self) @@ -1002,20 +781,22 @@ if libbe.TESTING == True: User('guest', 'Guest', password='guestpass')) def child_app(environ, start_response): pass - self.app = AdminApp( - child_app, users=self.users, logger=self.logger) - self.app = AuthenticationApp( - self.app, realm='Dummy Realm', users=self.users, + app = AdminApp( + app=child_app, users=self.users, logger=self.logger) + app = AuthenticationApp( + app=app, realm='Dummy Realm', users=self.users, logger=self.logger) - self.app = UppercaseHeaderApp(self.app, logger=self.logger) + self.app = UppercaseHeaderApp(app=app, logger=self.logger) + def basic_auth(self, uname, password): """HTTP basic authorization string""" - return 'Basic %s' % \ - ('%s:%s' % (uname, password)).encode('base64') + return 'Basic {}'.format( + '{}:{}'.format(uname, password).encode('base64')) + def test_new_name(self): self.getURL( self.app, '/admin/', method='POST', - data={'name':'Prince Al'}, + data_dict={'name':'Prince Al'}, environ={'HTTP_Authorization': self.basic_auth('Aladdin', 'open sesame')}) self.failUnless(self.status == '200 OK', self.status) @@ -1026,25 +807,27 @@ if libbe.TESTING == True: self.users['Aladdin'].name) self.failUnless(self.users.changed == True, self.users.changed) + def test_new_password(self): self.getURL( self.app, '/admin/', method='POST', - data={'password':'New Pass'}, + data_dict={'password':'New Pass'}, environ={'HTTP_Authorization': self.basic_auth('Aladdin', 'open sesame')}) self.failUnless(self.status == '200 OK', self.status) self.failUnless(self.response_headers == [], self.response_headers) self.failUnless(self.exc_info == None, self.exc_info) - self.failUnless(self.users['Aladdin'].passhash == \ - self.users['Aladdin'].hash('New Pass'), + self.failUnless((self.users['Aladdin'].passhash == + self.users['Aladdin'].hash('New Pass')), self.users['Aladdin'].passhash) self.failUnless(self.users.changed == True, self.users.changed) + def test_guest_name(self): self.getURL( self.app, '/admin/', method='POST', - data={'name':'SPAM'}, + data_dict={'name':'SPAM'}, environ={'HTTP_Authorization': self.basic_auth('guest', 'guestpass')}) self.failUnless(self.status.startswith('403 '), self.status) @@ -1056,10 +839,11 @@ if libbe.TESTING == True: self.users['guest'].name) self.failUnless(self.users.changed == False, self.users.changed) + def test_guest_password(self): self.getURL( self.app, '/admin/', method='POST', - data={'password':'SPAM'}, + data_dict={'password':'SPAM'}, environ={'HTTP_Authorization': self.basic_auth('guest', 'guestpass')}) self.failUnless(self.status.startswith('403 '), self.status) @@ -1072,33 +856,6 @@ if libbe.TESTING == True: self.failUnless(self.users.changed == False, self.users.changed) - class ServerAppTestCase (WSGITestCase): - def setUp(self): - WSGITestCase.setUp(self) - self.bd = libbe.bugdir.SimpleBugDir(memory=False) - self.app = ServerApp(self.bd.storage, logger=self.logger) - def tearDown(self): - self.bd.cleanup() - WSGITestCase.tearDown(self) - def test_add_get(self): - self.getURL(self.app, '/add/', method='GET') - self.failUnless(self.status.startswith('404 '), self.status) - self.failUnless(self.response_headers == [ - ('Content-Type', 'text/plain')], - self.response_headers) - self.failUnless(self.exc_info == None, self.exc_info) - def test_add_post(self): - self.getURL(self.app, '/add/', method='POST', - data={'id':'123456', 'parent':'abc123', - 'directory':'True'}) - self.failUnless(self.status == '200 OK', self.status) - self.failUnless(self.response_headers == [], - self.response_headers) - self.failUnless(self.exc_info == None, self.exc_info) - # Note: other methods tested in libbe.storage.http - - # TODO: integration tests on Serve? - unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) @@ -1106,20 +863,20 @@ if libbe.TESTING == True: # The following certificate-creation code is adapted From pyOpenSSL's # examples. -def get_cert_filenames(server_name, autogenerate=True, logger=None): +def _get_cert_filenames(server_name, autogenerate=True, logger=None): """ Generate private key and certification filenames. get_cert_filenames(server_name) -> (pkey_filename, cert_filename) """ - pkey_file = '%s.pkey' % server_name - cert_file = '%s.cert' % server_name - if autogenerate == True: + pkey_file = '{}.pkey'.format(server_name) + cert_file = '{}.cert'.format(server_name) + if autogenerate: for file in [pkey_file, cert_file]: if not os.path.exists(file): - make_certs(server_name, logger) + _make_certs(server_name, logger) return (pkey_file, cert_file) -def createKeyPair(type, bits): +def _create_key_pair(type, bits): """Create a public/private key pair. Returns the public/private key pair in a PKey object. @@ -1135,7 +892,7 @@ def createKeyPair(type, bits): pkey.generate_key(type, bits) return pkey -def createCertRequest(pkey, digest="md5", **name): +def _create_cert_request(pkey, digest="md5", **name): """Create a certificate request. Returns the certificate request in an X509Req object. @@ -1170,7 +927,8 @@ def createCertRequest(pkey, digest="md5", **name): req.sign(pkey, digest) return req -def createCertificate(req, (issuerCert, issuerKey), serial, (notBefore, notAfter), digest="md5"): +def _create_certificate(req, (issuerCert, issuerKey), serial, + (notBefore, notAfter), digest='md5'): """Generate a certificate given a certificate request. Returns the signed certificate in an X509 object. @@ -1204,22 +962,22 @@ def createCertificate(req, (issuerCert, issuerKey), serial, (notBefore, notAfter cert.sign(issuerKey, digest) return cert -def make_certs(server_name, logger=None) : +def _make_certs(server_name, logger=None) : """Generate private key and certification files. `mk_certs(server_name) -> (pkey_filename, cert_filename)` """ if OpenSSL == None: - raise libbe.command.UserError, \ - 'SSL certificate generation requires the OpenSSL module' + raise libbe.command.UserError( + 'SSL certificate generation requires the OpenSSL module') pkey_file,cert_file = get_cert_filenames( server_name, autogenerate=False) if logger != None: logger.log(logger._server_level, 'Generating certificates', pkey_file, cert_file) - cakey = createKeyPair(OpenSSL.crypto.TYPE_RSA, 1024) - careq = createCertRequest(cakey, CN='Certificate Authority') - cacert = createCertificate( + cakey = _create_key_pair(OpenSSL.crypto.TYPE_RSA, 1024) + careq = _create_cert_request(cakey, CN='Certificate Authority') + cacert = _create_certificate( careq, (careq, cakey), 0, (0, 60*60*24*365*5)) # five years open(pkey_file, 'w').write(OpenSSL.crypto.dump_privatekey( OpenSSL.crypto.FILETYPE_PEM, cakey)) |