# Bugs Everywhere, a distributed bugtracker # Copyright (C) 2008-2009 Chris Ball # Thomas Habets # W. Trevor King # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import base64 import os import os.path import sys import time import types try: # import core module, Python >= 2.5 from xml.etree import ElementTree except ImportError: # look for non-core module from elementtree import ElementTree import xml.sax.saxutils import doctest from beuuid import uuid_gen from properties import Property, doc_property, local_property, \ defaulting_property, checked_property, cached_property, \ primed_property, change_hook_property, settings_property import settings_object import mapfile from tree import Tree import utility class InvalidShortname(KeyError): def __init__(self, shortname, shortnames): msg = "Invalid shortname %s\n%s" % (shortname, shortnames) KeyError.__init__(self, msg) self.shortname = shortname self.shortnames = shortnames class InvalidXML(ValueError): def __init__(self, element, comment): msg = "Invalid comment xml: %s\n %s\n" \ % (comment, ElementTree.tostring(element)) ValueError.__init__(self, msg) self.element = element self.comment = comment class MissingReference(ValueError): def __init__(self, comment): msg = "Missing reference to %s" % (comment.in_reply_to) ValueError.__init__(self, msg) self.reference = comment.in_reply_to self.comment = comment class DiskAccessRequired (Exception): def __init__(self, goal): msg = "Cannot %s without accessing the disk" % goal Exception.__init__(self, msg) INVALID_UUID = "!!~~\n INVALID-UUID \n~~!!" def list_to_root(comments, bug, root=None, ignore_missing_references=False): """ Convert a raw list of comments to single root comment. We use a dummy root comment by default, because there can be several comment threads rooted on the same parent bug. To simplify comment interaction, we condense these threads into a single thread with a Comment dummy root. Can also be used to append a list of subcomments to a non-dummy root comment, so long as all the new comments are descendants of the root comment. No Comment method should use the dummy comment. """ root_comments = [] uuid_map = {} for comment in comments: assert comment.uuid != None uuid_map[comment.uuid] = comment for comment in comments: if comment.alt_id != None and comment.alt_id not in uuid_map: uuid_map[comment.alt_id] = comment if root == None: root = Comment(bug, uuid=INVALID_UUID) else: uuid_map[root.uuid] = root for comm in comments: if comm.in_reply_to == INVALID_UUID: comm.in_reply_to = None rep = comm.in_reply_to if rep == None or rep == bug.uuid: root_comments.append(comm) else: parentUUID = comm.in_reply_to try: parent = uuid_map[parentUUID] parent.add_reply(comm) except KeyError, e: if ignore_missing_references == True: print >> sys.stderr, \ "Ignoring missing reference to %s" % parentUUID comm.in_reply_to = None root_comments.append(comm) else: raise MissingReference(comm) root.extend(root_comments) return root def loadComments(bug, load_full=False): """ Set load_full=True when you want to load the comment completely from disk *now*, rather than waiting and lazy loading as required. """ if bug.sync_with_disk == False: raise DiskAccessRequired("load comments") path = bug.get_path("comments") if not os.path.isdir(path): return Comment(bug, uuid=INVALID_UUID) comments = [] for uuid in os.listdir(path): if uuid.startswith('.'): continue comm = Comment(bug, uuid, from_disk=True) comm.set_sync_with_disk(bug.sync_with_disk) if load_full == True: comm.load_settings() dummy = comm.body # force the body to load comments.append(comm) return list_to_root(comments, bug) def saveComments(bug): if bug.sync_with_disk == False: raise DiskAccessRequired("save comments") for comment in bug.comment_root.traverse(): comment.save() class Comment(Tree, settings_object.SavedSettingsObject): """ >>> c = Comment() >>> c.uuid != None True >>> c.uuid = "some-UUID" >>> print c.content_type text/plain """ settings_properties = [] required_saved_properties = [] _prop_save_settings = settings_object.prop_save_settings _prop_load_settings = settings_object.prop_load_settings def _versioned_property(settings_properties=settings_properties, required_saved_properties=required_saved_properties, **kwargs): if "settings_properties" not in kwargs: kwargs["settings_properties"] = settings_properties if "required_saved_properties" not in kwargs: kwargs["required_saved_properties"]=required_saved_properties return settings_object.versioned_property(**kwargs) @_versioned_property(name="Alt-id", doc="Alternate ID for linking imported comments. Internally comments are linked (via In-reply-to) to the parent's UUID. However, these UUIDs are generated internally, so Alt-id is provided as a user-controlled linking target.") def alt_id(): return {} @_versioned_property(name="Author", doc="The author of the comment") def author(): return {} @_versioned_property(name="In-reply-to", doc="UUID for parent comment or bug") def in_reply_to(): return {} @_versioned_property(name="Content-type", doc="Mime type for comment body", default="text/plain", require_save=True) def content_type(): return {} @_versioned_property(name="Date", doc="An RFC 2822 timestamp for comment creation") def date(): return {} def _get_time(self): if self.date == None: return None return utility.str_to_time(self.date) def _set_time(self, value): self.date = utility.time_to_str(value) time = property(fget=_get_time, fset=_set_time, doc="An integer version of .date") def _get_comment_body(self): if self.rcs != None and self.sync_with_disk == True: import rcs binary = not self.content_type.startswith("text/") return self.rcs.get_file_contents(self.get_path("body"), binary=binary) def _set_comment_body(self, old=None, new=None, force=False): if (self.rcs != None and self.sync_with_disk == True) or force==True: assert new != None, "Can't save empty comment" binary = not self.content_type.startswith("text/") self.rcs.set_file_contents(self.get_path("body"), new, binary=binary) @Property @change_hook_property(hook=_set_comment_body) @cached_property(generator=_get_comment_body) @local_property("body") @doc_property(doc="The meat of the comment") def body(): return {} def _get_rcs(self): if hasattr(self.bug, "rcs"): return self.bug.rcs @Property @cached_property(generator=_get_rcs) @local_property("rcs") @doc_property(doc="A revision control system instance.") def rcs(): return {} def _extra_strings_check_fn(value): return utility.iterable_full_of_strings(value, \ alternative=settings_object.EMPTY) def _extra_strings_change_hook(self, old, new): self.extra_strings.sort() # to make merging easier self._prop_save_settings(old, new) @_versioned_property(name="extra_strings", doc="Space for an array of extra strings. Useful for storing state for functionality implemented purely in becommands/.py.", default=[], check_fn=_extra_strings_check_fn, change_hook=_extra_strings_change_hook, mutable=True) def extra_strings(): return {} def __init__(self, bug=None, uuid=None, from_disk=False, in_reply_to=None, body=None): """ Set from_disk=True to load an old comment. Set from_disk=False to create a new comment. The uuid option is required when from_disk==True. The in_reply_to and body options are only used if from_disk==False (the default). When from_disk==True, they are loaded from the bug database. in_reply_to should be the uuid string of the parent comment. """ Tree.__init__(self) settings_object.SavedSettingsObject.__init__(self) self.bug = bug self.uuid = uuid if from_disk == True: self.sync_with_disk = True else: self.sync_with_disk = False if uuid == None: self.uuid = uuid_gen() self.time = int(time.time()) # only save to second precision if self.rcs != None: self.author = self.rcs.get_user_id() self.in_reply_to = in_reply_to self.body = body def __cmp__(self, other): return cmp_full(self, other) def __str__(self): """ >>> comm = Comment(bug=None, body="Some insightful remarks") >>> comm.uuid = "com-1" >>> comm.date = "Thu, 20 Nov 2008 15:55:11 +0000" >>> comm.author = "Jane Doe " >>> print comm --------- Comment --------- Name: com-1 From: Jane Doe Date: Thu, 20 Nov 2008 15:55:11 +0000 Some insightful remarks """ return self.string() def traverse(self, *args, **kwargs): """Avoid working with the possible dummy root comment""" for comment in Tree.traverse(self, *args, **kwargs): if comment.uuid == INVALID_UUID: continue yield comment # serializing methods def _setting_attr_string(self, setting): value = getattr(self, setting) if value == None: return "" return str(value) def xml(self, indent=0, shortname=None): """ >>> comm = Comment(bug=None, body="Some\\ninsightful\\nremarks\\n") >>> comm.uuid = "0123" >>> comm.date = "Thu, 01 Jan 1970 00:00:00 +0000" >>> print comm.xml(indent=2, shortname="com-1") 0123 com-1 Thu, 01 Jan 1970 00:00:00 +0000 text/plain Some insightful remarks """ if shortname == None: shortname = self.uuid if self.content_type.startswith("text/"): body = (self.body or "").rstrip('\n') else: maintype,subtype = self.content_type.split('/',1) msg = email.mime.base.MIMEBase(maintype, subtype) msg.set_payload(self.body or "") email.encoders.encode_base64(msg) body = base64.encodestring(self.body or "") info = [("uuid", self.uuid), ("alt-id", self.alt_id), ("short-name", shortname), ("in-reply-to", self.in_reply_to), ("author", self._setting_attr_string("author")), ("date", self.date), ("content-type", self.content_type), ("body", body)] lines = [""] for (k,v) in info: if v != None: lines.append(' <%s>%s' % (k,xml.sax.saxutils.escape(v),k)) lines.append("") istring = ' '*indent sep = '\n' + istring return istring + sep.join(lines).rstrip('\n') def from_xml(self, xml_string, verbose=True): """ Note: If alt-id is not given, translates any fields to fields. >>> commA = Comment(bug=None, body="Some\\ninsightful\\nremarks\\n") >>> commA.uuid = "0123" >>> commA.date = "Thu, 01 Jan 1970 00:00:00 +0000" >>> xml = commA.xml(shortname="com-1") >>> commB = Comment() >>> commB.from_xml(xml) >>> attrs=['uuid','alt_id','in_reply_to','author','date','content_type','body'] >>> for attr in attrs: # doctest: +ELLIPSIS ... if getattr(commB, attr) != getattr(commA, attr): ... estr = "Mismatch on %s: '%s' should be '%s'" ... args = (attr, getattr(commB, attr), getattr(commA, attr)) ... print estr % args Mismatch on uuid: '...' should be '0123' Mismatch on alt_id: '0123' should be 'None' >>> print commB.alt_id 0123 >>> commA.author >>> commB.author """ if type(xml_string) == types.UnicodeType: xml_string = xml_string.strip().encode("unicode_escape") comment = ElementTree.XML(xml_string) if comment.tag != "comment": raise InvalidXML(comment, "root element must be ") tags=['uuid','alt-id','in-reply-to','author','date','content-type','body'] uuid = None body = None for child in comment.getchildren(): if child.tag == "short-name": pass elif child.tag in tags: if child.text == None or len(child.text) == 0: text = settings_object.EMPTY else: text = xml.sax.saxutils.unescape(child.text) text = unicode(text).decode("unicode_escape").strip() if child.tag == "uuid": uuid = text continue # don't set the bug's uuid tag. if child.tag == "body": body = text continue # don't set the bug's body yet. else: attr_name = child.tag.replace('-','_') setattr(self, attr_name, text) elif verbose == True: print >> sys.stderr, "Ignoring unknown tag %s in %s" \ % (child.tag, comment.tag) if self.alt_id == None and uuid not in [None, self.uuid]: self.alt_id = uuid if body != None: if self.content_type.startswith("text/"): self.body = body+"\n" # restore trailing newline else: self.body = base64.decodestring(body) def string(self, indent=0, shortname=None): """ >>> comm = Comment(bug=None, body="Some\\ninsightful\\nremarks\\n") >>> comm.date = "Thu, 01 Jan 1970 00:00:00 +0000" >>> print comm.string(indent=2, shortname="com-1") --------- Comment --------- Name: com-1 From: Date: Thu, 01 Jan 1970 00:00:00 +0000 Some insightful remarks """ if shortname == None: shortname = self.uuid lines = [] lines.append("--------- Comment ---------") lines.append("Name: %s" % shortname) lines.append("From: %s" % (self._setting_attr_string("author"))) lines.append("Date: %s" % self.date) lines.append("") if self.content_type.startswith("text/"): lines.extend((self.body or "").splitlines()) else: lines.append("Content type %s not printable. Try XML output instead" % self.content_type) istring = ' '*indent sep = '\n' + istring return istring + sep.join(lines).rstrip('\n') def string_thread(self, string_method_name="string", name_map={}, indent=0, flatten=True, auto_name_map=False, bug_shortname=None): """ Return a string displaying a thread of comments. bug_shortname is only used if auto_name_map == True. string_method_name (defaults to "string") is the name of the Comment method used to generate the output string for each Comment in the thread. The method must take the arguments indent and shortname. SIDE-EFFECT: if auto_name_map==True, calls comment_shortnames() which will sort the tree by comment.time. Avoid by calling name_map = {} for shortname,comment in comm.comment_shortnames(bug_shortname): name_map[comment.uuid] = shortname comm.sort(key=lambda c : c.author) # your sort comm.string_thread(name_map=name_map) >>> a = Comment(bug=None, uuid="a", body="Insightful remarks") >>> a.time = utility.str_to_time("Thu, 20 Nov 2008 01:00:00 +0000") >>> b = a.new_reply("Critique original comment") >>> b.uuid = "b" >>> b.time = utility.str_to_time("Thu, 20 Nov 2008 02:00:00 +0000") >>> c = b.new_reply("Begin flamewar :p") >>> c.uuid = "c" >>> c.time = utility.str_to_time("Thu, 20 Nov 2008 03:00:00 +0000") >>> d = a.new_reply("Useful examples") >>> d.uuid = "d" >>> d.time = utility.str_to_time("Thu, 20 Nov 2008 04:00:00 +0000") >>> a.sort(key=lambda comm : comm.time) >>> print a.string_thread(flatten=True) --------- Comment --------- Name: a From: Date: Thu, 20 Nov 2008 01:00:00 +0000 Insightful remarks --------- Comment --------- Name: b From: Date: Thu, 20 Nov 2008 02:00:00 +0000 Critique original comment --------- Comment --------- Name: c From: Date: Thu, 20 Nov 2008 03:00:00 +0000 Begin flamewar :p --------- Comment --------- Name: d From: Date: Thu, 20 Nov 2008 04:00:00 +0000 Useful examples >>> print a.string_thread(auto_name_map=True, bug_shortname="bug-1") --------- Comment --------- Name: bug-1:1 From: Date: Thu, 20 Nov 2008 01:00:00 +0000 Insightful remarks --------- Comment --------- Name: bug-1:2 From: Date: Thu, 20 Nov 2008 02:00:00 +0000 Critique original comment --------- Comment --------- Name: bug-1:3 From: Date: Thu, 20 Nov 2008 03:00:00 +0000 Begin flamewar :p --------- Comment --------- Name: bug-1:4 From: Date: Thu, 20 Nov 2008 04:00:00 +0000 Useful examples """ if auto_name_map == True: name_map = {} for shortname,comment in self.comment_shortnames(bug_shortname): name_map[comment.uuid] = shortname stringlist = [] for depth,comment in self.thread(flatten=flatten): ind = 2*depth+indent if comment.uuid in name_map: sname = name_map[comment.uuid] else: sname = None string_fn = getattr(comment, string_method_name) stringlist.append(string_fn(indent=ind, shortname=sname)) return '\n'.join(stringlist) def xml_thread(self, name_map={}, indent=0, auto_name_map=False, bug_shortname=None): return self.string_thread(string_method_name="xml", name_map=name_map, indent=indent, auto_name_map=auto_name_map, bug_shortname=bug_shortname) # methods for saving/loading/acessing settings and properties. def get_path(self, name=None): my_dir = os.path.join(self.bug.get_path("comments"), self.uuid) if name is None: return my_dir assert name in ["values", "body"] return os.path.join(my_dir, name) def set_sync_with_disk(self, value): self.sync_with_disk = value def load_settings(self): if self.sync_with_disk == False: raise DiskAccessRequired("load settings") self.settings = mapfile.map_load(self.rcs, self.get_path("values")) # hack to deal with old BE comments: if "From" in self.settings: self.settings["Author"] = self.settings.pop("From") self._setup_saved_settings() def save_settings(self): if self.sync_with_disk == False: raise DiskAccessRequired("save settings") self.rcs.mkdir(self.get_path()) path = self.get_path("values") mapfile.map_save(self.rcs, path, self._get_saved_settings()) def save(self): """ Save any loaded contents to disk. However, if self.sync_with_disk = True, then any changes are automatically written to disk as soon as they happen, so calling this method will just waste time (unless something else has been messing with your on-disk files). """ sync_with_disk = self.sync_with_disk if sync_with_disk == False: self.set_sync_with_disk(True) assert self.body != None, "Can't save blank comment" self.save_settings() self._set_comment_body(new=self.body, force=True) if sync_with_disk == False: self.set_sync_with_disk(False) def remove(self): if self.sync_with_disk == False: raise DiskAccessRequired("remove") for comment in self.traverse(): path = comment.get_path() self.rcs.recursive_remove(path) def add_reply(self, reply, allow_time_inversion=False): if self.uuid != INVALID_UUID: reply.in_reply_to = self.uuid self.append(reply) def new_reply(self, body=None): """ >>> comm = Comment(bug=None, body="Some insightful remarks") >>> repA = comm.new_reply("Critique original comment") >>> repB = repA.new_reply("Begin flamewar :p") >>> repB.in_reply_to == repA.uuid True """ reply = Comment(self.bug, body=body) if self.bug != None: reply.set_sync_with_disk(self.bug.sync_with_disk) if reply.sync_with_disk == True: raise Exception, self.bug.sync_with_disk reply.save() self.add_reply(reply) return reply def comment_shortnames(self, bug_shortname=None): """ Iterate through (id, comment) pairs, in time order. (This is a user-friendly id, not the comment uuid). SIDE-EFFECT : will sort the comment tree by comment.time >>> a = Comment(bug=None, uuid="a") >>> b = a.new_reply() >>> b.uuid = "b" >>> c = b.new_reply() >>> c.uuid = "c" >>> d = a.new_reply() >>> d.uuid = "d" >>> for id,name in a.comment_shortnames("bug-1"): ... print id, name.uuid bug-1:1 a bug-1:2 b bug-1:3 c bug-1:4 d """ if bug_shortname == None: bug_shortname = "" self.sort(key=lambda comm : comm.time) for num,comment in enumerate(self.traverse()): yield ("%s:%d" % (bug_shortname, num+1), comment) def comment_from_shortname(self, comment_shortname, *args, **kwargs): """ Use a comment shortname to look up a comment. >>> a = Comment(bug=None, uuid="a") >>> b = a.new_reply() >>> b.uuid = "b" >>> c = b.new_reply() >>> c.uuid = "c" >>> d = a.new_reply() >>> d.uuid = "d" >>> comm = a.comment_from_shortname("bug-1:3", bug_shortname="bug-1") >>> id(comm) == id(c) True """ for cur_name, comment in self.comment_shortnames(*args, **kwargs): if comment_shortname == cur_name: return comment raise InvalidShortname(comment_shortname, list(self.comment_shortnames(*args, **kwargs))) def comment_from_uuid(self, uuid): """ Use a comment shortname to look up a comment. >>> a = Comment(bug=None, uuid="a") >>> b = a.new_reply() >>> b.uuid = "b" >>> c = b.new_reply() >>> c.uuid = "c" >>> d = a.new_reply() >>> d.uuid = "d" >>> comm = a.comment_from_uuid("d") >>> id(comm) == id(d) True """ for comment in self.traverse(): if comment.uuid == uuid: return comment raise KeyError(uuid) def cmp_attr(comment_1, comment_2, attr, invert=False): """ Compare a general attribute between two comments using the conventional comparison rule for that attribute type. If invert == True, sort *against* that convention. >>> attr="author" >>> commentA = Comment() >>> commentB = Comment() >>> commentA.author = "John Doe" >>> commentB.author = "Jane Doe" >>> cmp_attr(commentA, commentB, attr) < 0 True >>> cmp_attr(commentA, commentB, attr, invert=True) > 0 True >>> commentB.author = "John Doe" >>> cmp_attr(commentA, commentB, attr) == 0 True """ if not hasattr(comment_2, attr) : return 1 val_1 = getattr(comment_1, attr) val_2 = getattr(comment_2, attr) if val_1 == None: val_1 = None if val_2 == None: val_2 = None if invert == True : return -cmp(val_1, val_2) else : return cmp(val_1, val_2) # alphabetical rankings (a < z) cmp_uuid = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "uuid") cmp_author = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "author") cmp_in_reply_to = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "in_reply_to") cmp_content_type = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "content_type") cmp_body = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "body") # chronological rankings (newer < older) cmp_time = lambda comment_1, comment_2 : cmp_attr(comment_1, comment_2, "time", invert=True) DEFAULT_CMP_FULL_CMP_LIST = \ (cmp_time, cmp_author, cmp_content_type, cmp_body, cmp_in_reply_to, cmp_uuid) class CommentCompoundComparator (object): def __init__(self, cmp_list=DEFAULT_CMP_FULL_CMP_LIST): self.cmp_list = cmp_list def __call__(self, comment_1, comment_2): for comparison in self.cmp_list : val = comparison(comment_1, comment_2) if val != 0 : return val return 0 cmp_full = CommentCompoundComparator() suite = doctest.DocTestSuite()