From 3b168403ff5e50d767476c4c0f037d1841bb2bf9 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Nov 2009 10:53:04 -0500 Subject: Broke `be comment --xml` out and extended into `be import-xml`. It should currently do everything that `be comment --xml` did, but it still has a way to go before it lives up to it's longhelp string, mostly figuring out bug/comment merging. The allowed XML format also changed a bit, becoming a bit more structured. cmdutil.bug_from_shortname() renamed to cmdutil.bug_from_id(). New functions cmdutil.parse_id() and cmdutil.bug_comment_from_id(). Additional doctests in libbe.comment.Comment.comment_shortnames() to show example output if bug_shortname==None. Brought be-xml-to-mbox and be-mbox-to-xml up to speed on the current -rooted format. * Added handling to their comment handling. * Moved extra strings from email bodies to X-Extra-String headers (some comment bodies are not text, and we should keep the estr location consistent between bugs and comments.) --- becommands/import_xml.py | 257 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 becommands/import_xml.py (limited to 'becommands/import_xml.py') diff --git a/becommands/import_xml.py b/becommands/import_xml.py new file mode 100644 index 0000000..d28e97a --- /dev/null +++ b/becommands/import_xml.py @@ -0,0 +1,257 @@ +# Copyright (C) 2009 W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Import comments and bugs from XML""" +from libbe import cmdutil, bugdir, bug, comment, utility +from becommands.comment import complete +import os +import sys +try: # import core module, Python >= 2.5 + from xml.etree import ElementTree +except ImportError: # look for non-core module + from elementtree import ElementTree +__desc__ = __doc__ + +def execute(args, manipulate_encodings=True): + """ + >>> import time + >>> import StringIO + >>> bd = bugdir.SimpleBugDir() + >>> os.chdir(bd.root) + >>> orig_stdin = sys.stdin + >>> sys.stdin = StringIO.StringIO("cThis is a comment about a") + >>> execute(["-c", "a", "-"], manipulate_encodings=False) + >>> sys.stdin = orig_stdin + >>> bd._clear_bugs() + >>> bug = cmdutil.bug_from_id(bd, "a") + >>> bug.load_comments(load_full=False) + >>> comment = bug.comment_root[0] + >>> print comment.body + This is a comment about a + + >>> comment.author == bd.user_id + True + >>> comment.time <= int(time.time()) + True + >>> comment.in_reply_to is None + True + >>> bd.cleanup() + """ + parser = get_parser() + options, args = parser.parse_args(args) + complete(options, args, parser) + if len(args) < 1: + raise cmdutil.UsageError("Please specify an XML file.") + if len(args) > 1: + raise cmdutil.UsageError("Too many arguments.") + filename = args[0] + + bd = bugdir.BugDir(from_disk=True, + manipulate_encodings=manipulate_encodings) + if options.comment_root != None: + croot_bug,croot_comment = \ + cmdutil.bug_comment_from_id(bd, options.comment_root) + else: + croot_bug,croot_comment = (None, None) + + if filename == '-': + xml = sys.stdin.read() + else: + xml = bd.vcs.get_file_contents(filename, allow_no_vcs=True) + str_xml = xml.encode('unicode_escape').replace(r'\n', '\n') + # unicode read + encode to string so we know the encoding, + # which might not be given (?) in a binary string read? + + # parse the xml + root_bugs = [] + root_comments = [] + version = {} + be_xml = ElementTree.XML(str_xml) + if be_xml.tag != 'be-xml': + raise utility.InvalidXML( + 'import-xml', be_xml, 'root element must be ') + for child in be_xml.getchildren(): + if child.tag == 'bug': + new = bug.Bug(bugdir=bd) + new.from_xml(unicode(ElementTree.tostring(child)).decode('unicode_escape')) + root_bugs.append(new) + elif child.tag == 'comment': + new = comment.Comment(croot_bug) + new.from_xml(unicode(ElementTree.tostring(child)).decode("unicode_escape")) + root_comments.append(new) + elif child.tag == 'version': + for gchild in child.getchildren(): + if child.tag in ['tag', 'nick', 'revision', 'revision-id']: + text = xml.sax.saxutils.unescape(child.text) + text = text.decode('unicode_escape').strip() + version[child.tag] = text + else: + print >> sys.stderr, 'ignoring unknown tag %s in %s' \ + % (gchild.tag, child.tag) + else: + print >> sys.stderr, 'ignoring unknown tag %s in %s' \ + % (child.tag, comment_list.tag) + + # merge the new root_comments + croot_cids = [] + for c in croot_bug.comment_root.traverse(): + croot_cids.append(c.uuid) + if c.alt_id != None: + croot_cids.append(c.alt_id) + for new in root_comments: + if new.alt_id in croot_cids: + raise cmdutil.UserError( + 'clashing comment alt_id: %s' % new.alt_id) + croot_cids.append(new.uuid) + if new.alt_id != None: + croot_cids.append(new.alt_id) + if new.in_reply_to == None: + new.in_reply_to = croot_comment.uuid + try: + # link new comments + comment.list_to_root(root_comments,croot_bug,root=croot_comment, + ignore_missing_references= \ + options.ignore_missing_references) + except comment.MissingReference, e: + raise cmdutil.UserError(e) + + # merge the new croot_bugs + for new in root_bugs: + bd.append(new) + + # protect against programmer error causing data loss: + comms = [c.uuid for c in croot_comment.traverse()] + for new in root_comments: + assert new.uuid in comms, \ + "comment %s wasn't added to %s" % (new.uuid, croot_comment.uuid) + for new in root_bugs: + assert bd.has_bug(new.uuid), \ + "bug %s wasn't added" % (new.uuid) + + # save new information + for new in root_comments: + new.save() + for new in root_bugs: + new.save() + +def get_parser(): + parser = cmdutil.CmdOptionParser("be import-xml XMLFILE") + parser.add_option("-i", "--ignore-missing-references", action="store_true", + dest="ignore_missing_references", default=False, + help="If any comment's refers to a non-existent comment, ignore it (instead of raising an exception).") + parser.add_option("-a", "--add-only", action='store_true', + dest="add_only", default=False, + help="If any bug or comment listed in the XML file already exists in the bug repository, do not alter the repository version.") + parser.add_option("-c", "--comment-root", dest="comment_root", + help="Supply a bug or comment ID as the root of any elements that are direct children of the element. If any such elements exist, you are required to set this option.") + return parser + +longhelp=""" +Import comments and bugs from XMLFILE. If XMLFILE is '-', the file is +read from stdin. + +This command provides a fallback mechanism for passing bugs between +repositories, in case the repositories VCSs are incompatible. If the +VCSs are compatible, it's better to use their builtin merge/push/pull +to share this information, as that will preserve a more detailed +history. + +The XML file should be formatted similarly to + + + 1.0.0 + be + 446 + a@b.com-20091119214553-iqyw2cpqluww3zna + + + ... + ... + ... + + ... + ... + ... + +where the ellipses mark output commpatible with Bug.xml() and +Comment.xml(). Take a look at the output of `be show --xml --version` +for some explicit examples. Unrecognized tags are ignored. Missing +tags are left at the default value. The version tag is not required, +but is strongly recommended. + +The bug and comment UUIDs are always auto-generated, so if you set a + field, but no field, your will be used as the +comment's . An exception is raised if conflicts with +an existing comment. Bugs do not have a permantent alt-id, so they +the s you specify are not saved. The s _are_ used to +match agains prexisting bug and comment uuids, and comment alt-ids, +and fields explicitly given in the XML file will replace old versions +unless the --add-only flag. + +*.extra_strings recieves special treatment, and if --add-only is not +set, the resulting list concatenates both source lists and removes +repeats. + +Here's an example of import activity: + Repository + bug (uuid=B, author=John, status=open) + estr (don't forget your towel) + estr (helps with space travel) + com (uuid=C1, author=Jane, body=Hello) + com (uuid=C2, author=Jess, body=World) + XML + bug (uuid=B, status=fixed) + estr (don't forget your towel) + estr (watch out for flying dolphins) + com (uuid=C1, body=So long) + com (uuid=C3, author=Jed, body=And thanks) + Result + bug (uuid=B, author=John, status=fixed) + estr (don't forget your towel) + estr (helps with space travel) + estr (watch out for flying dolphins) + com (uuid=C1, author=Jane, body=So long) + com (uuid=C2, author=Jess, body=World) + com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) + Result, with --add-only + bug (uuid=B, author=John, status=open) + estr (don't forget your towel) + estr (helps with space travel) + com (uuid=C1, author=Jane, body=Hello) + com (uuid=C2, author=Jess, body=World) + com (uuid=C4, alt-id=C3, author=Jed, body=And thanks) + +Examples: + +Import comments (e.g. emails from an mbox) and append to bug XYZ + $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ - +Or you can append those emails underneath the prexisting comment XYZ-3 + $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ-3 - + +User creates a new bug + user$ be new "The demuxulizer is broken" + Created bug with ID 48f + user$ be comment 48f + + ... +User exports bug as xml and emails it to the developers + user$ be show --xml --version 48f > 48f.xml + user$ cat 48f.xml | mail -s "Demuxulizer bug xml" devs@b.com +Devs recieve email, and save it's contents as demux-bug.xml + dev$ cat demux-bug.xml | be import-xml - +""" + +def help(): + return get_parser().help_str() + longhelp -- cgit From a07c70a8cb30fa7295471490a7b7fdfbd48a99ec Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Nov 2009 12:23:47 -0500 Subject: Upgraded `be show --xml` to new format. --- becommands/import_xml.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'becommands/import_xml.py') diff --git a/becommands/import_xml.py b/becommands/import_xml.py index d28e97a..212c7a7 100644 --- a/becommands/import_xml.py +++ b/becommands/import_xml.py @@ -172,8 +172,8 @@ The XML file should be formatted similarly to 1.0.0 - be - 446 + be + 446

a@b.com-20091119214553-iqyw2cpqluww3zna

@@ -186,10 +186,10 @@ The XML file should be formatted similarly to ... where the ellipses mark output commpatible with Bug.xml() and -Comment.xml(). Take a look at the output of `be show --xml --version` -for some explicit examples. Unrecognized tags are ignored. Missing -tags are left at the default value. The version tag is not required, -but is strongly recommended. +Comment.xml(). Take a look at the output of `be show --xml` for some +explicit examples. Unrecognized tags are ignored. Missing tags are +left at the default value. The version tag is not required, but is +strongly recommended. The bug and comment UUIDs are always auto-generated, so if you set a field, but no field, your will be used as the -- cgit From cb6a9e819d05402ee8b9cde356d509ab22de4780 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Nov 2009 12:52:37 -0500 Subject: `be email-bugs` now uses `be show` internals to produce consistent XML. Broke the bulk of show.py out into new function show.output(), which is used by both show.py and email_bugs.py to reduce duplication of effort and increase consistency of the XML. Also a few relevant help string updates. --- becommands/import_xml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'becommands/import_xml.py') diff --git a/becommands/import_xml.py b/becommands/import_xml.py index 212c7a7..2572075 100644 --- a/becommands/import_xml.py +++ b/becommands/import_xml.py @@ -247,8 +247,11 @@ User creates a new bug ... User exports bug as xml and emails it to the developers - user$ be show --xml --version 48f > 48f.xml + user$ be show --xml 48f > 48f.xml user$ cat 48f.xml | mail -s "Demuxulizer bug xml" devs@b.com +or equivalently (with a slightly fancier be-handle-mail compatible +email): + user$ be email-bugs 48f Devs recieve email, and save it's contents as demux-bug.xml dev$ cat demux-bug.xml | be import-xml - """ -- cgit From 607c72ea7f3382f3a51598593b71eabbbdbef664 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Nov 2009 13:49:32 -0500 Subject: Adjusted `be import-xml` to properly handle croot_bug==None. If there aren't any root comments, then the user needn't specify --comment-root, in which case croot_bug will be None and we want to skip all the croot processing. --- becommands/import_xml.py | 57 ++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 26 deletions(-) (limited to 'becommands/import_xml.py') diff --git a/becommands/import_xml.py b/becommands/import_xml.py index 2572075..928ca46 100644 --- a/becommands/import_xml.py +++ b/becommands/import_xml.py @@ -105,37 +105,42 @@ def execute(args, manipulate_encodings=True): % (child.tag, comment_list.tag) # merge the new root_comments - croot_cids = [] - for c in croot_bug.comment_root.traverse(): - croot_cids.append(c.uuid) - if c.alt_id != None: - croot_cids.append(c.alt_id) - for new in root_comments: - if new.alt_id in croot_cids: - raise cmdutil.UserError( - 'clashing comment alt_id: %s' % new.alt_id) - croot_cids.append(new.uuid) - if new.alt_id != None: - croot_cids.append(new.alt_id) - if new.in_reply_to == None: - new.in_reply_to = croot_comment.uuid - try: - # link new comments - comment.list_to_root(root_comments,croot_bug,root=croot_comment, - ignore_missing_references= \ - options.ignore_missing_references) - except comment.MissingReference, e: - raise cmdutil.UserError(e) - + if len(root_comments) > 0: + if croot_bug == None: + raise UserError( + '--comment-root option is required for your root comments:\n%s' + % '\n\n'.join([c.string() for c in root_comments])) + croot_cids = [] + for c in croot_bug.comment_root.traverse(): + croot_cids.append(c.uuid) + if c.alt_id != None: + croot_cids.append(c.alt_id) + for new in root_comments: + if new.alt_id in croot_cids: + raise cmdutil.UserError( + 'clashing comment alt_id: %s' % new.alt_id) + croot_cids.append(new.uuid) + if new.alt_id != None: + croot_cids.append(new.alt_id) + if new.in_reply_to == None: + new.in_reply_to = croot_comment.uuid + try: + # link new comments + comment.list_to_root(root_comments,croot_bug,root=croot_comment, + ignore_missing_references= \ + options.ignore_missing_references) + except comment.MissingReference, e: + raise cmdutil.UserError(e) # merge the new croot_bugs for new in root_bugs: bd.append(new) # protect against programmer error causing data loss: - comms = [c.uuid for c in croot_comment.traverse()] - for new in root_comments: - assert new.uuid in comms, \ - "comment %s wasn't added to %s" % (new.uuid, croot_comment.uuid) + if croot_bug != None: + comms = [c.uuid for c in croot_comment.traverse()] + for new in root_comments: + assert new.uuid in comms, \ + "comment %s wasn't added to %s" % (new.uuid, croot_comment.uuid) for new in root_bugs: assert bd.has_bug(new.uuid), \ "bug %s wasn't added" % (new.uuid) -- cgit From 614d4e40e148520ac511cbe0606bcbdcf24c8a08 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 21 Nov 2009 15:18:02 -0500 Subject: Added restrict_file_access to becommands' execute() args. + associated adjustments in other files. See cmdutil.restrict_file_access.__doc__ for an explanation of the security hole this closes. --- becommands/import_xml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'becommands/import_xml.py') diff --git a/becommands/import_xml.py b/becommands/import_xml.py index 928ca46..a74d329 100644 --- a/becommands/import_xml.py +++ b/becommands/import_xml.py @@ -24,7 +24,7 @@ except ImportError: # look for non-core module from elementtree import ElementTree __desc__ = __doc__ -def execute(args, manipulate_encodings=True): +def execute(args, manipulate_encodings=True, restrict_file_access=False): """ >>> import time >>> import StringIO @@ -69,6 +69,8 @@ def execute(args, manipulate_encodings=True): if filename == '-': xml = sys.stdin.read() else: + if restrict_file_access == True: + cmdutil.restrict_file_access(bd, options.body) xml = bd.vcs.get_file_contents(filename, allow_no_vcs=True) str_xml = xml.encode('unicode_escape').replace(r'\n', '\n') # unicode read + encode to string so we know the encoding, -- cgit