diff options
| -rw-r--r-- | libbe/storage/__init__.py | 1 | ||||
| -rw-r--r-- | libbe/storage/util/mapfile.py | 130 | ||||
| -rw-r--r-- | libbe/storage/util/upgrade.py | 134 |
3 files changed, 181 insertions, 84 deletions
diff --git a/libbe/storage/__init__.py b/libbe/storage/__init__.py index c97d084..6189008 100644 --- a/libbe/storage/__init__.py +++ b/libbe/storage/__init__.py @@ -47,6 +47,7 @@ STORAGE_VERSIONS = ['Bugs Everywhere Tree 1 0', 'Bugs Everywhere Directory v1.2', 'Bugs Everywhere Directory v1.3', 'Bugs Everywhere Directory v1.4', + 'Bugs Everywhere Directory v1.5', ] # the current version diff --git a/libbe/storage/util/mapfile.py b/libbe/storage/util/mapfile.py index b33ce84..c744130 100644 --- a/libbe/storage/util/mapfile.py +++ b/libbe/storage/util/mapfile.py @@ -25,86 +25,79 @@ easy merging of independent/conflicting changes. """ import errno +import json import os.path -import types -import yaml import libbe if libbe.TESTING == True: import doctest -class IllegalKey(Exception): - def __init__(self, key): - Exception.__init__(self, 'Illegal key "%s"' % key) - self.key = key - -class IllegalValue(Exception): - def __init__(self, value): - Exception.__init__(self, 'Illegal value "%s"' % value) - self.value = value - -class InvalidMapfileContents(Exception): +class InvalidMapfileContents (Exception): def __init__(self, contents): - Exception.__init__(self, 'Invalid YAML contents') + super(InvalidMapfileContents, self).__init__('Invalid JSON contents') self.contents = contents + def generate(map): """Generate a YAML mapfile content string. Examples -------- - >>> generate({'q':'p'}) - 'q: p\\n\\n' + >>> import sys + >>> sys.stdout.write(generate({'q':'p'})) + { + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + "q": "p" + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + } >>> generate({'q':u'Fran\u00e7ais'}) - 'q: Fran\\xc3\\xa7ais\\n\\n' + '{\\n\\n\\n\\n\\n\\n\\n "q": "Fran\\\\u00e7ais"\\n\\n\\n\\n\\n\\n\\n}\\n' >>> generate({'q':u'hello'}) - 'q: hello\\n\\n' - >>> generate({'q=':'p'}) - Traceback (most recent call last): - IllegalKey: Illegal key "q=" - >>> generate({'q:':'p'}) - Traceback (most recent call last): - IllegalKey: Illegal key "q:" - >>> generate({'q\\n':'p'}) - Traceback (most recent call last): - IllegalKey: Illegal key "q\\n" - >>> generate({'':'p'}) - Traceback (most recent call last): - IllegalKey: Illegal key "" - >>> generate({'>q':'p'}) - Traceback (most recent call last): - IllegalKey: Illegal key ">q" - >>> generate({'q':'p\\n'}) - Traceback (most recent call last): - IllegalValue: Illegal value "p\\n" + '{\\n\\n\\n\\n\\n\\n\\n "q": "hello"\\n\\n\\n\\n\\n\\n\\n}\\n' + >>> sys.stdout.write(generate( + ... {'p':'really long line\\n'*10, 'q': 'the next entry'})) + { + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + "p": "really long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\nreally long line\\n", + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + "q": "the next entry" + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + <BLANKLINE> + } See Also -------- parse : inverse """ - keys = map.keys() - keys.sort() - for key in keys: - try: - assert not key.startswith('>') - assert('\n' not in key) - assert('=' not in key) - assert(':' not in key) - assert(len(key) > 0) - except AssertionError: - raise IllegalKey(unicode(key).encode('unicode_escape')) - if '\n' in map[key]: - raise IllegalValue(unicode(map[key]).encode('unicode_escape')) - - lines = [] - for key in keys: - lines.append(yaml.safe_dump({key: map[key]}, - default_flow_style=False, - allow_unicode=True)) - lines.append('') - return '\n'.join(lines) + lines = json.dumps(map, sort_keys=True, indent=4).splitlines() + # add blank lines for context-less merging + return '\n\n\n\n\n\n\n'.join(lines) + '\n' def parse(contents): """Parse a YAML mapfile string. @@ -112,18 +105,16 @@ def parse(contents): Examples -------- - >>> parse('q: p\\n\\n')['q'] - 'p' - >>> parse('q: \\'p\\'\\n\\n')['q'] - 'p' + >>> parse('{"q": "p"}')['q'] + u'p' >>> contents = generate({'a':'b', 'c':'d', 'e':'f'}) >>> dict = parse(contents) >>> dict['a'] - 'b' + u'b' >>> dict['c'] - 'd' + u'd' >>> dict['e'] - 'f' + u'f' >>> contents = generate({'q':u'Fran\u00e7ais'}) >>> dict = parse(contents) >>> dict['q'] @@ -131,18 +122,17 @@ def parse(contents): >>> dict = parse('a!') Traceback (most recent call last): ... - InvalidMapfileContents: Invalid YAML contents + InvalidMapfileContents: Invalid JSON contents See Also -------- generate : inverse """ - c = yaml.load(contents) - if type(c) == types.StringType: - raise InvalidMapfileContents( - 'Unable to parse YAML (BE format missmatch?):\n\n%s' % contents) - return c or {} + try: + return json.loads(contents) + except ValueError: + raise InvalidMapfileContents(contents) if libbe.TESTING == True: suite = doctest.DocTestSuite() diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py index 67842ef..66bf576 100644 --- a/libbe/storage/util/upgrade.py +++ b/libbe/storage/util/upgrade.py @@ -22,8 +22,12 @@ Handle conversion between the various BE storage formats. """ import codecs +import json import os, os.path import sys +import types + +import yaml import libbe import libbe.bug @@ -36,6 +40,66 @@ import libbe.util.encoding as encoding import libbe.util.id +def generate_yaml_mapfile(map): + """From v1.1 to v1.5, BE dirs used YAML mapfiles + + >>> generate_yaml_mapfile({'q':'p'}) + 'q: p\\n\\n' + >>> generate_yaml_mapfile({'q':u'Fran\u00e7ais'}) + 'q: Fran\\xc3\\xa7ais\\n\\n' + >>> generate_yaml_mapfile({'q':u'hello'}) + 'q: hello\\n\\n' + """ + keys = map.keys() + keys.sort() + for key in keys: + try: + assert not key.startswith('>') + assert('\n' not in key) + assert('=' not in key) + assert(':' not in key) + assert(len(key) > 0) + except AssertionError: + raise ValueError(unicode(key).encode('unicode_escape')) + if '\n' in map[key]: + raise ValueError(unicode(map[key]).encode('unicode_escape')) + + lines = [] + for key in keys: + lines.append(yaml.safe_dump({key: map[key]}, + default_flow_style=False, + allow_unicode=True)) + lines.append('') + return '\n'.join(lines) + + +def parse_yaml_mapfile(contents): + """From v1.1 to v1.5, BE dirs used YAML mapfiles + + >>> parse_yaml_mapfile('q: p\\n\\n')['q'] + 'p' + >>> parse_yaml_mapfile('q: \\'p\\'\\n\\n')['q'] + 'p' + >>> contents = generate_yaml_mapfile({'a':'b', 'c':'d', 'e':'f'}) + >>> dict = parse_yaml_mapfile(contents) + >>> dict['a'] + 'b' + >>> dict['c'] + 'd' + >>> dict['e'] + 'f' + >>> contents = generate_yaml_mapfile({'q':u'Fran\u00e7ais'}) + >>> dict = parse_yaml_mapfile(contents) + >>> dict['q'] + u'Fran\\xe7ais' + """ + c = yaml.safe_load(contents) + if type(c) == types.StringType: + raise mapfile.InvalidMapfileContents( + 'Unable to parse YAML (BE format missmatch?):\n\n%s' % contents) + return c or {} + + class Upgrader (object): "Class for converting between different on-disk BE storage formats." initial_version = None @@ -118,8 +182,10 @@ class Upgrade_1_0_to_1_1 (Upgrader): newlines.append(line) contents = '\n'.join(newlines) # load the YAML and save - map = mapfile.parse(contents) - contents = mapfile.generate(map) + map = parse_yaml_mapfile(contents) + if type(map) == types.StringType: + raise ValueError((path, contents)) + contents = generate_yaml_mapfile(map) encoding.set_file_contents(path, contents) self.vcs._vcs_update(path) @@ -145,7 +211,7 @@ class Upgrade_1_0_to_1_1 (Upgrader): if 'From' in settings: settings['Author'] = settings.pop('From') encoding.set_file_contents( - path, mapfile.generate(settings)) + path, generate_yaml_mapfile(settings)) self.vcs._vcs_update(path) @@ -154,7 +220,7 @@ class Upgrade_1_1_to_1_2 (Upgrader): final_version = "Bugs Everywhere Directory v1.2" def _get_vcs_name(self): path = self.get_path('settings') - settings = mapfile.parse(encoding.get_file_contents(path)) + settings = parse_yaml_mapfile(encoding.get_file_contents(path)) if 'rcs_name' in settings: return settings['rcs_name'] return None @@ -164,10 +230,10 @@ class Upgrade_1_1_to_1_2 (Upgrader): BugDir settings field "rcs_name" -> "vcs_name". """ path = self.get_path('settings') - settings = mapfile.parse(encoding.get_file_contents(path)) + settings = parse_yaml_mapfile(encoding.get_file_contents(path)) if 'rcs_name' in settings: settings['vcs_name'] = settings.pop('rcs_name') - encoding.set_file_contents(path, mapfile.generate(settings)) + encoding.set_file_contents(path, generate_yaml_mapfile(settings)) self.vcs._vcs_update(path) class Upgrade_1_2_to_1_3 (Upgrader): @@ -179,7 +245,7 @@ class Upgrade_1_2_to_1_3 (Upgrader): def _get_vcs_name(self): path = self.get_path('settings') - settings = mapfile.parse(encoding.get_file_contents(path)) + settings = parse_yaml_mapfile(encoding.get_file_contents(path)) if 'vcs_name' in settings: return settings['vcs_name'] return None @@ -190,7 +256,7 @@ class Upgrade_1_2_to_1_3 (Upgrader): if not os.path.exists(path): self.vcs._add_path(path, directory=False) path = self.get_path('bugs', bug.uuid, 'values') - mf = mapfile.generate(bug._get_saved_settings()) + mf = generate_yaml_mapfile(bug._get_saved_settings()) encoding.set_file_contents(path, mf) self.vcs._vcs_update(path) @@ -206,10 +272,10 @@ class Upgrade_1_2_to_1_3 (Upgrader): mf = encoding.get_file_contents(path) if mf == libbe.util.InvalidObject: return # settings file does not exist - settings = mapfile.parse(mf) + settings = parse_yaml_mapfile(mf) if 'target' in settings: settings['target'] = self._target_bug(settings['target']).uuid - mf = mapfile.generate(settings) + mf = generate_yaml_mapfile(settings) encoding.set_file_contents(path, mf) self.vcs._vcs_update(path) @@ -219,7 +285,7 @@ class Upgrade_1_2_to_1_3 (Upgrader): mf = encoding.get_file_contents(path) if mf == libbe.util.InvalidObject: return # settings file does not exist - settings = mapfile.parse(mf) + settings = parse_yaml_mapfile(mf) if 'target' in settings: target_bug = self._target_bug(settings['target']) @@ -231,7 +297,7 @@ class Upgrade_1_2_to_1_3 (Upgrader): settings['extra_strings'] = sorted(estrs) settings.pop('target') - mf = mapfile.generate(settings) + mf = generate_yaml_mapfile(settings) encoding.set_file_contents(path, mf) self.vcs._vcs_update(path) @@ -251,7 +317,7 @@ class Upgrade_1_3_to_1_4 (Upgrader): final_version = "Bugs Everywhere Directory v1.4" def _get_vcs_name(self): path = self.get_path('settings') - settings = mapfile.parse(encoding.get_file_contents(path)) + settings = parse_yaml_mapfile(encoding.get_file_contents(path)) if 'vcs_name' in settings: return settings['vcs_name'] return None @@ -288,10 +354,50 @@ class Upgrade_1_3_to_1_4 (Upgrader): raise Exception('Need user assistance\n%s' % '\n'.join(msg)) +class Upgrade_1_4_to_1_5 (Upgrader): + initial_version = "Bugs Everywhere Directory v1.4" + final_version = "Bugs Everywhere Directory v1.5" + def _get_vcs_name(self): + path = self.get_path('settings') + for p in os.listdir(self.get_path()): # check each bugdir's settings + path = os.path.join(self.get_path(), p) + if os.path.isdir(path): + settings_path = os.path.join(path, 'settings') + if os.path.isfile(settings_path): + settings = parse_yaml_mapfile(encoding.get_file_contents( + settings_path)) + if 'vcs_name' in settings: + return settings['vcs_name'] # first entry we found + return None + + def _upgrade(self): + """ + convert YAML settings to JSON (much faster parsing) + "./be/BUGDIR-UUID/settings" + "./be/BUGDIR-UUID/bugs/BUG-UUID/values" + "./be/BUGDIR-UUID/bugs/BUG-UUID/comments/COMMENT-UUID/values" + """ + self.repo = os.path.abspath(self.repo) + basenames = [p for p in os.listdir(self.get_path())] + for dirpath,dirnames,filenames in os.walk(self.get_path()): + for filename in filenames: + if filename in ['settings', 'values']: + self._upgrade_mapfile(os.path.join(dirpath, filename)) + + def _upgrade_mapfile(self, path): + contents = encoding.get_file_contents(path) + data = parse_yaml_mapfile(contents) + contents = mapfile.generate(data) + encoding.set_file_contents(path, contents) + self.vcs._vcs_update(path) + + upgraders = [Upgrade_1_0_to_1_1, Upgrade_1_1_to_1_2, Upgrade_1_2_to_1_3, - Upgrade_1_3_to_1_4] + Upgrade_1_3_to_1_4, + Upgrade_1_4_to_1_5] + upgrade_classes = {} for upgrader in upgraders: upgrade_classes[(upgrader.initial_version,upgrader.final_version)]=upgrader |