From eedd308ff46fb9d0529f4480d2d4ae17e435795d Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 8 Dec 2009 03:58:36 -0500 Subject: Use mapfile to only create & parse mapfile strings, not files --- libbe/storage/util/mapfile.py | 116 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 libbe/storage/util/mapfile.py (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/mapfile.py b/libbe/storage/util/mapfile.py new file mode 100644 index 0000000..a8d5516 --- /dev/null +++ b/libbe/storage/util/mapfile.py @@ -0,0 +1,116 @@ +# Copyright (C) 2005-2009 Aaron Bentley and Panometrics, Inc. +# Gianluca Montecchi +# W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Provide a means of saving and loading dictionaries of parameters. The +saved "mapfiles" should be clear, flat-text files, and allow easy merging of +independent/conflicting changes. +""" + +import errno +import os.path +import yaml + +import libbe +if libbe.TESTING == True: + import doctest + + +class IllegalKey(Exception): + def __init__(self, key): + Exception.__init__(self, 'Illegal key "%s"' % key) + self.key = key + +class IllegalValue(Exception): + def __init__(self, value): + Exception.__init__(self, 'Illegal value "%s"' % value) + self.value = value + +def generate(map): + """Generate a YAML mapfile content string. + >>> generate({"q":"p"}) + 'q: p\\n\\n' + >>> generate({"q":u"Fran\u00e7ais"}) + 'q: Fran\\xc3\\xa7ais\\n\\n' + >>> generate({"q":u"hello"}) + 'q: hello\\n\\n' + >>> generate({"q=":"p"}) + Traceback (most recent call last): + IllegalKey: Illegal key "q=" + >>> generate({"q:":"p"}) + Traceback (most recent call last): + IllegalKey: Illegal key "q:" + >>> generate({"q\\n":"p"}) + Traceback (most recent call last): + IllegalKey: Illegal key "q\\n" + >>> generate({"":"p"}) + Traceback (most recent call last): + IllegalKey: Illegal key "" + >>> generate({">q":"p"}) + Traceback (most recent call last): + IllegalKey: Illegal key ">q" + >>> generate({"q":"p\\n"}) + Traceback (most recent call last): + IllegalValue: Illegal value "p\\n" + """ + keys = map.keys() + keys.sort() + for key in keys: + try: + assert not key.startswith('>') + assert('\n' not in key) + assert('=' not in key) + assert(':' not in key) + assert(len(key) > 0) + except AssertionError: + raise IllegalKey(unicode(key).encode('unicode_escape')) + if '\n' in map[key]: + raise IllegalValue(unicode(map[key]).encode('unicode_escape')) + + lines = [] + for key in keys: + lines.append(yaml.safe_dump({key: map[key]}, + default_flow_style=False, + allow_unicode=True)) + lines.append('') + return '\n'.join(lines) + +def parse(contents): + """ + Parse a YAML mapfile string. + >>> parse('q: p\\n\\n')['q'] + 'p' + >>> parse('q: \\'p\\'\\n\\n')['q'] + 'p' + >>> contents = generate({"a":"b", "c":"d", "e":"f"}) + >>> dict = parse(contents) + >>> dict["a"] + 'b' + >>> dict["c"] + 'd' + >>> dict["e"] + 'f' + >>> contents = generate({"q":u"Fran\u00e7ais"}) + >>> dict = parse(contents) + >>> dict["q"] + u'Fran\\xe7ais' + """ + return yaml.load(contents) or {} + +if libbe.TESTING == True: + suite = doctest.DocTestSuite() -- cgit From df3a136010c3a05166a4a77028fd00a948d7cdeb Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 8 Dec 2009 04:10:10 -0500 Subject: Transitioned comment.py to new storage format. --- libbe/storage/util/__init__.py | 0 libbe/storage/util/config.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 libbe/storage/util/__init__.py create mode 100644 libbe/storage/util/config.py (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/__init__.py b/libbe/storage/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py new file mode 100644 index 0000000..ccd236b --- /dev/null +++ b/libbe/storage/util/config.py @@ -0,0 +1,94 @@ +# Copyright (C) 2005-2009 Aaron Bentley and Panometrics, Inc. +# Gianluca Montecchi +# W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Create, save, and load the per-user config file at path(). +""" + +import ConfigParser +import codecs +import locale +import os.path +import sys + +import libbe +if libbe.TESTING == True: + import doctest + + +default_encoding = sys.getfilesystemencoding() or locale.getpreferredencoding() + +def path(): + """Return the path to the per-user config file""" + return os.path.expanduser("~/.bugs_everywhere") + +def set_val(name, value, section="DEFAULT", encoding=None): + """Set a value in the per-user config file + + :param name: The name of the value to set + :param value: The new value to set (or None to delete the value) + :param section: The section to store the name/value in + """ + if encoding == None: + encoding = default_encoding + config = ConfigParser.ConfigParser() + if os.path.exists(path()) == False: # touch file or config + open(path(), "w").close() # read chokes on missing file + f = codecs.open(path(), "r", encoding) + config.readfp(f, path()) + f.close() + if value is not None: + config.set(section, name, value) + else: + config.remove_option(section, name) + f = codecs.open(path(), "w", encoding) + config.write(f) + f.close() + +def get_val(name, section="DEFAULT", default=None, encoding=None): + """ + Get a value from the per-user config file + + :param name: The name of the value to get + :section: The section that the name is in + :return: The value, or None + >>> get_val("junk") is None + True + >>> set_val("junk", "random") + >>> get_val("junk") + u'random' + >>> set_val("junk", None) + >>> get_val("junk") is None + True + """ + if os.path.exists(path()): + if encoding == None: + encoding = default_encoding + config = ConfigParser.ConfigParser() + f = codecs.open(path(), "r", encoding) + config.readfp(f, path()) + f.close() + try: + return config.get(section, name) + except ConfigParser.NoOptionError: + return default + else: + return default + +if libbe.TESTING == True: + suite = doctest.DocTestSuite() -- cgit From 79154201c1c012063aa3fe1881ff06a3f239fdc5 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 8 Dec 2009 09:01:26 -0500 Subject: Moved properties.py and settings_object.py to libbe/storage/util/ --- libbe/storage/util/properties.py | 642 ++++++++++++++++++++++++++++++++++ libbe/storage/util/settings_object.py | 433 +++++++++++++++++++++++ 2 files changed, 1075 insertions(+) create mode 100644 libbe/storage/util/properties.py create mode 100644 libbe/storage/util/settings_object.py (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/properties.py b/libbe/storage/util/properties.py new file mode 100644 index 0000000..ddd7b25 --- /dev/null +++ b/libbe/storage/util/properties.py @@ -0,0 +1,642 @@ +# Bugs Everywhere - a distributed bugtracker +# Copyright (C) 2008-2009 Gianluca Montecchi +# W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +This module provides a series of useful decorators for defining +various types of properties. For example usage, consider the +unittests at the end of the module. + +See + http://www.python.org/dev/peps/pep-0318/ +and + http://www.phyast.pitt.edu/~micheles/python/documentation.html +for more information on decorators. +""" + +import copy +import types + +import libbe +if libbe.TESTING == True: + import unittest + + +class ValueCheckError (ValueError): + def __init__(self, name, value, allowed): + action = "in" # some list of allowed values + if type(allowed) == types.FunctionType: + action = "allowed by" # some allowed-value check function + msg = "%s not %s %s for %s" % (value, action, allowed, name) + ValueError.__init__(self, msg) + self.name = name + self.value = value + self.allowed = allowed + +def Property(funcs): + """ + End a chain of property decorators, returning a property. + """ + args = {} + args["fget"] = funcs.get("fget", None) + args["fset"] = funcs.get("fset", None) + args["fdel"] = funcs.get("fdel", None) + args["doc"] = funcs.get("doc", None) + + #print "Creating a property with" + #for key, val in args.items(): print key, value + return property(**args) + +def doc_property(doc=None): + """ + Add a docstring to a chain of property decorators. + """ + def decorator(funcs=None): + """ + Takes either a dict of funcs {"fget":fnX, "fset":fnY, ...} + or a function fn() returning such a dict. + """ + if hasattr(funcs, "__call__"): + funcs = funcs() # convert from function-arg to dict + funcs["doc"] = doc + return funcs + return decorator + +def local_property(name, null=None, mutable_null=False): + """ + Define get/set access to per-parent-instance local storage. Uses + .__value to store the value for a particular owner instance. + If the .__value attribute does not exist, returns null. + + If mutable_null == True, we only release deepcopies of the null to + the outside world. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget", None) + fset = funcs.get("fset", None) + def _fget(self): + if fget is not None: + fget(self) + if mutable_null == True: + ret_null = copy.deepcopy(null) + else: + ret_null = null + value = getattr(self, "_%s_value" % name, ret_null) + return value + def _fset(self, value): + setattr(self, "_%s_value" % name, value) + if fset is not None: + fset(self, value) + funcs["fget"] = _fget + funcs["fset"] = _fset + funcs["name"] = name + return funcs + return decorator + +def settings_property(name, null=None): + """ + Similar to local_property, except where local_property stores the + value in instance.__value, settings_property stores the + value in instance.settings[name]. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget", None) + fset = funcs.get("fset", None) + def _fget(self): + if fget is not None: + fget(self) + value = self.settings.get(name, null) + return value + def _fset(self, value): + self.settings[name] = value + if fset is not None: + fset(self, value) + funcs["fget"] = _fget + funcs["fset"] = _fset + funcs["name"] = name + return funcs + return decorator + + +# Allow comparison and caching with _original_ values for mutables, +# since +# +# >>> a = [] +# >>> b = a +# >>> b.append(1) +# >>> a +# [1] +# >>> a==b +# True +def _hash_mutable_value(value): + return repr(value) +def _init_mutable_property_cache(self): + if not hasattr(self, "_mutable_property_cache_hash"): + # first call to _fget for any mutable property + self._mutable_property_cache_hash = {} + self._mutable_property_cache_copy = {} +def _set_cached_mutable_property(self, cacher_name, property_name, value): + _init_mutable_property_cache(self) + self._mutable_property_cache_hash[(cacher_name, property_name)] = \ + _hash_mutable_value(value) + self._mutable_property_cache_copy[(cacher_name, property_name)] = \ + copy.deepcopy(value) +def _get_cached_mutable_property(self, cacher_name, property_name, default=None): + _init_mutable_property_cache(self) + if (cacher_name, property_name) not in self._mutable_property_cache_copy: + return default + return self._mutable_property_cache_copy[(cacher_name, property_name)] +def _cmp_cached_mutable_property(self, cacher_name, property_name, value, default=None): + _init_mutable_property_cache(self) + if (cacher_name, property_name) not in self._mutable_property_cache_hash: + _set_cached_mutable_property(self, cacher_name, property_name, default) + old_hash = self._mutable_property_cache_hash[(cacher_name, property_name)] + return cmp(_hash_mutable_value(value), old_hash) + + +def defaulting_property(default=None, null=None, + mutable_default=False): + """ + Define a default value for get access to a property. + If the stored value is null, then default is returned. + + If mutable_default == True, we only release deepcopies of the + default to the outside world. + + null should never escape to the outside world, so don't worry + about it being a mutable. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + fset = funcs.get("fset") + name = funcs.get("name", "") + def _fget(self): + value = fget(self) + if value == null: + if mutable_default == True: + return copy.deepcopy(default) + else: + return default + return value + def _fset(self, value): + if value == default: + value = null + fset(self, value) + funcs["fget"] = _fget + funcs["fset"] = _fset + return funcs + return decorator + +def fn_checked_property(value_allowed_fn): + """ + Define allowed values for get/set access to a property. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + fset = funcs.get("fset") + name = funcs.get("name", "") + def _fget(self): + value = fget(self) + if value_allowed_fn(value) != True: + raise ValueCheckError(name, value, value_allowed_fn) + return value + def _fset(self, value): + if value_allowed_fn(value) != True: + raise ValueCheckError(name, value, value_allowed_fn) + fset(self, value) + funcs["fget"] = _fget + funcs["fset"] = _fset + return funcs + return decorator + +def checked_property(allowed=[]): + """ + Define allowed values for get/set access to a property. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + fset = funcs.get("fset") + name = funcs.get("name", "") + def _fget(self): + value = fget(self) + if value not in allowed: + raise ValueCheckError(name, value, allowed) + return value + def _fset(self, value): + if value not in allowed: + raise ValueCheckError(name, value, allowed) + fset(self, value) + funcs["fget"] = _fget + funcs["fset"] = _fset + return funcs + return decorator + +def cached_property(generator, initVal=None, mutable=False): + """ + Allow caching of values generated by generator(instance), where + instance is the instance to which this property belongs. Uses + .__cache to store a cache flag for a particular owner + instance. + + When the cache flag is True or missing and the stored value is + initVal, the first fget call triggers the generator function, + whose output is stored in __cached_value. That and + subsequent calls to fget will return this cached value. + + If the input value is no longer initVal (e.g. a value has been + loaded from disk or set with fset), that value overrides any + cached value, and this property has no effect. + + When the cache flag is False and the stored value is initVal, the + generator is not cached, but is called on every fget. + + The cache flag is missing on initialization. Particular instances + may override by setting their own flag. + + In the case that mutable == True, all caching is disabled and the + generator is called whenever the cached value would otherwise be + used. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + name = funcs.get("name", "") + def _fget(self): + cache = getattr(self, "_%s_cache" % name, True) + value = fget(self) + if value == initVal: + if cache == True and mutable == False: + if hasattr(self, "_%s_cached_value" % name): + value = getattr(self, "_%s_cached_value" % name) + else: + value = generator(self) + setattr(self, "_%s_cached_value" % name, value) + else: + value = generator(self) + return value + funcs["fget"] = _fget + return funcs + return decorator + +def primed_property(primer, initVal=None): + """ + Just like a cached_property, except that instead of returning a + new value and running fset to cache it, the primer performs some + background manipulation (e.g. loads data into instance.settings) + such that a _second_ pass through fget succeeds. + + The 'cache' flag becomes a 'prime' flag, with priming taking place + whenever .__prime is True, or is False or missing and + value == initVal. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + name = funcs.get("name", "") + def _fget(self): + prime = getattr(self, "_%s_prime" % name, False) + if prime == False: + value = fget(self) + if prime == True or (prime == False and value == initVal): + primer(self) + value = fget(self) + return value + funcs["fget"] = _fget + return funcs + return decorator + +def change_hook_property(hook, mutable=False, default=None): + """ + Call the function hook(instance, old_value, new_value) whenever a + value different from the current value is set (instance is a a + reference to the class instance to which this property belongs). + This is useful for saving changes to disk, etc. This function is + called _after_ the new value has been stored, allowing you to + change the stored value if you want. + + In the case of mutables, things are slightly trickier. Because + the property-owning class has no way of knowing when the value + changes. We work around this by caching a private deepcopy of the + mutable value, and checking for changes whenever the property is + set (obviously) or retrieved (to check for external changes). So + long as you're conscientious about accessing the property after + making external modifications, mutability won't be a problem. + t.x.append(5) # external modification + t.x # dummy access notices change and triggers hook + See testChangeHookMutableProperty for an example of the expected + behavior. + """ + def decorator(funcs): + if hasattr(funcs, "__call__"): + funcs = funcs() + fget = funcs.get("fget") + fset = funcs.get("fset") + name = funcs.get("name", "") + def _fget(self, new_value=None, from_fset=False): # only used if mutable == True + if from_fset == True: + value = new_value # compare new value with cached + else: + value = fget(self) # compare current value with cached + if _cmp_cached_mutable_property(self, "change hook property", name, value, default) != 0: + # there has been a change, cache new value + old_value = _get_cached_mutable_property(self, "change hook property", name, default) + _set_cached_mutable_property(self, "change hook property", name, value) + if from_fset == True: # return previously cached value + value = old_value + else: # the value changed while we weren't looking + hook(self, old_value, value) + return value + def _fset(self, value): + if mutable == True: # get cached previous value + old_value = _fget(self, new_value=value, from_fset=True) + else: + old_value = fget(self) + fset(self, value) + if value != old_value: + hook(self, old_value, value) + if mutable == True: + funcs["fget"] = _fget + funcs["fset"] = _fset + return funcs + return decorator + +if libbe.TESTING == True: + class DecoratorTests(unittest.TestCase): + def testLocalDoc(self): + class Test(object): + @Property + @doc_property("A fancy property") + def x(): + return {} + self.failUnless(Test.x.__doc__ == "A fancy property", + Test.x.__doc__) + def testLocalProperty(self): + class Test(object): + @Property + @local_property(name="LOCAL") + def x(): + return {} + t = Test() + self.failUnless(t.x == None, str(t.x)) + t.x = 'z' # the first set initializes ._LOCAL_value + self.failUnless(t.x == 'z', str(t.x)) + self.failUnless("_LOCAL_value" in dir(t), dir(t)) + self.failUnless(t._LOCAL_value == 'z', t._LOCAL_value) + def testSettingsProperty(self): + class Test(object): + @Property + @settings_property(name="attr") + def x(): + return {} + def __init__(self): + self.settings = {} + t = Test() + self.failUnless(t.x == None, str(t.x)) + t.x = 'z' # the first set initializes ._LOCAL_value + self.failUnless(t.x == 'z', str(t.x)) + self.failUnless("attr" in t.settings, t.settings) + self.failUnless(t.settings["attr"] == 'z', t.settings["attr"]) + def testDefaultingLocalProperty(self): + class Test(object): + @Property + @defaulting_property(default='y', null='x') + @local_property(name="DEFAULT", null=5) + def x(): return {} + t = Test() + self.failUnless(t.x == 5, str(t.x)) + t.x = 'x' + self.failUnless(t.x == 'y', str(t.x)) + t.x = 'y' + self.failUnless(t.x == 'y', str(t.x)) + t.x = 'z' + self.failUnless(t.x == 'z', str(t.x)) + t.x = 5 + self.failUnless(t.x == 5, str(t.x)) + def testCheckedLocalProperty(self): + class Test(object): + @Property + @checked_property(allowed=['x', 'y', 'z']) + @local_property(name="CHECKED") + def x(): return {} + def __init__(self): + self._CHECKED_value = 'x' + t = Test() + self.failUnless(t.x == 'x', str(t.x)) + try: + t.x = None + e = None + except ValueCheckError, e: + pass + self.failUnless(type(e) == ValueCheckError, type(e)) + def testTwoCheckedLocalProperties(self): + class Test(object): + @Property + @checked_property(allowed=['x', 'y', 'z']) + @local_property(name="X") + def x(): return {} + + @Property + @checked_property(allowed=['a', 'b', 'c']) + @local_property(name="A") + def a(): return {} + def __init__(self): + self._A_value = 'a' + self._X_value = 'x' + t = Test() + try: + t.x = 'a' + e = None + except ValueCheckError, e: + pass + self.failUnless(type(e) == ValueCheckError, type(e)) + t.x = 'x' + t.x = 'y' + t.x = 'z' + try: + t.a = 'x' + e = None + except ValueCheckError, e: + pass + self.failUnless(type(e) == ValueCheckError, type(e)) + t.a = 'a' + t.a = 'b' + t.a = 'c' + def testFnCheckedLocalProperty(self): + class Test(object): + @Property + @fn_checked_property(lambda v : v in ['x', 'y', 'z']) + @local_property(name="CHECKED") + def x(): return {} + def __init__(self): + self._CHECKED_value = 'x' + t = Test() + self.failUnless(t.x == 'x', str(t.x)) + try: + t.x = None + e = None + except ValueCheckError, e: + pass + self.failUnless(type(e) == ValueCheckError, type(e)) + def testCachedLocalProperty(self): + class Gen(object): + def __init__(self): + self.i = 0 + def __call__(self, owner): + self.i += 1 + return self.i + class Test(object): + @Property + @cached_property(generator=Gen(), initVal=None) + @local_property(name="CACHED") + def x(): return {} + t = Test() + self.failIf("_CACHED_cache" in dir(t), + getattr(t, "_CACHED_cache", None)) + self.failUnless(t.x == 1, t.x) + self.failUnless(t.x == 1, t.x) + self.failUnless(t.x == 1, t.x) + t.x = 8 + self.failUnless(t.x == 8, t.x) + self.failUnless(t.x == 8, t.x) + t._CACHED_cache = False # Caching is off, but the stored value + val = t.x # is 8, not the initVal (None), so we + self.failUnless(val == 8, val) # get 8. + t._CACHED_value = None # Now we've set the stored value to None + val = t.x # so future calls to fget (like this) + self.failUnless(val == 2, val) # will call the generator every time... + val = t.x + self.failUnless(val == 3, val) + val = t.x + self.failUnless(val == 4, val) + t._CACHED_cache = True # We turn caching back on, and get + self.failUnless(t.x == 1, str(t.x)) # the original cached value. + del t._CACHED_cached_value # Removing that value forces a + self.failUnless(t.x == 5, str(t.x)) # single cache-regenerating call + self.failUnless(t.x == 5, str(t.x)) # to the genenerator, after which + self.failUnless(t.x == 5, str(t.x)) # we get the new cached value. + def testPrimedLocalProperty(self): + class Test(object): + def prime(self): + self.settings["PRIMED"] = "initialized" + @Property + @primed_property(primer=prime, initVal=None) + @settings_property(name="PRIMED") + def x(): return {} + def __init__(self): + self.settings={} + t = Test() + self.failIf("_PRIMED_prime" in dir(t), + getattr(t, "_PRIMED_prime", None)) + self.failUnless(t.x == "initialized", t.x) + t.x = 1 + self.failUnless(t.x == 1, t.x) + t.x = None + self.failUnless(t.x == "initialized", t.x) + t._PRIMED_prime = True + t.x = 3 + self.failUnless(t.x == "initialized", t.x) + t._PRIMED_prime = False + t.x = 3 + self.failUnless(t.x == 3, t.x) + def testChangeHookLocalProperty(self): + class Test(object): + def _hook(self, old, new): + self.old = old + self.new = new + + @Property + @change_hook_property(_hook) + @local_property(name="HOOKED") + def x(): return {} + t = Test() + t.x = 1 + self.failUnless(t.old == None, t.old) + self.failUnless(t.new == 1, t.new) + t.x = 1 + self.failUnless(t.old == None, t.old) + self.failUnless(t.new == 1, t.new) + t.x = 2 + self.failUnless(t.old == 1, t.old) + self.failUnless(t.new == 2, t.new) + def testChangeHookMutableProperty(self): + class Test(object): + def _hook(self, old, new): + self.old = old + self.new = new + self.hook_calls += 1 + + @Property + @change_hook_property(_hook, mutable=True) + @local_property(name="HOOKED") + def x(): return {} + t = Test() + t.hook_calls = 0 + t.x = [] + self.failUnless(t.old == None, t.old) + self.failUnless(t.new == [], t.new) + self.failUnless(t.hook_calls == 1, t.hook_calls) + a = t.x + a.append(5) + t.x = a + self.failUnless(t.old == [], t.old) + self.failUnless(t.new == [5], t.new) + self.failUnless(t.hook_calls == 2, t.hook_calls) + t.x = [] + self.failUnless(t.old == [5], t.old) + self.failUnless(t.new == [], t.new) + self.failUnless(t.hook_calls == 3, t.hook_calls) + # now append without reassigning. this doesn't trigger the + # change, since we don't ever set t.x, only get it and mess + # with it. It does, however, update our t.new, since t.new = + # t.x and is not a static copy. + t.x.append(5) + self.failUnless(t.old == [5], t.old) + self.failUnless(t.new == [5], t.new) + self.failUnless(t.hook_calls == 3, t.hook_calls) + # however, the next t.x get _will_ notice the change... + a = t.x + self.failUnless(t.old == [], t.old) + self.failUnless(t.new == [5], t.new) + self.failUnless(t.hook_calls == 4, t.hook_calls) + t.x.append(6) # this append(6) is not noticed yet + self.failUnless(t.old == [], t.old) + self.failUnless(t.new == [5,6], t.new) + self.failUnless(t.hook_calls == 4, t.hook_calls) + # this append(7) is not noticed, but the t.x get causes the + # append(6) to be noticed + t.x.append(7) + self.failUnless(t.old == [5], t.old) + self.failUnless(t.new == [5,6,7], t.new) + self.failUnless(t.hook_calls == 5, t.hook_calls) + a = t.x # now the append(7) is noticed + self.failUnless(t.old == [5,6], t.old) + self.failUnless(t.new == [5,6,7], t.new) + self.failUnless(t.hook_calls == 6, t.hook_calls) + + suite = unittest.TestLoader().loadTestsFromTestCase(DecoratorTests) diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py new file mode 100644 index 0000000..be119dd --- /dev/null +++ b/libbe/storage/util/settings_object.py @@ -0,0 +1,433 @@ +# Bugs Everywhere - a distributed bugtracker +# Copyright (C) 2008-2009 Gianluca Montecchi +# W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +This module provides a base class implementing settings-dict based +property storage useful for BE objects with saved properties +(e.g. BugDir, Bug, Comment). For example usage, consider the +unittests at the end of the module. +""" + +import libbe +from properties import Property, doc_property, local_property, \ + defaulting_property, checked_property, fn_checked_property, \ + cached_property, primed_property, change_hook_property, \ + settings_property +if libbe.TESTING == True: + import doctest + import unittest + +class _Token (object): + """ + `Control' value class for properties. We want values that only + mean something to the settings_object module. + """ + pass + +class UNPRIMED (_Token): + "Property has not been primed." + pass + +class EMPTY (_Token): + """ + Property has been primed but has no user-set value, so use + default/generator value. + """ + pass + + +def prop_save_settings(self, old, new): + """ + The default action undertaken when a property changes. + """ + if self.storage != None and self.storage.is_writeable(): + self.save_settings() + +def prop_load_settings(self): + """ + The default action undertaken when an UNPRIMED property is accessed. + """ + if self.storage != None and self.storage.is_readable() \ + and self._settings_loaded==False: + self.load_settings() + else: + self._setup_saved_settings(flag_as_loaded=False) + +# Some name-mangling routines for pretty printing setting names +def setting_name_to_attr_name(self, name): + """ + Convert keys to the .settings dict into their associated + SavedSettingsObject attribute names. + >>> print setting_name_to_attr_name(None,"User-id") + user_id + """ + return name.lower().replace('-', '_') + +def attr_name_to_setting_name(self, name): + """ + The inverse of setting_name_to_attr_name. + >>> print attr_name_to_setting_name(None, "user_id") + User-id + """ + return name.capitalize().replace('_', '-') + + +def versioned_property(name, doc, + default=None, generator=None, + change_hook=prop_save_settings, + mutable=False, + primer=prop_load_settings, + allowed=None, check_fn=None, + settings_properties=[], + required_saved_properties=[], + require_save=False): + """ + Combine the common decorators in a single function. + + Use zero or one (but not both) of default or generator, since a + working default will keep the generator from functioning. Use the + default if you know what you want the default value to be at + 'coding time'. Use the generator if you can write a function to + determine a valid default at run time. If both default and + generator are None, then the property will be a defaulting + property which defaults to None. + + allowed and check_fn have a similar relationship, although you can + use both of these if you want. allowed compares the proposed + value against a list determined at 'coding time' and check_fn + allows more flexible comparisons to take place at run time. + + Set require_save to True if you want to save the default/generated + value for a property, to protect against future changes. E.g., we + currently expect all comments to be 'text/plain' but in the future + we may want to default to 'text/html'. If we don't want the old + comments to be interpreted as 'text/html', we would require that + the content type be saved. + + change_hook, primer, settings_properties, and + required_saved_properties are only options to get their defaults + into our local scope. Don't mess with them. + + Set mutable=True if: + * default is a mutable + * your generator function may return mutables + * you set change_hook and might have mutable property values + See the docstrings in libbe.properties for details on how each of + these cases are handled. + """ + settings_properties.append(name) + if require_save == True: + required_saved_properties.append(name) + def decorator(funcs): + fulldoc = doc + if default != None or generator == None: + defaulting = defaulting_property(default=default, null=EMPTY, + mutable_default=mutable) + fulldoc += "\n\nThis property defaults to %s." % default + if generator != None: + cached = cached_property(generator=generator, initVal=EMPTY, + mutable=mutable) + fulldoc += "\n\nThis property is generated with %s." % generator + if check_fn != None: + fn_checked = fn_checked_property(value_allowed_fn=check_fn) + fulldoc += "\n\nThis property is checked with %s." % check_fn + if allowed != None: + checked = checked_property(allowed=allowed) + fulldoc += "\n\nThe allowed values for this property are: %s." \ + % (', '.join(allowed)) + hooked = change_hook_property(hook=change_hook, mutable=mutable, + default=EMPTY) + primed = primed_property(primer=primer, initVal=UNPRIMED) + settings = settings_property(name=name, null=UNPRIMED) + docp = doc_property(doc=fulldoc) + deco = hooked(primed(settings(docp(funcs)))) + if default != None or generator == None: + deco = defaulting(deco) + if generator != None: + deco = cached(deco) + if check_fn != None: + deco = fn_checked(deco) + if allowed != None: + deco = checked(deco) + return Property(deco) + return decorator + +class SavedSettingsObject(object): + + # Keep a list of properties that may be stored in the .settings dict. + #settings_properties = [] + + # A list of properties that we save to disk, even if they were + # never set (in which case we save the default value). This + # protects against future changes in default values. + #required_saved_properties = [] + + _setting_name_to_attr_name = setting_name_to_attr_name + _attr_name_to_setting_name = attr_name_to_setting_name + + def __init__(self): + self._settings_loaded = False + self.storage = None + self.settings = {} + + def load_settings(self): + """Load the settings from disk.""" + # Override. Must call ._setup_saved_settings() after loading. + self.settings = {} + self._setup_saved_settings() + + def _setup_saved_settings(self, flag_as_loaded=True): + """ + To be run after setting self.settings up from disk. Marks all + settings as primed. + """ + for property in self.settings_properties: + if property not in self.settings: + self.settings[property] = EMPTY + elif self.settings[property] == UNPRIMED: + self.settings[property] = EMPTY + if flag_as_loaded == True: + self._settings_loaded = True + + def save_settings(self): + """Load the settings from disk.""" + # Override. Should save the dict output of ._get_saved_settings() + settings = self._get_saved_settings() + pass # write settings to disk.... + + def _get_saved_settings(self): + settings = {} + for k,v in self.settings.items(): + if v != None and v != EMPTY: + settings[k] = v + for k in self.required_saved_properties: + settings[k] = getattr(self, self._setting_name_to_attr_name(k)) + return settings + + def clear_cached_setting(self, setting=None): + "If setting=None, clear *all* cached settings" + if setting != None: + if hasattr(self, "_%s_cached_value" % setting): + delattr(self, "_%s_cached_value" % setting) + else: + for setting in settings_properties: + self.clear_cached_setting(setting) + + +if libbe.TESTING == True: + class SavedSettingsObjectTests(unittest.TestCase): + def testSimpleProperty(self): + """Testing a minimal versioned property""" + class Test(SavedSettingsObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property(name="Content-type", + doc="A test property", + settings_properties=settings_properties, + required_saved_properties= \ + required_saved_properties) + def content_type(): return {} + def __init__(self): + SavedSettingsObject.__init__(self) + t = Test() + # access missing setting + self.failUnless(t._settings_loaded == False, t._settings_loaded) + self.failUnless(len(t.settings) == 0, len(t.settings)) + self.failUnless(t.content_type == None, t.content_type) + # accessing t.content_type triggers the priming, which runs + # t._setup_saved_settings, which fills out t.settings with + # EMPTY data. t._settings_loaded is still false though, since + # the default priming does not do any of the `official' loading + # that occurs in t.load_settings. + self.failUnless(len(t.settings) == 1, len(t.settings)) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + self.failUnless(t._settings_loaded == False, t._settings_loaded) + # load settings creates an EMPTY value in the settings array + t.load_settings() + self.failUnless(t._settings_loaded == True, t._settings_loaded) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + self.failUnless(t.content_type == None, t.content_type) + self.failUnless(len(t.settings) == 1, len(t.settings)) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + # now we set a value + t.content_type = 5 + self.failUnless(t.settings["Content-type"] == 5, + t.settings["Content-type"]) + self.failUnless(t.content_type == 5, t.content_type) + self.failUnless(t.settings["Content-type"] == 5, + t.settings["Content-type"]) + # now we set another value + t.content_type = "text/plain" + self.failUnless(t.content_type == "text/plain", t.content_type) + self.failUnless(t.settings["Content-type"] == "text/plain", + t.settings["Content-type"]) + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/plain"}, + t._get_saved_settings()) + # now we clear to the post-primed value + t.content_type = EMPTY + self.failUnless(t._settings_loaded == True, t._settings_loaded) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + self.failUnless(t.content_type == None, t.content_type) + self.failUnless(len(t.settings) == 1, len(t.settings)) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + def testDefaultingProperty(self): + """Testing a defaulting versioned property""" + class Test(SavedSettingsObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property(name="Content-type", + doc="A test property", + default="text/plain", + settings_properties=settings_properties, + required_saved_properties= \ + required_saved_properties) + def content_type(): return {} + def __init__(self): + SavedSettingsObject.__init__(self) + t = Test() + self.failUnless(t._settings_loaded == False, t._settings_loaded) + self.failUnless(t.content_type == "text/plain", t.content_type) + self.failUnless(t._settings_loaded == False, t._settings_loaded) + t.load_settings() + self.failUnless(t._settings_loaded == True, t._settings_loaded) + self.failUnless(t.content_type == "text/plain", t.content_type) + self.failUnless(t.settings["Content-type"] == EMPTY, + t.settings["Content-type"]) + self.failUnless(t._get_saved_settings() == {}, + t._get_saved_settings()) + t.content_type = "text/html" + self.failUnless(t.content_type == "text/html", + t.content_type) + self.failUnless(t.settings["Content-type"] == "text/html", + t.settings["Content-type"]) + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/html"}, + t._get_saved_settings()) + def testRequiredDefaultingProperty(self): + """Testing a required defaulting versioned property""" + class Test(SavedSettingsObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property(name="Content-type", + doc="A test property", + default="text/plain", + settings_properties=settings_properties, + required_saved_properties= \ + required_saved_properties, + require_save=True) + def content_type(): return {} + def __init__(self): + SavedSettingsObject.__init__(self) + t = Test() + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/plain"}, + t._get_saved_settings()) + t.content_type = "text/html" + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/html"}, + t._get_saved_settings()) + def testClassVersionedPropertyDefinition(self): + """Testing a class-specific _versioned property decorator""" + class Test(SavedSettingsObject): + settings_properties = [] + required_saved_properties = [] + def _versioned_property(settings_properties= \ + settings_properties, + required_saved_properties= \ + required_saved_properties, + **kwargs): + if "settings_properties" not in kwargs: + kwargs["settings_properties"] = settings_properties + if "required_saved_properties" not in kwargs: + kwargs["required_saved_properties"] = \ + required_saved_properties + return versioned_property(**kwargs) + @_versioned_property(name="Content-type", + doc="A test property", + default="text/plain", + require_save=True) + def content_type(): return {} + def __init__(self): + SavedSettingsObject.__init__(self) + t = Test() + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/plain"}, + t._get_saved_settings()) + t.content_type = "text/html" + self.failUnless(t._get_saved_settings() == \ + {"Content-type":"text/html"}, + t._get_saved_settings()) + def testMutableChangeHookedProperty(self): + """Testing a mutable change-hooked property""" + SAVES = [] + def prop_log_save_settings(self, old, new, saves=SAVES): + saves.append("'%s' -> '%s'" % (str(old), str(new))) + prop_save_settings(self, old, new) + class Test(SavedSettingsObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property(name="List-type", + doc="A test property", + mutable=True, + change_hook=prop_log_save_settings, + settings_properties=settings_properties, + required_saved_properties= \ + required_saved_properties) + def list_type(): return {} + def __init__(self): + SavedSettingsObject.__init__(self) + t = Test() + self.failUnless(t._settings_loaded == False, t._settings_loaded) + t.load_settings() + self.failUnless(SAVES == [], SAVES) + self.failUnless(t._settings_loaded == True, t._settings_loaded) + self.failUnless(t.list_type == None, t.list_type) + self.failUnless(SAVES == [], SAVES) + self.failUnless(t.settings["List-type"]==EMPTY, + t.settings["List-type"]) + t.list_type = [] + self.failUnless(t.settings["List-type"] == [], + t.settings["List-type"]) + self.failUnless(SAVES == [ + "'' -> '[]'" + ], SAVES) + t.list_type.append(5) + self.failUnless(SAVES == [ + "'' -> '[]'", + ], SAVES) + self.failUnless(t.settings["List-type"] == [5], + t.settings["List-type"]) + self.failUnless(SAVES == [ # the append(5) has not yet been saved + "'' -> '[]'", + ], SAVES) + self.failUnless(t.list_type == [5], t.list_type)#get triggers saved + + self.failUnless(SAVES == [ # now the append(5) has been saved. + "'' -> '[]'", + "'[]' -> '[5]'" + ], SAVES) + + unitsuite = unittest.TestLoader().loadTestsFromTestCase( \ + SavedSettingsObjectTests) + suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) -- cgit From f8a498f76d7bbcb42cf7bbc80164d98bfe57f8ab Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 12 Dec 2009 01:43:20 -0500 Subject: Added libbe.ui.util.user for managing user ids. --- libbe/storage/util/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py index ccd236b..a0fea0c 100644 --- a/libbe/storage/util/config.py +++ b/libbe/storage/util/config.py @@ -22,16 +22,15 @@ Create, save, and load the per-user config file at path(). import ConfigParser import codecs -import locale import os.path -import sys import libbe +import libbe.util.encoding if libbe.TESTING == True: import doctest -default_encoding = sys.getfilesystemencoding() or locale.getpreferredencoding() +default_encoding = libbe.util.encoding.get_filesystem_encoding() def path(): """Return the path to the per-user config file""" -- cgit From dff6bd9bf89ca80e2265696a478e540476718c9c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 12 Dec 2009 20:57:59 -0500 Subject: Moved be to libbe.ui.command_line and transitioned to Command format. --- libbe/storage/util/settings_object.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index be119dd..760df03 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -410,21 +410,21 @@ if libbe.TESTING == True: self.failUnless(t.settings["List-type"] == [], t.settings["List-type"]) self.failUnless(SAVES == [ - "'' -> '[]'" + "'' -> '[]'" ], SAVES) t.list_type.append(5) self.failUnless(SAVES == [ - "'' -> '[]'", + "'' -> '[]'", ], SAVES) self.failUnless(t.settings["List-type"] == [5], t.settings["List-type"]) self.failUnless(SAVES == [ # the append(5) has not yet been saved - "'' -> '[]'", + "'' -> '[]'", ], SAVES) self.failUnless(t.list_type == [5], t.list_type)#get triggers saved self.failUnless(SAVES == [ # now the append(5) has been saved. - "'' -> '[]'", + "'' -> '[]'", "'[]' -> '[5]'" ], SAVES) -- cgit From 4d057dab603f42ec40b911dbee6792dcf107bd14 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 13 Dec 2009 06:19:23 -0500 Subject: Converted libbe.storage.vcs.base to new Storage format. --- libbe/storage/util/upgrade.py | 242 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 libbe/storage/util/upgrade.py (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py new file mode 100644 index 0000000..7ef760e --- /dev/null +++ b/libbe/storage/util/upgrade.py @@ -0,0 +1,242 @@ +# Copyright (C) 2009 W. Trevor King +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Handle conversion between the various BE storage formats. +""" + +import codecs +import os, os.path +import sys + +import libbe +import libbe.bug as bug +import libbe.util.encoding as encoding +import libbe.storage.util.mapfile as mapfile + +if libbe.TESTING == True: + import doctest + +# a list of all past versions +BUGDIR_DISK_VERSIONS = ["Bugs Everywhere Tree 1 0", + "Bugs Everywhere Directory v1.1", + "Bugs Everywhere Directory v1.2", + "Bugs Everywhere Directory v1.3"] + +# the current version +BUGDIR_DISK_VERSION = BUGDIR_DISK_VERSIONS[-1] + +class Upgrader (object): + "Class for converting between different on-disk BE storage formats." + initial_version = None + final_version = None + def __init__(self, root): + self.root = root + + def get_path(self, *args): + """ + Return a path relative to .root. + """ + dir = os.path.join(self.root, '.be') + if len(args) == 0: + return dir + assert args[0] in ['version', 'settings', 'bugs'], str(args) + return os.path.join(dir, *args) + + def check_initial_version(self): + path = self.get_path('version') + version = self.vcs.get_file_contents(path).rstrip('\n') + assert version == self.initial_version, version + + def set_version(self): + path = self.get_path("version") + self.vcs.set_file_contents(path, self.final_version+"\n") + + def upgrade(self): + print >> sys.stderr, "upgrading bugdir from '%s' to '%s'" \ + % (self.initial_version, self.final_version) + self.check_initial_version() + self.set_version() + self._upgrade() + + def _upgrade(self): + raise NotImplementedError + + +class Upgrade_1_0_to_1_1 (Upgrader): + initial_version = "Bugs Everywhere Tree 1 0" + final_version = "Bugs Everywhere Directory v1.1" + def _upgrade_mapfile(self, path): + contents = self.vcs.get_file_contents(path) + old_format = False + for line in contents.splitlines(): + if len(line.split("=")) == 2: + old_format = True + break + if old_format == True: + # translate to YAML. + newlines = [] + for line in contents.splitlines(): + line = line.rstrip('\n') + if len(line) == 0: + continue + fields = line.split("=") + if len(fields) == 2: + key,value = fields + newlines.append('%s: "%s"' % (key, value.replace('"','\\"'))) + else: + newlines.append(line) + contents = '\n'.join(newlines) + # load the YAML and save + map = mapfile.parse(contents) + mapfile.map_save(self.vcs, path, map) + + def _upgrade(self): + """ + Comment value field "From" -> "Author". + Homegrown mapfile -> YAML. + """ + path = self.get_path("settings") + self._upgrade_mapfile(path) + for bug_uuid in os.listdir(self.get_path("bugs")): + path = self.get_path("bugs", bug_uuid, "values") + self._upgrade_mapfile(path) + c_path = ["bugs", bug_uuid, "comments"] + if not os.path.exists(self.get_path(*c_path)): + continue # no comments for this bug + for comment_uuid in os.listdir(self.get_path(*c_path)): + path_list = c_path + [comment_uuid, "values"] + path = self.get_path(*path_list) + self._upgrade_mapfile(path) + settings = mapfile.map_load(self.vcs, path) + if "From" in settings: + settings["Author"] = settings.pop("From") + mapfile.map_save(self.vcs, path, settings) + + +class Upgrade_1_1_to_1_2 (Upgrader): + initial_version = "Bugs Everywhere Directory v1.1" + final_version = "Bugs Everywhere Directory v1.2" + def _upgrade(self): + """ + BugDir settings field "rcs_name" -> "vcs_name". + """ + path = self.get_path("settings") + settings = mapfile.map_load(self.vcs, path) + if "rcs_name" in settings: + settings["vcs_name"] = settings.pop("rcs_name") + mapfile.map_save(self.vcs, path, settings) + +class Upgrade_1_2_to_1_3 (Upgrader): + initial_version = "Bugs Everywhere Directory v1.2" + final_version = "Bugs Everywhere Directory v1.3" + def __init__(self, *args, **kwargs): + Upgrader.__init__(self, *args, **kwargs) + self._targets = {} # key: target text,value: new target bug + path = self.get_path('settings') + settings = mapfile.map_load(self.vcs, path) + if 'vcs_name' in settings: + old_vcs = self.vcs + self.vcs = vcs.vcs_by_name(settings['vcs_name']) + self.vcs.root(self.root) + self.vcs.encoding = old_vcs.encoding + + def _target_bug(self, target_text): + if target_text not in self._targets: + _bug = bug.Bug(bugdir=self, summary=target_text) + # note: we're not a bugdir, but all Bug.save() needs is + # .root, .vcs, and .get_path(), which we have. + _bug.severity = 'target' + self._targets[target_text] = _bug + return self._targets[target_text] + + def _upgrade_bugdir_mapfile(self): + path = self.get_path('settings') + settings = mapfile.map_load(self.vcs, path) + if 'target' in settings: + settings['target'] = self._target_bug(settings['target']).uuid + mapfile.map_save(self.vcs, path, settings) + + def _upgrade_bug_mapfile(self, bug_uuid): + import becommands.depend + path = self.get_path('bugs', bug_uuid, 'values') + settings = mapfile.map_load(self.vcs, path) + if 'target' in settings: + target_bug = self._target_bug(settings['target']) + _bug = bug.Bug(bugdir=self, uuid=bug_uuid, from_disk=True) + # note: we're not a bugdir, but all Bug.load_settings() + # needs is .root, .vcs, and .get_path(), which we have. + becommands.depend.add_block(target_bug, _bug) + _bug.settings.pop('target') + _bug.save() + + def _upgrade(self): + """ + Bug value field "target" -> target bugs. + Bugdir value field "target" -> pointer to current target bug. + """ + for bug_uuid in os.listdir(self.get_path('bugs')): + self._upgrade_bug_mapfile(bug_uuid) + self._upgrade_bugdir_mapfile() + for _bug in self._targets.values(): + _bug.save() + +upgraders = [Upgrade_1_0_to_1_1, + Upgrade_1_1_to_1_2, + Upgrade_1_2_to_1_3] +upgrade_classes = {} +for upgrader in upgraders: + upgrade_classes[(upgrader.initial_version,upgrader.final_version)]=upgrader + +def upgrade(path, current_version, + target_version=BUGDIR_DISK_VERSION): + """ + Call the appropriate upgrade function to convert current_version + to target_version. If a direct conversion function does not exist, + use consecutive conversion functions. + """ + if current_version not in BUGDIR_DISK_VERSIONS: + raise NotImplementedError, \ + "Cannot handle version '%s' yet." % version + if target_version not in BUGDIR_DISK_VERSIONS: + raise NotImplementedError, \ + "Cannot handle version '%s' yet." % version + + if (current_version, target_version) in upgrade_classes: + # direct conversion + upgrade_class = upgrade_classes[(current_version, target_version)] + u = upgrade_class(path) + u.upgrade() + else: + # consecutive single-step conversion + i = BUGDIR_DISK_VERSIONS.index(current_version) + while True: + version_a = BUGDIR_DISK_VERSIONS[i] + version_b = BUGDIR_DISK_VERSIONS[i+1] + try: + upgrade_class = upgrade_classes[(version_a, version_b)] + except KeyError: + raise NotImplementedError, \ + "Cannot convert version '%s' to '%s' yet." \ + % (version_a, version_b) + u = upgrade_class(path) + u.upgrade() + if version_b == target_version: + break + i += 1 + +if libbe.TESTING == True: + suite = doctest.DocTestSuite() -- cgit From 19fe0817ba7c2cd04caea3adfa82d4490288a548 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 14 Dec 2009 07:37:51 -0500 Subject: Transitioned comment to Command format --- libbe/storage/util/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py index a0fea0c..a0a252e 100644 --- a/libbe/storage/util/config.py +++ b/libbe/storage/util/config.py @@ -47,15 +47,15 @@ def set_val(name, value, section="DEFAULT", encoding=None): encoding = default_encoding config = ConfigParser.ConfigParser() if os.path.exists(path()) == False: # touch file or config - open(path(), "w").close() # read chokes on missing file - f = codecs.open(path(), "r", encoding) + open(path(), 'w').close() # read chokes on missing file + f = codecs.open(path(), 'r', encoding) config.readfp(f, path()) f.close() if value is not None: config.set(section, name, value) else: config.remove_option(section, name) - f = codecs.open(path(), "w", encoding) + f = codecs.open(path(), 'w', encoding) config.write(f) f.close() @@ -79,7 +79,7 @@ def get_val(name, section="DEFAULT", default=None, encoding=None): if encoding == None: encoding = default_encoding config = ConfigParser.ConfigParser() - f = codecs.open(path(), "r", encoding) + f = codecs.open(path(), 'r', encoding) config.readfp(f, path()) f.close() try: -- cgit From 89b7a1411e4658e831f5d635534b24355dbb941d Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 15 Dec 2009 06:44:20 -0500 Subject: Fixed libbe.command.diff + ugly BugDir.duplicate_bugdir implementation duplicate_bugdir() works, but for the vcs backends, it could require shelling out for _every_ file read. This could, and probably will, be horribly slow. Still it works ;). I'm not sure what a better implementation would be. The old implementation checked out the entire earlier state into a temporary directory pros: single shell out, simple upgrade implementation cons: wouldn't work well for HTTP backens I think a good solution would run along the lines of the currently commented out code in duplicate_bugdir(), where a VersionedStorage.changed_since(revision) call would give you a list of changed files. diff could work off of that directly, without the need to generate a whole duplicate bugdir. I'm stuck on how to handle upgrades though... Also removed trailing whitespace from all python files. --- libbe/storage/util/config.py | 2 +- libbe/storage/util/mapfile.py | 48 ++++++++++++++++++++++------------- libbe/storage/util/settings_object.py | 5 ++-- libbe/storage/util/upgrade.py | 36 +++++++++++++------------- 4 files changed, 52 insertions(+), 39 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py index a0a252e..9f95d14 100644 --- a/libbe/storage/util/config.py +++ b/libbe/storage/util/config.py @@ -46,7 +46,7 @@ def set_val(name, value, section="DEFAULT", encoding=None): if encoding == None: encoding = default_encoding config = ConfigParser.ConfigParser() - if os.path.exists(path()) == False: # touch file or config + if os.path.exists(path()) == False: # touch file or config open(path(), 'w').close() # read chokes on missing file f = codecs.open(path(), 'r', encoding) config.readfp(f, path()) diff --git a/libbe/storage/util/mapfile.py b/libbe/storage/util/mapfile.py index a8d5516..35ae1a0 100644 --- a/libbe/storage/util/mapfile.py +++ b/libbe/storage/util/mapfile.py @@ -24,6 +24,7 @@ independent/conflicting changes. import errno import os.path +import types import yaml import libbe @@ -39,32 +40,37 @@ class IllegalKey(Exception): class IllegalValue(Exception): def __init__(self, value): Exception.__init__(self, 'Illegal value "%s"' % value) - self.value = value + self.value = value + +class InvalidMapfileContents(Exception): + def __init__(self, contents): + Exception.__init__(self, 'Invalid YAML contents') + self.contents = contents def generate(map): """Generate a YAML mapfile content string. - >>> generate({"q":"p"}) + >>> generate({'q':'p'}) 'q: p\\n\\n' - >>> generate({"q":u"Fran\u00e7ais"}) + >>> generate({'q':u'Fran\u00e7ais'}) 'q: Fran\\xc3\\xa7ais\\n\\n' - >>> generate({"q":u"hello"}) + >>> generate({'q':u'hello'}) 'q: hello\\n\\n' - >>> generate({"q=":"p"}) + >>> generate({'q=':'p'}) Traceback (most recent call last): IllegalKey: Illegal key "q=" - >>> generate({"q:":"p"}) + >>> generate({'q:':'p'}) Traceback (most recent call last): IllegalKey: Illegal key "q:" - >>> generate({"q\\n":"p"}) + >>> generate({'q\\n':'p'}) Traceback (most recent call last): IllegalKey: Illegal key "q\\n" - >>> generate({"":"p"}) + >>> generate({'':'p'}) Traceback (most recent call last): IllegalKey: Illegal key "" - >>> generate({">q":"p"}) + >>> generate({'>q':'p'}) Traceback (most recent call last): IllegalKey: Illegal key ">q" - >>> generate({"q":"p\\n"}) + >>> generate({'q':'p\\n'}) Traceback (most recent call last): IllegalValue: Illegal value "p\\n" """ @@ -97,20 +103,28 @@ def parse(contents): 'p' >>> parse('q: \\'p\\'\\n\\n')['q'] 'p' - >>> contents = generate({"a":"b", "c":"d", "e":"f"}) + >>> contents = generate({'a':'b', 'c':'d', 'e':'f'}) >>> dict = parse(contents) - >>> dict["a"] + >>> dict['a'] 'b' - >>> dict["c"] + >>> dict['c'] 'd' - >>> dict["e"] + >>> dict['e'] 'f' - >>> contents = generate({"q":u"Fran\u00e7ais"}) + >>> contents = generate({'q':u'Fran\u00e7ais'}) >>> dict = parse(contents) - >>> dict["q"] + >>> dict['q'] u'Fran\\xe7ais' + >>> dict = parse('a!') + Traceback (most recent call last): + ... + InvalidMapfileContents: Invalid YAML contents """ - return yaml.load(contents) or {} + c = yaml.load(contents) + if type(c) == types.StringType: + raise InvalidMapfileContents( + 'Unable to parse YAML (BE format missmatch?):\n\n%s' % contents) + return c or {} if libbe.TESTING == True: suite = doctest.DocTestSuite() diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 760df03..8b86829 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -197,9 +197,8 @@ class SavedSettingsObject(object): settings as primed. """ for property in self.settings_properties: - if property not in self.settings: - self.settings[property] = EMPTY - elif self.settings[property] == UNPRIMED: + if property not in self.settings \ + or self.settings[property] == UNPRIMED: self.settings[property] = EMPTY if flag_as_loaded == True: self._settings_loaded = True diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py index 7ef760e..c94f171 100644 --- a/libbe/storage/util/upgrade.py +++ b/libbe/storage/util/upgrade.py @@ -24,17 +24,16 @@ import sys import libbe import libbe.bug as bug -import libbe.util.encoding as encoding import libbe.storage.util.mapfile as mapfile +import libbe.util.encoding as encoding +import libbe.util.id -if libbe.TESTING == True: - import doctest # a list of all past versions -BUGDIR_DISK_VERSIONS = ["Bugs Everywhere Tree 1 0", - "Bugs Everywhere Directory v1.1", - "Bugs Everywhere Directory v1.2", - "Bugs Everywhere Directory v1.3"] +BUGDIR_DISK_VERSIONS = ['Bugs Everywhere Tree 1 0', + 'Bugs Everywhere Directory v1.1', + 'Bugs Everywhere Directory v1.2', + 'Bugs Everywhere Directory v1.3'] # the current version BUGDIR_DISK_VERSION = BUGDIR_DISK_VERSIONS[-1] @@ -43,13 +42,17 @@ class Upgrader (object): "Class for converting between different on-disk BE storage formats." initial_version = None final_version = None - def __init__(self, root): - self.root = root + def __init__(self, repo): + self.repo = repo - def get_path(self, *args): + def get_path(self, id): """ - Return a path relative to .root. + Return a path relative to .repo. """ + if id == 'version': + return os.path.join(self.repo, id) + +TODO dir = os.path.join(self.root, '.be') if len(args) == 0: return dir @@ -58,15 +61,15 @@ class Upgrader (object): def check_initial_version(self): path = self.get_path('version') - version = self.vcs.get_file_contents(path).rstrip('\n') + version = encoding.get_file_contents(path).rstrip('\n') assert version == self.initial_version, version def set_version(self): - path = self.get_path("version") - self.vcs.set_file_contents(path, self.final_version+"\n") + path = self.get_path('version') + encoding.set_file_contents(path, self.final_version+'\n') def upgrade(self): - print >> sys.stderr, "upgrading bugdir from '%s' to '%s'" \ + print >> sys.stderr, 'upgrading bugdir from "%s" to "%s"' \ % (self.initial_version, self.final_version) self.check_initial_version() self.set_version() @@ -237,6 +240,3 @@ def upgrade(path, current_version, if version_b == target_version: break i += 1 - -if libbe.TESTING == True: - suite = doctest.DocTestSuite() -- cgit From 214c4317bb90684dcfdab4d2402daa66fbad2e77 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 27 Dec 2009 15:58:29 -0500 Subject: Fixed libbe.storage.util.upgrade Note that it only upgrades on-disk versions, so you can't use a non-VCS storage backend whose version isn't your command's current storage version. See #bea/110/bd1# for reasoning. To see the on-disk storage version, look at .be/version To see your command's supported storage version, look at be --full-version I added test_upgrade.sh to exercise the upgrade mechanism on BE's own repository. --- libbe/storage/util/upgrade.py | 207 +++++++++++++++++++++++++++++++----------- 1 file changed, 153 insertions(+), 54 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py index c94f171..ce6831d 100644 --- a/libbe/storage/util/upgrade.py +++ b/libbe/storage/util/upgrade.py @@ -23,8 +23,11 @@ import os, os.path import sys import libbe -import libbe.bug as bug +import libbe.bug import libbe.storage.util.mapfile as mapfile +#import libbe.storage.vcs # delay import to avoid cyclic dependency +import libbe.ui.util.editor +import libbe.util import libbe.util.encoding as encoding import libbe.util.id @@ -33,7 +36,9 @@ import libbe.util.id BUGDIR_DISK_VERSIONS = ['Bugs Everywhere Tree 1 0', 'Bugs Everywhere Directory v1.1', 'Bugs Everywhere Directory v1.2', - 'Bugs Everywhere Directory v1.3'] + 'Bugs Everywhere Directory v1.3', + 'Bugs Everywhere Directory v1.4', + ] # the current version BUGDIR_DISK_VERSION = BUGDIR_DISK_VERSIONS[-1] @@ -43,30 +48,37 @@ class Upgrader (object): initial_version = None final_version = None def __init__(self, repo): + import libbe.storage.vcs + self.repo = repo + vcs_name = self._get_vcs_name() + if vcs_name == None: + vcs_name = 'None' + self.vcs = libbe.storage.vcs.vcs_by_name(vcs_name) + self.vcs.repo = self.repo + self.vcs.root() - def get_path(self, id): + def get_path(self, *args): """ - Return a path relative to .repo. + Return the absolute path using args relative to .be. """ - if id == 'version': - return os.path.join(self.repo, id) - -TODO - dir = os.path.join(self.root, '.be') + dir = os.path.join(self.repo, '.be') if len(args) == 0: return dir - assert args[0] in ['version', 'settings', 'bugs'], str(args) return os.path.join(dir, *args) + def _get_vcs_name(self): + return None + def check_initial_version(self): path = self.get_path('version') - version = encoding.get_file_contents(path).rstrip('\n') - assert version == self.initial_version, version + version = encoding.get_file_contents(path, decode=True).rstrip('\n') + assert version == self.initial_version, '%s: %s' % (path, version) def set_version(self): path = self.get_path('version') encoding.set_file_contents(path, self.final_version+'\n') + self.vcs._vcs_update(path) def upgrade(self): print >> sys.stderr, 'upgrading bugdir from "%s" to "%s"' \ @@ -82,11 +94,20 @@ TODO class Upgrade_1_0_to_1_1 (Upgrader): initial_version = "Bugs Everywhere Tree 1 0" final_version = "Bugs Everywhere Directory v1.1" + def _get_vcs_name(self): + path = self.get_path('settings') + settings = encoding.get_file_contents(path) + for line in settings.splitlines(False): + fields = line.split('=') + if len(fields) == 2 and fields[0] == 'rcs_name': + return fields[1] + return None + def _upgrade_mapfile(self, path): - contents = self.vcs.get_file_contents(path) + contents = encoding.get_file_contents(path, decode=True) old_format = False for line in contents.splitlines(): - if len(line.split("=")) == 2: + if len(line.split('=')) == 2: old_format = True break if old_format == True: @@ -105,43 +126,56 @@ class Upgrade_1_0_to_1_1 (Upgrader): contents = '\n'.join(newlines) # load the YAML and save map = mapfile.parse(contents) - mapfile.map_save(self.vcs, path, map) + contents = mapfile.generate(map) + encoding.set_file_contents(path, contents) + self.vcs._vcs_update(path) def _upgrade(self): """ Comment value field "From" -> "Author". Homegrown mapfile -> YAML. """ - path = self.get_path("settings") + path = self.get_path('settings') self._upgrade_mapfile(path) - for bug_uuid in os.listdir(self.get_path("bugs")): - path = self.get_path("bugs", bug_uuid, "values") + for bug_uuid in os.listdir(self.get_path('bugs')): + path = self.get_path('bugs', bug_uuid, 'values') self._upgrade_mapfile(path) - c_path = ["bugs", bug_uuid, "comments"] + c_path = ['bugs', bug_uuid, 'comments'] if not os.path.exists(self.get_path(*c_path)): continue # no comments for this bug for comment_uuid in os.listdir(self.get_path(*c_path)): - path_list = c_path + [comment_uuid, "values"] + path_list = c_path + [comment_uuid, 'values'] path = self.get_path(*path_list) self._upgrade_mapfile(path) - settings = mapfile.map_load(self.vcs, path) - if "From" in settings: - settings["Author"] = settings.pop("From") - mapfile.map_save(self.vcs, path, settings) + settings = mapfile.parse( + encoding.get_file_contents(path)) + if 'From' in settings: + settings['Author'] = settings.pop('From') + encoding.set_file_contents( + path, mapfile.generate(settings)) + self.vcs._vcs_update(path) class Upgrade_1_1_to_1_2 (Upgrader): initial_version = "Bugs Everywhere Directory v1.1" final_version = "Bugs Everywhere Directory v1.2" + def _get_vcs_name(self): + path = self.get_path('settings') + settings = mapfile.parse(encoding.get_file_contents(path)) + if 'rcs_name' in settings: + return settings['rcs_name'] + return None + def _upgrade(self): """ BugDir settings field "rcs_name" -> "vcs_name". """ - path = self.get_path("settings") - settings = mapfile.map_load(self.vcs, path) - if "rcs_name" in settings: - settings["vcs_name"] = settings.pop("rcs_name") - mapfile.map_save(self.vcs, path, settings) + path = self.get_path('settings') + settings = mapfile.parse(encoding.get_file_contents(path)) + if 'rcs_name' in settings: + settings['vcs_name'] = settings.pop('rcs_name') + encoding.set_file_contents(path, mapfile.generate(settings)) + self.vcs._vcs_update(path) class Upgrade_1_2_to_1_3 (Upgrader): initial_version = "Bugs Everywhere Directory v1.2" @@ -149,42 +183,64 @@ class Upgrade_1_2_to_1_3 (Upgrader): def __init__(self, *args, **kwargs): Upgrader.__init__(self, *args, **kwargs) self._targets = {} # key: target text,value: new target bug + + def _get_vcs_name(self): path = self.get_path('settings') - settings = mapfile.map_load(self.vcs, path) + settings = mapfile.parse(encoding.get_file_contents(path)) if 'vcs_name' in settings: - old_vcs = self.vcs - self.vcs = vcs.vcs_by_name(settings['vcs_name']) - self.vcs.root(self.root) - self.vcs.encoding = old_vcs.encoding + return settings['vcs_name'] + return None + + def _save_bug_settings(self, bug): + # The target bugs don't have comments + path = self.get_path('bugs', bug.uuid, 'values') + if not os.path.exists(path): + self.vcs._add_path(path, directory=False) + path = self.get_path('bugs', bug.uuid, 'values') + mf = mapfile.generate(bug._get_saved_settings()) + encoding.set_file_contents(path, mf) + self.vcs._vcs_update(path) def _target_bug(self, target_text): if target_text not in self._targets: - _bug = bug.Bug(bugdir=self, summary=target_text) - # note: we're not a bugdir, but all Bug.save() needs is - # .root, .vcs, and .get_path(), which we have. - _bug.severity = 'target' - self._targets[target_text] = _bug + bug = libbe.bug.Bug(summary=target_text) + bug.severity = 'target' + self._targets[target_text] = bug return self._targets[target_text] def _upgrade_bugdir_mapfile(self): path = self.get_path('settings') - settings = mapfile.map_load(self.vcs, path) + mf = encoding.get_file_contents(path) + if mf == libbe.util.InvalidObject: + return # settings file does not exist + settings = mapfile.parse(mf) if 'target' in settings: settings['target'] = self._target_bug(settings['target']).uuid - mapfile.map_save(self.vcs, path, settings) + mf = mapfile.generate(settings) + encoding.set_file_contents(path, mf) + self.vcs._vcs_update(path) def _upgrade_bug_mapfile(self, bug_uuid): - import becommands.depend + import libbe.command.depend as dep path = self.get_path('bugs', bug_uuid, 'values') - settings = mapfile.map_load(self.vcs, path) + mf = encoding.get_file_contents(path) + if mf == libbe.util.InvalidObject: + return # settings file does not exist + settings = mapfile.parse(mf) if 'target' in settings: target_bug = self._target_bug(settings['target']) - _bug = bug.Bug(bugdir=self, uuid=bug_uuid, from_disk=True) - # note: we're not a bugdir, but all Bug.load_settings() - # needs is .root, .vcs, and .get_path(), which we have. - becommands.depend.add_block(target_bug, _bug) - _bug.settings.pop('target') - _bug.save() + + blocked_by_string = '%s%s' % (dep.BLOCKED_BY_TAG, bug_uuid) + dep._add_remove_extra_string(target_bug, blocked_by_string, add=True) + blocks_string = dep._generate_blocks_string(target_bug) + estrs = settings.get('extra_strings', []) + estrs.append(blocks_string) + settings['extra_strings'] = sorted(estrs) + + settings.pop('target') + mf = mapfile.generate(settings) + encoding.set_file_contents(path, mf) + self.vcs._vcs_update(path) def _upgrade(self): """ @@ -194,12 +250,55 @@ class Upgrade_1_2_to_1_3 (Upgrader): for bug_uuid in os.listdir(self.get_path('bugs')): self._upgrade_bug_mapfile(bug_uuid) self._upgrade_bugdir_mapfile() - for _bug in self._targets.values(): - _bug.save() + for bug in self._targets.values(): + self._save_bug_settings(bug) + +class Upgrade_1_3_to_1_4 (Upgrader): + initial_version = "Bugs Everywhere Directory v1.3" + final_version = "Bugs Everywhere Directory v1.4" + def _get_vcs_name(self): + path = self.get_path('settings') + settings = mapfile.parse(encoding.get_file_contents(path)) + if 'vcs_name' in settings: + return settings['vcs_name'] + return None + + def _upgrade(self): + """ + add new directory "./be/BUGDIR-UUID" + "./be/bugs" -> "./be/BUGDIR-UUID/bugs" + "./be/settings" -> "./be/BUGDIR-UUID/settings" + """ + self.repo = os.path.abspath(self.repo) + basenames = [p for p in os.listdir(self.get_path())] + if not 'bugs' in basenames and not 'settings' in basenames \ + and len([p for p in basenames if len(p)==36]) == 1: + return # the user has upgraded the directory. + basenames = [p for p in basenames if p in ['bugs','settings']] + uuid = libbe.util.id.uuid_gen() + add = [self.get_path(uuid)] + move = [(self.get_path(p), self.get_path(uuid, p)) for p in basenames] + msg = ['Upgrading BE directory version v1.3 to v1.4', + '', + "Because BE's VCS drivers don't support 'move',", + 'please make the following changes with your VCS', + 'and re-run BE. Note that you can choose a different', + 'bugdir UUID to preserve uniformity across branches', + 'of a distributed repository.' + '', + 'add', + ' ' + '\n '.join(add), + 'move', + ' ' + '\n '.join(['%s %s' % (a,b) for a,b in move]), + ] + self.vcs._cached_path_id.destroy() + raise Exception('Need user assistance\n%s' % '\n'.join(msg)) + upgraders = [Upgrade_1_0_to_1_1, Upgrade_1_1_to_1_2, - Upgrade_1_2_to_1_3] + Upgrade_1_2_to_1_3, + Upgrade_1_3_to_1_4] upgrade_classes = {} for upgrader in upgraders: upgrade_classes[(upgrader.initial_version,upgrader.final_version)]=upgrader @@ -213,10 +312,10 @@ def upgrade(path, current_version, """ if current_version not in BUGDIR_DISK_VERSIONS: raise NotImplementedError, \ - "Cannot handle version '%s' yet." % version + "Cannot handle version '%s' yet." % current_version if target_version not in BUGDIR_DISK_VERSIONS: raise NotImplementedError, \ - "Cannot handle version '%s' yet." % version + "Cannot handle version '%s' yet." % current_version if (current_version, target_version) in upgrade_classes: # direct conversion -- cgit From dff704764d77bffbf6cc94c5ba4bb03309da45f8 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 27 Dec 2009 16:30:54 -0500 Subject: Added storage.Storage.storage_version() and command.InvalidStorageVersion. Now commands automatically check for storage version compatibility. --- libbe/storage/util/upgrade.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py index ce6831d..20ef1e4 100644 --- a/libbe/storage/util/upgrade.py +++ b/libbe/storage/util/upgrade.py @@ -25,6 +25,7 @@ import sys import libbe import libbe.bug import libbe.storage.util.mapfile as mapfile +from libbe.storage import STORAGE_VERSIONS, STORAGE_VERSION #import libbe.storage.vcs # delay import to avoid cyclic dependency import libbe.ui.util.editor import libbe.util @@ -32,17 +33,6 @@ import libbe.util.encoding as encoding import libbe.util.id -# a list of all past versions -BUGDIR_DISK_VERSIONS = ['Bugs Everywhere Tree 1 0', - 'Bugs Everywhere Directory v1.1', - 'Bugs Everywhere Directory v1.2', - 'Bugs Everywhere Directory v1.3', - 'Bugs Everywhere Directory v1.4', - ] - -# the current version -BUGDIR_DISK_VERSION = BUGDIR_DISK_VERSIONS[-1] - class Upgrader (object): "Class for converting between different on-disk BE storage formats." initial_version = None @@ -304,16 +294,16 @@ for upgrader in upgraders: upgrade_classes[(upgrader.initial_version,upgrader.final_version)]=upgrader def upgrade(path, current_version, - target_version=BUGDIR_DISK_VERSION): + target_version=STORAGE_VERSION): """ Call the appropriate upgrade function to convert current_version to target_version. If a direct conversion function does not exist, use consecutive conversion functions. """ - if current_version not in BUGDIR_DISK_VERSIONS: + if current_version not in STORAGE_VERSIONS: raise NotImplementedError, \ "Cannot handle version '%s' yet." % current_version - if target_version not in BUGDIR_DISK_VERSIONS: + if target_version not in STORAGE_VERSIONS: raise NotImplementedError, \ "Cannot handle version '%s' yet." % current_version @@ -324,10 +314,10 @@ def upgrade(path, current_version, u.upgrade() else: # consecutive single-step conversion - i = BUGDIR_DISK_VERSIONS.index(current_version) + i = STORAGE_VERSIONS.index(current_version) while True: - version_a = BUGDIR_DISK_VERSIONS[i] - version_b = BUGDIR_DISK_VERSIONS[i+1] + version_a = STORAGE_VERSIONS[i] + version_b = STORAGE_VERSIONS[i+1] try: upgrade_class = upgrade_classes[(version_a, version_b)] except KeyError: -- cgit From 4d4283ecd654f1efb058cd7f7dba6be88b70ee92 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 1 Jan 2010 08:11:08 -0500 Subject: Updated copyright information --- libbe/storage/util/config.py | 2 +- libbe/storage/util/mapfile.py | 2 +- libbe/storage/util/properties.py | 2 +- libbe/storage/util/settings_object.py | 2 +- libbe/storage/util/upgrade.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py index 9f95d14..8526724 100644 --- a/libbe/storage/util/config.py +++ b/libbe/storage/util/config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2005-2009 Aaron Bentley and Panometrics, Inc. +# Copyright (C) 2005-2010 Aaron Bentley and Panometrics, Inc. # Gianluca Montecchi # W. Trevor King # diff --git a/libbe/storage/util/mapfile.py b/libbe/storage/util/mapfile.py index 35ae1a0..0b8af23 100644 --- a/libbe/storage/util/mapfile.py +++ b/libbe/storage/util/mapfile.py @@ -1,4 +1,4 @@ -# Copyright (C) 2005-2009 Aaron Bentley and Panometrics, Inc. +# Copyright (C) 2005-2010 Aaron Bentley and Panometrics, Inc. # Gianluca Montecchi # W. Trevor King # diff --git a/libbe/storage/util/properties.py b/libbe/storage/util/properties.py index ddd7b25..f48cfa0 100644 --- a/libbe/storage/util/properties.py +++ b/libbe/storage/util/properties.py @@ -1,5 +1,5 @@ # Bugs Everywhere - a distributed bugtracker -# Copyright (C) 2008-2009 Gianluca Montecchi +# Copyright (C) 2008-2010 Gianluca Montecchi # W. Trevor King # # This program is free software; you can redistribute it and/or modify diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 8b86829..9f2b7af 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -1,5 +1,5 @@ # Bugs Everywhere - a distributed bugtracker -# Copyright (C) 2008-2009 Gianluca Montecchi +# Copyright (C) 2008-2010 Gianluca Montecchi # W. Trevor King # # This program is free software; you can redistribute it and/or modify diff --git a/libbe/storage/util/upgrade.py b/libbe/storage/util/upgrade.py index 20ef1e4..f3c4912 100644 --- a/libbe/storage/util/upgrade.py +++ b/libbe/storage/util/upgrade.py @@ -1,4 +1,4 @@ -# Copyright (C) 2009 W. Trevor King +# Copyright (C) 2009-2010 W. Trevor King # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit From be2de86f947b7bf2bb44f415df0a6f685633a64b Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 20 Jan 2010 19:15:02 -0500 Subject: Fix not-yet-loaded bug in SavedSettingsObject._get_saved_settings() The earlier implementation only copied in the currently loaded properties and the required ones. The new implementation copies in _all_ the non-default properties. --- libbe/storage/util/settings_object.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 9f2b7af..181d4ab 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -204,18 +204,31 @@ class SavedSettingsObject(object): self._settings_loaded = True def save_settings(self): - """Load the settings from disk.""" + """Save the settings to disk.""" # Override. Should save the dict output of ._get_saved_settings() settings = self._get_saved_settings() pass # write settings to disk.... def _get_saved_settings(self): + """ + In order to avoid overwriting unread on-disk data, make sure + we've loaded anything sitting on the disk. In the current + implementation, all the settings are stored in a single file, + so we need to load _all_ the saved settings. Another approach + would be per-setting saves, in which case you could skip this + step, since any setting changes would have forced that setting + load already. + """ settings = {} - for k,v in self.settings.items(): - if v != None and v != EMPTY: - settings[k] = v - for k in self.required_saved_properties: - settings[k] = getattr(self, self._setting_name_to_attr_name(k)) + for k in self.settings_properties: + if k in self.settings and \ + not self.settings[k] in [None, EMPTY]: + settings[k] = self.settings[k] + else: + value = getattr( + self, self._setting_name_to_attr_name(k)) + if value not in [None, EMPTY, []]: + settings[k] = value return settings def clear_cached_setting(self, setting=None): -- cgit From 7fae8599ba74ecf3d36c9466feec25b3c3f36529 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 21 Jan 2010 12:47:38 -0500 Subject: Update libbe.storage.util.settings_object tests for new ._get_saved_settings --- libbe/storage/util/settings_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 181d4ab..ca94f23 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -326,7 +326,8 @@ if libbe.TESTING == True: self.failUnless(t.content_type == "text/plain", t.content_type) self.failUnless(t.settings["Content-type"] == EMPTY, t.settings["Content-type"]) - self.failUnless(t._get_saved_settings() == {}, + self.failUnless(t._get_saved_settings() == + {"Content-type":"text/plain"}, t._get_saved_settings()) t.content_type = "text/html" self.failUnless(t.content_type == "text/html", -- cgit From a818e71b0c3ae00e7c67ce745e49c7cef7ceac55 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 22 Jan 2010 09:17:08 -0500 Subject: Added unprimableVal option to primed_property. For clearer handling of the "Tried to load but failed" case. --- libbe/storage/util/properties.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/properties.py b/libbe/storage/util/properties.py index f48cfa0..55bac85 100644 --- a/libbe/storage/util/properties.py +++ b/libbe/storage/util/properties.py @@ -303,12 +303,14 @@ def cached_property(generator, initVal=None, mutable=False): return funcs return decorator -def primed_property(primer, initVal=None): +def primed_property(primer, initVal=None, unprimeableVal=None): """ Just like a cached_property, except that instead of returning a - new value and running fset to cache it, the primer performs some + new value and running fset to cache it, the primer attempts some background manipulation (e.g. loads data into instance.settings) - such that a _second_ pass through fget succeeds. + such that a _second_ pass through fget succeeds. If the second + pass doesn't succeed (e.g. no readable storage), we give up and + return unprimeableVal. The 'cache' flag becomes a 'prime' flag, with priming taking place whenever .__prime is True, or is False or missing and @@ -326,6 +328,8 @@ def primed_property(primer, initVal=None): if prime == True or (prime == False and value == initVal): primer(self) value = fget(self) + if prime == False and value == initVal: + return unprimeableVal return value funcs["fget"] = _fget return funcs @@ -543,13 +547,14 @@ if libbe.TESTING == True: def testPrimedLocalProperty(self): class Test(object): def prime(self): - self.settings["PRIMED"] = "initialized" + self.settings["PRIMED"] = self.primeVal @Property - @primed_property(primer=prime, initVal=None) + @primed_property(primer=prime, initVal=None, unprimeableVal=2) @settings_property(name="PRIMED") def x(): return {} def __init__(self): self.settings={} + self.primeVal = "initialized" t = Test() self.failIf("_PRIMED_prime" in dir(t), getattr(t, "_PRIMED_prime", None)) @@ -564,6 +569,10 @@ if libbe.TESTING == True: t._PRIMED_prime = False t.x = 3 self.failUnless(t.x == 3, t.x) + # test unprimableVal + t.x = None + t.primeVal = None + self.failUnless(t.x == 2, t.x) def testChangeHookLocalProperty(self): class Test(object): def _hook(self, old, new): -- cgit From bda68bb5d93f4b608fb1dd17c5a0cf1bb406daf9 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 22 Jan 2010 11:30:26 -0500 Subject: Reworked settings_object module, but command.init tests still fail: $ python test.py libbe.command.init Doctest: libbe.command.init.Init ... FAIL ... ----------------------- File ".../libbe/command/init.py", line 47, in libbe.command.init.Init Failed example: ui.run(cmd) Exception raised: Traceback (most recent call last): ... File "/tmp/be.wtk/libbe/command/init.py", line 97, in _run bd = libbe.bugdir.BugDir(storage, from_storage=False) File "/tmp/be.wtk/libbe/bugdir.py", line 185, in __init__ self.save() File "/tmp/be.wtk/libbe/bugdir.py", line 228, in save self.save_settings() File "/tmp/be.wtk/libbe/bugdir.py", line 204, in save_settings mf = mapfile.generate(self._get_saved_settings()) File "/tmp/be.wtk/libbe/storage/util/settings_object.py", line 230, in _get_saved_settings self, self._setting_name_to_attr_name(k)) File "/tmp/be.wtk/libbe/storage/util/properties.py", line 194, in _fget value = fget(self) File "/tmp/be.wtk/libbe/storage/util/properties.py", line 329, in _fget primer(self) File "/tmp/be.wtk/libbe/storage/util/settings_object.py", line 69, in prop_load_settings self.load_settings() File "/tmp/be.wtk/libbe/bugdir.py", line 194, in load_settings self.settings = mapfile.parse(settings_mapfile) File "/tmp/be.wtk/libbe/storage/util/mapfile.py", line 123, in parse c = yaml.load(contents) ... File "/usr/lib/python2.6/site-packages/yaml/reader.py", line 213, in update_raw data = self.stream.read(size) AttributeError: 'NoneType' object has no attribute 'read' ... --- libbe/storage/util/settings_object.py | 345 ++++++++++++++++++++++------------ 1 file changed, 226 insertions(+), 119 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index ca94f23..655e0ed 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -40,7 +40,7 @@ class _Token (object): pass class UNPRIMED (_Token): - "Property has not been primed." + "Property has not been primed (loaded)." pass class EMPTY (_Token): @@ -60,13 +60,13 @@ def prop_save_settings(self, old, new): def prop_load_settings(self): """ - The default action undertaken when an UNPRIMED property is accessed. + The default action undertaken when an UNPRIMED property is + accessed. Attempt to run .load_settings(), which calls + ._setup_saved_settings() internally. If .storage is inaccessible, + don't do anything. """ - if self.storage != None and self.storage.is_readable() \ - and self._settings_loaded==False: + if self.storage != None and self.storage.is_readable(): self.load_settings() - else: - self._setup_saved_settings(flag_as_loaded=False) # Some name-mangling routines for pretty printing setting names def setting_name_to_attr_name(self, name): @@ -129,6 +129,12 @@ def versioned_property(name, doc, * you set change_hook and might have mutable property values See the docstrings in libbe.properties for details on how each of these cases are handled. + + The value stored in .settings[name] will be + * no value (or UNPRIMED) if the property has been neither set, + nor loaded as blank. + * EMPTY if the value has been loaded as blank. + * some value if the property has been either loaded or set. """ settings_properties.append(name) if require_save == True: @@ -152,7 +158,8 @@ def versioned_property(name, doc, % (', '.join(allowed)) hooked = change_hook_property(hook=change_hook, mutable=mutable, default=EMPTY) - primed = primed_property(primer=primer, initVal=UNPRIMED) + primed = primed_property(primer=primer, initVal=UNPRIMED, + unprimeableVal=EMPTY) settings = settings_property(name=name, null=UNPRIMED) docp = doc_property(doc=fulldoc) deco = hooked(primed(settings(docp(funcs)))) @@ -181,7 +188,6 @@ class SavedSettingsObject(object): _attr_name_to_setting_name = attr_name_to_setting_name def __init__(self): - self._settings_loaded = False self.storage = None self.settings = {} @@ -191,17 +197,15 @@ class SavedSettingsObject(object): self.settings = {} self._setup_saved_settings() - def _setup_saved_settings(self, flag_as_loaded=True): + def _setup_saved_settings(self): """ - To be run after setting self.settings up from disk. Marks all - settings as primed. + To be run after setting self.settings up from disk. Fills in + all missing settings entries with EMPTY. """ for property in self.settings_properties: if property not in self.settings \ or self.settings[property] == UNPRIMED: self.settings[property] = EMPTY - if flag_as_loaded == True: - self._settings_loaded = True def save_settings(self): """Save the settings to disk.""" @@ -220,15 +224,16 @@ class SavedSettingsObject(object): load already. """ settings = {} + for k in self.settings_properties: # force full load + if not k in self.settings or self.settings[k] == UNPRIMED: + value = getattr( + self, self._setting_name_to_attr_name(k)) for k in self.settings_properties: - if k in self.settings and \ - not self.settings[k] in [None, EMPTY]: + if k in self.settings and self.settings[k] != EMPTY: settings[k] = self.settings[k] - else: - value = getattr( + elif k in self.required_saved_properties: + settings[k] = getattr( self, self._setting_name_to_attr_name(k)) - if value not in [None, EMPTY, []]: - settings[k] = value return settings def clear_cached_setting(self, setting=None): @@ -242,134 +247,245 @@ class SavedSettingsObject(object): if libbe.TESTING == True: + import copy + + class TestStorage (list): + def __init__(self): + list.__init__(self) + self.readable = True + self.writeable = True + def is_readable(self): + return self.readable + def is_writeable(self): + return self.writeable + + class TestObject (SavedSettingsObject): + def load_settings(self): + self.load_count += 1 + if len(self.storage) == 0: + self.settings = {} + else: + self.settings = copy.deepcopy(self.storage[-1]) + self._setup_saved_settings() + def save_settings(self): + settings = self._get_saved_settings() + self.storage.append(copy.deepcopy(settings)) + def __init__(self): + SavedSettingsObject.__init__(self) + self.load_count = 0 + self.storage = TestStorage() + class SavedSettingsObjectTests(unittest.TestCase): - def testSimpleProperty(self): - """Testing a minimal versioned property""" - class Test(SavedSettingsObject): + def testSimplePropertyDoc(self): + """Testing a minimal versioned property docstring""" + class Test (TestObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property( + name="Content-type", + doc="A test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) + def content_type(): return {} + expected = "A test property\n\nThis property defaults to None." + self.failUnless(Test.content_type.__doc__ == expected, + Test.content_type.__doc__) + def testSimplePropertyFromMemory(self): + """Testing a minimal versioned property from memory""" + class Test (TestObject): settings_properties = [] required_saved_properties = [] - @versioned_property(name="Content-type", - doc="A test property", - settings_properties=settings_properties, - required_saved_properties= \ - required_saved_properties) + @versioned_property( + name="Content-type", + doc="A test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) def content_type(): return {} - def __init__(self): - SavedSettingsObject.__init__(self) t = Test() - # access missing setting - self.failUnless(t._settings_loaded == False, t._settings_loaded) self.failUnless(len(t.settings) == 0, len(t.settings)) + # accessing t.content_type triggers the priming, but + # t.storage.is_readable() == False, so nothing happens. + t.storage.readable = False + self.failUnless(t.content_type == None, t.content_type) + self.failUnless(t.settings == {}, t.settings) + self.failUnless(len(t.settings) == 0, len(t.settings)) + self.failUnless(t.content_type == None, t.content_type) + # accessing t.content_type triggers the priming again, and + # now that t.storage.is_readable() == True, this fills out + # t.settings with EMPTY data. At this point there should + # be one load and no saves. + t.storage.readable = True self.failUnless(t.content_type == None, t.content_type) - # accessing t.content_type triggers the priming, which runs - # t._setup_saved_settings, which fills out t.settings with - # EMPTY data. t._settings_loaded is still false though, since - # the default priming does not do any of the `official' loading - # that occurs in t.load_settings. self.failUnless(len(t.settings) == 1, len(t.settings)) - self.failUnless(t.settings["Content-type"] == EMPTY, - t.settings["Content-type"]) - self.failUnless(t._settings_loaded == False, t._settings_loaded) - # load settings creates an EMPTY value in the settings array - t.load_settings() - self.failUnless(t._settings_loaded == True, t._settings_loaded) self.failUnless(t.settings["Content-type"] == EMPTY, t.settings["Content-type"]) self.failUnless(t.content_type == None, t.content_type) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 0, len(t.storage)) + # an explicit call to load settings forces a reload, + # but nothing else changes. + t.load_settings() self.failUnless(len(t.settings) == 1, len(t.settings)) self.failUnless(t.settings["Content-type"] == EMPTY, t.settings["Content-type"]) + self.failUnless(t.content_type == None, t.content_type) + self.failUnless(t.load_count == 2, t.load_count) + self.failUnless(len(t.storage) == 0, len(t.storage)) # now we set a value t.content_type = 5 self.failUnless(t.settings["Content-type"] == 5, t.settings["Content-type"]) + self.failUnless(t.load_count == 2, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':5}], t.storage) + # getting its value changes nothing self.failUnless(t.content_type == 5, t.content_type) self.failUnless(t.settings["Content-type"] == 5, t.settings["Content-type"]) + self.failUnless(t.load_count == 2, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':5}], t.storage) # now we set another value t.content_type = "text/plain" self.failUnless(t.content_type == "text/plain", t.content_type) self.failUnless(t.settings["Content-type"] == "text/plain", t.settings["Content-type"]) + self.failUnless(t.load_count == 2, t.load_count) + self.failUnless(len(t.storage) == 2, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':5}, + {'Content-type':'text/plain'}], + t.storage) + # t._get_saved_settings() returns a dict of required or + # non-default values. self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/plain"}, t._get_saved_settings()) # now we clear to the post-primed value t.content_type = EMPTY - self.failUnless(t._settings_loaded == True, t._settings_loaded) self.failUnless(t.settings["Content-type"] == EMPTY, t.settings["Content-type"]) self.failUnless(t.content_type == None, t.content_type) self.failUnless(len(t.settings) == 1, len(t.settings)) self.failUnless(t.settings["Content-type"] == EMPTY, t.settings["Content-type"]) + self.failUnless(t._get_saved_settings() == {}, + t._get_saved_settings()) + self.failUnless(t.storage == [{'Content-type':5}, + {'Content-type':'text/plain'}, + {}], + t.storage) + def testSimplePropertyFromStorage(self): + """Testing a minimal versioned property from storage""" + class Test (TestObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property( + name="prop-a", + doc="A test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) + def prop_a(): return {} + @versioned_property( + name="prop-b", + doc="Another test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) + def prop_b(): return {} + t = Test() + t.storage.append({'prop-a':'saved'}) + # setting prop-b forces a load (to check for changes), + # which also pulls in prop-a. + t.prop_b = 'new-b' + settings = {'prop-b':'new-b', 'prop-a':'saved'} + self.failUnless(t.settings == settings, t.settings) + self.failUnless(t._get_saved_settings() == settings, + t._get_saved_settings()) + # test that _get_saved_settings() works even when settings + # were _not_ loaded beforehand + t = Test() + t.storage.append({'prop-a':'saved'}) + settings ={'prop-a':'saved'} + self.failUnless(t.settings == {}, t.settings) + self.failUnless(t._get_saved_settings() == settings, + t._get_saved_settings()) + def testDefaultingProperty(self): """Testing a defaulting versioned property""" - class Test(SavedSettingsObject): + class Test (TestObject): settings_properties = [] required_saved_properties = [] - @versioned_property(name="Content-type", - doc="A test property", - default="text/plain", - settings_properties=settings_properties, - required_saved_properties= \ - required_saved_properties) + @versioned_property( + name="Content-type", + doc="A test property", + default="text/plain", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) def content_type(): return {} - def __init__(self): - SavedSettingsObject.__init__(self) t = Test() - self.failUnless(t._settings_loaded == False, t._settings_loaded) + self.failUnless(t.settings == {}, t.settings) self.failUnless(t.content_type == "text/plain", t.content_type) - self.failUnless(t._settings_loaded == False, t._settings_loaded) - t.load_settings() - self.failUnless(t._settings_loaded == True, t._settings_loaded) - self.failUnless(t.content_type == "text/plain", t.content_type) - self.failUnless(t.settings["Content-type"] == EMPTY, - t.settings["Content-type"]) - self.failUnless(t._get_saved_settings() == - {"Content-type":"text/plain"}, + self.failUnless(t.settings == {"Content-type":EMPTY}, + t.settings) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 0, len(t.storage)) + self.failUnless(t._get_saved_settings() == {}, t._get_saved_settings()) t.content_type = "text/html" self.failUnless(t.content_type == "text/html", t.content_type) - self.failUnless(t.settings["Content-type"] == "text/html", - t.settings["Content-type"]) + self.failUnless(t.settings == {"Content-type":"text/html"}, + t.settings) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':'text/html'}], + t.storage) self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/html"}, t._get_saved_settings()) def testRequiredDefaultingProperty(self): """Testing a required defaulting versioned property""" - class Test(SavedSettingsObject): + class Test (TestObject): settings_properties = [] required_saved_properties = [] - @versioned_property(name="Content-type", - doc="A test property", - default="text/plain", - settings_properties=settings_properties, - required_saved_properties= \ - required_saved_properties, - require_save=True) + @versioned_property( + name="Content-type", + doc="A test property", + default="text/plain", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties, + require_save=True) def content_type(): return {} - def __init__(self): - SavedSettingsObject.__init__(self) t = Test() + self.failUnless(t.settings == {}, t.settings) + self.failUnless(t.content_type == "text/plain", t.content_type) + self.failUnless(t.settings == {"Content-type":EMPTY}, + t.settings) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 0, len(t.storage)) self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/plain"}, t._get_saved_settings()) t.content_type = "text/html" + self.failUnless(t.content_type == "text/html", + t.content_type) + self.failUnless(t.settings == {"Content-type":"text/html"}, + t.settings) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':'text/html'}], + t.storage) self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/html"}, t._get_saved_settings()) def testClassVersionedPropertyDefinition(self): """Testing a class-specific _versioned property decorator""" - class Test(SavedSettingsObject): + class Test (TestObject): settings_properties = [] required_saved_properties = [] - def _versioned_property(settings_properties= \ - settings_properties, - required_saved_properties= \ - required_saved_properties, - **kwargs): + def _versioned_property( + settings_properties=settings_properties, + required_saved_properties=required_saved_properties, + **kwargs): if "settings_properties" not in kwargs: kwargs["settings_properties"] = settings_properties if "required_saved_properties" not in kwargs: @@ -377,69 +493,60 @@ if libbe.TESTING == True: required_saved_properties return versioned_property(**kwargs) @_versioned_property(name="Content-type", - doc="A test property", - default="text/plain", - require_save=True) + doc="A test property", + default="text/plain", + require_save=True) def content_type(): return {} - def __init__(self): - SavedSettingsObject.__init__(self) t = Test() self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/plain"}, t._get_saved_settings()) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 0, len(t.storage)) t.content_type = "text/html" self.failUnless(t._get_saved_settings() == \ {"Content-type":"text/html"}, t._get_saved_settings()) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'Content-type':'text/html'}], + t.storage) def testMutableChangeHookedProperty(self): """Testing a mutable change-hooked property""" - SAVES = [] - def prop_log_save_settings(self, old, new, saves=SAVES): - saves.append("'%s' -> '%s'" % (str(old), str(new))) - prop_save_settings(self, old, new) - class Test(SavedSettingsObject): + class Test (TestObject): settings_properties = [] required_saved_properties = [] - @versioned_property(name="List-type", - doc="A test property", - mutable=True, - change_hook=prop_log_save_settings, - settings_properties=settings_properties, - required_saved_properties= \ - required_saved_properties) + @versioned_property( + name="List-type", + doc="A test property", + mutable=True, + change_hook=prop_save_settings, + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) def list_type(): return {} - def __init__(self): - SavedSettingsObject.__init__(self) t = Test() - self.failUnless(t._settings_loaded == False, t._settings_loaded) - t.load_settings() - self.failUnless(SAVES == [], SAVES) - self.failUnless(t._settings_loaded == True, t._settings_loaded) + self.failUnless(len(t.storage) == 0, len(t.storage)) self.failUnless(t.list_type == None, t.list_type) - self.failUnless(SAVES == [], SAVES) + self.failUnless(len(t.storage) == 0, len(t.storage)) self.failUnless(t.settings["List-type"]==EMPTY, t.settings["List-type"]) t.list_type = [] self.failUnless(t.settings["List-type"] == [], t.settings["List-type"]) - self.failUnless(SAVES == [ - "'' -> '[]'" - ], SAVES) - t.list_type.append(5) - self.failUnless(SAVES == [ - "'' -> '[]'", - ], SAVES) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'List-type':[]}], + t.storage) + t.list_type.append(5) # external modification not detected yet + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'List-type':[]}], + t.storage) self.failUnless(t.settings["List-type"] == [5], t.settings["List-type"]) - self.failUnless(SAVES == [ # the append(5) has not yet been saved - "'' -> '[]'", - ], SAVES) - self.failUnless(t.list_type == [5], t.list_type)#get triggers saved - - self.failUnless(SAVES == [ # now the append(5) has been saved. - "'' -> '[]'", - "'[]' -> '[5]'" - ], SAVES) + self.failUnless(t.list_type == [5], t.list_type)# get triggers save + self.failUnless(len(t.storage) == 2, len(t.storage)) + self.failUnless(t.storage == [{'List-type':[]}, + {'List-type':[5]}], + t.storage) unitsuite = unittest.TestLoader().loadTestsFromTestCase( \ SavedSettingsObjectTests) -- cgit From 4d925a688394b7840ec81165bbf2cdd43536f01b Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 22 Jan 2010 20:35:23 -0500 Subject: Added testSimplePropertySetStorageSave and relavant rewrites to settings_object It hadn't been handling the "attach storage after initializing" technique that BugDir, Bug, and Comment use when from_memory==True. Now it does, by refusing to overwrite self.settings with the newly-loaded settings. --- libbe/storage/util/settings_object.py | 55 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 655e0ed..8434952 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -193,19 +193,24 @@ class SavedSettingsObject(object): def load_settings(self): """Load the settings from disk.""" - # Override. Must call ._setup_saved_settings() after loading. - self.settings = {} - self._setup_saved_settings() + # Override. Must call ._setup_saved_settings({}) with + # from-storage settings. + self._setup_saved_settings({}) - def _setup_saved_settings(self): + def _setup_saved_settings(self, settings=None): """ - To be run after setting self.settings up from disk. Fills in + Sets up a settings dict loaded from storage. Fills in all missing settings entries with EMPTY. """ + if settings == None: + settings = {} for property in self.settings_properties: if property not in self.settings \ or self.settings[property] == UNPRIMED: - self.settings[property] = EMPTY + if property in settings: + self.settings[property] = settings[property] + else: + self.settings[property] = EMPTY def save_settings(self): """Save the settings to disk.""" @@ -263,10 +268,10 @@ if libbe.TESTING == True: def load_settings(self): self.load_count += 1 if len(self.storage) == 0: - self.settings = {} + settings = {} else: - self.settings = copy.deepcopy(self.storage[-1]) - self._setup_saved_settings() + settings = copy.deepcopy(self.storage[-1]) + self._setup_saved_settings(settings) def save_settings(self): settings = self._get_saved_settings() self.storage.append(copy.deepcopy(settings)) @@ -408,7 +413,37 @@ if libbe.TESTING == True: self.failUnless(t.settings == {}, t.settings) self.failUnless(t._get_saved_settings() == settings, t._get_saved_settings()) - + def testSimplePropertySetStorageSave(self): + """Set a property, then attach storage and save""" + class Test (TestObject): + settings_properties = [] + required_saved_properties = [] + @versioned_property( + name="prop-a", + doc="A test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) + def prop_a(): return {} + @versioned_property( + name="prop-b", + doc="Another test property", + settings_properties=settings_properties, + required_saved_properties=required_saved_properties) + def prop_b(): return {} + t = Test() + storage = t.storage + t.storage = None + t.prop_a = 'text/html' + t.storage = storage + t.save_settings() + self.failUnless(t.prop_a == 'text/html', t.prop_a) + self.failUnless(t.settings == {'prop-a':'text/html', + 'prop-b':EMPTY}, + t.settings) + self.failUnless(t.load_count == 1, t.load_count) + self.failUnless(len(t.storage) == 1, len(t.storage)) + self.failUnless(t.storage == [{'prop-a':'text/html'}], + t.storage) def testDefaultingProperty(self): """Testing a defaulting versioned property""" class Test (TestObject): -- cgit From 977eff5af10b50ba6e6edb6abc4f40804c418b12 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 7 Feb 2010 17:53:53 -0500 Subject: Fixed docstrings so only Sphinx errors are "autosummary" and "missing attribute" --- libbe/storage/util/config.py | 45 ++++++++++++----- libbe/storage/util/mapfile.py | 28 ++++++++--- libbe/storage/util/properties.py | 51 ++++++++++++------- libbe/storage/util/settings_object.py | 95 +++++++++++++++++++++++------------ 4 files changed, 150 insertions(+), 69 deletions(-) (limited to 'libbe/storage/util') diff --git a/libbe/storage/util/config.py b/libbe/storage/util/config.py index 8526724..724d2d3 100644 --- a/libbe/storage/util/config.py +++ b/libbe/storage/util/config.py @@ -16,8 +16,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -""" -Create, save, and load the per-user config file at path(). +"""Create, save, and load the per-user config file at :func:`path`. """ import ConfigParser @@ -31,17 +30,29 @@ if libbe.TESTING == True: default_encoding = libbe.util.encoding.get_filesystem_encoding() +"""Default filesystem encoding. + +Initialized with :func:`libbe.util.encoding.get_filesystem_encoding`. +""" def path(): - """Return the path to the per-user config file""" + """Return the path to the per-user config file. + """ return os.path.expanduser("~/.bugs_everywhere") def set_val(name, value, section="DEFAULT", encoding=None): - """Set a value in the per-user config file + """Set a value in the per-user config file. - :param name: The name of the value to set - :param value: The new value to set (or None to delete the value) - :param section: The section to store the name/value in + Parameters + ---------- + name : str + The name of the value to set. + value : str or None + The new value to set (or None to delete the value). + section : str + The section to store the name/value in. + encoding : str + The config file's encoding, defaults to :data:`default_encoding`. """ if encoding == None: encoding = default_encoding @@ -60,12 +71,22 @@ def set_val(name, value, section="DEFAULT", encoding=None): f.close() def get_val(name, section="DEFAULT", default=None, encoding=None): - """ - Get a value from the per-user config file + """Get a value from the per-user config file + + Parameters + ---------- + name : str + The name of the value to set. + section : str + The section to store the name/value in. + default : + The value to return if `name` is not set. + encoding : str + The config file's encoding, defaults to :data:`default_encoding`. + + Examples + -------- - :param name: The name of the value to get - :section: The section that the name is in - :return: The value, or None >>> get_val("junk") is None True >>> set_val("junk", "random") diff --git a/libbe/storage/util/mapfile.py b/libbe/storage/util/mapfile.py index 0b8af23..55863d7 100644 --- a/libbe/storage/util/mapfile.py +++ b/libbe/storage/util/mapfile.py @@ -16,10 +16,10 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -""" -Provide a means of saving and loading dictionaries of parameters. The -saved "mapfiles" should be clear, flat-text files, and allow easy merging of -independent/conflicting changes. +"""Serializing and deserializing dictionaries of parameters. + +The serialized "mapfiles" should be clear, flat-text strings, and allow +easy merging of independent/conflicting changes. """ import errno @@ -49,6 +49,10 @@ class InvalidMapfileContents(Exception): def generate(map): """Generate a YAML mapfile content string. + + Examples + -------- + >>> generate({'q':'p'}) 'q: p\\n\\n' >>> generate({'q':u'Fran\u00e7ais'}) @@ -73,6 +77,10 @@ def generate(map): >>> generate({'q':'p\\n'}) Traceback (most recent call last): IllegalValue: Illegal value "p\\n" + + See Also + -------- + parse : inverse """ keys = map.keys() keys.sort() @@ -97,8 +105,11 @@ def generate(map): return '\n'.join(lines) def parse(contents): - """ - Parse a YAML mapfile string. + """Parse a YAML mapfile string. + + Examples + -------- + >>> parse('q: p\\n\\n')['q'] 'p' >>> parse('q: \\'p\\'\\n\\n')['q'] @@ -119,6 +130,11 @@ def parse(contents): Traceback (most recent call last): ... InvalidMapfileContents: Invalid YAML contents + + See Also + -------- + generate : inverse + """ c = yaml.load(contents) if type(c) == types.StringType: diff --git a/libbe/storage/util/properties.py b/libbe/storage/util/properties.py index 55bac85..b5681b1 100644 --- a/libbe/storage/util/properties.py +++ b/libbe/storage/util/properties.py @@ -16,16 +16,24 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -""" -This module provides a series of useful decorators for defining -various types of properties. For example usage, consider the -unittests at the end of the module. - -See - http://www.python.org/dev/peps/pep-0318/ -and - http://www.phyast.pitt.edu/~micheles/python/documentation.html -for more information on decorators. +"""Provides a series of useful decorators for defining various types +of properties. + +For example usage, consider the unittests at the end of the module. + +Notes +----- + +See `PEP 318` and Michele Simionato's `decorator documentation` for +more information on decorators. + +.. _PEP 318: http://www.python.org/dev/peps/pep-0318/ +.. _decorator documentation: http://www.phyast.pitt.edu/~micheles/python/documentation.html + +See Also +-------- +:mod:`libbe.storage.util.settings_object` : bundle properties into a convenient package + """ import copy @@ -336,12 +344,11 @@ def primed_property(primer, initVal=None, unprimeableVal=None): return decorator def change_hook_property(hook, mutable=False, default=None): - """ - Call the function hook(instance, old_value, new_value) whenever a - value different from the current value is set (instance is a a - reference to the class instance to which this property belongs). + """Call the function `hook` whenever a value different from the + current value is set. + This is useful for saving changes to disk, etc. This function is - called _after_ the new value has been stored, allowing you to + called *after* the new value has been stored, allowing you to change the stored value if you want. In the case of mutables, things are slightly trickier. Because @@ -350,11 +357,19 @@ def change_hook_property(hook, mutable=False, default=None): mutable value, and checking for changes whenever the property is set (obviously) or retrieved (to check for external changes). So long as you're conscientious about accessing the property after - making external modifications, mutability won't be a problem. + making external modifications, mutability won't be a problem:: + t.x.append(5) # external modification t.x # dummy access notices change and triggers hook - See testChangeHookMutableProperty for an example of the expected - behavior. + + See :class:`testChangeHookMutableProperty` for an example of the + expected behavior. + + Parameters + ---------- + hook : fn + `hook(instance, old_value, new_value)`, where `instance` is a + reference to the class instance to which this property belongs. """ def decorator(funcs): if hasattr(funcs, "__call__"): diff --git a/libbe/storage/util/settings_object.py b/libbe/storage/util/settings_object.py index 8434952..6e4da55 100644 --- a/libbe/storage/util/settings_object.py +++ b/libbe/storage/util/settings_object.py @@ -16,11 +16,12 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -""" -This module provides a base class implementing settings-dict based -property storage useful for BE objects with saved properties -(e.g. BugDir, Bug, Comment). For example usage, consider the -unittests at the end of the module. +"""Provides :class:`SavedSettingsObject` implementing settings-dict +based property storage. + +See Also +-------- +:mod:`libbe.storage.util.properties` : underlying property definitions """ import libbe @@ -33,9 +34,10 @@ if libbe.TESTING == True: import unittest class _Token (object): - """ - `Control' value class for properties. We want values that only - mean something to the settings_object module. + """`Control' value class for properties. + + We want values that only mean something to the `settings_object` + module. """ pass @@ -44,45 +46,58 @@ class UNPRIMED (_Token): pass class EMPTY (_Token): - """ - Property has been primed but has no user-set value, so use + """Property has been primed but has no user-set value, so use default/generator value. """ pass def prop_save_settings(self, old, new): - """ - The default action undertaken when a property changes. + """The default action undertaken when a property changes. """ if self.storage != None and self.storage.is_writeable(): self.save_settings() def prop_load_settings(self): - """ - The default action undertaken when an UNPRIMED property is - accessed. Attempt to run .load_settings(), which calls - ._setup_saved_settings() internally. If .storage is inaccessible, - don't do anything. + """The default action undertaken when an UNPRIMED property is + accessed. + + Attempt to run `.load_settings()`, which calls + `._setup_saved_settings()` internally. If `.storage` is + inaccessible, don't do anything. """ if self.storage != None and self.storage.is_readable(): self.load_settings() # Some name-mangling routines for pretty printing setting names def setting_name_to_attr_name(self, name): - """ - Convert keys to the .settings dict into their associated + """Convert keys to the `.settings` dict into their associated SavedSettingsObject attribute names. + + Examples + -------- + >>> print setting_name_to_attr_name(None,"User-id") user_id + + See Also + -------- + attr_name_to_setting_name : inverse """ return name.lower().replace('-', '_') def attr_name_to_setting_name(self, name): - """ - The inverse of setting_name_to_attr_name. + """Convert SavedSettingsObject attribute names to `.settings` dict + keys. + + Examples: + >>> print attr_name_to_setting_name(None, "user_id") User-id + + See Also + -------- + setting_name_to_attr_name : inverse """ return name.capitalize().replace('_', '-') @@ -96,8 +111,7 @@ def versioned_property(name, doc, settings_properties=[], required_saved_properties=[], require_save=False): - """ - Combine the common decorators in a single function. + """Combine the common decorators in a single function. Use zero or one (but not both) of default or generator, since a working default will keep the generator from functioning. Use the @@ -124,17 +138,20 @@ def versioned_property(name, doc, into our local scope. Don't mess with them. Set mutable=True if: - * default is a mutable - * your generator function may return mutables - * you set change_hook and might have mutable property values - See the docstrings in libbe.properties for details on how each of + + * default is a mutable + * your generator function may return mutables + * you set change_hook and might have mutable property values + + See the docstrings in `libbe.properties` for details on how each of these cases are handled. - The value stored in .settings[name] will be - * no value (or UNPRIMED) if the property has been neither set, - nor loaded as blank. - * EMPTY if the value has been loaded as blank. - * some value if the property has been either loaded or set. + The value stored in `.settings[name]` will be + + * no value (or UNPRIMED) if the property has been neither set, + nor loaded as blank. + * EMPTY if the value has been loaded as blank. + * some value if the property has been either loaded or set. """ settings_properties.append(name) if require_save == True: @@ -175,7 +192,19 @@ def versioned_property(name, doc, return decorator class SavedSettingsObject(object): - + """Setup a framework for lazy saving and loading of `.settings` + properties. + + This is useful for BE objects with saved properties + (e.g. :class:`~libbe.bugdir.BugDir`, :class:`~libbe.bug.Bug`, + :class:`~libbe.comment.Comment`). For example usage, consider the + unittests at the end of the module. + + See Also + -------- + versioned_property, prop_save_settings, prop_load_settings + setting_name_to_attr_name, attr_name_to_setting_name + """ # Keep a list of properties that may be stored in the .settings dict. #settings_properties = [] -- cgit