Merged be.html-storage

Added HTTP storage backend and server
Serve a local repo on http://localhost:8000
  be --repo REPO serve
Then connect from other be calls, for example
  be --repo http://localhost:8000 list

6 files changed, 651 insertions, 8 deletions

diff --git a/libbe/command/serve.py b/libbe/command/serve.py
new file mode 100644
index 0000000..5dbd2b4
--- /dev/null
+++ b/libbe/command/serve.py
@@ -0,0 +1,367 @@
+# Copyright (C) 2010 W. Trevor King <wking@drexel.edu>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import BaseHTTPServer as server
+import posixpath
+import urllib
+import urlparse
+
+import libbe
+import libbe.command
+import libbe.command.util
+import libbe.version
+
+HTTP_USER_ERROR = 418
+STORAGE = None
+COMMAND = None
+
+# Maximum input we will accept when REQUEST_METHOD is POST
+# 0 ==> unlimited input
+MAXLEN = 0
+
+
+class _HandlerError (Exception):
+    pass
+
+class BERequestHandler (server.BaseHTTPRequestHandler):
+    """Simple HTTP request handler for serving the
+    libbe.storage.http.HTTP backend with GET, POST, and HEAD commands.
+
+    This serves files from a connected storage instance, usually
+    a VCS-based repository located on the local machine.
+
+    The GET and HEAD requests are identical except that the HEAD
+    request omits the actual content of the file.
+    """
+
+    server_version = "BE-server/" + libbe.version.version()
+
+    def do_GET(self, head=False):
+        """Serve a GET (or HEAD, if head==True) request."""
+        self.s = STORAGE
+        self.c = COMMAND
+        request = 'GET'
+        if head == True:
+            request = 'HEAD'
+        self.log_request(request)
+        path,query,fragment = self.parse_path(self.path)
+        if fragment != '':
+            self.send_error(406,
+                '%s implementation does not allow fragment URL portion'
+                % request)
+            return None
+        data = self.parse_query(query)
+
+        try:
+            if path == ['children']:
+                content,ctype = self.handle_children(data)
+            elif len(path) > 1 and path[0] == 'get':
+                content,ctype = self.handle_get('/'.join(path[1:]), data)
+            elif path == ['revision-id']:
+                content,ctype = self.handle_revision_id(data)
+            elif path == ['version']:
+                content,ctype = self.handle_version(data)
+            else:
+                self.send_error(400, 'File not found')
+                return None
+        except libbe.storage.NotReadable, e:
+            self.send_error(403, 'Read permission denied')
+            return None
+        except libbe.storage.InvalidID, e:
+            self.send_error(HTTP_USER_ERROR, 'InvalidID %s' % e)
+            return None
+        except _HandlerError:
+            return None
+
+        if content != None:
+            self.send_header('Content-type', ctype)
+            self.send_header('Content-Length', len(content))
+        self.end_headers()
+        if request == 'GET' and content != None:
+            self.wfile.write(content)
+
+    def do_HEAD(self):
+        """Serve a HEAD request."""
+        return self.do_GET(head=True)
+
+    def do_POST(self):
+        """Serve a POST request."""
+        self.s = STORAGE
+        self.c = COMMAND
+        self.log_request('POST')
+        post_data = self.read_post_data()
+        data = self.parse_post(post_data)
+        path,query,fragment = self.parse_path(self.path)
+        if query != '':
+            self.send_error(
+                406, 'POST implementation does not allow query URL portion')
+            return None
+        if fragment != '':
+            self.send_error(
+                406, 'POST implementation does not allow fragment URL portion')
+            return None
+        try:
+            if path == ['add']:
+                content,ctype = self.handle_add(data)
+            elif path == ['remove']:
+                content,ctype = self.handle_remove(data)
+            elif len(path) > 1 and path[0] == 'set':
+                content,ctype = self.handle_set('/'.join(path[1:]), data)
+            elif path == ['commit']:
+                content,ctype = self.handle_commit(data)
+            else:
+                self.send_error(400, 'File not found')
+                return None
+        except libbe.storage.NotWriteable, e:
+            self.send_error(403, 'Write permission denied')
+            return None
+        except libbe.storage.InvalidID, e:
+            self.send_error(HTTP_USER_ERROR, 'InvalidID %s' % e)
+            return None
+        except _HandlerError:
+            return None
+        if content != None:
+            self.send_header('Content-type', ctype)
+            self.send_header('Content-Length', len(content))
+        self.end_headers()
+        if content != None:
+            self.wfile.write(content)
+
+    def handle_add(self, data):
+        if not 'id' in data:
+            self.send_error(406, 'Missing query key id')
+            raise _HandlerError()
+        elif data['id'] == 'None':
+            data['id'] = None
+        id = data['id']
+        if not 'parent' in data or data['parent'] == None:
+            data['parent'] = None
+        parent = data['parent']
+        if not 'directory' in data:
+            directory = False
+        elif data['directory'] == 'True':
+            directory = True
+        else:
+            directory = False
+        self.s.add(id, parent=parent, directory=directory)
+        self.send_response(200)
+        return (None,None)
+
+    def handle_remove(self, data):
+        if not 'id' in data:
+            self.send_error(406, 'Missing query key id')
+            raise _HandlerError()
+        elif data['id'] == 'None':
+            data['id'] = None
+        id = data['id']
+        if not 'recursive' in data:
+            recursive = False
+        elif data['recursive'] == 'True':
+            recursive = True
+        else:
+            recursive = False
+        if recursive == True:
+            self.s.recursive_remove(id)
+        else:
+            self.s.remove(id)
+        self.send_response(200)
+        return (None,None)
+
+    def handle_children(self, data):
+        if not 'id' in data:
+            self.send_error(406, 'Missing query key id')
+            raise _HandlerError()
+        elif data['id'] == 'None':
+            data['id'] = None
+        id = data['id']
+        if not 'revision' in data or data['revision'] == 'None':
+            data['revision'] = None
+        revision = data['revision']
+        content = '\n'.join(self.s.children(id, revision))
+        ctype = 'application/octet-stream'
+        self.send_response(200)
+        return content,ctype
+
+    def handle_get(self, id, data):
+        if not 'revision' in data or data['revision'] == 'None':
+            data['revision'] = None
+        revision = data['revision']
+        content = self.s.get(id, revision=revision)
+        be_version = self.s.storage_version(revision)
+        ctype = 'application/octet-stream'
+        self.send_response(200)
+        self.send_header('X-BE-Version', be_version)
+        return content,ctype
+
+    def handle_set(self, id, data):
+        if not 'value' in data:
+            self.send_error(406, 'Missing query key value')
+            raise _HandlerError()
+        value = data['value']
+        self.s.set(id, value)
+        self.send_response(200)
+        return (None,None)
+
+    def handle_commit(self, data):
+        if not 'summary' in data:
+            self.send_error(406, 'Missing query key summary')
+            raise _HandlerError()
+        summary = data['summary']
+        if not 'body' in data or data['body'] == 'None':
+            data['body'] = None
+        body = data['body']
+        if not 'allow_empty' in data \
+                or data['allow_empty'] == 'True':
+            allow_empty = True
+        else:
+            allow_empty = False
+        try:
+            self.s.commit(summary, body, allow_empty)
+        except libbe.storage.EmptyCommit, e:
+            self.send_error(HTTP_USER_ERROR, 'EmptyCommit')
+            raise _HandlerError()
+        self.send_response(200)
+        return (None,None)
+
+    def handle_revision_id(self, data):
+        if not 'index' in data:
+            self.send_error(406, 'Missing query key index')
+            raise _HandlerError()
+        index = int(data['index'])
+        content = self.s.revision_id(index)
+        ctype = 'application/octet-stream'
+        self.send_response(200)
+        return content,ctype
+
+    def handle_version(self, data):
+        if not 'revision' in data or data['revision'] == 'None':
+            data['revision'] = None
+        revision = data['revision']
+        content = self.s.storage_version(revision)
+        ctype = 'application/octet-stream'
+        self.send_response(200)
+        return content,ctype
+
+    def parse_path(self, path):
+        """Parse a url to path,query,fragment parts."""
+        # abandon query parameters
+        scheme,netloc,path,query,fragment = urlparse.urlsplit(path)
+        path = posixpath.normpath(urllib.unquote(path)).split('/')
+        assert path[0] == '', path
+        path = path[1:]
+        return (path,query,fragment)
+
+    def log_request(self, request):
+        print >> self.c.stdout, request, self.path
+
+    def parse_query(self, query):
+        if len(query) == 0:
+            return {}
+        data = urlparse.parse_qs(
+            query, keep_blank_values=True, strict_parsing=True)
+        for k,v in data.items():
+            if len(v) == 1:
+                data[k] = v[0]
+        return data
+
+    def parse_post(self, post):
+        return self.parse_query(post)
+
+    def read_post_data(self):
+        clen = -1
+        if 'content-length' in self.headers:
+            try:
+                clen = int(self.headers['content-length'])
+            except ValueError:
+                pass
+            if MAXLEN > 0 and clen > MAXLEN:
+                raise ValueError, 'Maximum content length exceeded'
+        post_data = self.rfile.read(clen)
+        return post_data
+        
+
+class Serve (libbe.command.Command):
+    """Serve a Storage backend for the HTTP storage client
+
+    >>> import libbe.bugdir
+    >>> bd = libbe.bugdir.SimpleBugDir(memory=False)
+    >>> io = libbe.command.StringInputOutput()
+    >>> io.stdout = sys.stdout
+    >>> ui = libbe.command.UserInterface(io=io)
+    >>> ui.storage_callbacks.set_storage(bd.storage)
+    >>> cmd = List(ui=ui)
+
+    >>> ret = ui.run(cmd)
+    abc/a:om: Bug A
+    >>> ret = ui.run(cmd, {'status':'closed'})
+    abc/b:cm: Bug B
+    >>> bd.storage.writeable
+    True
+    >>> ui.cleanup()
+    >>> bd.cleanup()
+    """
+
+    name = 'serve'
+
+    def __init__(self, *args, **kwargs):
+        libbe.command.Command.__init__(self, *args, **kwargs)
+        self.options.extend([
+                libbe.command.Option(name='port',
+                    help='Bind server to port (%default)',
+                    arg=libbe.command.Argument(
+                        name='port', metavar='INT', type='int', default=8000)),
+                libbe.command.Option(name='host',
+                    help='Set host string (blank for localhost, %default)',
+                    arg=libbe.command.Argument(
+                        name='host', metavar='HOST', default='')),
+                libbe.command.Option(name='read-only', short_name='r',
+                    help='Dissable operations that require writing'),
+                ])
+
+    def _run(self, **params):
+        global STORAGE, COMMAND
+        COMMAND = self
+        STORAGE = self._get_storage()
+        if params['read-only'] == True:
+            writeable = STORAGE.writeable
+            STORAGE.writeable = False
+        server_class = server.HTTPServer
+        handler_class = BERequestHandler
+        httpd = server_class(
+            (params['host'], params['port']), handler_class)
+        sa = httpd.socket.getsockname()
+        print >> self.stdout, 'Serving HTTP on', sa[0], 'port', sa[1], '...'
+        print >> self.stdout, 'BE repository', STORAGE.repo
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            pass
+        print >> self.stdout, 'Closing server'
+        httpd.server_close()
+        if params['read-only'] == True:
+            STORAGE.writeable = writeable
+
+    def _long_help(self):
+        return """
+Example usage:
+  $ be serve
+And in another terminal (or after backgrounding the server)
+  $ be --repo http://localhost:8000 list
+
+If you bind your server to a public interface, you should probably use
+the --read-only option so other people can't mess with your
+repository.
+"""
diff --git a/libbe/storage/__init__.py b/libbe/storage/__init__.py
index 7abe791..b6b0ac1 100644
--- a/libbe/storage/__init__.py
+++ b/libbe/storage/__init__.py
@@ -36,15 +36,24 @@ STORAGE_VERSIONS = ['Bugs Everywhere Tree 1 0',
 # the current version
 STORAGE_VERSION = STORAGE_VERSIONS[-1]
 
-def get_storage(location):
-    """
-    Return a Storage instance from a repo location string.
-    """
+def get_http_storage(location):
+    import http
+    return http.HTTP(location)
+
+def get_vcs_storage(location):
     import vcs
     s = vcs.detect_vcs(location)
     s.repo = location
     return s
 
+def get_storage(location):
+    """
+    Return a Storage instance from a repo location string.
+    """
+    if location.startswith('http://'):
+        return get_http_storage(location)
+    return get_vcs_storage(location)
+
 __all__ = [ConnectionError, InvalidStorageVersion, InvalidID,
            InvalidRevision, InvalidDirectory, NotWriteable, NotReadable,
            EmptyCommit, STORAGE_VERSIONS, STORAGE_VERSION,
diff --git a/libbe/storage/base.py b/libbe/storage/base.py
index d85627f..aa32ea9 100644
--- a/libbe/storage/base.py
+++ b/libbe/storage/base.py
@@ -27,6 +27,7 @@ from libbe.error import NotSupported
 import libbe.storage
 from libbe.util.tree import Tree
 from libbe.util import InvalidObject
+import libbe.version
 from libbe import TESTING
 
 if TESTING == True:
@@ -163,7 +164,7 @@ class Storage (object):
 
     def version(self):
         """Return a version string for this backend."""
-        return '0'
+        return libbe.version.version()
 
     def storage_version(self, revision=None):
         """Return the storage format for this backend."""
diff --git a/libbe/storage/http.py b/libbe/storage/http.py
new file mode 100644
index 0000000..0792b1e
--- /dev/null
+++ b/libbe/storage/http.py
@@ -0,0 +1,262 @@
+# Copyright (C) 2010 W. Trevor King <wking@drexel.edu>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# For urllib2 information, see
+#   urllib2, from urllib2 - The Missing Manual
+#   http://www.voidspace.org.uk/python/articles/urllib2.shtml
+# 
+# A dictionary of response codes is available in
+#   httplib.responses
+
+"""
+Access bug repository data over HTTP.
+"""
+
+import sys
+import urllib
+import urllib2
+import urlparse
+
+import libbe
+import libbe.version
+import base
+from libbe import TESTING
+
+if TESTING == True:
+    import doctest
+    import unittest
+
+
+USER_AGENT = 'BE-HTTP-Storage'
+HTTP_OK = 200
+HTTP_FOUND = 302
+HTTP_TEMP_REDIRECT = 307
+HTTP_USER_ERROR = 418
+HTTP_VALID = [HTTP_OK, HTTP_FOUND, HTTP_TEMP_REDIRECT, HTTP_USER_ERROR]
+
+class InvalidURL (Exception):
+    def __init__(self, error=None, url=None, msg=None):
+        Exception.__init__(self, msg)
+        self.url = url
+        self.error = error
+        self.msg = msg
+    def __str__(self):
+        if self.msg == None:
+            if self.error == None:
+                return "Unknown URL error: %s" % self.url
+            return self.error.__str__()
+        return self.msg
+
+def get_post_url(url, get=True, data_dict=None):
+    """
+    get:        use GET if True, otherwise use POST.
+    data_dict:  dict of data to send.
+    """
+    if data_dict == None:
+        data_dict = {}
+    if get == True:
+        if data_dict != {}:
+            # encode get parameters in the url
+            param_string = urllib.urlencode(data_dict)
+            url = "%s?%s" % (url, param_string)
+        data = None
+    else:
+        data = urllib.urlencode(data_dict)
+    headers = {'User-Agent':USER_AGENT}
+    req = urllib2.Request(url, data=data, headers=headers)
+    try:
+        response = urllib2.urlopen(req)
+    except urllib2.HTTPError, e:
+        if hasattr(e, 'reason'):
+            msg = 'We failed to reach a server.\nURL: %s\nReason: %s' \
+                % (url, e.reason)
+        elif hasattr(e, 'code'):
+            msg = "The server couldn't fulfill the request.\nURL: %s\nError code: %s" \
+                % (url, e.code)
+        raise InvalidURL(error=e, url=url, msg=msg)
+    page = response.read()
+    final_url = response.geturl()
+    info = response.info()
+    response.close()
+    return (page, final_url, info)
+
+
+class HTTP (base.VersionedStorage):
+    """
+    This class implements a Storage interface over HTTP, using GET to
+    retrieve information and POST to set information.
+    """
+    name = 'HTTP'
+
+    def __init__(self, *args, **kwargs):
+        base.VersionedStorage.__init__(self, *args, **kwargs)
+
+    def storage_version(self, revision=None):
+        """Return the storage format for this backend."""
+        return libbe.storage.STORAGE_VERSION
+
+    def _init(self):
+        """Create a new storage repository."""
+        raise base.NotSupported(
+            'init', 'Cannot initialize this repository format.')
+
+    def _destroy(self):
+        """Remove the storage repository."""
+        raise base.NotSupported(
+            'destroy', 'Cannot destroy this repository format.')
+
+    def _connect(self):
+        self.check_storage_version()
+
+    def _disconnect(self):
+        pass
+
+    def _add(self, id, parent=None, directory=False):
+        url = urlparse.urljoin(self.repo, 'add')
+        page,final_url,info = get_post_url(
+            url, get=False,
+            data_dict={'id':id, 'parent':parent, 'directory':directory})
+
+    def _remove(self, id):
+        url = urlparse.urljoin(self.repo, 'remove')
+        page,final_url,info = get_post_url(
+            url, get=False,
+            data_dict={'id':id, 'recursive':False})
+
+    def _recursive_remove(self, id):
+        url = urlparse.urljoin(self.repo, 'remove')
+        page,final_url,info = get_post_url(
+            url, get=False,
+            data_dict={'id':id, 'recursive':True})
+
+    def _children(self, id=None, revision=None):
+        url = urlparse.urljoin(self.repo, 'children')
+        page,final_url,info = get_post_url(
+            url, get=True,
+            data_dict={'id':id, 'revision':revision})
+        return page.strip('\n').splitlines()
+
+    def _get(self, id, default=base.InvalidObject, revision=None):
+        url = urlparse.urljoin(self.repo, '/'.join(['get', id]))
+        try:
+            page,final_url,info = get_post_url(
+                url, get=True,
+                data_dict={'revision':revision})
+        except InvalidURL, e:
+            if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+                raise
+            elif default == base.InvalidObject:
+                raise base.InvalidID(id)
+            return default
+        version = info['X-BE-Version']
+        if version != libbe.storage.STORAGE_VERSION:
+            raise base.InvalidStorageVersion(
+                version, libbe.storage.STORAGE_VERSION)
+        return page
+
+    def _set(self, id, value):
+        url = urlparse.urljoin(self.repo, '/'.join(['set', id]))
+        try:
+            page,final_url,info = get_post_url(
+                url, get=False,
+                data_dict={'value':value})
+        except InvalidURL, e:
+            if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+                raise
+            if e.error.code == HTTP_USER_ERROR:
+                raise base.InvalidDirectory(
+                    'Directory %s cannot have data' % id)
+            raise base.InvalidID(id)
+
+    def _commit(self, summary, body=None, allow_empty=False):
+        url = urlparse.urljoin(self.repo, 'commit')
+        try:
+            page,final_url,info = get_post_url(
+                url, get=False,
+                data_dict={'summary':summary, 'body':body,
+                           'allow_empty':allow_empty})
+        except InvalidURL, e:
+            if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+                raise
+            if e.error.code == HTTP_USER_ERROR:
+                raise base.EmptyCommit
+            raise base.InvalidID(id)
+        return page.rstrip('\n')
+
+    def revision_id(self, index=None):
+        """
+        Return the name of the <index>th revision.  The choice of
+        which branch to follow when crossing branches/merges is not
+        defined.  Revision indices start at 1; ID 0 is the blank
+        repository.
+
+        Return None if index==None.
+
+        If the specified revision does not exist, raise InvalidRevision.
+        """
+        if index == None:
+            return None
+        try:
+            if int(index) != index:
+                raise InvalidRevision(index)
+        except ValueError:
+            raise InvalidRevision(index)
+        url = urlparse.urljoin(self.repo, 'revision-id')
+        try:
+            page,final_url,info = get_post_url(
+                url, get=True,
+                data_dict={'index':index})
+        except InvalidURL, e:
+            if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+                raise
+            if e.error.code == HTTP_USER_ERROR:
+                raise base.InvalidRevision(index)
+            raise base.InvalidID(id)
+        return page.rstrip('\n')
+
+    def check_storage_version(self):
+        version = self.storage_version()
+        if version != libbe.storage.STORAGE_VERSION:
+            raise base.InvalidStorageVersion(
+                version, libbe.storage.STORAGE_VERSION)
+
+    def storage_version(self, revision=None):
+        url = urlparse.urljoin(self.repo, 'version')
+        page,final_url,info = get_post_url(
+            url, get=True, data_dict={'revision':revision})
+        return page.rstrip('\n')
+
+if TESTING == True:
+    class GetPostUrlTestCase (unittest.TestCase):
+        """Test cases for get_post_url()"""
+        def test_get(self):
+            url = 'http://bugseverywhere.org/be/show/HomePage'
+            page,final_url,info = get_post_url(url=url)
+            self.failUnless(final_url == url,
+                'Redirect?\n  Expected: "%s"\n  Got:      "%s"'
+                % (url, final_url))
+        def test_get_redirect(self):
+            url = 'http://bugseverywhere.org'
+            expected = 'http://bugseverywhere.org/be/show/HomePage'
+            page,final_url,info = get_post_url(url=url)
+            self.failUnless(final_url == expected,
+                'Redirect?\n  Expected: "%s"\n  Got:      "%s"'
+                % (expected, final_url))
+
+    #make_storage_testcase_subclasses(VersionedStorage, sys.modules[__name__])
+
+    unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+    suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()])
diff --git a/libbe/storage/vcs/base.py b/libbe/storage/vcs/base.py
index 39f5082..8390cbc 100644
--- a/libbe/storage/vcs/base.py
+++ b/libbe/storage/vcs/base.py
@@ -800,7 +800,7 @@ os.listdir(self.get_path("bugs")):
         try:
             path = self._cached_path_id.path(id)
         except InvalidID, e:
-            raise e
+            raise
         if not os.path.exists(path):
             raise InvalidID(id)
         if os.path.isdir(path):
@@ -877,7 +877,11 @@ os.listdir(self.get_path("bugs")):
           /.be
         or None if none of those files exist.
         """
-        return search_parent_directories(path, filename)
+        try:
+            ret = search_parent_directories(path, filename)
+        except AssertionError, e:
+            return None
+        return ret
 
     def _u_find_id(self, id, revision):
         """
diff --git a/libbe/ui/command_line.py b/libbe/ui/command_line.py
index f57983f..7ba6cf5 100644
--- a/libbe/ui/command_line.py
+++ b/libbe/ui/command_line.py
@@ -299,7 +299,7 @@ def main():
     command = Class(ui=ui)
     ui.setup_command(command)
 
-    if command.name in ['comment', 'commit']:
+    if command.name in ['comment', 'commit', 'serve']:
         paginate = 'never'
     else:
         paginate = 'auto'