diff options
Diffstat (limited to 'libbe')
| -rw-r--r-- | libbe/command/serve.py | 367 | ||||
| -rw-r--r-- | libbe/storage/__init__.py | 17 | ||||
| -rw-r--r-- | libbe/storage/base.py | 3 | ||||
| -rw-r--r-- | libbe/storage/http.py | 262 | ||||
| -rw-r--r-- | libbe/storage/vcs/base.py | 8 | ||||
| -rw-r--r-- | libbe/ui/command_line.py | 2 |
6 files changed, 651 insertions, 8 deletions
diff --git a/libbe/command/serve.py b/libbe/command/serve.py new file mode 100644 index 0000000..5dbd2b4 --- /dev/null +++ b/libbe/command/serve.py @@ -0,0 +1,367 @@ +# Copyright (C) 2010 W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import BaseHTTPServer as server +import posixpath +import urllib +import urlparse + +import libbe +import libbe.command +import libbe.command.util +import libbe.version + +HTTP_USER_ERROR = 418 +STORAGE = None +COMMAND = None + +# Maximum input we will accept when REQUEST_METHOD is POST +# 0 ==> unlimited input +MAXLEN = 0 + + +class _HandlerError (Exception): + pass + +class BERequestHandler (server.BaseHTTPRequestHandler): + """Simple HTTP request handler for serving the + libbe.storage.http.HTTP backend with GET, POST, and HEAD commands. + + This serves files from a connected storage instance, usually + a VCS-based repository located on the local machine. + + The GET and HEAD requests are identical except that the HEAD + request omits the actual content of the file. + """ + + server_version = "BE-server/" + libbe.version.version() + + def do_GET(self, head=False): + """Serve a GET (or HEAD, if head==True) request.""" + self.s = STORAGE + self.c = COMMAND + request = 'GET' + if head == True: + request = 'HEAD' + self.log_request(request) + path,query,fragment = self.parse_path(self.path) + if fragment != '': + self.send_error(406, + '%s implementation does not allow fragment URL portion' + % request) + return None + data = self.parse_query(query) + + try: + if path == ['children']: + content,ctype = self.handle_children(data) + elif len(path) > 1 and path[0] == 'get': + content,ctype = self.handle_get('/'.join(path[1:]), data) + elif path == ['revision-id']: + content,ctype = self.handle_revision_id(data) + elif path == ['version']: + content,ctype = self.handle_version(data) + else: + self.send_error(400, 'File not found') + return None + except libbe.storage.NotReadable, e: + self.send_error(403, 'Read permission denied') + return None + except libbe.storage.InvalidID, e: + self.send_error(HTTP_USER_ERROR, 'InvalidID %s' % e) + return None + except _HandlerError: + return None + + if content != None: + self.send_header('Content-type', ctype) + self.send_header('Content-Length', len(content)) + self.end_headers() + if request == 'GET' and content != None: + self.wfile.write(content) + + def do_HEAD(self): + """Serve a HEAD request.""" + return self.do_GET(head=True) + + def do_POST(self): + """Serve a POST request.""" + self.s = STORAGE + self.c = COMMAND + self.log_request('POST') + post_data = self.read_post_data() + data = self.parse_post(post_data) + path,query,fragment = self.parse_path(self.path) + if query != '': + self.send_error( + 406, 'POST implementation does not allow query URL portion') + return None + if fragment != '': + self.send_error( + 406, 'POST implementation does not allow fragment URL portion') + return None + try: + if path == ['add']: + content,ctype = self.handle_add(data) + elif path == ['remove']: + content,ctype = self.handle_remove(data) + elif len(path) > 1 and path[0] == 'set': + content,ctype = self.handle_set('/'.join(path[1:]), data) + elif path == ['commit']: + content,ctype = self.handle_commit(data) + else: + self.send_error(400, 'File not found') + return None + except libbe.storage.NotWriteable, e: + self.send_error(403, 'Write permission denied') + return None + except libbe.storage.InvalidID, e: + self.send_error(HTTP_USER_ERROR, 'InvalidID %s' % e) + return None + except _HandlerError: + return None + if content != None: + self.send_header('Content-type', ctype) + self.send_header('Content-Length', len(content)) + self.end_headers() + if content != None: + self.wfile.write(content) + + def handle_add(self, data): + if not 'id' in data: + self.send_error(406, 'Missing query key id') + raise _HandlerError() + elif data['id'] == 'None': + data['id'] = None + id = data['id'] + if not 'parent' in data or data['parent'] == None: + data['parent'] = None + parent = data['parent'] + if not 'directory' in data: + directory = False + elif data['directory'] == 'True': + directory = True + else: + directory = False + self.s.add(id, parent=parent, directory=directory) + self.send_response(200) + return (None,None) + + def handle_remove(self, data): + if not 'id' in data: + self.send_error(406, 'Missing query key id') + raise _HandlerError() + elif data['id'] == 'None': + data['id'] = None + id = data['id'] + if not 'recursive' in data: + recursive = False + elif data['recursive'] == 'True': + recursive = True + else: + recursive = False + if recursive == True: + self.s.recursive_remove(id) + else: + self.s.remove(id) + self.send_response(200) + return (None,None) + + def handle_children(self, data): + if not 'id' in data: + self.send_error(406, 'Missing query key id') + raise _HandlerError() + elif data['id'] == 'None': + data['id'] = None + id = data['id'] + if not 'revision' in data or data['revision'] == 'None': + data['revision'] = None + revision = data['revision'] + content = '\n'.join(self.s.children(id, revision)) + ctype = 'application/octet-stream' + self.send_response(200) + return content,ctype + + def handle_get(self, id, data): + if not 'revision' in data or data['revision'] == 'None': + data['revision'] = None + revision = data['revision'] + content = self.s.get(id, revision=revision) + be_version = self.s.storage_version(revision) + ctype = 'application/octet-stream' + self.send_response(200) + self.send_header('X-BE-Version', be_version) + return content,ctype + + def handle_set(self, id, data): + if not 'value' in data: + self.send_error(406, 'Missing query key value') + raise _HandlerError() + value = data['value'] + self.s.set(id, value) + self.send_response(200) + return (None,None) + + def handle_commit(self, data): + if not 'summary' in data: + self.send_error(406, 'Missing query key summary') + raise _HandlerError() + summary = data['summary'] + if not 'body' in data or data['body'] == 'None': + data['body'] = None + body = data['body'] + if not 'allow_empty' in data \ + or data['allow_empty'] == 'True': + allow_empty = True + else: + allow_empty = False + try: + self.s.commit(summary, body, allow_empty) + except libbe.storage.EmptyCommit, e: + self.send_error(HTTP_USER_ERROR, 'EmptyCommit') + raise _HandlerError() + self.send_response(200) + return (None,None) + + def handle_revision_id(self, data): + if not 'index' in data: + self.send_error(406, 'Missing query key index') + raise _HandlerError() + index = int(data['index']) + content = self.s.revision_id(index) + ctype = 'application/octet-stream' + self.send_response(200) + return content,ctype + + def handle_version(self, data): + if not 'revision' in data or data['revision'] == 'None': + data['revision'] = None + revision = data['revision'] + content = self.s.storage_version(revision) + ctype = 'application/octet-stream' + self.send_response(200) + return content,ctype + + def parse_path(self, path): + """Parse a url to path,query,fragment parts.""" + # abandon query parameters + scheme,netloc,path,query,fragment = urlparse.urlsplit(path) + path = posixpath.normpath(urllib.unquote(path)).split('/') + assert path[0] == '', path + path = path[1:] + return (path,query,fragment) + + def log_request(self, request): + print >> self.c.stdout, request, self.path + + def parse_query(self, query): + if len(query) == 0: + return {} + data = urlparse.parse_qs( + query, keep_blank_values=True, strict_parsing=True) + for k,v in data.items(): + if len(v) == 1: + data[k] = v[0] + return data + + def parse_post(self, post): + return self.parse_query(post) + + def read_post_data(self): + clen = -1 + if 'content-length' in self.headers: + try: + clen = int(self.headers['content-length']) + except ValueError: + pass + if MAXLEN > 0 and clen > MAXLEN: + raise ValueError, 'Maximum content length exceeded' + post_data = self.rfile.read(clen) + return post_data + + +class Serve (libbe.command.Command): + """Serve a Storage backend for the HTTP storage client + + >>> import libbe.bugdir + >>> bd = libbe.bugdir.SimpleBugDir(memory=False) + >>> io = libbe.command.StringInputOutput() + >>> io.stdout = sys.stdout + >>> ui = libbe.command.UserInterface(io=io) + >>> ui.storage_callbacks.set_storage(bd.storage) + >>> cmd = List(ui=ui) + + >>> ret = ui.run(cmd) + abc/a:om: Bug A + >>> ret = ui.run(cmd, {'status':'closed'}) + abc/b:cm: Bug B + >>> bd.storage.writeable + True + >>> ui.cleanup() + >>> bd.cleanup() + """ + + name = 'serve' + + def __init__(self, *args, **kwargs): + libbe.command.Command.__init__(self, *args, **kwargs) + self.options.extend([ + libbe.command.Option(name='port', + help='Bind server to port (%default)', + arg=libbe.command.Argument( + name='port', metavar='INT', type='int', default=8000)), + libbe.command.Option(name='host', + help='Set host string (blank for localhost, %default)', + arg=libbe.command.Argument( + name='host', metavar='HOST', default='')), + libbe.command.Option(name='read-only', short_name='r', + help='Dissable operations that require writing'), + ]) + + def _run(self, **params): + global STORAGE, COMMAND + COMMAND = self + STORAGE = self._get_storage() + if params['read-only'] == True: + writeable = STORAGE.writeable + STORAGE.writeable = False + server_class = server.HTTPServer + handler_class = BERequestHandler + httpd = server_class( + (params['host'], params['port']), handler_class) + sa = httpd.socket.getsockname() + print >> self.stdout, 'Serving HTTP on', sa[0], 'port', sa[1], '...' + print >> self.stdout, 'BE repository', STORAGE.repo + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + print >> self.stdout, 'Closing server' + httpd.server_close() + if params['read-only'] == True: + STORAGE.writeable = writeable + + def _long_help(self): + return """ +Example usage: + $ be serve +And in another terminal (or after backgrounding the server) + $ be --repo http://localhost:8000 list + +If you bind your server to a public interface, you should probably use +the --read-only option so other people can't mess with your +repository. +""" diff --git a/libbe/storage/__init__.py b/libbe/storage/__init__.py index 7abe791..b6b0ac1 100644 --- a/libbe/storage/__init__.py +++ b/libbe/storage/__init__.py @@ -36,15 +36,24 @@ STORAGE_VERSIONS = ['Bugs Everywhere Tree 1 0', # the current version STORAGE_VERSION = STORAGE_VERSIONS[-1] -def get_storage(location): - """ - Return a Storage instance from a repo location string. - """ +def get_http_storage(location): + import http + return http.HTTP(location) + +def get_vcs_storage(location): import vcs s = vcs.detect_vcs(location) s.repo = location return s +def get_storage(location): + """ + Return a Storage instance from a repo location string. + """ + if location.startswith('http://'): + return get_http_storage(location) + return get_vcs_storage(location) + __all__ = [ConnectionError, InvalidStorageVersion, InvalidID, InvalidRevision, InvalidDirectory, NotWriteable, NotReadable, EmptyCommit, STORAGE_VERSIONS, STORAGE_VERSION, diff --git a/libbe/storage/base.py b/libbe/storage/base.py index d85627f..aa32ea9 100644 --- a/libbe/storage/base.py +++ b/libbe/storage/base.py @@ -27,6 +27,7 @@ from libbe.error import NotSupported import libbe.storage from libbe.util.tree import Tree from libbe.util import InvalidObject +import libbe.version from libbe import TESTING if TESTING == True: @@ -163,7 +164,7 @@ class Storage (object): def version(self): """Return a version string for this backend.""" - return '0' + return libbe.version.version() def storage_version(self, revision=None): """Return the storage format for this backend.""" diff --git a/libbe/storage/http.py b/libbe/storage/http.py new file mode 100644 index 0000000..0792b1e --- /dev/null +++ b/libbe/storage/http.py @@ -0,0 +1,262 @@ +# Copyright (C) 2010 W. Trevor King <wking@drexel.edu> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# For urllib2 information, see +# urllib2, from urllib2 - The Missing Manual +# http://www.voidspace.org.uk/python/articles/urllib2.shtml +# +# A dictionary of response codes is available in +# httplib.responses + +""" +Access bug repository data over HTTP. +""" + +import sys +import urllib +import urllib2 +import urlparse + +import libbe +import libbe.version +import base +from libbe import TESTING + +if TESTING == True: + import doctest + import unittest + + +USER_AGENT = 'BE-HTTP-Storage' +HTTP_OK = 200 +HTTP_FOUND = 302 +HTTP_TEMP_REDIRECT = 307 +HTTP_USER_ERROR = 418 +HTTP_VALID = [HTTP_OK, HTTP_FOUND, HTTP_TEMP_REDIRECT, HTTP_USER_ERROR] + +class InvalidURL (Exception): + def __init__(self, error=None, url=None, msg=None): + Exception.__init__(self, msg) + self.url = url + self.error = error + self.msg = msg + def __str__(self): + if self.msg == None: + if self.error == None: + return "Unknown URL error: %s" % self.url + return self.error.__str__() + return self.msg + +def get_post_url(url, get=True, data_dict=None): + """ + get: use GET if True, otherwise use POST. + data_dict: dict of data to send. + """ + if data_dict == None: + data_dict = {} + if get == True: + if data_dict != {}: + # encode get parameters in the url + param_string = urllib.urlencode(data_dict) + url = "%s?%s" % (url, param_string) + data = None + else: + data = urllib.urlencode(data_dict) + headers = {'User-Agent':USER_AGENT} + req = urllib2.Request(url, data=data, headers=headers) + try: + response = urllib2.urlopen(req) + except urllib2.HTTPError, e: + if hasattr(e, 'reason'): + msg = 'We failed to reach a server.\nURL: %s\nReason: %s' \ + % (url, e.reason) + elif hasattr(e, 'code'): + msg = "The server couldn't fulfill the request.\nURL: %s\nError code: %s" \ + % (url, e.code) + raise InvalidURL(error=e, url=url, msg=msg) + page = response.read() + final_url = response.geturl() + info = response.info() + response.close() + return (page, final_url, info) + + +class HTTP (base.VersionedStorage): + """ + This class implements a Storage interface over HTTP, using GET to + retrieve information and POST to set information. + """ + name = 'HTTP' + + def __init__(self, *args, **kwargs): + base.VersionedStorage.__init__(self, *args, **kwargs) + + def storage_version(self, revision=None): + """Return the storage format for this backend.""" + return libbe.storage.STORAGE_VERSION + + def _init(self): + """Create a new storage repository.""" + raise base.NotSupported( + 'init', 'Cannot initialize this repository format.') + + def _destroy(self): + """Remove the storage repository.""" + raise base.NotSupported( + 'destroy', 'Cannot destroy this repository format.') + + def _connect(self): + self.check_storage_version() + + def _disconnect(self): + pass + + def _add(self, id, parent=None, directory=False): + url = urlparse.urljoin(self.repo, 'add') + page,final_url,info = get_post_url( + url, get=False, + data_dict={'id':id, 'parent':parent, 'directory':directory}) + + def _remove(self, id): + url = urlparse.urljoin(self.repo, 'remove') + page,final_url,info = get_post_url( + url, get=False, + data_dict={'id':id, 'recursive':False}) + + def _recursive_remove(self, id): + url = urlparse.urljoin(self.repo, 'remove') + page,final_url,info = get_post_url( + url, get=False, + data_dict={'id':id, 'recursive':True}) + + def _children(self, id=None, revision=None): + url = urlparse.urljoin(self.repo, 'children') + page,final_url,info = get_post_url( + url, get=True, + data_dict={'id':id, 'revision':revision}) + return page.strip('\n').splitlines() + + def _get(self, id, default=base.InvalidObject, revision=None): + url = urlparse.urljoin(self.repo, '/'.join(['get', id])) + try: + page,final_url,info = get_post_url( + url, get=True, + data_dict={'revision':revision}) + except InvalidURL, e: + if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): + raise + elif default == base.InvalidObject: + raise base.InvalidID(id) + return default + version = info['X-BE-Version'] + if version != libbe.storage.STORAGE_VERSION: + raise base.InvalidStorageVersion( + version, libbe.storage.STORAGE_VERSION) + return page + + def _set(self, id, value): + url = urlparse.urljoin(self.repo, '/'.join(['set', id])) + try: + page,final_url,info = get_post_url( + url, get=False, + data_dict={'value':value}) + except InvalidURL, e: + if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): + raise + if e.error.code == HTTP_USER_ERROR: + raise base.InvalidDirectory( + 'Directory %s cannot have data' % id) + raise base.InvalidID(id) + + def _commit(self, summary, body=None, allow_empty=False): + url = urlparse.urljoin(self.repo, 'commit') + try: + page,final_url,info = get_post_url( + url, get=False, + data_dict={'summary':summary, 'body':body, + 'allow_empty':allow_empty}) + except InvalidURL, e: + if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): + raise + if e.error.code == HTTP_USER_ERROR: + raise base.EmptyCommit + raise base.InvalidID(id) + return page.rstrip('\n') + + def revision_id(self, index=None): + """ + Return the name of the <index>th revision. The choice of + which branch to follow when crossing branches/merges is not + defined. Revision indices start at 1; ID 0 is the blank + repository. + + Return None if index==None. + + If the specified revision does not exist, raise InvalidRevision. + """ + if index == None: + return None + try: + if int(index) != index: + raise InvalidRevision(index) + except ValueError: + raise InvalidRevision(index) + url = urlparse.urljoin(self.repo, 'revision-id') + try: + page,final_url,info = get_post_url( + url, get=True, + data_dict={'index':index}) + except InvalidURL, e: + if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID): + raise + if e.error.code == HTTP_USER_ERROR: + raise base.InvalidRevision(index) + raise base.InvalidID(id) + return page.rstrip('\n') + + def check_storage_version(self): + version = self.storage_version() + if version != libbe.storage.STORAGE_VERSION: + raise base.InvalidStorageVersion( + version, libbe.storage.STORAGE_VERSION) + + def storage_version(self, revision=None): + url = urlparse.urljoin(self.repo, 'version') + page,final_url,info = get_post_url( + url, get=True, data_dict={'revision':revision}) + return page.rstrip('\n') + +if TESTING == True: + class GetPostUrlTestCase (unittest.TestCase): + """Test cases for get_post_url()""" + def test_get(self): + url = 'http://bugseverywhere.org/be/show/HomePage' + page,final_url,info = get_post_url(url=url) + self.failUnless(final_url == url, + 'Redirect?\n Expected: "%s"\n Got: "%s"' + % (url, final_url)) + def test_get_redirect(self): + url = 'http://bugseverywhere.org' + expected = 'http://bugseverywhere.org/be/show/HomePage' + page,final_url,info = get_post_url(url=url) + self.failUnless(final_url == expected, + 'Redirect?\n Expected: "%s"\n Got: "%s"' + % (expected, final_url)) + + #make_storage_testcase_subclasses(VersionedStorage, sys.modules[__name__]) + + unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) + suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()]) diff --git a/libbe/storage/vcs/base.py b/libbe/storage/vcs/base.py index 39f5082..8390cbc 100644 --- a/libbe/storage/vcs/base.py +++ b/libbe/storage/vcs/base.py @@ -800,7 +800,7 @@ os.listdir(self.get_path("bugs")): try: path = self._cached_path_id.path(id) except InvalidID, e: - raise e + raise if not os.path.exists(path): raise InvalidID(id) if os.path.isdir(path): @@ -877,7 +877,11 @@ os.listdir(self.get_path("bugs")): /.be or None if none of those files exist. """ - return search_parent_directories(path, filename) + try: + ret = search_parent_directories(path, filename) + except AssertionError, e: + return None + return ret def _u_find_id(self, id, revision): """ diff --git a/libbe/ui/command_line.py b/libbe/ui/command_line.py index f57983f..7ba6cf5 100644 --- a/libbe/ui/command_line.py +++ b/libbe/ui/command_line.py @@ -299,7 +299,7 @@ def main(): command = Class(ui=ui) ui.setup_command(command) - if command.name in ['comment', 'commit']: + if command.name in ['comment', 'commit', 'serve']: paginate = 'never' else: paginate = 'auto' |