aboutsummaryrefslogtreecommitdiffstats
path: root/libbe/storage/http.py
diff options
context:
space:
mode:
Diffstat (limited to 'libbe/storage/http.py')
-rw-r--r--libbe/storage/http.py446
1 files changed, 446 insertions, 0 deletions
diff --git a/libbe/storage/http.py b/libbe/storage/http.py
new file mode 100644
index 0000000..7ec9f54
--- /dev/null
+++ b/libbe/storage/http.py
@@ -0,0 +1,446 @@
+# Copyright (C) 2010 W. Trevor King <wking@drexel.edu>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# For urllib2 information, see
+# urllib2, from urllib2 - The Missing Manual
+# http://www.voidspace.org.uk/python/articles/urllib2.shtml
+#
+# A dictionary of response codes is available in
+# httplib.responses
+
+"""Define an HTTP-based :class:`~libbe.storage.base.VersionedStorage`
+implementation.
+
+See Also
+--------
+:mod:`libbe.command.serve` : the associated server
+
+"""
+
+import sys
+import urllib
+import urllib2
+import urlparse
+
+import libbe
+import libbe.version
+import base
+from libbe import TESTING
+
+if TESTING == True:
+ import copy
+ import doctest
+ import StringIO
+ import unittest
+
+ import libbe.bugdir
+ import libbe.command.serve
+
+
+USER_AGENT = 'BE-HTTP-Storage'
+HTTP_OK = 200
+HTTP_FOUND = 302
+HTTP_TEMP_REDIRECT = 307
+HTTP_USER_ERROR = 418
+"""Status returned to indicate exceptions on the server side.
+
+A BE-specific extension to the HTTP/1.1 protocol (See `RFC 2616`_).
+
+.. _RFC 2616: http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1
+"""
+
+HTTP_VALID = [HTTP_OK, HTTP_FOUND, HTTP_TEMP_REDIRECT, HTTP_USER_ERROR]
+
+class InvalidURL (Exception):
+ def __init__(self, error=None, url=None, msg=None):
+ Exception.__init__(self, msg)
+ self.url = url
+ self.error = error
+ self.msg = msg
+ def __str__(self):
+ if self.msg == None:
+ if self.error == None:
+ return "Unknown URL error: %s" % self.url
+ return self.error.__str__()
+ return self.msg
+
+def get_post_url(url, get=True, data_dict=None, headers=[]):
+ """Execute a GET or POST transaction.
+
+ Parameters
+ ----------
+ url : str
+ The base URL (query portion added internally, if necessary).
+ get : bool
+ Use GET if True, otherwise use POST.
+ data_dict : dict
+ Data to send, either by URL query (if GET) or by POST (if POST).
+ headers : list
+ Extra HTTP headers to add to the request.
+ """
+ if data_dict == None:
+ data_dict = {}
+ if get == True:
+ if data_dict != {}:
+ # encode get parameters in the url
+ param_string = urllib.urlencode(data_dict)
+ url = "%s?%s" % (url, param_string)
+ data = None
+ else:
+ data = urllib.urlencode(data_dict)
+ headers = dict(headers)
+ headers['User-Agent'] = USER_AGENT
+ req = urllib2.Request(url, data=data, headers=headers)
+ try:
+ response = urllib2.urlopen(req)
+ except urllib2.HTTPError, e:
+ if hasattr(e, 'reason'):
+ msg = 'We failed to reach a server.\nURL: %s\nReason: %s' \
+ % (url, e.reason)
+ elif hasattr(e, 'code'):
+ msg = "The server couldn't fulfill the request.\nURL: %s\nError code: %s" \
+ % (url, e.code)
+ raise InvalidURL(error=e, url=url, msg=msg)
+ page = response.read()
+ final_url = response.geturl()
+ info = response.info()
+ response.close()
+ return (page, final_url, info)
+
+
+class HTTP (base.VersionedStorage):
+ """:class:`~libbe.storage.base.VersionedStorage` implementation over
+ HTTP.
+
+ Uses GET to retrieve information and POST to set information.
+ """
+ name = 'HTTP'
+
+ def __init__(self, repo, *args, **kwargs):
+ repo,self.uname,self.password = self.parse_repo(repo)
+ base.VersionedStorage.__init__(self, repo, *args, **kwargs)
+
+ def parse_repo(self, repo):
+ """Grab username and password (if any) from the repo URL.
+
+ Examples
+ --------
+
+ >>> s = HTTP('http://host.com/path/to/repo')
+ >>> s.repo
+ 'http://host.com/path/to/repo'
+ >>> s.uname == None
+ True
+ >>> s.password == None
+ True
+ >>> s.parse_repo('http://joe:secret@host.com/path/to/repo')
+ ('http://host.com/path/to/repo', 'joe', 'secret')
+ """
+ scheme,netloc,path,params,query,fragment = urlparse.urlparse(repo)
+ parts = netloc.split('@', 1)
+ if len(parts) == 2:
+ uname,password = parts[0].split(':')
+ repo = urlparse.urlunparse(
+ (scheme, parts[1], path, params, query, fragment))
+ else:
+ uname,password = (None, None)
+ return (repo, uname, password)
+
+ def get_post_url(self, url, get=True, data_dict=None, headers=[]):
+ if self.uname != None and self.password != None:
+ headers.append(('Authorization','Basic %s' % \
+ ('%s:%s' % (self.uname, self.password)).encode('base64')))
+ return get_post_url(url, get, data_dict, headers)
+
+ def storage_version(self, revision=None):
+ """Return the storage format for this backend."""
+ return libbe.storage.STORAGE_VERSION
+
+ def _init(self):
+ """Create a new storage repository."""
+ raise base.NotSupported(
+ 'init', 'Cannot initialize this repository format.')
+
+ def _destroy(self):
+ """Remove the storage repository."""
+ raise base.NotSupported(
+ 'destroy', 'Cannot destroy this repository format.')
+
+ def _connect(self):
+ self.check_storage_version()
+
+ def _disconnect(self):
+ pass
+
+ def _add(self, id, parent=None, directory=False):
+ url = urlparse.urljoin(self.repo, 'add')
+ page,final_url,info = self.get_post_url(
+ url, get=False,
+ data_dict={'id':id, 'parent':parent, 'directory':directory})
+
+ def _exists(self, id, revision=None):
+ url = urlparse.urljoin(self.repo, 'exists')
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'id':id, 'revision':revision})
+ if page == 'True':
+ return True
+ return False
+
+ def _remove(self, id):
+ url = urlparse.urljoin(self.repo, 'remove')
+ page,final_url,info = self.get_post_url(
+ url, get=False,
+ data_dict={'id':id, 'recursive':False})
+
+ def _recursive_remove(self, id):
+ url = urlparse.urljoin(self.repo, 'remove')
+ page,final_url,info = self.get_post_url(
+ url, get=False,
+ data_dict={'id':id, 'recursive':True})
+
+ def _ancestors(self, id=None, revision=None):
+ url = urlparse.urljoin(self.repo, 'ancestors')
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'id':id, 'revision':revision})
+ return page.strip('\n').splitlines()
+
+ def _children(self, id=None, revision=None):
+ url = urlparse.urljoin(self.repo, 'children')
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'id':id, 'revision':revision})
+ return page.strip('\n').splitlines()
+
+ def _get(self, id, default=base.InvalidObject, revision=None):
+ url = urlparse.urljoin(self.repo, '/'.join(['get', id]))
+ try:
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'revision':revision})
+ except InvalidURL, e:
+ if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+ raise
+ elif default == base.InvalidObject:
+ raise base.InvalidID(id)
+ return default
+ version = info['X-BE-Version']
+ if version != libbe.storage.STORAGE_VERSION:
+ raise base.InvalidStorageVersion(
+ version, libbe.storage.STORAGE_VERSION)
+ return page
+
+ def _set(self, id, value):
+ url = urlparse.urljoin(self.repo, '/'.join(['set', id]))
+ try:
+ page,final_url,info = self.get_post_url(
+ url, get=False,
+ data_dict={'value':value})
+ except InvalidURL, e:
+ if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+ raise
+ if e.error.code == HTTP_USER_ERROR \
+ and not 'InvalidID' in str(e.error):
+ raise base.InvalidDirectory(
+ 'Directory %s cannot have data' % id)
+ raise base.InvalidID(id)
+
+ def _commit(self, summary, body=None, allow_empty=False):
+ url = urlparse.urljoin(self.repo, 'commit')
+ try:
+ page,final_url,info = self.get_post_url(
+ url, get=False,
+ data_dict={'summary':summary, 'body':body,
+ 'allow_empty':allow_empty})
+ except InvalidURL, e:
+ if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+ raise
+ if e.error.code == HTTP_USER_ERROR:
+ raise base.EmptyCommit
+ raise base.InvalidID(id)
+ return page.rstrip('\n')
+
+ def revision_id(self, index=None):
+ """Return the name of the <index>th revision.
+
+ The choice of which branch to follow when crossing
+ branches/merges is not defined. Revision indices start at 1;
+ ID 0 is the blank repository.
+
+ Return None if index==None.
+
+ Raises
+ ------
+ InvalidRevision
+ If the specified revision does not exist.
+ """
+ if index == None:
+ return None
+ try:
+ if int(index) != index:
+ raise base.InvalidRevision(index)
+ except ValueError:
+ raise base.InvalidRevision(index)
+ url = urlparse.urljoin(self.repo, 'revision-id')
+ try:
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'index':index})
+ except InvalidURL, e:
+ if not (hasattr(e.error, 'code') and e.error.code in HTTP_VALID):
+ raise
+ if e.error.code == HTTP_USER_ERROR:
+ raise base.InvalidRevision(index)
+ raise base.InvalidID(id)
+ return page.rstrip('\n')
+
+ def changed(self, revision=None):
+ url = urlparse.urljoin(self.repo, 'changed')
+ page,final_url,info = self.get_post_url(
+ url, get=True,
+ data_dict={'revision':revision})
+ lines = page.strip('\n')
+ new,mod,rem = [p.splitlines() for p in page.split('\n\n')]
+ return (new, mod, rem)
+
+ def check_storage_version(self):
+ version = self.storage_version()
+ if version != libbe.storage.STORAGE_VERSION:
+ raise base.InvalidStorageVersion(
+ version, libbe.storage.STORAGE_VERSION)
+
+ def storage_version(self, revision=None):
+ url = urlparse.urljoin(self.repo, 'version')
+ page,final_url,info = self.get_post_url(
+ url, get=True, data_dict={'revision':revision})
+ return page.rstrip('\n')
+
+if TESTING == True:
+ class GetPostUrlTestCase (unittest.TestCase):
+ """Test cases for get_post_url()"""
+ def test_get(self):
+ url = 'http://bugseverywhere.org/be/show/HomePage'
+ page,final_url,info = get_post_url(url=url)
+ self.failUnless(final_url == url,
+ 'Redirect?\n Expected: "%s"\n Got: "%s"'
+ % (url, final_url))
+ def test_get_redirect(self):
+ url = 'http://bugseverywhere.org'
+ expected = 'http://bugseverywhere.org/be/show/HomePage'
+ page,final_url,info = get_post_url(url=url)
+ self.failUnless(final_url == expected,
+ 'Redirect?\n Expected: "%s"\n Got: "%s"'
+ % (expected, final_url))
+
+ class TestingHTTP (HTTP):
+ name = 'TestingHTTP'
+ def __init__(self, repo, *args, **kwargs):
+ self._storage_backend = base.VersionedStorage(repo)
+ self.app = libbe.command.serve.ServerApp(
+ storage=self._storage_backend)
+ HTTP.__init__(self, repo='http://localhost:8000/', *args, **kwargs)
+ self.intitialized = False
+ # duplicated from libbe.storage.serve.WSGITestCase
+ self.default_environ = {
+ 'REQUEST_METHOD': 'GET', # 'POST', 'HEAD'
+ 'SCRIPT_NAME':'',
+ 'PATH_INFO': '',
+ #'QUERY_STRING':'', # may be empty or absent
+ #'CONTENT_TYPE':'', # may be empty or absent
+ #'CONTENT_LENGTH':'', # may be empty or absent
+ 'SERVER_NAME':'example.com',
+ 'SERVER_PORT':'80',
+ 'SERVER_PROTOCOL':'HTTP/1.1',
+ 'wsgi.version':(1,0),
+ 'wsgi.url_scheme':'http',
+ 'wsgi.input':StringIO.StringIO(),
+ 'wsgi.errors':StringIO.StringIO(),
+ 'wsgi.multithread':False,
+ 'wsgi.multiprocess':False,
+ 'wsgi.run_once':False,
+ }
+ def getURL(self, app, path='/', method='GET', data=None,
+ scheme='http', environ={}):
+ # duplicated from libbe.storage.serve.WSGITestCase
+ env = copy.copy(self.default_environ)
+ env['PATH_INFO'] = path
+ env['REQUEST_METHOD'] = method
+ env['scheme'] = scheme
+ if data != None:
+ enc_data = urllib.urlencode(data)
+ if method == 'POST':
+ env['CONTENT_LENGTH'] = len(enc_data)
+ env['wsgi.input'] = StringIO.StringIO(enc_data)
+ else:
+ assert method in ['GET', 'HEAD'], method
+ env['QUERY_STRING'] = enc_data
+ for key,value in environ.items():
+ env[key] = value
+ return ''.join(app(env, self.start_response))
+ def start_response(self, status, response_headers, exc_info=None):
+ self.status = status
+ self.response_headers = response_headers
+ self.exc_info = exc_info
+ def get_post_url(self, url, get=True, data_dict=None, headers=[]):
+ if get == True:
+ method = 'GET'
+ else:
+ method = 'POST'
+ scheme,netloc,path,params,query,fragment = urlparse.urlparse(url)
+ environ = {}
+ for header_name,header_value in headers:
+ environ['HTTP_%s' % header_name] = header_value
+ output = self.getURL(
+ self.app, path, method, data_dict, scheme, environ)
+ if self.status != '200 OK':
+ class __estr (object):
+ def __init__(self, string):
+ self.string = string
+ self.code = int(string.split()[0])
+ def __str__(self):
+ return self.string
+ error = __estr(self.status)
+ raise InvalidURL(error=error, url=url, msg=output)
+ info = dict(self.response_headers)
+ return (output, url, info)
+ def _init(self):
+ try:
+ HTTP._init(self)
+ raise AssertionError
+ except base.NotSupported:
+ pass
+ self._storage_backend._init()
+ def _destroy(self):
+ try:
+ HTTP._destroy(self)
+ raise AssertionError
+ except base.NotSupported:
+ pass
+ self._storage_backend._destroy()
+ def _connect(self):
+ self._storage_backend._connect()
+ HTTP._connect(self)
+ def _disconnect(self):
+ HTTP._disconnect(self)
+ self._storage_backend._disconnect()
+
+
+ base.make_versioned_storage_testcase_subclasses(
+ TestingHTTP, sys.modules[__name__])
+
+ unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+ suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()])