From 5e73f01cb2e027a8f02801635b79d3a9bc866914 Mon Sep 17 00:00:00 2001 From: Alberto Cortés Date: Mon, 4 Jul 2016 17:09:22 +0200 Subject: Adds support to open local repositories and to use file-based object storage (#55) * remove some comments * idx writer/reader * Shut up ssh tests, they are annoying * Add file scheme test to clients * Add dummy file client * Add test fot file client * Make tests use fixture endpoint * add parser for packed-refs format * add parser for packed-refs format * WIP adding dir.Refs() tests * Add test for fixture refs * refs parser for the refs directory * Documentation * Add Capabilities to file client * tgz.Exatract now accpets a path instead of a Reader * fix bug in idxfile fanout calculation * remove dead code * packfile documentation * clean packfile parser code * add core.Object.Content() and returns errors for core.ObjectStorage.Iter() * add seekable storage * add dir repos to NewRepository * clean prints * Add dir client documentation to README * Organize the README * README * Clean tgz package * Clean temp dirs after tgz tests * Gometalinter on gitdir * Clean pattern function * metalinter tgz * metalinter gitdir * gitdir coverage and remove seekable packfile filedescriptor leak * gitdir Idxfile tests and remove file descriptor leak * gitdir Idxfile tests when no idx is found * clean storage/seekable/internal/index and some formats/idxfile API issues * clean storage/seekable * clean formats/idx * turn packfile/doc.go into packfile/doc.txt * move formats/packfile/reader to decoder * fix packfile decoder error names * improve documentation * comment packfile decoder errors * comment public API (format/packfile) * remve duplicated code in packfile decoder test * move tracking_reader into an internal package and clean it * use iota for packfile format * rename packfile parse.go to packfile object_at.go * clean packfile deltas * fix delta header size bug * improve delta documentation * clean packfile deltas * clean packfiles deltas * clean repository.go * Remove go 1.5 from Travis CI Because go 1.5 does not suport internal packages. * change local repo scheme to local:// * change "local://" to "file://" as the local scheme * fix broken indentation * shortens names of variables in short scopes * more shortening of variable names * more shortening of variable names * Rename git dir client to "file", as the scheme used for it * Fix file format ctor name, now that the package name has change * Sortcut local repo constructor to not use remotes The object storage is build directly in the repository ctor, instead of creating a remote and waiting for the user to pull it. * update README and fix some errors in it * remove file scheme client * Local respositories has now a new ctor This is, they are no longer identified by the scheme of the URL, but are created different from inception. * remove unused URL field form Repository * move all git dir logic to seekable sotrage ctor * fix documentation * Make formats/file/dir an internal package to storage/seekable * change package storage/seekable to storage/fs * clean storage/fs * overall storage/fs clean * more cleaning * some metalinter fixes * upgrade cshared to last changes * remove dead code * fix test error info * remove file scheme check from clients * fix test error message * fix test error message * fix error messages * style changes * fix comments everywhere * style changes * style changes * scaffolding and tests for local packfiles without ifx files * outsource index building from packfile to the packfile decoder * refactor packfile header reading into a new function * move code to generate index from packfile back to index package * add header parsing * fix documentation errata * add undeltified and OFS delta support for index building from the packfile * add tests for packfile with ref-deltas * support for packfiles with ref-deltas and no idx * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * WIP refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * remove prints from tests * remove prints from tests * refactor packfile.core into packfile.parser * rename packfile reader to something that shows it is a recaller * rename cannot recall error * rename packfile.Reader to packfile.ReadRecaller and document * speed up test by using StreamReader instead of SeekableReader when possible * clean packfile StreamReader * stream_reader tests * refactor packfile.StreamReader into packfile.StreamReadRecaller * refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it * generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations * speed up storage/fs tests * speed up tests in . by loading packfiles in memory * speed up repository tests by using and smaller fixture * restore doc.go files * rename packfile.ReadRecaller implementations to shorter names * update comments to type changes * packfile.Parser test (WIP) * packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller * add test for packfile.ReadRecaller.ForgetAll() * clarify seekable being able to recallByOffset forgetted objects * use better names for internal maps * metalinter packfile package * speed up some tests * documentation fixes * change storage.fs package name to storage.proxy to avoid confusion with new filesystem support * New fs package and os transparent implementation Now NewRepositoryFromFS receives a fs and a path and tests are modified accordingly, but it is still not using for anything. * add fs to gitdir and proxy.store * reduce fs interface for easier implementation * remove garbage dirs from tgz tests * change file name gitdir/dir.go to gitdir/gitdir.go * fs.OS tests * metalinter utils/fs * add NewRepositoryFromFS documentation to README * Readability fixes to README * move tgz to an external dependency * move filesystem impl. example to example dir * rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go * rename proxy package to seekable --- storage/memory/object.go | 35 ++++++++++++++++++++++++----------- storage/memory/object_test.go | 4 ++-- storage/memory/storage.go | 9 ++------- storage/memory/storage_test.go | 16 ++-------------- 4 files changed, 30 insertions(+), 34 deletions(-) (limited to 'storage/memory') diff --git a/storage/memory/object.go b/storage/memory/object.go index 125e887..1720ebd 100644 --- a/storage/memory/object.go +++ b/storage/memory/object.go @@ -9,19 +9,29 @@ import ( // Object on memory core.Object implementation type Object struct { - t core.ObjectType - h core.Hash - content []byte - size int64 + t core.ObjectType + h core.Hash + cont []byte + sz int64 +} + +// NewObject creates a new object with the given type and content +func NewObject(typ core.ObjectType, size int64, cont []byte) *Object { + return &Object{ + t: typ, + h: core.ComputeHash(typ, cont), + cont: cont, + sz: int64(len(cont)), + } } // Hash return the object Hash, the hash is calculated on-the-fly the first -// time is called, the subsequent calls the same Hash is returned even in the +// time is called, the subsequent calls the same Hash is returned even if the // type or the content has changed. The Hash is only generated if the size of // the content is exactly the Object.Size func (o *Object) Hash() core.Hash { - if o.h == core.ZeroHash && int64(len(o.content)) == o.size { - o.h = core.ComputeHash(o.t, o.content) + if o.h == core.ZeroHash && int64(len(o.cont)) == o.sz { + o.h = core.ComputeHash(o.t, o.cont) } return o.h @@ -34,14 +44,17 @@ func (o *Object) Type() core.ObjectType { return o.t } func (o *Object) SetType(t core.ObjectType) { o.t = t } // Size return the size of the object -func (o *Object) Size() int64 { return o.size } +func (o *Object) Size() int64 { return o.sz } // SetSize set the object size, the given size should be written afterwards -func (o *Object) SetSize(s int64) { o.size = s } +func (o *Object) SetSize(s int64) { o.sz = s } + +// Content returns the contents of the object +func (o *Object) Content() []byte { return o.cont } // Reader returns a core.ObjectReader used to read the object's content. func (o *Object) Reader() (core.ObjectReader, error) { - return ioutil.NopCloser(bytes.NewBuffer(o.content)), nil + return ioutil.NopCloser(bytes.NewBuffer(o.cont)), nil } // Writer returns a core.ObjectWriter used to write the object's content. @@ -50,7 +63,7 @@ func (o *Object) Writer() (core.ObjectWriter, error) { } func (o *Object) Write(p []byte) (n int, err error) { - o.content = append(o.content, p...) + o.cont = append(o.cont, p...) return len(p), nil } diff --git a/storage/memory/object_test.go b/storage/memory/object_test.go index f2873fa..3271254 100644 --- a/storage/memory/object_test.go +++ b/storage/memory/object_test.go @@ -49,7 +49,7 @@ func (s *ObjectSuite) TestSize(c *C) { } func (s *ObjectSuite) TestReader(c *C) { - o := &Object{content: []byte("foo")} + o := &Object{cont: []byte("foo")} reader, err := o.Reader() c.Assert(err, IsNil) @@ -71,5 +71,5 @@ func (s *ObjectSuite) TestWriter(c *C) { c.Assert(err, IsNil) c.Assert(n, Equals, 3) - c.Assert(o.content, DeepEquals, []byte("foo")) + c.Assert(o.cont, DeepEquals, []byte("foo")) } diff --git a/storage/memory/storage.go b/storage/memory/storage.go index fbf4bc4..84de980 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -28,11 +28,6 @@ func NewObjectStorage() *ObjectStorage { } } -// New returns a new empty memory.Object -func (o *ObjectStorage) New() (core.Object, error) { - return &Object{}, nil -} - // Set stores an object, the object should be properly filled before set it. func (o *ObjectStorage) Set(obj core.Object) (core.Hash, error) { h := obj.Hash() @@ -65,7 +60,7 @@ func (o *ObjectStorage) Get(h core.Hash) (core.Object, error) { } // Iter returns a core.ObjectIter for the given core.ObjectTybe -func (o *ObjectStorage) Iter(t core.ObjectType) core.ObjectIter { +func (o *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) { var series []core.Object switch t { case core.CommitObject: @@ -77,7 +72,7 @@ func (o *ObjectStorage) Iter(t core.ObjectType) core.ObjectIter { case core.TagObject: series = flattenObjectMap(o.Tags) } - return core.NewObjectSliceIter(series) + return core.NewObjectSliceIter(series), nil } func flattenObjectMap(m map[core.Hash]core.Object) []core.Object { diff --git a/storage/memory/storage_test.go b/storage/memory/storage_test.go index df6d5cf..19f4476 100644 --- a/storage/memory/storage_test.go +++ b/storage/memory/storage_test.go @@ -9,20 +9,10 @@ type ObjectStorageSuite struct{} var _ = Suite(&ObjectStorageSuite{}) -func (s *ObjectStorageSuite) TestNew(c *C) { - os := NewObjectStorage() - - o, err := os.New() - c.Assert(err, IsNil) - c.Assert(o.Size(), Equals, int64(0)) -} - func (s *ObjectStorageSuite) TestSet(c *C) { os := NewObjectStorage() - o, err := os.New() - c.Assert(err, IsNil) - + o := &Object{} o.SetType(core.CommitObject) o.SetSize(3) @@ -40,9 +30,7 @@ func (s *ObjectStorageSuite) TestSet(c *C) { func (s *ObjectStorageSuite) TestGet(c *C) { os := NewObjectStorage() - o, err := os.New() - c.Assert(err, IsNil) - + o := &Object{} o.SetType(core.CommitObject) o.SetSize(3) -- cgit