From 19da30a8ff233285a79df9f1da8b0d88d99eb626 Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Mon, 5 Sep 2016 19:55:01 +0200 Subject: core: ObjectStorage.Writer --- core/storage.go | 26 ++++++++++---- examples/storage/aerospike/storage.go | 5 +++ storage/filesystem/internal/index/index.go | 24 +++++++++---- storage/filesystem/object.go | 54 ++++++++++++++++++++++++++++++ storage/filesystem/object_test.go | 17 ++++++++++ storage/memory/storage.go | 6 ++++ 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/core/storage.go b/core/storage.go index 09b2450..607e5c6 100644 --- a/core/storage.go +++ b/core/storage.go @@ -1,15 +1,28 @@ package core -import "errors" +import ( + "errors" + "io" +) -//ErrStop is used to stop a ForEach function in an Iter -var ErrStop = errors.New("stop iter") +var ( + //ErrStop is used to stop a ForEach function in an Iter + ErrStop = errors.New("stop iter") + ErrNotImplemented = errors.New("method not-implemented") +) // ObjectStorage generic storage of objects type ObjectStorage interface { // NewObject returns a new Object, the real type of the object can be a // custom implementation or the defaul one, MemoryObject NewObject() Object + // Writer retuns a writer for writing a packfile to the Storage, this method + // is optional, if not implemented the ObjectStorage should return a + // ErrNotImplemented error. + // + // If the implementation not implements Writer the objects should be written + // using the Set method. + Writer() (io.WriteCloser, error) // Set save an object into the storage, the object shuld be create with // the NewObject, method, and file if the type is not supported. Set(Object) (Hash, error) @@ -17,15 +30,14 @@ type ObjectStorage interface { // return (nil, ErrObjectNotFound) if an object doesn't exist with both the // given hash and object type. // - // Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject - // and AnyObject. + // Valid ObjectType values are CommitObject, BlobObject, TagObject, + // TreeObject and AnyObject. // // If AnyObject is given, the object must be looked up regardless of its type. Get(ObjectType, Hash) (Object, error) // Iter returns a custom ObjectIter over all the object on the storage. // - // Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject - // and AnyObject. + // Valid ObjectType values are CommitObject, BlobObject, TagObject, Iter(ObjectType) (ObjectIter, error) // Begin starts a transaction. Begin() TxObjectStorage diff --git a/examples/storage/aerospike/storage.go b/examples/storage/aerospike/storage.go index b57d932..a62d1dc 100644 --- a/examples/storage/aerospike/storage.go +++ b/examples/storage/aerospike/storage.go @@ -70,6 +70,11 @@ func (s *ObjectStorage) NewObject() core.Object { return &core.MemoryObject{} } +// Writer method not supported, this method is optional to implemented. +func (s *ObjectStorage) Writer() (io.WriteCloser, error) { + return nil, core.ErrNotImplemented +} + func (s *ObjectStorage) Set(obj core.Object) (core.Hash, error) { key, err := s.buildKey(obj.Hash(), obj.Type()) if err != nil { diff --git a/storage/filesystem/internal/index/index.go b/storage/filesystem/internal/index/index.go index 737aca6..233dcbd 100644 --- a/storage/filesystem/internal/index/index.go +++ b/storage/filesystem/internal/index/index.go @@ -35,18 +35,26 @@ func NewFromIdx(r io.Reader) (Index, error) { // NewFrompackfile returns a new index from a packfile reader. func NewFromPackfile(rs io.ReadSeeker) (Index, error) { - index := make(Index) + s := packfile.NewSeekable(rs) + return newFromPackfile(rs, s) +} - r := packfile.NewSeekable(rs) - p := packfile.NewParser(r) +func NewFromPackfileInMemory(rs io.Reader) (Index, error) { + s := packfile.NewStream(rs) + return newFromPackfile(rs, s) +} +func newFromPackfile(r io.Reader, s packfile.ReadRecaller) (Index, error) { + index := make(Index) + + p := packfile.NewParser(s) count, err := p.ReadHeader() if err != nil { return nil, err } for i := 0; i < int(count); i++ { - offset, err := r.Offset() + offset, err := s.Offset() if err != nil { return nil, err } @@ -56,17 +64,19 @@ func NewFromPackfile(rs io.ReadSeeker) (Index, error) { return nil, err } - err = r.Remember(offset, obj) + err = s.Remember(offset, obj) if err != nil { return nil, err } - err = index.Set(obj.Hash(), offset) - if err != nil { + if err = index.Set(obj.Hash(), offset); err != nil { return nil, err } } + //The trailer records 20-byte SHA-1 checksum of all of the above. + p.ReadHash() + return index, nil } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 0696c70..fbcc9ae 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -1,7 +1,9 @@ package filesystem import ( + "bytes" "fmt" + "io" "os" "gopkg.in/src-d/go-git.v4/core" @@ -30,6 +32,58 @@ func (s *ObjectStorage) NewObject() core.Object { return &core.MemoryObject{} } +// Writer method not supported on Memory storage +func (o *ObjectStorage) Writer() (io.WriteCloser, error) { + file := bytes.NewBuffer(nil) + return newPackWrite(o, file), nil +} + +type packWriter struct { + writer io.Writer + pipeReader io.ReadCloser + pipeWriter io.WriteCloser + file io.Writer + result chan error +} + +func newPackWrite(o *ObjectStorage, file io.Writer) io.WriteCloser { + r, w := io.Pipe() + + ch := make(chan error) + go func(r io.ReadCloser) { + defer r.Close() + index, err := index.NewFromPackfileInMemory(r) + o.index = index + + ch <- err + }(r) + + return &packWriter{ + writer: io.MultiWriter(w, file), + pipeReader: r, + pipeWriter: w, + file: file, + result: ch, + } + +} + +func (w *packWriter) Write(p []byte) (int, error) { + return w.writer.Write(p) +} + +func (w *packWriter) Close() error { + defer func() { + close(w.result) + }() + + if err := w.pipeWriter.Close(); err != nil { + return err + } + + return <-w.result +} + // Set adds a new object to the storage. As this functionality is not // yet supported, this method always returns a "not implemented yet" // error an zero hash. diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 5752f79..78ec20c 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -2,6 +2,7 @@ package filesystem import ( "fmt" + "io" "io/ioutil" "os" "reflect" @@ -62,6 +63,22 @@ func (s *FsSuite) TearDownSuite(c *C) { } } +func (s *FsSuite) TestWriter(c *C) { + r, err := os.Open("../../formats/packfile/fixtures/git-fixture.ofs-delta") + c.Assert(err, IsNil) + + o := &ObjectStorage{} + w, err := o.Writer() + c.Assert(err, IsNil) + + n, err := io.Copy(w, r) + c.Assert(err, IsNil) + c.Check(n, Equals, int64(85300)) + + c.Assert(o.index, HasLen, 28) + c.Assert(w.Close(), IsNil) +} + func (s *FsSuite) TestHashNotFound(c *C) { sto := s.newObjectStorage(c, "binary-relations") types := []core.ObjectType{core.AnyObject, core.TagObject, core.CommitObject, core.BlobObject, core.TreeObject} diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 01d5fa5..216336c 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -3,6 +3,7 @@ package memory import ( "fmt" + "io" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/core" @@ -109,6 +110,11 @@ func (o *ObjectStorage) NewObject() core.Object { return &core.MemoryObject{} } +// Writer method not supported on Memory storage +func (o *ObjectStorage) Writer() (io.WriteCloser, error) { + return nil, core.ErrNotImplemented +} + // Set stores an object, the object should be properly filled before set it. func (o *ObjectStorage) Set(obj core.Object) (core.Hash, error) { h := obj.Hash() -- cgit