aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMáximo Cuadros <mcuadros@gmail.com>2016-09-05 19:55:01 +0200
committerMáximo Cuadros <mcuadros@gmail.com>2016-09-05 19:55:01 +0200
commit19da30a8ff233285a79df9f1da8b0d88d99eb626 (patch)
treebdb2313cc29381afbad9af75bd273c5ec6d9f24b
parent65427f64599e9383ab926aed75fc02f2db32022d (diff)
downloadgo-git-19da30a8ff233285a79df9f1da8b0d88d99eb626.tar.gz
core: ObjectStorage.Writer
-rw-r--r--core/storage.go26
-rw-r--r--examples/storage/aerospike/storage.go5
-rw-r--r--storage/filesystem/internal/index/index.go24
-rw-r--r--storage/filesystem/object.go54
-rw-r--r--storage/filesystem/object_test.go17
-rw-r--r--storage/memory/storage.go6
6 files changed, 118 insertions, 14 deletions
diff --git a/core/storage.go b/core/storage.go
index 09b2450..607e5c6 100644
--- a/core/storage.go
+++ b/core/storage.go
@@ -1,15 +1,28 @@
package core
-import "errors"
+import (
+ "errors"
+ "io"
+)
-//ErrStop is used to stop a ForEach function in an Iter
-var ErrStop = errors.New("stop iter")
+var (
+ //ErrStop is used to stop a ForEach function in an Iter
+ ErrStop = errors.New("stop iter")
+ ErrNotImplemented = errors.New("method not-implemented")
+)
// ObjectStorage generic storage of objects
type ObjectStorage interface {
// NewObject returns a new Object, the real type of the object can be a
// custom implementation or the defaul one, MemoryObject
NewObject() Object
+ // Writer retuns a writer for writing a packfile to the Storage, this method
+ // is optional, if not implemented the ObjectStorage should return a
+ // ErrNotImplemented error.
+ //
+ // If the implementation not implements Writer the objects should be written
+ // using the Set method.
+ Writer() (io.WriteCloser, error)
// Set save an object into the storage, the object shuld be create with
// the NewObject, method, and file if the type is not supported.
Set(Object) (Hash, error)
@@ -17,15 +30,14 @@ type ObjectStorage interface {
// return (nil, ErrObjectNotFound) if an object doesn't exist with both the
// given hash and object type.
//
- // Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject
- // and AnyObject.
+ // Valid ObjectType values are CommitObject, BlobObject, TagObject,
+ // TreeObject and AnyObject.
//
// If AnyObject is given, the object must be looked up regardless of its type.
Get(ObjectType, Hash) (Object, error)
// Iter returns a custom ObjectIter over all the object on the storage.
//
- // Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject
- // and AnyObject.
+ // Valid ObjectType values are CommitObject, BlobObject, TagObject,
Iter(ObjectType) (ObjectIter, error)
// Begin starts a transaction.
Begin() TxObjectStorage
diff --git a/examples/storage/aerospike/storage.go b/examples/storage/aerospike/storage.go
index b57d932..a62d1dc 100644
--- a/examples/storage/aerospike/storage.go
+++ b/examples/storage/aerospike/storage.go
@@ -70,6 +70,11 @@ func (s *ObjectStorage) NewObject() core.Object {
return &core.MemoryObject{}
}
+// Writer method not supported, this method is optional to implemented.
+func (s *ObjectStorage) Writer() (io.WriteCloser, error) {
+ return nil, core.ErrNotImplemented
+}
+
func (s *ObjectStorage) Set(obj core.Object) (core.Hash, error) {
key, err := s.buildKey(obj.Hash(), obj.Type())
if err != nil {
diff --git a/storage/filesystem/internal/index/index.go b/storage/filesystem/internal/index/index.go
index 737aca6..233dcbd 100644
--- a/storage/filesystem/internal/index/index.go
+++ b/storage/filesystem/internal/index/index.go
@@ -35,18 +35,26 @@ func NewFromIdx(r io.Reader) (Index, error) {
// NewFrompackfile returns a new index from a packfile reader.
func NewFromPackfile(rs io.ReadSeeker) (Index, error) {
- index := make(Index)
+ s := packfile.NewSeekable(rs)
+ return newFromPackfile(rs, s)
+}
- r := packfile.NewSeekable(rs)
- p := packfile.NewParser(r)
+func NewFromPackfileInMemory(rs io.Reader) (Index, error) {
+ s := packfile.NewStream(rs)
+ return newFromPackfile(rs, s)
+}
+func newFromPackfile(r io.Reader, s packfile.ReadRecaller) (Index, error) {
+ index := make(Index)
+
+ p := packfile.NewParser(s)
count, err := p.ReadHeader()
if err != nil {
return nil, err
}
for i := 0; i < int(count); i++ {
- offset, err := r.Offset()
+ offset, err := s.Offset()
if err != nil {
return nil, err
}
@@ -56,17 +64,19 @@ func NewFromPackfile(rs io.ReadSeeker) (Index, error) {
return nil, err
}
- err = r.Remember(offset, obj)
+ err = s.Remember(offset, obj)
if err != nil {
return nil, err
}
- err = index.Set(obj.Hash(), offset)
- if err != nil {
+ if err = index.Set(obj.Hash(), offset); err != nil {
return nil, err
}
}
+ //The trailer records 20-byte SHA-1 checksum of all of the above.
+ p.ReadHash()
+
return index, nil
}
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 0696c70..fbcc9ae 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -1,7 +1,9 @@
package filesystem
import (
+ "bytes"
"fmt"
+ "io"
"os"
"gopkg.in/src-d/go-git.v4/core"
@@ -30,6 +32,58 @@ func (s *ObjectStorage) NewObject() core.Object {
return &core.MemoryObject{}
}
+// Writer method not supported on Memory storage
+func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
+ file := bytes.NewBuffer(nil)
+ return newPackWrite(o, file), nil
+}
+
+type packWriter struct {
+ writer io.Writer
+ pipeReader io.ReadCloser
+ pipeWriter io.WriteCloser
+ file io.Writer
+ result chan error
+}
+
+func newPackWrite(o *ObjectStorage, file io.Writer) io.WriteCloser {
+ r, w := io.Pipe()
+
+ ch := make(chan error)
+ go func(r io.ReadCloser) {
+ defer r.Close()
+ index, err := index.NewFromPackfileInMemory(r)
+ o.index = index
+
+ ch <- err
+ }(r)
+
+ return &packWriter{
+ writer: io.MultiWriter(w, file),
+ pipeReader: r,
+ pipeWriter: w,
+ file: file,
+ result: ch,
+ }
+
+}
+
+func (w *packWriter) Write(p []byte) (int, error) {
+ return w.writer.Write(p)
+}
+
+func (w *packWriter) Close() error {
+ defer func() {
+ close(w.result)
+ }()
+
+ if err := w.pipeWriter.Close(); err != nil {
+ return err
+ }
+
+ return <-w.result
+}
+
// Set adds a new object to the storage. As this functionality is not
// yet supported, this method always returns a "not implemented yet"
// error an zero hash.
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index 5752f79..78ec20c 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -2,6 +2,7 @@ package filesystem
import (
"fmt"
+ "io"
"io/ioutil"
"os"
"reflect"
@@ -62,6 +63,22 @@ func (s *FsSuite) TearDownSuite(c *C) {
}
}
+func (s *FsSuite) TestWriter(c *C) {
+ r, err := os.Open("../../formats/packfile/fixtures/git-fixture.ofs-delta")
+ c.Assert(err, IsNil)
+
+ o := &ObjectStorage{}
+ w, err := o.Writer()
+ c.Assert(err, IsNil)
+
+ n, err := io.Copy(w, r)
+ c.Assert(err, IsNil)
+ c.Check(n, Equals, int64(85300))
+
+ c.Assert(o.index, HasLen, 28)
+ c.Assert(w.Close(), IsNil)
+}
+
func (s *FsSuite) TestHashNotFound(c *C) {
sto := s.newObjectStorage(c, "binary-relations")
types := []core.ObjectType{core.AnyObject, core.TagObject, core.CommitObject, core.BlobObject, core.TreeObject}
diff --git a/storage/memory/storage.go b/storage/memory/storage.go
index 01d5fa5..216336c 100644
--- a/storage/memory/storage.go
+++ b/storage/memory/storage.go
@@ -3,6 +3,7 @@ package memory
import (
"fmt"
+ "io"
"gopkg.in/src-d/go-git.v4/config"
"gopkg.in/src-d/go-git.v4/core"
@@ -109,6 +110,11 @@ func (o *ObjectStorage) NewObject() core.Object {
return &core.MemoryObject{}
}
+// Writer method not supported on Memory storage
+func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
+ return nil, core.ErrNotImplemented
+}
+
// Set stores an object, the object should be properly filled before set it.
func (o *ObjectStorage) Set(obj core.Object) (core.Hash, error) {
h := obj.Hash()