From 009f1069a1248c1e9189a9e4c342f6d017156ec4 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 19 Jul 2018 15:20:10 +0200 Subject: plumbing/format/idxfile: add new Index and MemoryIndex Signed-off-by: Miguel Molina --- storage/filesystem/index.go | 47 -------------------------------------------- storage/filesystem/object.go | 2 +- 2 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 storage/filesystem/index.go (limited to 'storage') diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go deleted file mode 100644 index 2ebf57e..0000000 --- a/storage/filesystem/index.go +++ /dev/null @@ -1,47 +0,0 @@ -package filesystem - -import ( - "os" - - "gopkg.in/src-d/go-git.v4/plumbing/format/index" - "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" - "gopkg.in/src-d/go-git.v4/utils/ioutil" -) - -type IndexStorage struct { - dir *dotgit.DotGit -} - -func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { - f, err := s.dir.IndexWriter() - if err != nil { - return err - } - - defer ioutil.CheckClose(f, &err) - - e := index.NewEncoder(f) - err = e.Encode(idx) - return err -} - -func (s *IndexStorage) Index() (i *index.Index, err error) { - idx := &index.Index{ - Version: 2, - } - - f, err := s.dir.Index() - if err != nil { - if os.IsNotExist(err) { - return idx, nil - } - - return nil, err - } - - defer ioutil.CheckClose(f, &err) - - d := index.NewDecoder(f) - err = d.Decode(idx) - return idx, err -} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 9ffe4dc..ef67f50 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -63,7 +63,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) - idxf := idxfile.NewIdxfile() + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { return err -- cgit From 79f249465b24104b73c9dc220d9098cecdab4d77 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 13:42:51 +0200 Subject: plumbing, storage: integrate new index Now dotgit.PackWriter uses the new packfile.Parser and index. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/writers.go | 33 ++++++++++------------ storage/filesystem/dotgit/writers_test.go | 3 +- storage/filesystem/object.go | 46 +++++++++++++++++++------------ 3 files changed, 46 insertions(+), 36 deletions(-) (limited to 'storage') diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index c2b420f..e1ede3c 100644 --- a/storage/filesystem/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -20,13 +20,14 @@ import ( // is renamed/moved (depends on the Filesystem implementation) to the final // location, if the PackWriter is not used, nothing is written type PackWriter struct { - Notify func(plumbing.Hash, *packfile.Index) + Notify func(plumbing.Hash, *idxfile.Writer) fs billy.Filesystem fr, fw billy.File synced *syncedReader checksum plumbing.Hash - index *packfile.Index + parser *packfile.Parser + writer *idxfile.Writer result chan error } @@ -55,20 +56,16 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) - d, err := packfile.NewDecoder(s, nil) - if err != nil { - w.result <- err - return - } + w.writer = new(idxfile.Writer) + w.parser = packfile.NewParser(s, w.writer) - checksum, err := d.Decode() + checksum, err := w.parser.Parse() if err != nil { w.result <- err return } w.checksum = checksum - w.index = d.Index() w.result <- err } @@ -92,8 +89,8 @@ func (w *PackWriter) Write(p []byte) (int, error) { // was written, the tempfiles are deleted without writing a packfile. func (w *PackWriter) Close() error { defer func() { - if w.Notify != nil && w.index != nil && w.index.Size() > 0 { - w.Notify(w.checksum, w.index) + if w.Notify != nil && w.writer != nil && w.writer.Finished() { + w.Notify(w.checksum, w.writer) } close(w.result) @@ -115,7 +112,7 @@ func (w *PackWriter) Close() error { return err } - if w.index == nil || w.index.Size() == 0 { + if w.writer == nil || !w.writer.Finished() { return w.clean() } @@ -145,11 +142,13 @@ func (w *PackWriter) save() error { } func (w *PackWriter) encodeIdx(writer io.Writer) error { - idx := w.index.ToIdxFile() - idx.PackfileChecksum = w.checksum - idx.Version = idxfile.VersionSupported + idx, err := w.writer.Index() + if err != nil { + return err + } + e := idxfile.NewEncoder(writer) - _, err := e.Encode(idx) + _, err = e.Encode(idx) return err } @@ -209,7 +208,6 @@ func (s *syncedReader) isBlocked() bool { func (s *syncedReader) wake() { if s.isBlocked() { - // fmt.Println("wake") atomic.StoreUint32(&s.blocked, 0) s.news <- true } @@ -220,7 +218,6 @@ func (s *syncedReader) sleep() { written := atomic.LoadUint64(&s.written) if read >= written { atomic.StoreUint32(&s.blocked, 1) - // fmt.Println("sleep", read, written) <-s.news } diff --git a/storage/filesystem/dotgit/writers_test.go b/storage/filesystem/dotgit/writers_test.go index bf00762..5a5f7b4 100644 --- a/storage/filesystem/dotgit/writers_test.go +++ b/storage/filesystem/dotgit/writers_test.go @@ -9,6 +9,7 @@ import ( "strconv" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" . "gopkg.in/check.v1" @@ -148,7 +149,7 @@ func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) { w, err := newPackWrite(fs) c.Assert(err, IsNil) - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { + w.Notify = func(h plumbing.Hash, idx *idxfile.Writer) { c.Fatal("unexpected call to PackWriter.Notify") } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index ef67f50..b73b309 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -23,7 +23,7 @@ type ObjectStorage struct { deltaBaseCache cache.Object dir *dotgit.DotGit - index map[plumbing.Hash]*packfile.Index + index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory. @@ -41,7 +41,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index) + s.index = make(map[plumbing.Hash]idxfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -69,7 +69,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { return err } - s.index[h] = packfile.NewIndexFromIdxFile(idxf) + s.index[h] = idxf return err } @@ -87,8 +87,11 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { return nil, err } - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { - s.index[h] = idx + w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { + index, err := writer.Index() + if err == nil { + s.index[h] = index + } } return w, nil @@ -278,7 +281,7 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err @@ -299,7 +302,7 @@ func (s *ObjectStorage) decodeObjectAt( func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64, hash plumbing.Hash) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { @@ -324,12 +327,10 @@ func (s *ObjectStorage) decodeDeltaObjectAt( case plumbing.REFDeltaObject: base = header.Reference case plumbing.OFSDeltaObject: - e, ok := idx.LookupOffset(uint64(header.OffsetReference)) - if !ok { - return nil, plumbing.ErrObjectNotFound + base, err = idx.FindHash(header.OffsetReference) + if err != nil { + return nil, err } - - base = e.Hash default: return s.decodeObjectAt(f, idx, offset) } @@ -350,8 +351,9 @@ func (s *ObjectStorage) decodeDeltaObjectAt( func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { for packfile, index := range s.index { - if e, ok := index.LookupHash(h); ok { - return packfile, e.Hash, int64(e.Offset) + offset, err := index.FindOffset(h) + if err == nil { + return packfile, h, offset } } @@ -460,12 +462,22 @@ type packfileIter struct { total uint32 } -func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) { +// NewPackfileIter returns a new EncodedObjectIter for the provided packfile +// and object type. +func NewPackfileIter( + f billy.File, + t plumbing.ObjectType, +) (storer.EncodedObjectIter, error) { return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil) } -func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, - index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) { +func newPackfileIter( + f billy.File, + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, + index idxfile.Index, + cache cache.Object, +) (storer.EncodedObjectIter, error) { s := packfile.NewScanner(f) _, total, err := s.Header() if err != nil { -- cgit From ffdfb7dbabb78090b27ca29b762b803969c89fd7 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 20 Jul 2018 15:51:15 +0200 Subject: plumbing: packfile, new Packfile representation Signed-off-by: Miguel Molina --- storage/filesystem/storage.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'storage') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 622bb4a..6af906d 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,9 @@ package filesystem import ( + "fmt" + + "gopkg.in/src-d/go-git.v4/plumbing/format/index" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -51,3 +54,15 @@ func (s *Storage) Filesystem() billy.Filesystem { func (s *Storage) Init() error { return s.dir.Initialize() } + +type IndexStorage struct { + dir *dotgit.DotGit +} + +func (IndexStorage) SetIndex(*index.Index) error { + return fmt.Errorf("not implemented") +} + +func (IndexStorage) Index() (*index.Index, error) { + return nil, fmt.Errorf("not implemented") +} -- cgit From 3657a32e0ead55601a2af578abecd65dd2d8b64b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 12:24:09 +0200 Subject: storage/filesystem: add back IndexStorage Signed-off-by: Javi Fontan --- storage/filesystem/index.go | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 storage/filesystem/index.go (limited to 'storage') diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go new file mode 100644 index 0000000..2ebf57e --- /dev/null +++ b/storage/filesystem/index.go @@ -0,0 +1,47 @@ +package filesystem + +import ( + "os" + + "gopkg.in/src-d/go-git.v4/plumbing/format/index" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +type IndexStorage struct { + dir *dotgit.DotGit +} + +func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { + f, err := s.dir.IndexWriter() + if err != nil { + return err + } + + defer ioutil.CheckClose(f, &err) + + e := index.NewEncoder(f) + err = e.Encode(idx) + return err +} + +func (s *IndexStorage) Index() (i *index.Index, err error) { + idx := &index.Index{ + Version: 2, + } + + f, err := s.dir.Index() + if err != nil { + if os.IsNotExist(err) { + return idx, nil + } + + return nil, err + } + + defer ioutil.CheckClose(f, &err) + + d := index.NewDecoder(f) + err = d.Decode(idx) + return idx, err +} -- cgit From ccd0fa0bc17f0680038529b00f5c5a44f8e77b41 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 27 Jul 2018 15:07:25 +0200 Subject: plumbing: packfile, lazy object reads with DiskObjects Signed-off-by: Miguel Molina --- storage/memory/storage.go | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'storage') diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 2e32509..a950a62 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -91,6 +91,16 @@ type ObjectStorage struct { Tags map[plumbing.Hash]plumbing.EncodedObject } +func NewObjectStorage() *ObjectStorage { + return &ObjectStorage{ + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), + Commits: make(map[plumbing.Hash]plumbing.EncodedObject), + Trees: make(map[plumbing.Hash]plumbing.EncodedObject), + Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), + Tags: make(map[plumbing.Hash]plumbing.EncodedObject), + } +} + func (o *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { return &plumbing.MemoryObject{} } -- cgit From 6f8f2ed229cc88a175d6ea47a53135b6dcef6912 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 18:17:43 +0200 Subject: storage/filesystem: remove duplicated IndexStorage Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'storage') diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 6af906d..622bb4a 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,9 +2,6 @@ package filesystem import ( - "fmt" - - "gopkg.in/src-d/go-git.v4/plumbing/format/index" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -54,15 +51,3 @@ func (s *Storage) Filesystem() billy.Filesystem { func (s *Storage) Init() error { return s.dir.Initialize() } - -type IndexStorage struct { - dir *dotgit.DotGit -} - -func (IndexStorage) SetIndex(*index.Index) error { - return fmt.Errorf("not implemented") -} - -func (IndexStorage) Index() (*index.Index, error) { - return nil, fmt.Errorf("not implemented") -} -- cgit From 6f7fc05543861ee074aa17f75e1d1b5c1b948d48 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Mon, 30 Jul 2018 17:11:01 +0200 Subject: plumbing: packfile, fix package tests Signed-off-by: Miguel Molina --- storage/memory/storage.go | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'storage') diff --git a/storage/memory/storage.go b/storage/memory/storage.go index a950a62..2e32509 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -91,16 +91,6 @@ type ObjectStorage struct { Tags map[plumbing.Hash]plumbing.EncodedObject } -func NewObjectStorage() *ObjectStorage { - return &ObjectStorage{ - Objects: make(map[plumbing.Hash]plumbing.EncodedObject), - Commits: make(map[plumbing.Hash]plumbing.EncodedObject), - Trees: make(map[plumbing.Hash]plumbing.EncodedObject), - Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), - Tags: make(map[plumbing.Hash]plumbing.EncodedObject), - } -} - func (o *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { return &plumbing.MemoryObject{} } -- cgit From 6a24b4c1f0cb9e5daf30fa7979f2643a967af1ad Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Tue, 7 Aug 2018 18:41:19 +0200 Subject: *: use parser to populate non writable storages and bug fixes Signed-off-by: Miguel Molina --- storage/filesystem/object.go | 77 +++++++++++++++------------------------ storage/filesystem/object_test.go | 6 ++- 2 files changed, 35 insertions(+), 48 deletions(-) (limited to 'storage') diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index b73b309..2032eac 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -12,7 +12,6 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" - "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -282,29 +281,34 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, idx idxfile.Index, - offset int64) (plumbing.EncodedObject, error) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, err + offset int64, +) (plumbing.EncodedObject, error) { + hash, err := idx.FindHash(offset) + if err == nil { + obj, ok := s.deltaBaseCache.Get(hash) + if ok { + return obj, nil + } } - p := packfile.NewScanner(f) + if err != nil && err != plumbing.ErrObjectNotFound { + return nil, err + } - d, err := packfile.NewDecoderWithCache(p, memory.NewStorage(), - s.deltaBaseCache) + obj, err := packfile.NewPackfile(idx, f).GetByOffset(offset) if err != nil { return nil, err } - d.SetIndex(idx) - obj, err := d.DecodeObjectAt(offset) - return obj, err + return packfile.MemoryObjectFromDisk(obj) } func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, idx idxfile.Index, offset int64, - hash plumbing.Hash) (plumbing.EncodedObject, error) { + hash plumbing.Hash, +) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err } @@ -453,22 +457,23 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { - f billy.File - d *packfile.Decoder - t plumbing.ObjectType - - seen map[plumbing.Hash]struct{} - position uint32 - total uint32 + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile // and object type. func NewPackfileIter( f billy.File, + idxFile billy.File, t plumbing.ObjectType, ) (storer.EncodedObjectIter, error) { - return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil) + idx := idxfile.NewMemoryIndex() + if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { + return nil, err + } + + return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) } func newPackfileIter( @@ -478,47 +483,26 @@ func newPackfileIter( index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - s := packfile.NewScanner(f) - _, total, err := s.Header() + iter, err := packfile.NewPackfile(index, f).GetByType(t) if err != nil { return nil, err } - d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t, cache) - if err != nil { - return nil, err - } - - d.SetIndex(index) - return &packfileIter{ - f: f, - d: d, - t: t, - - total: total, - seen: seen, + iter: iter, + seen: seen, }, nil } func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { for { - if iter.position >= iter.total { - return nil, io.EOF - } - - obj, err := iter.d.DecodeObject() + obj, err := iter.iter.Next() if err != nil { return nil, err } - iter.position++ - if obj == nil { - continue - } - if _, ok := iter.seen[obj.Hash()]; ok { - return iter.Next() + continue } return obj, nil @@ -531,8 +515,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { } func (iter *packfileIter) Close() { - iter.f.Close() - iter.d.Close() + iter.iter.Close() } type objectsIter struct { diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index ecd6beb..ae11c3b 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -115,7 +115,11 @@ func (s *FsSuite) TestPackfileIter(c *C) { for _, h := range ph { f, err := dg.ObjectPack(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(f, t) + + idxf, err := dg.ObjectPackIdx(h) + c.Assert(err, IsNil) + + iter, err := NewPackfileIter(f, idxf, t) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) -- cgit From 5889a3b669f0f515ff445aa040afc1e7eeb2bbd1 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Wed, 8 Aug 2018 16:56:20 +0200 Subject: plumbing: packfile, allow non-seekable sources on Parser Signed-off-by: Miguel Molina --- storage/filesystem/dotgit/writers.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index e1ede3c..93d2d8c 100644 --- a/storage/filesystem/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -57,7 +57,12 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) w.writer = new(idxfile.Writer) - w.parser = packfile.NewParser(s, w.writer) + var err error + w.parser, err = packfile.NewParser(s, w.writer) + if err != nil { + w.result <- err + return + } checksum, err := w.parser.Parse() if err != nil { -- cgit From 34cc506735ee0cd29362da51592b49a217df8159 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 12:16:57 +0200 Subject: storage: filesystem, benchmark PackfileIter Signed-off-by: Miguel Molina --- storage/filesystem/object.go | 30 +++++++++++++-- storage/filesystem/object_test.go | 79 +++++++++++++++++++++++++++++++++------ 2 files changed, 94 insertions(+), 15 deletions(-) (limited to 'storage') diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 2032eac..4757938 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -509,9 +509,20 @@ func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { } } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *packfileIter) Close() { @@ -543,9 +554,20 @@ func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { return obj, err } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *objectsIter) Close() { diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index ae11c3b..0dc19fe 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,6 +1,8 @@ package filesystem import ( + "testing" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" @@ -10,17 +12,16 @@ import ( type FsSuite struct { fixtures.Suite - Types []plumbing.ObjectType } -var _ = Suite(&FsSuite{ - Types: []plumbing.ObjectType{ - plumbing.CommitObject, - plumbing.TagObject, - plumbing.TreeObject, - plumbing.BlobObject, - }, -}) +var objectTypes = []plumbing.ObjectType{ + plumbing.CommitObject, + plumbing.TagObject, + plumbing.TreeObject, + plumbing.BlobObject, +} + +var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() @@ -84,7 +85,7 @@ func (s *FsSuite) TestIter(c *C) { func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { - for _, t := range s.Types { + for _, t := range objectTypes { fs := f.DotGit() o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) @@ -108,7 +109,7 @@ func (s *FsSuite) TestPackfileIter(c *C) { fs := f.DotGit() dg := dotgit.New(fs) - for _, t := range s.Types { + for _, t := range objectTypes { ph, err := dg.ObjectPacks() c.Assert(err, IsNil) @@ -132,3 +133,59 @@ func (s *FsSuite) TestPackfileIter(c *C) { }) } + +func BenchmarkPackfileIter(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(f, idxf, t) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + return nil + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} -- cgit From 038cf238e6250094c7aeb387fd7ea92438719699 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 12:36:37 +0200 Subject: storage: filesystem, close Packfile after iterating objects Signed-off-by: Miguel Molina --- storage/filesystem/object.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 4757938..86d0da9 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -457,12 +457,14 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { + pack billy.File iter storer.EncodedObjectIter seen map[plumbing.Hash]struct{} } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile -// and object type. +// and object type. Packfile and index file will be closed after they're +// used. func NewPackfileIter( f billy.File, idxFile billy.File, @@ -473,6 +475,10 @@ func NewPackfileIter( return nil, err } + if err := idxFile.Close(); err != nil { + return nil, err + } + return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) } @@ -489,6 +495,7 @@ func newPackfileIter( } return &packfileIter{ + pack: f, iter: iter, seen: seen, }, nil @@ -514,6 +521,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { o, err := iter.Next() if err != nil { if err == io.EOF { + iter.Close() return nil } return err @@ -527,6 +535,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { func (iter *packfileIter) Close() { iter.iter.Close() + _ = iter.pack.Close() } type objectsIter struct { -- cgit From d93b3869f366df7488286614b0205968bc6263df Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 13:11:37 +0200 Subject: storage: filesystem, add PackfileIter benchmark reading object content Signed-off-by: Miguel Molina --- storage/filesystem/object_test.go | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) (limited to 'storage') diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 0dc19fe..88f22bf 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,6 +1,7 @@ package filesystem import ( + "io/ioutil" "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -189,3 +190,69 @@ func BenchmarkPackfileIter(b *testing.B) { }) } } + +func BenchmarkPackfileIterReadContent(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(f, idxf, t) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + + r, err := o.Reader() + if err != nil { + b.Fatal(err) + } + + if _, err := ioutil.ReadAll(r); err != nil { + b.Fatal(err) + } + + return r.Close() + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} -- cgit From 56c5e91b158bc4569b38bfd5d27d4b4be5e06a27 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 16:53:00 +0200 Subject: plumbing: packfile, open and close packfile on FSObject reads Signed-off-by: Miguel Molina --- storage/filesystem/dotgit/dotgit.go | 5 +++++ storage/filesystem/object.go | 15 ++++++-------- storage/filesystem/object_test.go | 40 ++++++++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 12 deletions(-) (limited to 'storage') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index dc12f23..af07eb5 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -784,6 +784,11 @@ func (d *DotGit) Alternates() ([]*DotGit, error) { return alternates, nil } +// Fs returns the underlying filesystem of the DotGit folder. +func (d *DotGit) Fs() billy.Filesystem { + return d.fs +} + func isHex(s string) bool { for _, b := range []byte(s) { if isNum(b) { diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 86d0da9..6958e32 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -295,12 +295,7 @@ func (s *ObjectStorage) decodeObjectAt( return nil, err } - obj, err := packfile.NewPackfile(idx, f).GetByOffset(offset) - if err != nil { - return nil, err - } - - return packfile.MemoryObjectFromDisk(obj) + return packfile.NewPackfile(idx, s.dir.Fs(), f).GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( @@ -404,7 +399,7 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb if err != nil { return nil, err } - return newPackfileIter(pack, t, seen, s.index[h], s.deltaBaseCache) + return newPackfileIter(s.dir.Fs(), pack, t, seen, s.index[h], s.deltaBaseCache) }, }, nil } @@ -466,6 +461,7 @@ type packfileIter struct { // and object type. Packfile and index file will be closed after they're // used. func NewPackfileIter( + fs billy.Filesystem, f billy.File, idxFile billy.File, t plumbing.ObjectType, @@ -479,17 +475,18 @@ func NewPackfileIter( return nil, err } - return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) + return newPackfileIter(fs, f, t, make(map[plumbing.Hash]struct{}), idx, nil) } func newPackfileIter( + fs billy.Filesystem, f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - iter, err := packfile.NewPackfile(index, f).GetByType(t) + iter, err := packfile.NewPackfile(index, fs, f).GetByType(t) if err != nil { return nil, err } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 88f22bf..b1408b7 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -121,7 +121,7 @@ func (s *FsSuite) TestPackfileIter(c *C) { idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) @@ -169,7 +169,7 @@ func BenchmarkPackfileIter(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) if err != nil { b.Fatal(err) } @@ -225,7 +225,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) if err != nil { b.Fatal(err) } @@ -256,3 +256,37 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { }) } } + +func BenchmarkGetObjectFromPackfile(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.Basic() { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + o, err := NewObjectStorage(dotgit.New(fs)) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) + if err != nil { + b.Fatal(err) + } + + if obj.Hash() != expected { + b.Errorf("expecting %s, got %s", expected, obj.Hash()) + } + } + }) + } +} -- cgit