storage: filessytem read multiple packfiles support and index decoding

1 files changed, 93 insertions, 109 deletions

diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 65fc6e0..1da24e5 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -3,13 +3,14 @@ package filesystem
 import (
 	"fmt"
 	"io"
+	"os"
 
 	"gopkg.in/src-d/go-git.v4/core"
+	"gopkg.in/src-d/go-git.v4/formats/idxfile"
 	"gopkg.in/src-d/go-git.v4/formats/objfile"
 	"gopkg.in/src-d/go-git.v4/formats/packfile"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
-	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index"
-	"gopkg.in/src-d/go-git.v4/utils/fs"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
 // ObjectStorage is an implementation of core.ObjectStorage that stores
@@ -23,7 +24,40 @@ import (
 // Gitdir values will get outdated as soon as repositories change on disk.
 type ObjectStorage struct {
 	dir   *dotgit.DotGit
-	index index.Index
+	index map[core.Hash]index
+}
+
+func newObjectStorage(dir *dotgit.DotGit) (*ObjectStorage, error) {
+	s := &ObjectStorage{
+		dir:   dir,
+		index: make(map[core.Hash]index, 0),
+	}
+	return s, s.loadIdxFiles()
+}
+
+func (s *ObjectStorage) loadIdxFiles() error {
+	packs, err := s.dir.ObjectPacks()
+	if err != nil {
+		return err
+	}
+
+	for _, h := range packs {
+		if err := s.loadIdxFile(h); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *ObjectStorage) loadIdxFile(h core.Hash) error {
+	idx, err := s.dir.ObjectPackIdx(h)
+	if err != nil {
+		return err
+	}
+
+	s.index[h] = make(index)
+	return s.index[h].Decode(idx)
 }
 
 func (s *ObjectStorage) NewObject() core.Object {
@@ -31,8 +65,8 @@ func (s *ObjectStorage) NewObject() core.Object {
 }
 
 // Writer method not supported on Memory storage
-func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
-	return o.dir.NewObjectPack()
+func (s *ObjectStorage) Writer() (io.WriteCloser, error) {
+	return s.dir.NewObjectPack()
 }
 
 // Set adds a new object to the storage. As this functionality is not
@@ -45,20 +79,29 @@ func (s *ObjectStorage) Set(core.Object) (core.Hash, error) {
 // Get returns the object with the given hash, by searching for it in
 // the packfile and the git object directories.
 func (s *ObjectStorage) Get(t core.ObjectType, h core.Hash) (core.Object, error) {
-	obj, err := s.getFromUnpacked(t, h)
-	if err == dotgit.ErrObjfileNotFound {
-		if s.index == nil {
-			return nil, core.ErrObjectNotFound
-		}
-		return s.getFromPackfile(t, h)
+	obj, err := s.getFromUnpacked(h)
+	if err == core.ErrObjectNotFound {
+		obj, err = s.getFromPackfile(h)
+	}
+
+	if err != nil {
+		return nil, err
 	}
 
-	return obj, err
+	if core.AnyObject != t && obj.Type() != t {
+		return nil, core.ErrObjectNotFound
+	}
+
+	return obj, nil
 }
 
-func (s *ObjectStorage) getFromUnpacked(t core.ObjectType, h core.Hash) (obj core.Object, err error) {
+func (s *ObjectStorage) getFromUnpacked(h core.Hash) (obj core.Object, err error) {
 	f, err := s.dir.Object(h)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, core.ErrObjectNotFound
+		}
+
 		return nil, err
 	}
 
@@ -74,6 +117,7 @@ func (s *ObjectStorage) getFromUnpacked(t core.ObjectType, h core.Hash) (obj cor
 	if err != nil {
 		return nil, err
 	}
+
 	defer func() {
 		errClose := objReader.Close()
 		if err == nil {
@@ -81,54 +125,41 @@ func (s *ObjectStorage) getFromUnpacked(t core.ObjectType, h core.Hash) (obj cor
 		}
 	}()
 
-	err = objReader.FillObject(obj)
-	if err != nil {
+	if err := objReader.FillObject(obj); err != nil {
 		return nil, err
 	}
-	if core.AnyObject != t && obj.Type() != t {
-		return nil, core.ErrObjectNotFound
-	}
+
 	return obj, nil
 }
 
 // Get returns the object with the given hash, by searching for it in
 // the packfile.
-func (s *ObjectStorage) getFromPackfile(t core.ObjectType, h core.Hash) (core.Object, error) {
-	offset, err := s.index.Get(h)
-	if err != nil {
-		return nil, err
+func (s *ObjectStorage) getFromPackfile(h core.Hash) (core.Object, error) {
+	pack, offset := s.findObjectInPackfile(h)
+	if offset == -1 {
+		return nil, core.ErrObjectNotFound
 	}
 
-	packs, err := s.dir.ObjectsPacks()
+	f, err := s.dir.ObjectPack(pack)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(packs) == 0 {
-		return nil, nil
-	}
-
-	f, _, err := s.dir.ObjectPack(packs[0].Name())
-	defer func() {
-		errClose := f.Close()
-		if err == nil {
-			err = errClose
-		}
-	}()
+	defer f.Close()
 
 	p := packfile.NewScanner(f)
-	d := packfile.NewDecoder(p, nil)
-
-	obj, err := d.ReadObjectAt(offset)
-	if err != nil {
-		return nil, err
-	}
+	d := packfile.NewDecoder(p, memory.NewStorage().ObjectStorage())
+	return d.ReadObjectAt(offset)
+}
 
-	if core.AnyObject != t && obj.Type() != t {
-		return nil, core.ErrObjectNotFound
+func (s *ObjectStorage) findObjectInPackfile(h core.Hash) (core.Hash, int64) {
+	for packfile, index := range s.index {
+		if offset, ok := index[h]; ok {
+			return packfile, offset
+		}
 	}
 
-	return obj, nil
+	return core.ZeroHash, -1
 }
 
 // Iter returns an iterator for all the objects in the packfile with the
@@ -142,7 +173,7 @@ func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	}
 
 	for _, hash := range hashes {
-		object, err := s.getFromUnpacked(core.AnyObject, hash)
+		object, err := s.getFromUnpacked(hash)
 		if err != nil {
 			return nil, err
 		}
@@ -152,7 +183,7 @@ func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	}
 
 	for hash := range s.index {
-		object, err := s.getFromPackfile(core.AnyObject, hash)
+		object, err := s.getFromPackfile(hash)
 		if err != nil {
 			return nil, err
 		}
@@ -164,70 +195,6 @@ func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	return core.NewObjectSliceIter(objects), nil
 }
 
-func buildIndex(fs fs.Filesystem, dir *dotgit.DotGit) (index.Index, error) {
-	packs, err := dir.ObjectsPacks()
-	if err != nil {
-		return nil, err
-	}
-
-	if len(packs) == 0 {
-		return nil, nil
-	}
-
-	_, _, err = dir.ObjectPack(packs[0].Name())
-	if err != nil {
-		if err == dotgit.ErrIdxNotFound {
-			return buildIndexFromPackfile(dir)
-		}
-		return nil, err
-	}
-
-	return buildIndexFromIdxfile(fs, "")
-}
-
-func buildIndexFromPackfile(dir *dotgit.DotGit) (index.Index, error) {
-	packs, err := dir.ObjectsPacks()
-	if err != nil {
-		return nil, err
-	}
-
-	if len(packs) == 0 {
-		return nil, nil
-	}
-
-	f, _, err := dir.ObjectPack(packs[0].Name())
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		errClose := f.Close()
-		if err == nil {
-			err = errClose
-		}
-	}()
-
-	index, _, err := index.NewFromPackfile(f)
-	return index, err
-}
-
-func buildIndexFromIdxfile(fs fs.Filesystem, path string) (index.Index, error) {
-	f, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		errClose := f.Close()
-		if err == nil {
-			err = errClose
-		}
-	}()
-
-	i := index.New()
-	return i, i.Decode(f)
-}
-
 func (o *ObjectStorage) Begin() core.TxObjectStorage {
 	return &TxObjectStorage{}
 }
@@ -249,3 +216,20 @@ func (tx *TxObjectStorage) Commit() error {
 func (tx *TxObjectStorage) Rollback() error {
 	return fmt.Errorf("not implemented yet")
 }
+
+type index map[core.Hash]int64
+
+func (i index) Decode(r io.Reader) error {
+	idx := &idxfile.Idxfile{}
+
+	d := idxfile.NewDecoder(r)
+	if err := d.Decode(idx); err != nil {
+		return err
+	}
+
+	for _, e := range idx.Entries {
+		i[e.Hash] = int64(e.Offset)
+	}
+
+	return nil
+}