Merge pull request #904 from jfontan/feature/new-packfile-parser

Tests and indexes in packfile decoder

4 files changed, 132 insertions, 32 deletions

diff --git a/plumbing/format/idxfile/idxfile_test.go b/plumbing/format/idxfile/idxfile_test.go
index f42a419..d15accf 100644
--- a/plumbing/format/idxfile/idxfile_test.go
+++ b/plumbing/format/idxfile/idxfile_test.go
@@ -3,15 +3,22 @@ package idxfile_test
 import (
 	"bytes"
 	"encoding/base64"
+	"fmt"
 	"io"
 	"testing"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
+
+	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git-fixtures.v3"
 )
 
 func BenchmarkFindOffset(b *testing.B) {
-	idx := fixtureIndex(b)
+	idx, err := fixtureIndex()
+	if err != nil {
+		b.Fatalf(err.Error())
+	}
 
 	for i := 0; i < b.N; i++ {
 		for _, h := range fixtureHashes {
@@ -24,7 +31,10 @@ func BenchmarkFindOffset(b *testing.B) {
 }
 
 func BenchmarkFindCRC32(b *testing.B) {
-	idx := fixtureIndex(b)
+	idx, err := fixtureIndex()
+	if err != nil {
+		b.Fatalf(err.Error())
+	}
 
 	for i := 0; i < b.N; i++ {
 		for _, h := range fixtureHashes {
@@ -37,7 +47,10 @@ func BenchmarkFindCRC32(b *testing.B) {
 }
 
 func BenchmarkContains(b *testing.B) {
-	idx := fixtureIndex(b)
+	idx, err := fixtureIndex()
+	if err != nil {
+		b.Fatalf(err.Error())
+	}
 
 	for i := 0; i < b.N; i++ {
 		for _, h := range fixtureHashes {
@@ -54,7 +67,10 @@ func BenchmarkContains(b *testing.B) {
 }
 
 func BenchmarkEntries(b *testing.B) {
-	idx := fixtureIndex(b)
+	idx, err := fixtureIndex()
+	if err != nil {
+		b.Fatalf(err.Error())
+	}
 
 	for i := 0; i < b.N; i++ {
 		iter, err := idx.Entries()
@@ -82,6 +98,23 @@ func BenchmarkEntries(b *testing.B) {
 	}
 }
 
+type IndexSuite struct {
+	fixtures.Suite
+}
+
+var _ = Suite(&IndexSuite{})
+
+func (s *IndexSuite) TestFindHash(c *C) {
+	idx, err := fixtureIndex()
+	c.Assert(err, IsNil)
+
+	for i, pos := range fixtureOffsets {
+		hash, err := idx.FindHash(pos)
+		c.Assert(err, IsNil)
+		c.Assert(hash, Equals, fixtureHashes[i])
+	}
+}
+
 var fixtureHashes = []plumbing.Hash{
 	plumbing.NewHash("303953e5aa461c203a324821bc1717f9b4fff895"),
 	plumbing.NewHash("5296768e3d9f661387ccbff18c4dea6c997fd78c"),
@@ -94,7 +127,19 @@ var fixtureHashes = []plumbing.Hash{
 	plumbing.NewHash("35858be9c6f5914cbe6768489c41eb6809a2bceb"),
 }
 
-func fixtureIndex(t testing.TB) *idxfile.MemoryIndex {
+var fixtureOffsets = []int64{
+	12,
+	142,
+	1601322837,
+	2646996529,
+	3452385606,
+	3707047470,
+	5323223332,
+	5894072943,
+	5924278919,
+}
+
+func fixtureIndex() (*idxfile.MemoryIndex, error) {
 	f := bytes.NewBufferString(fixtureLarge4GB)
 
 	idx := new(idxfile.MemoryIndex)
@@ -102,8 +147,8 @@ func fixtureIndex(t testing.TB) *idxfile.MemoryIndex {
 	d := idxfile.NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
 	err := d.Decode(idx)
 	if err != nil {
-		t.Fatalf("unexpected error decoding index: %s", err)
+		return nil, fmt.Errorf("unexpected error decoding index: %s", err)
 	}
 
-	return idx
+	return idx, nil
 }
diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go
index 51273a3..780acd9 100644
--- a/plumbing/format/idxfile/writer_test.go
+++ b/plumbing/format/idxfile/writer_test.go
@@ -2,8 +2,10 @@ package idxfile_test
 
 import (
 	"bytes"
+	"encoding/base64"
 	"io/ioutil"
 
+	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
 
@@ -11,13 +13,13 @@ import (
 	"gopkg.in/src-d/go-git-fixtures.v3"
 )
 
-type IndexSuite struct {
+type WriterSuite struct {
 	fixtures.Suite
 }
 
-var _ = Suite(&IndexSuite{})
+var _ = Suite(&WriterSuite{})
 
-func (s *IndexSuite) TestIndexWriter(c *C) {
+func (s *WriterSuite) TestWriter(c *C) {
 	f := fixtures.Basic().One()
 	scanner := packfile.NewScanner(f.Packfile())
 
@@ -43,3 +45,53 @@ func (s *IndexSuite) TestIndexWriter(c *C) {
 
 	c.Assert(buf.Bytes(), DeepEquals, expected)
 }
+
+func (s *WriterSuite) TestWriterLarge(c *C) {
+	writer := new(idxfile.Writer)
+	err := writer.OnHeader(uint32(len(fixture4GbEntries)))
+	c.Assert(err, IsNil)
+
+	for _, o := range fixture4GbEntries {
+		err = writer.OnInflatedObjectContent(plumbing.NewHash(o.hash), o.offset, o.crc)
+		c.Assert(err, IsNil)
+	}
+
+	err = writer.OnFooter(fixture4GbChecksum)
+	c.Assert(err, IsNil)
+
+	idx, err := writer.Index()
+	c.Assert(err, IsNil)
+
+	// load fixture index
+	f := bytes.NewBufferString(fixtureLarge4GB)
+	expected, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, f))
+	c.Assert(err, IsNil)
+
+	buf := new(bytes.Buffer)
+	encoder := idxfile.NewEncoder(buf)
+	n, err := encoder.Encode(idx)
+	c.Assert(err, IsNil)
+	c.Assert(n, Equals, len(expected))
+
+	c.Assert(buf.Bytes(), DeepEquals, expected)
+}
+
+var (
+	fixture4GbChecksum = plumbing.NewHash("afabc2269205cf85da1bf7e2fdff42f73810f29b")
+
+	fixture4GbEntries = []struct {
+		offset int64
+		hash   string
+		crc    uint32
+	}{
+		{12, "303953e5aa461c203a324821bc1717f9b4fff895", 0xbc347c4c},
+		{142, "5296768e3d9f661387ccbff18c4dea6c997fd78c", 0xcdc22842},
+		{1601322837, "03fc8d58d44267274edef4585eaeeb445879d33f", 0x929dfaaa},
+		{2646996529, "8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772", 0xa61def8a},
+		{3452385606, "e0d1d625010087f79c9e01ad9d8f95e1628dda02", 0x06bea180},
+		{3707047470, "90eba326cdc4d1d61c5ad25224ccbf08731dd041", 0x7193f3ba},
+		{5323223332, "bab53055add7bc35882758a922c54a874d6b1272", 0xac269b8e},
+		{5894072943, "1b8995f51987d8a449ca5ea4356595102dc2fbd4", 0x2187c056},
+		{5924278919, "35858be9c6f5914cbe6768489c41eb6809a2bceb", 0x9c89d9d2},
+	}
+)
diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go
index edf386b..d6bc0ef 100644
--- a/plumbing/format/packfile/decoder.go
+++ b/plumbing/format/packfile/decoder.go
@@ -66,6 +66,7 @@ type Decoder struct {
 	// will be built incrementally while decoding.
 	hasBuiltIndex bool
 	idx           idxfile.Index
+	writer        *idxfile.Writer
 
 	offsetToType map[int64]plumbing.ObjectType
 	decoderType  plumbing.ObjectType
@@ -144,7 +145,17 @@ func (d *Decoder) Decode() (checksum plumbing.Hash, err error) {
 		return plumbing.ZeroHash, err
 	}
 
-	return d.s.Checksum()
+	checksum, err = d.s.Checksum()
+	if err != nil {
+		return plumbing.ZeroHash, err
+	}
+
+	if !d.hasBuiltIndex {
+		d.writer.OnFooter(checksum)
+		d.idx = d.Index()
+	}
+
+	return checksum, err
 }
 
 func (d *Decoder) fillOffsetsToHashes() error {
@@ -177,6 +188,8 @@ func (d *Decoder) doDecode() error {
 	if !d.hasBuiltIndex {
 		// TODO: MemoryIndex is not writable, change to something else
 		d.idx = idxfile.NewMemoryIndex()
+		d.writer = new(idxfile.Writer)
+		d.writer.OnHeader(count)
 	}
 	defer func() { d.hasBuiltIndex = true }()
 
@@ -329,14 +342,15 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error
 	obj.SetSize(h.Length)
 	obj.SetType(h.Type)
 
+	var crc uint32
 	var err error
 	switch h.Type {
 	case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject:
-		_, err = d.fillRegularObjectContent(obj)
+		crc, err = d.fillRegularObjectContent(obj)
 	case plumbing.REFDeltaObject:
-		_, err = d.fillREFDeltaObjectContent(obj, h.Reference)
+		crc, err = d.fillREFDeltaObjectContent(obj, h.Reference)
 	case plumbing.OFSDeltaObject:
-		_, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference)
+		crc, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference)
 	default:
 		err = ErrInvalidObject.AddDetails("type %q", h.Type)
 	}
@@ -345,6 +359,10 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error
 		return obj, err
 	}
 
+	if !d.hasBuiltIndex {
+		d.writer.Add(obj.Hash(), uint64(h.Offset), crc)
+	}
+
 	d.offsetToHash[h.Offset] = obj.Hash()
 
 	return obj, nil
@@ -468,9 +486,9 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) {
 		return d.DecodeObjectAt(o)
 	}
 
-	hash, err := d.idx.FindHash(o)
-	if err != nil {
-		return nil, err
+	hash, ok := d.offsetToHash[o]
+	if !ok {
+		return nil, plumbing.ErrObjectNotFound
 	}
 
 	return d.recallByHashNonSeekable(hash)
diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go
index 6af906d..622bb4a 100644
--- a/storage/filesystem/storage.go
+++ b/storage/filesystem/storage.go
@@ -2,9 +2,6 @@
 package filesystem
 
 import (
-	"fmt"
-
-	"gopkg.in/src-d/go-git.v4/plumbing/format/index"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
 
 	"gopkg.in/src-d/go-billy.v4"
@@ -54,15 +51,3 @@ func (s *Storage) Filesystem() billy.Filesystem {
 func (s *Storage) Init() error {
 	return s.dir.Initialize()
 }
-
-type IndexStorage struct {
-	dir *dotgit.DotGit
-}
-
-func (IndexStorage) SetIndex(*index.Index) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (IndexStorage) Index() (*index.Index, error) {
-	return nil, fmt.Errorf("not implemented")
-}