format: packfile based on ObjectStorage and CRC32 calculation

5 files changed, 86 insertions, 65 deletions

diff --git a/formats/idxfile/decoder.go b/formats/idxfile/decoder.go
index 9cd249d..617c0f4 100644
--- a/formats/idxfile/decoder.go
+++ b/formats/idxfile/decoder.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 
 	"gopkg.in/src-d/go-git.v4/core"
@@ -95,7 +96,7 @@ func readFanout(idx *Idxfile, r io.Reader) error {
 	}
 
 	idx.ObjectCount, err = readInt32(r)
-
+	fmt.Println(idx.ObjectCount)
 	return err
 }
 
@@ -116,7 +117,7 @@ func readObjectNames(idx *Idxfile, r io.Reader) error {
 func readCRC32(idx *Idxfile, r io.Reader) error {
 	c := int(idx.ObjectCount)
 	for i := 0; i < c; i++ {
-		if _, err := r.Read(idx.Entries[i].CRC32[:]); err != nil {
+		if err := binary.Read(r, binary.BigEndian, &idx.Entries[i].CRC32); err != nil {
 			return err
 		}
 	}
diff --git a/formats/idxfile/decoder_test.go b/formats/idxfile/decoder_test.go
index 597a002..5a844a3 100644
--- a/formats/idxfile/decoder_test.go
+++ b/formats/idxfile/decoder_test.go
@@ -1,11 +1,14 @@
 package idxfile
 
 import (
+	"bytes"
 	"fmt"
-	"os"
 	"testing"
 
 	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/fixtures"
+	"gopkg.in/src-d/go-git.v4/formats/packfile"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -15,26 +18,50 @@ type IdxfileSuite struct{}
 var _ = Suite(&IdxfileSuite{})
 
 func (s *IdxfileSuite) TestDecode(c *C) {
-	f, err := os.Open("fixtures/git-fixture.idx")
-	c.Assert(err, IsNil)
+	f := fixtures.Basic().One()
 
-	d := NewDecoder(f)
+	d := NewDecoder(f.Idx())
 	idx := &Idxfile{}
-	err = d.Decode(idx)
+	err := d.Decode(idx)
 	c.Assert(err, IsNil)
 
-	err = f.Close()
-	c.Assert(err, IsNil)
-
-	c.Assert(int(idx.ObjectCount), Equals, 31)
 	c.Assert(idx.Entries, HasLen, 31)
-	c.Assert(idx.Entries[0].Hash.String(), Equals,
-		"1669dce138d9b841a518c64b10914d88f5e488ea")
+	c.Assert(idx.Entries[0].Hash.String(), Equals, "1669dce138d9b841a518c64b10914d88f5e488ea")
 	c.Assert(idx.Entries[0].Offset, Equals, uint64(615))
+	c.Assert(idx.Entries[0].CRC32, Equals, uint32(3645019190))
+
+	c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9")
+	c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String())
+}
+
+func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
+	f := fixtures.Basic().ByTag("ofs-delta")
+
+	scanner := packfile.NewScanner(f.Packfile())
+	storage := memory.NewStorage()
+
+	pd := packfile.NewDecoder(scanner, storage.ObjectStorage())
+	checksum, err := pd.Decode()
+	c.Assert(err, IsNil)
+
+	i := &Idxfile{Version: VersionSupported}
+	i.PackfileChecksum = checksum
+
+	offsets := pd.Offsets()
+	for h, crc := range pd.CRCs() {
+		i.Add(h, uint64(offsets[h]), crc)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	e := NewEncoder(buf)
+	_, err = e.Encode(i)
+	c.Assert(err, IsNil)
 
-	c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals,
-		"bba9b7a9895724819225a044c857d391bb9d61d9")
-	c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals,
-		"54bb61360ab2dad1a3e344a8cd3f82b848518cba")
+	idx := &Idxfile{}
+
+	d := NewDecoder(buf)
+	err = d.Decode(idx)
+	c.Assert(err, IsNil)
 
+	c.Assert(idx, DeepEquals, i)
 }
diff --git a/formats/idxfile/encoder.go b/formats/idxfile/encoder.go
index 6a61137..e9b0338 100644
--- a/formats/idxfile/encoder.go
+++ b/formats/idxfile/encoder.go
@@ -65,25 +65,24 @@ func (e *Encoder) encodeFanout(idx *Idxfile) (int, error) {
 }
 
 func (e *Encoder) encodeHashes(idx *Idxfile) (int, error) {
-	return e.encodeEntryField(idx, true)
-}
+	sz := 0
+	for _, ent := range idx.Entries {
+		i, err := e.Write(ent.Hash[:])
+		sz += i
 
-func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
-	return e.encodeEntryField(idx, false)
+		if err != nil {
+			return sz, err
+		}
+	}
+
+	return sz, nil
 }
 
-func (e *Encoder) encodeEntryField(idx *Idxfile, isHash bool) (int, error) {
+func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
 	sz := 0
 	for _, ent := range idx.Entries {
-		var data []byte
-		if isHash {
-			data = ent.Hash[:]
-		} else {
-			data = ent.CRC32[:]
-		}
-
-		i, err := e.Write(data)
-		sz += i
+		err := binary.Write(e, binary.BigEndian, ent.CRC32)
+		sz += 4
 
 		if err != nil {
 			return sz, err
diff --git a/formats/idxfile/encoder_test.go b/formats/idxfile/encoder_test.go
index bfb9f91..11ecaee 100644
--- a/formats/idxfile/encoder_test.go
+++ b/formats/idxfile/encoder_test.go
@@ -2,46 +2,32 @@ package idxfile
 
 import (
 	"bytes"
-	"io"
-	"os"
+	"io/ioutil"
 
 	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/fixtures"
 )
 
-func (s *IdxfileSuite) TestEncode(c *C) {
-	for i, path := range [...]string{
-		"fixtures/git-fixture.idx",
-		"../packfile/fixtures/spinnaker-spinnaker.idx",
-	} {
-		com := Commentf("subtest %d: path = %s", i, path)
-
-		exp, idx, err := decode(path)
-		c.Assert(err, IsNil, com)
-
-		obt := new(bytes.Buffer)
-		e := NewEncoder(obt)
-		size, err := e.Encode(idx)
-		c.Assert(err, IsNil, com)
-
-		c.Assert(size, Equals, exp.Len(), com)
-		c.Assert(obt, DeepEquals, exp, com)
-	}
+func (s *IdxfileSuite) SetUpSuite(c *C) {
+	fixtures.RootFolder = "../../fixtures"
 }
 
-func decode(path string) (*bytes.Buffer, *Idxfile, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, nil, err
-	}
+func (s *IdxfileSuite) TestEncode(c *C) {
+	fixtures.All().Test(c, func(f *fixtures.Fixture) {
+		expected, err := ioutil.ReadAll(f.Idx())
+		c.Assert(err, IsNil)
 
-	cont := new(bytes.Buffer)
-	tee := io.TeeReader(f, cont)
+		idx := &Idxfile{}
+		d := NewDecoder(bytes.NewBuffer(expected))
+		err = d.Decode(idx)
+		c.Assert(err, IsNil)
 
-	d := NewDecoder(tee)
-	idx := &Idxfile{}
-	if err = d.Decode(idx); err != nil {
-		return nil, nil, err
-	}
+		result := bytes.NewBuffer(nil)
+		e := NewEncoder(result)
+		size, err := e.Encode(idx)
+		c.Assert(err, IsNil)
 
-	return cont, idx, f.Close()
+		c.Assert(size, Equals, len(expected))
+		c.Assert(result.Bytes(), DeepEquals, expected)
+	})
 }
diff --git a/formats/idxfile/idxfile.go b/formats/idxfile/idxfile.go
index 217027d..ce4c976 100644
--- a/formats/idxfile/idxfile.go
+++ b/formats/idxfile/idxfile.go
@@ -25,10 +25,18 @@ type Idxfile struct {
 // offset and CRC32 checksum.
 type Entry struct {
 	Hash   core.Hash
-	CRC32  [4]byte
+	CRC32  uint32
 	Offset uint64
 }
 
+func (idx *Idxfile) Add(h core.Hash, offset uint64, crc32 uint32) {
+	idx.Entries = append(idx.Entries, Entry{
+		Hash:   h,
+		Offset: offset,
+		CRC32:  crc32,
+	})
+}
+
 func (idx *Idxfile) isValid() bool {
 	fanout := idx.calculateFanout()
 	for k, c := range idx.Fanout {