format: idxfile, support for >2Gb packfiles

4 files changed, 106 insertions, 1 deletions

diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go
index 4243f76..f361213 100644
--- a/plumbing/format/idxfile/decoder.go
+++ b/plumbing/format/idxfile/decoder.go
@@ -123,6 +123,7 @@ func readCRC32(idx *Idxfile, r io.Reader) error {
 
 func readOffsets(idx *Idxfile, r io.Reader) error {
 	c := int(idx.ObjectCount)
+
 	for i := 0; i < c; i++ {
 		o, err := binary.ReadUint32(r)
 		if err != nil {
@@ -132,6 +133,19 @@ func readOffsets(idx *Idxfile, r io.Reader) error {
 		idx.Entries[i].Offset = uint64(o)
 	}
 
+	for i := 0; i < c; i++ {
+		if idx.Entries[i].Offset <= offsetLimit {
+			continue
+		}
+
+		o, err := binary.ReadUint64(r)
+		if err != nil {
+			return err
+		}
+
+		idx.Entries[i].Offset = o
+	}
+
 	return nil
 }
 
diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go
index 991232d..c7decb2 100644
--- a/plumbing/format/idxfile/decoder_test.go
+++ b/plumbing/format/idxfile/decoder_test.go
@@ -2,6 +2,7 @@ package idxfile_test
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"testing"
 
@@ -65,3 +66,76 @@ func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
 
 	c.Assert(idx.Entries, DeepEquals, i.Entries)
 }
+
+func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
+	f := bytes.NewBufferString(fixtureLarge4GB)
+
+	idx := &Idxfile{}
+
+	d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+	err := d.Decode(idx)
+	c.Assert(err, IsNil)
+
+	expected := map[string]uint64{
+		"303953e5aa461c203a324821bc1717f9b4fff895": 12,
+		"5296768e3d9f661387ccbff18c4dea6c997fd78c": 142,
+		"03fc8d58d44267274edef4585eaeeb445879d33f": 1601322837,
+		"8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772": 2646996529,
+		"e0d1d625010087f79c9e01ad9d8f95e1628dda02": 3452385606,
+		"90eba326cdc4d1d61c5ad25224ccbf08731dd041": 3707047470,
+		"bab53055add7bc35882758a922c54a874d6b1272": 5323223332,
+		"1b8995f51987d8a449ca5ea4356595102dc2fbd4": 5894072943,
+		"35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919,
+	}
+
+	for _, e := range idx.Entries {
+		c.Assert(expected[e.Hash.String()], Equals, e.Offset)
+	}
+}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) {
+	f := bytes.NewBufferString(fixtureLarge4GB)
+
+	expected := &Idxfile{}
+
+	d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+	err := d.Decode(expected)
+	c.Assert(err, IsNil)
+
+	buf := bytes.NewBuffer(nil)
+	_, err = NewEncoder(buf).Encode(expected)
+	c.Assert(err, IsNil)
+
+	idx := &Idxfile{}
+	err = NewDecoder(buf).Decode(idx)
+	c.Assert(err, IsNil)
+
+	c.Assert(idx.Entries, DeepEquals, expected.Entries)
+}
+
+const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA
+AAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA
+AAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAA
+AgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAQAAAAE
+AAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQA
+AAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABQAA
+AAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAA
+BQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAF
+AAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUA
+AAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAA
+AAUAAAAFAAAABQAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAA
+BwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAH
+AAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcA
+AAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAA
+AAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAA
+CAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAkAAAAJ
+AAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkA
+AAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAA
+AAkAAAAJA/yNWNRCZydO3vRYXq7rRFh50z8biZX1GYfYpEnKXqQ1ZZUQLcL71DA5U+WqRhwgOjJI
+IbwXF/m0//iVNYWL6cb1kUy+Z2hInEHraAmivOtSlnaOPZ9mE4fMv/GMTepsmX/XjI88606ky55K
+D3UXletByaTwe+dykOujJs3E0dYcWtJSJMy/CHMd0EG6tTBVrde8NYgnWKkixUqHTWsScuDR1iUB
+AIf3nJ4BrZ2PleFijdoCkp36qiGHwFa8NHxMnInZ0s3CKEKmHe+KcZPzuqwmm44GvqGAX3I/VYAA
+AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA
+ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp
+ch2xUA==
+`
diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go
index d8f4d94..40abfb8 100644
--- a/plumbing/format/idxfile/encoder.go
+++ b/plumbing/format/idxfile/encoder.go
@@ -98,13 +98,28 @@ func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
 
 func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) {
 	sz := 0
+
+	var o64bits []uint64
 	for _, ent := range idx.Entries {
-		if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil {
+		o := ent.Offset
+		if o > offsetLimit {
+			o64bits = append(o64bits, o)
+			o = offsetLimit + uint64(len(o64bits))
+		}
+
+		if err := binary.WriteUint32(e, uint32(o)); err != nil {
 			return sz, err
 		}
 
 		sz += 4
+	}
+
+	for _, o := range o64bits {
+		if err := binary.WriteUint64(e, o); err != nil {
+			return sz, err
+		}
 
+		sz += 8
 	}
 
 	return sz, nil
diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go
index b9bb1c2..6b05eaa 100644
--- a/plumbing/format/idxfile/idxfile.go
+++ b/plumbing/format/idxfile/idxfile.go
@@ -5,6 +5,8 @@ import "gopkg.in/src-d/go-git.v4/plumbing"
 const (
 	// VersionSupported is the only idx version supported.
 	VersionSupported = 2
+
+	offsetLimit = 0x7fffffff
 )
 
 var (