aboutsummaryrefslogtreecommitdiffstats
path: root/plumbing/format
diff options
context:
space:
mode:
authorMáximo Cuadros <mcuadros@gmail.com>2017-07-27 09:34:58 +0200
committerMáximo Cuadros <mcuadros@gmail.com>2017-07-27 09:34:58 +0200
commit48900a1c810dd2dac9cb44511e6a2412d3064da6 (patch)
treefb0042909e7bc55b9862ef2515d84eb1758b5ec7 /plumbing/format
parente19163e22eb19b352dd022f6edc9d81e1cd7a7ed (diff)
downloadgo-git-48900a1c810dd2dac9cb44511e6a2412d3064da6.tar.gz
format: idxfile, support for >2Gb packfiles
Diffstat (limited to 'plumbing/format')
-rw-r--r--plumbing/format/idxfile/decoder.go14
-rw-r--r--plumbing/format/idxfile/decoder_test.go74
-rw-r--r--plumbing/format/idxfile/encoder.go17
-rw-r--r--plumbing/format/idxfile/idxfile.go2
4 files changed, 106 insertions, 1 deletions
diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go
index 4243f76..f361213 100644
--- a/plumbing/format/idxfile/decoder.go
+++ b/plumbing/format/idxfile/decoder.go
@@ -123,6 +123,7 @@ func readCRC32(idx *Idxfile, r io.Reader) error {
func readOffsets(idx *Idxfile, r io.Reader) error {
c := int(idx.ObjectCount)
+
for i := 0; i < c; i++ {
o, err := binary.ReadUint32(r)
if err != nil {
@@ -132,6 +133,19 @@ func readOffsets(idx *Idxfile, r io.Reader) error {
idx.Entries[i].Offset = uint64(o)
}
+ for i := 0; i < c; i++ {
+ if idx.Entries[i].Offset <= offsetLimit {
+ continue
+ }
+
+ o, err := binary.ReadUint64(r)
+ if err != nil {
+ return err
+ }
+
+ idx.Entries[i].Offset = o
+ }
+
return nil
}
diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go
index 991232d..c7decb2 100644
--- a/plumbing/format/idxfile/decoder_test.go
+++ b/plumbing/format/idxfile/decoder_test.go
@@ -2,6 +2,7 @@ package idxfile_test
import (
"bytes"
+ "encoding/base64"
"fmt"
"testing"
@@ -65,3 +66,76 @@ func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
c.Assert(idx.Entries, DeepEquals, i.Entries)
}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) {
+ f := bytes.NewBufferString(fixtureLarge4GB)
+
+ idx := &Idxfile{}
+
+ d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+ err := d.Decode(idx)
+ c.Assert(err, IsNil)
+
+ expected := map[string]uint64{
+ "303953e5aa461c203a324821bc1717f9b4fff895": 12,
+ "5296768e3d9f661387ccbff18c4dea6c997fd78c": 142,
+ "03fc8d58d44267274edef4585eaeeb445879d33f": 1601322837,
+ "8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772": 2646996529,
+ "e0d1d625010087f79c9e01ad9d8f95e1628dda02": 3452385606,
+ "90eba326cdc4d1d61c5ad25224ccbf08731dd041": 3707047470,
+ "bab53055add7bc35882758a922c54a874d6b1272": 5323223332,
+ "1b8995f51987d8a449ca5ea4356595102dc2fbd4": 5894072943,
+ "35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919,
+ }
+
+ for _, e := range idx.Entries {
+ c.Assert(expected[e.Hash.String()], Equals, e.Offset)
+ }
+}
+
+func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) {
+ f := bytes.NewBufferString(fixtureLarge4GB)
+
+ expected := &Idxfile{}
+
+ d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f))
+ err := d.Decode(expected)
+ c.Assert(err, IsNil)
+
+ buf := bytes.NewBuffer(nil)
+ _, err = NewEncoder(buf).Encode(expected)
+ c.Assert(err, IsNil)
+
+ idx := &Idxfile{}
+ err = NewDecoder(buf).Decode(idx)
+ c.Assert(err, IsNil)
+
+ c.Assert(idx.Entries, DeepEquals, expected.Entries)
+}
+
+const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA
+AAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA
+AAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAA
+AgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAQAAAAE
+AAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQA
+AAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABQAA
+AAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAA
+BQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAF
+AAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUA
+AAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAA
+AAUAAAAFAAAABQAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAA
+BwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAH
+AAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcA
+AAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAA
+AAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAA
+CAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAkAAAAJ
+AAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkA
+AAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAA
+AAkAAAAJA/yNWNRCZydO3vRYXq7rRFh50z8biZX1GYfYpEnKXqQ1ZZUQLcL71DA5U+WqRhwgOjJI
+IbwXF/m0//iVNYWL6cb1kUy+Z2hInEHraAmivOtSlnaOPZ9mE4fMv/GMTepsmX/XjI88606ky55K
+D3UXletByaTwe+dykOujJs3E0dYcWtJSJMy/CHMd0EG6tTBVrde8NYgnWKkixUqHTWsScuDR1iUB
+AIf3nJ4BrZ2PleFijdoCkp36qiGHwFa8NHxMnInZ0s3CKEKmHe+KcZPzuqwmm44GvqGAX3I/VYAA
+AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA
+ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp
+ch2xUA==
+`
diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go
index d8f4d94..40abfb8 100644
--- a/plumbing/format/idxfile/encoder.go
+++ b/plumbing/format/idxfile/encoder.go
@@ -98,13 +98,28 @@ func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) {
sz := 0
+
+ var o64bits []uint64
for _, ent := range idx.Entries {
- if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil {
+ o := ent.Offset
+ if o > offsetLimit {
+ o64bits = append(o64bits, o)
+ o = offsetLimit + uint64(len(o64bits))
+ }
+
+ if err := binary.WriteUint32(e, uint32(o)); err != nil {
return sz, err
}
sz += 4
+ }
+
+ for _, o := range o64bits {
+ if err := binary.WriteUint64(e, o); err != nil {
+ return sz, err
+ }
+ sz += 8
}
return sz, nil
diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go
index b9bb1c2..6b05eaa 100644
--- a/plumbing/format/idxfile/idxfile.go
+++ b/plumbing/format/idxfile/idxfile.go
@@ -5,6 +5,8 @@ import "gopkg.in/src-d/go-git.v4/plumbing"
const (
// VersionSupported is the only idx version supported.
VersionSupported = 2
+
+ offsetLimit = 0x7fffffff
)
var (