From 48900a1c810dd2dac9cb44511e6a2412d3064da6 Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Thu, 27 Jul 2017 09:34:58 +0200 Subject: format: idxfile, support for >2Gb packfiles --- plumbing/format/idxfile/decoder.go | 14 +++++++ plumbing/format/idxfile/decoder_test.go | 74 +++++++++++++++++++++++++++++++++ plumbing/format/idxfile/encoder.go | 17 +++++++- plumbing/format/idxfile/idxfile.go | 2 + 4 files changed, 106 insertions(+), 1 deletion(-) (limited to 'plumbing/format') diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 4243f76..f361213 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -123,6 +123,7 @@ func readCRC32(idx *Idxfile, r io.Reader) error { func readOffsets(idx *Idxfile, r io.Reader) error { c := int(idx.ObjectCount) + for i := 0; i < c; i++ { o, err := binary.ReadUint32(r) if err != nil { @@ -132,6 +133,19 @@ func readOffsets(idx *Idxfile, r io.Reader) error { idx.Entries[i].Offset = uint64(o) } + for i := 0; i < c; i++ { + if idx.Entries[i].Offset <= offsetLimit { + continue + } + + o, err := binary.ReadUint64(r) + if err != nil { + return err + } + + idx.Entries[i].Offset = o + } + return nil } diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go index 991232d..c7decb2 100644 --- a/plumbing/format/idxfile/decoder_test.go +++ b/plumbing/format/idxfile/decoder_test.go @@ -2,6 +2,7 @@ package idxfile_test import ( "bytes" + "encoding/base64" "fmt" "testing" @@ -65,3 +66,76 @@ func (s *IdxfileSuite) TestDecodeCRCs(c *C) { c.Assert(idx.Entries, DeepEquals, i.Entries) } + +func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) { + f := bytes.NewBufferString(fixtureLarge4GB) + + idx := &Idxfile{} + + d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) + err := d.Decode(idx) + c.Assert(err, IsNil) + + expected := map[string]uint64{ + "303953e5aa461c203a324821bc1717f9b4fff895": 12, + "5296768e3d9f661387ccbff18c4dea6c997fd78c": 142, + "03fc8d58d44267274edef4585eaeeb445879d33f": 1601322837, + "8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772": 2646996529, + "e0d1d625010087f79c9e01ad9d8f95e1628dda02": 3452385606, + "90eba326cdc4d1d61c5ad25224ccbf08731dd041": 3707047470, + "bab53055add7bc35882758a922c54a874d6b1272": 5323223332, + "1b8995f51987d8a449ca5ea4356595102dc2fbd4": 5894072943, + "35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919, + } + + for _, e := range idx.Entries { + c.Assert(expected[e.Hash.String()], Equals, e.Offset) + } +} + +func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) { + f := bytes.NewBufferString(fixtureLarge4GB) + + expected := &Idxfile{} + + d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) + err := d.Decode(expected) + c.Assert(err, IsNil) + + buf := bytes.NewBuffer(nil) + _, err = NewEncoder(buf).Encode(expected) + c.Assert(err, IsNil) + + idx := &Idxfile{} + err = NewDecoder(buf).Decode(idx) + c.Assert(err, IsNil) + + c.Assert(idx.Entries, DeepEquals, expected.Entries) +} + +const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA +AAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAA +AAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAA +AgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAADAAAAAwAAAAMAAAADAAAAAwAAAAQAAAAE +AAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQA +AAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABQAA +AAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAA +BQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAF +AAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUA +AAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAAAAUAAAAFAAAABQAA +AAUAAAAFAAAABQAAAAYAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAA +BwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAH +AAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcAAAAHAAAABwAAAAcA +AAAHAAAABwAAAAcAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAA +AAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAA +CAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAkAAAAJ +AAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkA +AAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAAAAkAAAAJAAAACQAA +AAkAAAAJA/yNWNRCZydO3vRYXq7rRFh50z8biZX1GYfYpEnKXqQ1ZZUQLcL71DA5U+WqRhwgOjJI +IbwXF/m0//iVNYWL6cb1kUy+Z2hInEHraAmivOtSlnaOPZ9mE4fMv/GMTepsmX/XjI88606ky55K +D3UXletByaTwe+dykOujJs3E0dYcWtJSJMy/CHMd0EG6tTBVrde8NYgnWKkixUqHTWsScuDR1iUB +AIf3nJ4BrZ2PleFijdoCkp36qiGHwFa8NHxMnInZ0s3CKEKmHe+KcZPzuqwmm44GvqGAX3I/VYAA +AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA +ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp +ch2xUA== +` diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index d8f4d94..40abfb8 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -98,13 +98,28 @@ func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) { func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) { sz := 0 + + var o64bits []uint64 for _, ent := range idx.Entries { - if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil { + o := ent.Offset + if o > offsetLimit { + o64bits = append(o64bits, o) + o = offsetLimit + uint64(len(o64bits)) + } + + if err := binary.WriteUint32(e, uint32(o)); err != nil { return sz, err } sz += 4 + } + + for _, o := range o64bits { + if err := binary.WriteUint64(e, o); err != nil { + return sz, err + } + sz += 8 } return sz, nil diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index b9bb1c2..6b05eaa 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -5,6 +5,8 @@ import "gopkg.in/src-d/go-git.v4/plumbing" const ( // VersionSupported is the only idx version supported. VersionSupported = 2 + + offsetLimit = 0x7fffffff ) var ( -- cgit