diff options
Diffstat (limited to 'plumbing/format')
35 files changed, 458 insertions, 191 deletions
diff --git a/plumbing/format/commitgraph/encoder.go b/plumbing/format/commitgraph/encoder.go index d34076f..f61025b 100644 --- a/plumbing/format/commitgraph/encoder.go +++ b/plumbing/format/commitgraph/encoder.go @@ -1,11 +1,10 @@ package commitgraph
import (
- "crypto/sha1"
- "hash"
"io"
"github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/hash"
"github.com/go-git/go-git/v5/utils/binary"
)
@@ -17,7 +16,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w.
func NewEncoder(w io.Writer) *Encoder {
- h := sha1.New()
+ h := hash.New(hash.CryptoType)
mw := io.MultiWriter(w, h)
return &Encoder{mw, h}
}
@@ -31,7 +30,7 @@ func (e *Encoder) Encode(idx Index) error { hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes)
chunkSignatures := [][]byte{oidFanoutSignature, oidLookupSignature, commitDataSignature}
- chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * 20, uint64(len(hashes)) * 36}
+ chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * 36}
if extraEdgesCount > 0 {
chunkSignatures = append(chunkSignatures, extraEdgeListSignature)
chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4)
@@ -183,6 +182,6 @@ func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) { }
func (e *Encoder) encodeChecksum() error {
- _, err := e.Write(e.hash.Sum(nil)[:20])
+ _, err := e.Write(e.hash.Sum(nil)[:hash.Size])
return err
}
diff --git a/plumbing/format/config/encoder.go b/plumbing/format/config/encoder.go index 4eac896..de069ae 100644 --- a/plumbing/format/config/encoder.go +++ b/plumbing/format/config/encoder.go @@ -11,6 +11,10 @@ type Encoder struct { w io.Writer } +var ( + subsectionReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) + valueReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`, "\n", `\n`, "\t", `\t`, "\b", `\b`) +) // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { return &Encoder{w} @@ -48,8 +52,7 @@ func (e *Encoder) encodeSection(s *Section) error { } func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error { - //TODO: escape - if err := e.printf("[%s \"%s\"]\n", sectionName, s.Name); err != nil { + if err := e.printf("[%s \"%s\"]\n", sectionName, subsectionReplacer.Replace(s.Name)); err != nil { return err } @@ -58,12 +61,14 @@ func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error { func (e *Encoder) encodeOptions(opts Options) error { for _, o := range opts { - pattern := "\t%s = %s\n" - if strings.Contains(o.Value, "\\") { - pattern = "\t%s = %q\n" + var value string + if strings.ContainsAny(o.Value, "#;\"\t\n\\") || strings.HasPrefix(o.Value, " ") || strings.HasSuffix(o.Value, " ") { + value = `"`+valueReplacer.Replace(o.Value)+`"` + } else { + value = o.Value } - if err := e.printf(pattern, o.Key, o.Value); err != nil { + if err := e.printf("\t%s = %s\n", o.Key, value); err != nil { return err } } diff --git a/plumbing/format/config/fixtures_test.go b/plumbing/format/config/fixtures_test.go index f3533df..2fa7840 100644 --- a/plumbing/format/config/fixtures_test.go +++ b/plumbing/format/config/fixtures_test.go @@ -43,6 +43,41 @@ var fixtures = []*Fixture{ Config: New().AddOption("core", "", "repositoryformatversion", "0"), }, { + Raw: `[section] + option1 = "has # hash" + option2 = "has \" quote" + option3 = "has \\ backslash" + option4 = "has ; semicolon" + option5 = "has \n line-feed" + option6 = "has \t tab" + option7 = " has leading spaces" + option8 = "has trailing spaces " + option9 = has no special characters + option10 = has unusual ` + "\x01\x7f\xc8\x80 characters\n", + Text: `[section] + option1 = "has # hash" + option2 = "has \" quote" + option3 = "has \\ backslash" + option4 = "has ; semicolon" + option5 = "has \n line-feed" + option6 = "has \t tab" + option7 = " has leading spaces" + option8 = "has trailing spaces " + option9 = has no special characters + option10 = has unusual ` + "\x01\x7f\xc8\x80 characters\n", + Config: New(). + AddOption("section", "", "option1", `has # hash`). + AddOption("section", "", "option2", `has " quote`). + AddOption("section", "", "option3", `has \ backslash`). + AddOption("section", "", "option4", `has ; semicolon`). + AddOption("section", "", "option5", "has \n line-feed"). + AddOption("section", "", "option6", "has \t tab"). + AddOption("section", "", "option7", ` has leading spaces`). + AddOption("section", "", "option8", `has trailing spaces `). + AddOption("section", "", "option9", `has no special characters`). + AddOption("section", "", "option10", "has unusual \x01\x7f\u0200 characters"), + }, + { Raw: ` [sect1] opt1 = value1 diff --git a/plumbing/format/config/format.go b/plumbing/format/config/format.go new file mode 100644 index 0000000..4873ea9 --- /dev/null +++ b/plumbing/format/config/format.go @@ -0,0 +1,53 @@ +package config + +// RepositoryFormatVersion represents the repository format version, +// as per defined at: +// +// https://git-scm.com/docs/repository-version +type RepositoryFormatVersion string + +const ( + // Version_0 is the format defined by the initial version of git, + // including but not limited to the format of the repository + // directory, the repository configuration file, and the object + // and ref storage. + // + // Specifying the complete behavior of git is beyond the scope + // of this document. + Version_0 = "0" + + // Version_1 is identical to version 0, with the following exceptions: + // + // 1. When reading the core.repositoryformatversion variable, a git + // implementation which supports version 1 MUST also read any + // configuration keys found in the extensions section of the + // configuration file. + // + // 2. If a version-1 repository specifies any extensions.* keys that + // the running git has not implemented, the operation MUST NOT proceed. + // Similarly, if the value of any known key is not understood by the + // implementation, the operation MUST NOT proceed. + // + // Note that if no extensions are specified in the config file, then + // core.repositoryformatversion SHOULD be set to 0 (setting it to 1 provides + // no benefit, and makes the repository incompatible with older + // implementations of git). + Version_1 = "1" + + // DefaultRepositoryFormatVersion holds the default repository format version. + DefaultRepositoryFormatVersion = Version_0 +) + +// ObjectFormat defines the object format. +type ObjectFormat string + +const ( + // SHA1 represents the object format used for SHA1. + SHA1 ObjectFormat = "sha1" + + // SHA256 represents the object format used for SHA256. + SHA256 ObjectFormat = "sha256" + + // DefaultObjectFormat holds the default object format. + DefaultObjectFormat = SHA1 +) diff --git a/plumbing/format/config/section.go b/plumbing/format/config/section.go index 07f72f3..4625ac5 100644 --- a/plumbing/format/config/section.go +++ b/plumbing/format/config/section.go @@ -103,7 +103,7 @@ func (s *Section) RemoveSubsection(name string) *Section { return s } -// Option return the value for the specified key. Empty string is returned if +// Option returns the value for the specified key. Empty string is returned if // key does not exists. func (s *Section) Option(key string) string { return s.Options.Get(key) diff --git a/plumbing/format/diff/patch.go b/plumbing/format/diff/patch.go index 39a66a1..c7678b0 100644 --- a/plumbing/format/diff/patch.go +++ b/plumbing/format/diff/patch.go @@ -9,7 +9,7 @@ import ( type Operation int const ( - // Equal item represents a equals diff. + // Equal item represents an equals diff. Equal Operation = iota // Add item represents an insert diff. Add @@ -26,15 +26,15 @@ type Patch interface { Message() string } -// FilePatch represents the necessary steps to transform one file to another. +// FilePatch represents the necessary steps to transform one file into another. type FilePatch interface { // IsBinary returns true if this patch is representing a binary file. IsBinary() bool - // Files returns the from and to Files, with all the necessary metadata to + // Files returns the from and to Files, with all the necessary metadata // about them. If the patch creates a new file, "from" will be nil. // If the patch deletes a file, "to" will be nil. Files() (from, to File) - // Chunks returns a slice of ordered changes to transform "from" File to + // Chunks returns a slice of ordered changes to transform "from" File into // "to" File. If the file is a binary one, Chunks will be empty. Chunks() []Chunk } @@ -49,7 +49,7 @@ type File interface { Path() string } -// Chunk represents a portion of a file transformation to another. +// Chunk represents a portion of a file transformation into another. type Chunk interface { // Content contains the portion of the file. Content() string diff --git a/plumbing/format/gitattributes/attributes.go b/plumbing/format/gitattributes/attributes.go index 329e667..d36ec1b 100644 --- a/plumbing/format/gitattributes/attributes.go +++ b/plumbing/format/gitattributes/attributes.go @@ -3,7 +3,6 @@ package gitattributes import ( "errors" "io" - "io/ioutil" "strings" ) @@ -89,7 +88,7 @@ func (a attribute) String() string { // ReadAttributes reads patterns and attributes from the gitattributes format. func ReadAttributes(r io.Reader, domain []string, allowMacro bool) (attributes []MatchAttribute, err error) { - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/plumbing/format/gitattributes/pattern.go b/plumbing/format/gitattributes/pattern.go index d961aba..f101f47 100644 --- a/plumbing/format/gitattributes/pattern.go +++ b/plumbing/format/gitattributes/pattern.go @@ -52,6 +52,11 @@ func (p *pattern) Match(path []string) bool { var match, doublestar bool var err error for _, part := range path { + // path is deeper than pattern + if len(pattern) == 0 { + return false + } + // skip empty if pattern[0] == "" { pattern = pattern[1:] diff --git a/plumbing/format/gitattributes/pattern_test.go b/plumbing/format/gitattributes/pattern_test.go index f95be6e..981d56f 100644 --- a/plumbing/format/gitattributes/pattern_test.go +++ b/plumbing/format/gitattributes/pattern_test.go @@ -174,6 +174,12 @@ func (s *PatternSuite) TestGlobMatch_tailingAsterisks_single(c *C) { c.Assert(r, Equals, true) } +func (s *PatternSuite) TestGlobMatch_tailingAsterisk_single(c *C) { + p := ParsePattern("/*lue/*", nil) + r := p.Match([]string{"value", "volcano", "tail"}) + c.Assert(r, Equals, false) +} + func (s *PatternSuite) TestGlobMatch_tailingAsterisks_exactMatch(c *C) { p := ParsePattern("/*lue/vol?ano/**", nil) r := p.Match([]string{"value", "volcano"}) diff --git a/plumbing/format/gitignore/dir.go b/plumbing/format/gitignore/dir.go index 15bc9c7..d8fb30c 100644 --- a/plumbing/format/gitignore/dir.go +++ b/plumbing/format/gitignore/dir.go @@ -3,11 +3,12 @@ package gitignore import ( "bufio" "bytes" - "io/ioutil" + "io" "os" "strings" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5/internal/path_util" "github.com/go-git/go-git/v5/plumbing/format/config" gioutil "github.com/go-git/go-git/v5/utils/ioutil" ) @@ -25,6 +26,9 @@ const ( // readIgnoreFile reads a specific git ignore file. func readIgnoreFile(fs billy.Filesystem, path []string, ignoreFile string) (ps []Pattern, err error) { + + ignoreFile, _ = path_util.ReplaceTildeWithHome(ignoreFile) + f, err := fs.Open(fs.Join(append(path, ignoreFile)...)) if err == nil { defer f.Close() @@ -86,7 +90,7 @@ func loadPatterns(fs billy.Filesystem, path string) (ps []Pattern, err error) { defer gioutil.CheckClose(f, &err) - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return } diff --git a/plumbing/format/gitignore/dir_test.go b/plumbing/format/gitignore/dir_test.go index facc36d..465c571 100644 --- a/plumbing/format/gitignore/dir_test.go +++ b/plumbing/format/gitignore/dir_test.go @@ -2,7 +2,9 @@ package gitignore import ( "os" + "os/user" "strconv" + "strings" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" @@ -12,6 +14,8 @@ import ( type MatcherSuite struct { GFS billy.Filesystem // git repository root RFS billy.Filesystem // root that contains user home + RFSR billy.Filesystem // root that contains user home, but with relative ~/.gitignore_global + RFSU billy.Filesystem // root that contains user home, but with relative ~user/.gitignore_global MCFS billy.Filesystem // root that contains user home, but missing ~/.gitconfig MEFS billy.Filesystem // root that contains user home, but missing excludesfile entry MIFS billy.Filesystem // root that contains user home, but missing .gitignore @@ -63,6 +67,27 @@ func (s *MatcherSuite) SetUpTest(c *C) { err = fs.MkdirAll("vendor/gopkg.in", os.ModePerm) c.Assert(err, IsNil) + err = fs.MkdirAll("multiple/sub/ignores/first", os.ModePerm) + c.Assert(err, IsNil) + err = fs.MkdirAll("multiple/sub/ignores/second", os.ModePerm) + c.Assert(err, IsNil) + f, err = fs.Create("multiple/sub/ignores/first/.gitignore") + c.Assert(err, IsNil) + _, err = f.Write([]byte("ignore_dir\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + f, err = fs.Create("multiple/sub/ignores/second/.gitignore") + c.Assert(err, IsNil) + _, err = f.Write([]byte("ignore_dir\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + err = fs.MkdirAll("multiple/sub/ignores/first/ignore_dir", os.ModePerm) + c.Assert(err, IsNil) + err = fs.MkdirAll("multiple/sub/ignores/second/ignore_dir", os.ModePerm) + c.Assert(err, IsNil) + s.GFS = fs // setup root that contains user home @@ -95,6 +120,64 @@ func (s *MatcherSuite) SetUpTest(c *C) { s.RFS = fs + // root that contains user home, but with relative ~/.gitignore_global + fs = memfs.New() + err = fs.MkdirAll(home, os.ModePerm) + c.Assert(err, IsNil) + + f, err = fs.Create(fs.Join(home, gitconfigFile)) + c.Assert(err, IsNil) + _, err = f.Write([]byte("[core]\n")) + c.Assert(err, IsNil) + _, err = f.Write([]byte(" excludesfile = ~/.gitignore_global" + "\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + + f, err = fs.Create(fs.Join(home, ".gitignore_global")) + c.Assert(err, IsNil) + _, err = f.Write([]byte("# IntelliJ\n")) + c.Assert(err, IsNil) + _, err = f.Write([]byte(".idea/\n")) + c.Assert(err, IsNil) + _, err = f.Write([]byte("*.iml\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + + s.RFSR = fs + + // root that contains user home, but with relative ~user/.gitignore_global + fs = memfs.New() + err = fs.MkdirAll(home, os.ModePerm) + c.Assert(err, IsNil) + + f, err = fs.Create(fs.Join(home, gitconfigFile)) + c.Assert(err, IsNil) + _, err = f.Write([]byte("[core]\n")) + c.Assert(err, IsNil) + currentUser, err := user.Current() + c.Assert(err, IsNil) + // remove domain for windows + username := currentUser.Username[strings.Index(currentUser.Username, "\\")+1:] + _, err = f.Write([]byte(" excludesfile = ~" + username + "/.gitignore_global" + "\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + + f, err = fs.Create(fs.Join(home, ".gitignore_global")) + c.Assert(err, IsNil) + _, err = f.Write([]byte("# IntelliJ\n")) + c.Assert(err, IsNil) + _, err = f.Write([]byte(".idea/\n")) + c.Assert(err, IsNil) + _, err = f.Write([]byte("*.iml\n")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + + s.RFSU = fs + // root that contains user home, but missing ~/.gitconfig fs = memfs.New() err = fs.MkdirAll(home, os.ModePerm) @@ -183,15 +266,39 @@ func (s *MatcherSuite) SetUpTest(c *C) { } func (s *MatcherSuite) TestDir_ReadPatterns(c *C) { + checkPatterns := func(ps []Pattern) { + c.Assert(ps, HasLen, 6) + m := NewMatcher(ps) + + c.Assert(m.Match([]string{"exclude.crlf"}, true), Equals, true) + c.Assert(m.Match([]string{"ignore.crlf"}, true), Equals, true) + c.Assert(m.Match([]string{"vendor", "gopkg.in"}, true), Equals, true) + c.Assert(m.Match([]string{"vendor", "github.com"}, true), Equals, false) + c.Assert(m.Match([]string{"multiple", "sub", "ignores", "first", "ignore_dir"}, true), Equals, true) + c.Assert(m.Match([]string{"multiple", "sub", "ignores", "second", "ignore_dir"}, true), Equals, true) + } + ps, err := ReadPatterns(s.GFS, nil) c.Assert(err, IsNil) - c.Assert(ps, HasLen, 4) + checkPatterns(ps) - m := NewMatcher(ps) - c.Assert(m.Match([]string{"exclude.crlf"}, true), Equals, true) - c.Assert(m.Match([]string{"ignore.crlf"}, true), Equals, true) - c.Assert(m.Match([]string{"vendor", "gopkg.in"}, true), Equals, true) - c.Assert(m.Match([]string{"vendor", "github.com"}, true), Equals, false) + // passing an empty slice with capacity to check we don't hit a bug where the extra capacity is reused incorrectly + ps, err = ReadPatterns(s.GFS, make([]string, 0, 6)) + c.Assert(err, IsNil) + checkPatterns(ps) +} + +func (s *MatcherSuite) TestDir_ReadRelativeGlobalGitIgnore(c *C) { + for _, fs := range []billy.Filesystem{s.RFSR, s.RFSU} { + ps, err := LoadGlobalPatterns(fs) + c.Assert(err, IsNil) + c.Assert(ps, HasLen, 2) + + m := NewMatcher(ps) + c.Assert(m.Match([]string{".idea/"}, true), Equals, false) + c.Assert(m.Match([]string{"*.iml"}, true), Equals, true) + c.Assert(m.Match([]string{"IntelliJ"}, true), Equals, false) + } } func (s *MatcherSuite) TestDir_LoadGlobalPatterns(c *C) { diff --git a/plumbing/format/gitignore/pattern.go b/plumbing/format/gitignore/pattern.go index 098cb50..450b3cd 100644 --- a/plumbing/format/gitignore/pattern.go +++ b/plumbing/format/gitignore/pattern.go @@ -39,6 +39,8 @@ type pattern struct { // ParsePattern parses a gitignore pattern string into the Pattern structure. func ParsePattern(p string, domain []string) Pattern { + // storing domain, copy it to ensure it isn't changed externally + domain = append([]string(nil), domain...) res := pattern{domain: domain} if strings.HasPrefix(p, inclusionPrefix) { diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 51a3904..9afdce3 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -6,6 +6,7 @@ import ( "errors" "io" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -19,7 +20,7 @@ var ( const ( fanout = 256 - objectIDLength = 20 + objectIDLength = hash.Size ) // Decoder reads and decodes idx files from an input stream. diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go index 94059cc..2c4a801 100644 --- a/plumbing/format/idxfile/decoder_test.go +++ b/plumbing/format/idxfile/decoder_test.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "testing" "github.com/go-git/go-git/v5/plumbing" @@ -119,7 +118,7 @@ ch2xUA== func BenchmarkDecode(b *testing.B) { f := fixtures.Basic().One() - fixture, err := ioutil.ReadAll(f.Idx()) + fixture, err := io.ReadAll(f.Idx()) if err != nil { b.Errorf("unexpected error reading idx file: %s", err) } diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index 26b2e4d..7514737 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -1,10 +1,9 @@ package idxfile import ( - "crypto/sha1" - "hash" "io" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -16,7 +15,7 @@ type Encoder struct { // NewEncoder returns a new stream encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := sha1.New() + h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } @@ -133,10 +132,10 @@ func (e *Encoder) encodeChecksums(idx *MemoryIndex) (int, error) { return 0, err } - copy(idx.IdxChecksum[:], e.hash.Sum(nil)[:20]) + copy(idx.IdxChecksum[:], e.hash.Sum(nil)[:hash.Size]) if _, err := e.Write(idx.IdxChecksum[:]); err != nil { return 0, err } - return 40, nil + return hash.HexSize, nil } diff --git a/plumbing/format/idxfile/encoder_test.go b/plumbing/format/idxfile/encoder_test.go index 32b60f9..b8ece83 100644 --- a/plumbing/format/idxfile/encoder_test.go +++ b/plumbing/format/idxfile/encoder_test.go @@ -2,7 +2,7 @@ package idxfile_test import ( "bytes" - "io/ioutil" + "io" . "github.com/go-git/go-git/v5/plumbing/format/idxfile" @@ -12,7 +12,7 @@ import ( func (s *IdxfileSuite) TestDecodeEncode(c *C) { fixtures.ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - expected, err := ioutil.ReadAll(f.Idx()) + expected, err := io.ReadAll(f.Idx()) c.Assert(err, IsNil) idx := new(MemoryIndex) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index 64dd8dc..9237a74 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -8,6 +8,7 @@ import ( encbin "encoding/binary" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" ) const ( @@ -53,8 +54,8 @@ type MemoryIndex struct { Offset32 [][]byte CRC32 [][]byte Offset64 []byte - PackfileChecksum [20]byte - IdxChecksum [20]byte + PackfileChecksum [hash.Size]byte + IdxChecksum [hash.Size]byte offsetHash map[int64]plumbing.Hash offsetHashIsFull bool diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index daa1605..c4c21e1 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -84,11 +84,8 @@ func (w *Writer) OnFooter(h plumbing.Hash) error { w.checksum = h w.finished = true _, err := w.createIndex() - if err != nil { - return err - } - return nil + return err } // creatIndex returns a filled MemoryIndex with the information filled by @@ -139,15 +136,23 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { offset := o.Offset if offset > math.MaxInt32 { - offset = w.addOffset64(offset) + var err error + offset, err = w.addOffset64(offset) + if err != nil { + return nil, err + } } buf.Truncate(0) - binary.WriteUint32(buf, uint32(offset)) + if err := binary.WriteUint32(buf, uint32(offset)); err != nil { + return nil, err + } idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) - binary.WriteUint32(buf, o.CRC32) + if err := binary.WriteUint32(buf, o.CRC32); err != nil { + return nil, err + } idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...) } @@ -161,15 +166,17 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { return idx, nil } -func (w *Writer) addOffset64(pos uint64) uint64 { +func (w *Writer) addOffset64(pos uint64) (uint64, error) { buf := new(bytes.Buffer) - binary.WriteUint64(buf, pos) - w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) + if err := binary.WriteUint64(buf, pos); err != nil { + return 0, err + } + w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) index := uint64(w.offset64 | (1 << 31)) w.offset64++ - return index + return index, nil } func (o objects) Len() int { diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index fba3e42..eaa8605 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -3,7 +3,7 @@ package idxfile_test import ( "bytes" "encoding/base64" - "io/ioutil" + "io" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" @@ -34,7 +34,7 @@ func (s *WriterSuite) TestWriter(c *C) { c.Assert(err, IsNil) idxFile := f.Idx() - expected, err := ioutil.ReadAll(idxFile) + expected, err := io.ReadAll(idxFile) c.Assert(err, IsNil) idxFile.Close() @@ -65,7 +65,7 @@ func (s *WriterSuite) TestWriterLarge(c *C) { // load fixture index f := bytes.NewBufferString(fixtureLarge4GB) - expected, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) + expected, err := io.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) c.Assert(err, IsNil) buf := new(bytes.Buffer) diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index 036b636..6778cf7 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -3,15 +3,14 @@ package index import ( "bufio" "bytes" - "crypto/sha1" "errors" - "hash" "io" - "io/ioutil" + "strconv" "time" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -49,7 +48,7 @@ type Decoder struct { // NewDecoder returns a new decoder that reads from r. func NewDecoder(r io.Reader) *Decoder { - h := sha1.New() + h := hash.New(hash.CryptoType) return &Decoder{ r: io.TeeReader(r, h), hash: h, @@ -202,7 +201,7 @@ func (d *Decoder) padEntry(idx *Index, e *Entry, read int) error { entrySize := read + len(e.Name) padLen := 8 - entrySize%8 - _, err := io.CopyN(ioutil.Discard, d.r, int64(padLen)) + _, err := io.CopyN(io.Discard, d.r, int64(padLen)) return err } diff --git a/plumbing/format/index/encoder.go b/plumbing/format/index/encoder.go index 2c94d93..fa2d814 100644 --- a/plumbing/format/index/encoder.go +++ b/plumbing/format/index/encoder.go @@ -2,13 +2,12 @@ package index import ( "bytes" - "crypto/sha1" "errors" - "hash" "io" "sort" "time" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/utils/binary" ) @@ -29,7 +28,7 @@ type Encoder struct { // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - h := sha1.New() + h := hash.New(hash.CryptoType) mw := io.MultiWriter(w, h) return &Encoder{mw, h} } diff --git a/plumbing/format/objfile/reader.go b/plumbing/format/objfile/reader.go index b6b2ca0..d7932f4 100644 --- a/plumbing/format/objfile/reader.go +++ b/plumbing/format/objfile/reader.go @@ -1,13 +1,13 @@ package objfile import ( - "compress/zlib" "errors" "io" "strconv" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -20,20 +20,22 @@ var ( // Reader implements io.ReadCloser. Close should be called when finished with // the Reader. Close will not close the underlying io.Reader. type Reader struct { - multi io.Reader - zlib io.ReadCloser - hasher plumbing.Hasher + multi io.Reader + zlib io.Reader + zlibref sync.ZLibReader + hasher plumbing.Hasher } // NewReader returns a new Reader reading from r. func NewReader(r io.Reader) (*Reader, error) { - zlib, err := zlib.NewReader(r) + zlib, err := sync.GetZlibReader(r) if err != nil { return nil, packfile.ErrZLib.AddDetails(err.Error()) } return &Reader{ - zlib: zlib, + zlib: zlib.Reader, + zlibref: zlib, }, nil } @@ -110,5 +112,6 @@ func (r *Reader) Hash() plumbing.Hash { // Close releases any resources consumed by the Reader. Calling Close does not // close the wrapped io.Reader originally passed to NewReader. func (r *Reader) Close() error { - return r.zlib.Close() + sync.PutZlibReader(r.zlibref) + return nil } diff --git a/plumbing/format/objfile/reader_test.go b/plumbing/format/objfile/reader_test.go index d697d54..5526f7f 100644 --- a/plumbing/format/objfile/reader_test.go +++ b/plumbing/format/objfile/reader_test.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "github.com/go-git/go-git/v5/plumbing" @@ -36,7 +35,7 @@ func testReader(c *C, source io.Reader, hash plumbing.Hash, t plumbing.ObjectTyp c.Assert(typ, Equals, t) c.Assert(content, HasLen, int(size)) - rc, err := ioutil.ReadAll(r) + rc, err := io.ReadAll(r) c.Assert(err, IsNil) c.Assert(rc, DeepEquals, content, Commentf("%scontent=%s, expected=%s", base64.StdEncoding.EncodeToString(rc), base64.StdEncoding.EncodeToString(content))) diff --git a/plumbing/format/objfile/writer.go b/plumbing/format/objfile/writer.go index 2a96a43..0d0f154 100644 --- a/plumbing/format/objfile/writer.go +++ b/plumbing/format/objfile/writer.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -18,9 +19,9 @@ var ( // not close the underlying io.Writer. type Writer struct { raw io.Writer - zlib io.WriteCloser hasher plumbing.Hasher multi io.Writer + zlib *zlib.Writer closed bool pending int64 // number of unwritten bytes @@ -31,9 +32,10 @@ type Writer struct { // The returned Writer implements io.WriteCloser. Close should be called when // finished with the Writer. Close will not close the underlying io.Writer. func NewWriter(w io.Writer) *Writer { + zlib := sync.GetZlibWriter(w) return &Writer{ raw: w, - zlib: zlib.NewWriter(w), + zlib: zlib, } } @@ -100,6 +102,7 @@ func (w *Writer) Hash() plumbing.Hash { // Calling Close does not close the wrapped io.Writer originally passed to // NewWriter. func (w *Writer) Close() error { + defer sync.PutZlibWriter(w.zlib) if err := w.zlib.Close(); err != nil { return err } diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index df423ad..36c5ef5 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -1,10 +1,7 @@ package packfile import ( - "bytes" - "compress/zlib" "io" - "sync" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" @@ -61,18 +58,3 @@ func WritePackfileToObjectStorage( return err } - -var bufPool = sync.Pool{ - New: func() interface{} { - return bytes.NewBuffer(nil) - }, -} - -var zlibInitBytes = []byte{0x78, 0x9c, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01} - -var zlibReaderPool = sync.Pool{ - New: func() interface{} { - r, _ := zlib.NewReader(bytes.NewReader(zlibInitBytes)) - return r - }, -} diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go index 137e485..e8f5ea6 100644 --- a/plumbing/format/packfile/delta_test.go +++ b/plumbing/format/packfile/delta_test.go @@ -2,7 +2,7 @@ package packfile import ( "bytes" - "io/ioutil" + "io" "math/rand" "github.com/go-git/go-git/v5/plumbing" @@ -109,14 +109,14 @@ func (s *DeltaSuite) TestAddDeltaReader(c *C) { targetBuf := genBytes(t.target) delta := DiffDelta(baseBuf, targetBuf) - deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + deltaRC := io.NopCloser(bytes.NewReader(delta)) c.Log("Executing test case:", t.description) resultRC, err := ReaderFromDelta(baseObj, deltaRC) c.Assert(err, IsNil) - result, err := ioutil.ReadAll(resultRC) + result, err := io.ReadAll(resultRC) c.Assert(err, IsNil) err = resultRC.Close() @@ -164,12 +164,12 @@ func (s *DeltaSuite) TestMaxCopySizeDeltaReader(c *C) { targetBuf = append(targetBuf, byte(1)) delta := DiffDelta(baseBuf, targetBuf) - deltaRC := ioutil.NopCloser(bytes.NewReader(delta)) + deltaRC := io.NopCloser(bytes.NewReader(delta)) resultRC, err := ReaderFromDelta(baseObj, deltaRC) c.Assert(err, IsNil) - result, err := ioutil.ReadAll(resultRC) + result, err := io.ReadAll(resultRC) c.Assert(err, IsNil) err = resultRC.Close() diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 1951b34..2c7a335 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -5,6 +5,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) // See https://github.com/jelmer/dulwich/blob/master/dulwich/pack.py and @@ -43,18 +44,16 @@ func getDelta(index *deltaIndex, base, target plumbing.EncodedObject) (o plumbin defer ioutil.CheckClose(tr, &err) - bb := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(bb) - bb.Reset() + bb := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(bb) _, err = bb.ReadFrom(br) if err != nil { return nil, err } - tb := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(tb) - tb.Reset() + tb := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(tb) _, err = tb.ReadFrom(tr) if err != nil { @@ -80,9 +79,8 @@ func DiffDelta(src, tgt []byte) []byte { } func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) buf.Write(deltaEncodeSize(len(src))) buf.Write(deltaEncodeSize(len(tgt))) @@ -90,9 +88,8 @@ func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { index.init(src) } - ibuf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(ibuf) - ibuf.Reset() + ibuf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(ibuf) for i := 0; i < len(tgt); i++ { offset, l := index.findMatch(src, tgt, i) diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index 5501f88..804f5a8 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -2,11 +2,11 @@ package packfile import ( "compress/zlib" - "crypto/sha1" "fmt" "io" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/binary" "github.com/go-git/go-git/v5/utils/ioutil" @@ -28,7 +28,7 @@ type Encoder struct { // OFSDeltaObject. To use Reference deltas, set useRefDeltas to true. func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *Encoder { h := plumbing.Hasher{ - Hash: sha1.New(), + Hash: hash.New(hash.CryptoType), } mw := io.MultiWriter(w, h) ow := newOffsetWriter(mw) @@ -131,11 +131,7 @@ func (e *Encoder) entry(o *ObjectToPack) (err error) { defer ioutil.CheckClose(or, &err) _, err = io.Copy(e.zw, or) - if err != nil { - return err - } - - return nil + return err } func (e *Encoder) writeBaseIfDelta(o *ObjectToPack) error { diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index c9d49c3..6719f37 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -3,10 +3,10 @@ package packfile import ( "bytes" "io" - stdioutil "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/format/idxfile" + "github.com/go-git/go-git/v5/plumbing/hash" "github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-billy/v5/memfs" @@ -30,10 +30,10 @@ func (s *EncoderSuite) SetUpTest(c *C) { } func (s *EncoderSuite) TestCorrectPackHeader(c *C) { - hash, err := s.enc.Encode([]plumbing.Hash{}, 10) + h, err := s.enc.Encode([]plumbing.Hash{}, 10) c.Assert(err, IsNil) - hb := [20]byte(hash) + hb := [hash.Size]byte(h) // PACK + VERSION + OBJECTS + HASH expectedResult := []byte{'P', 'A', 'C', 'K', 0, 0, 0, 2, 0, 0, 0, 0} @@ -51,7 +51,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) + h, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) // PACK + VERSION(2) + OBJECT NUMBER(1) @@ -64,7 +64,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { []byte{120, 156, 1, 0, 0, 255, 255, 0, 0, 0, 1}...) // + HASH - hb := [20]byte(hash) + hb := [hash.Size]byte(h) expectedResult = append(expectedResult, hb[:]...) result := s.buf.Bytes() @@ -277,13 +277,13 @@ func objectsEqual(c *C, o1, o2 plumbing.EncodedObject) { r1, err := o1.Reader() c.Assert(err, IsNil) - b1, err := stdioutil.ReadAll(r1) + b1, err := io.ReadAll(r1) c.Assert(err, IsNil) r2, err := o2.Reader() c.Assert(err, IsNil) - b2, err := stdioutil.ReadAll(r2) + b2, err := io.ReadAll(r2) c.Assert(err, IsNil) c.Assert(bytes.Compare(b1, b2), Equals, 0) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 8dd6041..6852702 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -2,7 +2,6 @@ package packfile import ( "bytes" - "compress/zlib" "fmt" "io" "os" @@ -13,6 +12,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/idxfile" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -138,9 +138,8 @@ func (p *Packfile) getObjectSize(h *ObjectHeader) (int64, error) { case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: return h.Length, nil case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) if _, _, err := p.s.NextObject(buf); err != nil { return 0, err @@ -227,9 +226,9 @@ func (p *Packfile) getNextObject(h *ObjectHeader, hash plumbing.Hash) (plumbing. // For delta objects we read the delta data and apply the small object // optimization only if the expanded version of the object still meets // the small object threshold condition. - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + if _, _, err := p.s.NextObject(buf); err != nil { return nil, err } @@ -290,14 +289,13 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { func asyncReader(p *Packfile) (io.ReadCloser, error) { reader := ioutil.NewReaderUsingReaderAt(p.file, p.s.r.offset) - zr := zlibReaderPool.Get().(io.ReadCloser) - - if err := zr.(zlib.Resetter).Reset(reader, nil); err != nil { + zr, err := sync.GetZlibReader(reader) + if err != nil { return nil, fmt.Errorf("zlib reset error: %s", err) } - return ioutil.NewReadCloserWithCloser(zr, func() error { - zlibReaderPool.Put(zr) + return ioutil.NewReadCloserWithCloser(zr.Reader, func() error { + sync.PutZlibReader(zr) return nil }), nil @@ -373,9 +371,9 @@ func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) (err err } func (p *Packfile) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) error { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + _, _, err := p.s.NextObject(buf) if err != nil { return err @@ -417,9 +415,9 @@ func (p *Packfile) fillREFDeltaObjectContentWithBuffer(obj plumbing.EncodedObjec } func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) error { - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + _, _, err := p.s.NextObject(buf) if err != nil { return err diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index ee5c289..edbc0e7 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -4,12 +4,12 @@ import ( "bytes" "errors" "io" - stdioutil "io/ioutil" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -175,7 +175,8 @@ func (p *Parser) init() error { } func (p *Parser) indexObjects() error { - buf := new(bytes.Buffer) + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) for i := uint32(0); i < p.count; i++ { buf.Reset() @@ -219,6 +220,7 @@ func (p *Parser) indexObjects() error { ota = newBaseObject(oh.Offset, oh.Length, t) } + buf.Grow(int(oh.Length)) _, crc, err := p.scanner.NextObject(buf) if err != nil { return err @@ -234,6 +236,15 @@ func (p *Parser) indexObjects() error { return err } + // Move children of placeholder parent into actual parent, in case this + // was a non-external delta reference. + if placeholder, ok := p.oiByHash[sha1]; ok { + ota.Children = placeholder.Children + for _, c := range ota.Children { + c.Parent = ota + } + } + ota.SHA1 = sha1 p.oiByHash[ota.SHA1] = ota } @@ -264,7 +275,9 @@ func (p *Parser) indexObjects() error { } func (p *Parser) resolveDeltas() error { - buf := &bytes.Buffer{} + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) + for _, obj := range p.oi { buf.Reset() err := p.get(obj, buf) @@ -283,9 +296,10 @@ func (p *Parser) resolveDeltas() error { if !obj.IsDelta() && len(obj.Children) > 0 { for _, child := range obj.Children { - if err := p.resolveObject(stdioutil.Discard, child, content); err != nil { + if err := p.resolveObject(io.Discard, child, content); err != nil { return err } + p.resolveExternalRef(child) } // Remove the delta from the cache. @@ -298,6 +312,16 @@ func (p *Parser) resolveDeltas() error { return nil } +func (p *Parser) resolveExternalRef(o *objectInfo) { + if ref, ok := p.oiByHash[o.SHA1]; ok && ref.ExternalRef { + p.oiByHash[o.SHA1] = o + o.Children = ref.Children + for _, c := range o.Children { + c.Parent = o + } + } +} + func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { if !o.ExternalRef { // skip cache check for placeholder parents b, ok := p.cache.Get(o.Offset) @@ -335,9 +359,8 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) { } if o.DiskType.IsDelta() { - b := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(b) - b.Reset() + b := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(b) err := p.get(o.Parent, b) if err != nil { return err @@ -371,9 +394,8 @@ func (p *Parser) resolveObject( if !o.DiskType.IsDelta() { return nil } - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) err := p.readData(buf, o) if err != nil { return err diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index b0b4af8..b8d080f 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -10,8 +10,10 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/packfile" "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage/filesystem" . "gopkg.in/check.v1" ) @@ -132,6 +134,32 @@ func (s *ParserSuite) TestThinPack(c *C) { } +func (s *ParserSuite) TestResolveExternalRefsInThinPack(c *C) { + extRefsThinPack := fixtures.ByTag("codecommit").One() + + scanner := packfile.NewScanner(extRefsThinPack.Packfile()) + + obs := new(testObserver) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) + + _, err = parser.Parse() + c.Assert(err, IsNil) +} + +func (s *ParserSuite) TestResolveExternalRefs(c *C) { + extRefsThinPack := fixtures.ByTag("delta-before-base").One() + + scanner := packfile.NewScanner(extRefsThinPack.Packfile()) + + obs := new(testObserver) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) + + _, err = parser.Parse() + c.Assert(err, IsNil) +} + type observerObject struct { hash string otype plumbing.ObjectType @@ -235,3 +263,29 @@ func BenchmarkParseBasic(b *testing.B) { } } } + +func BenchmarkParser(b *testing.B) { + f := fixtures.Basic().One() + defer fixtures.Clean() + + b.ResetTimer() + for n := 0; n < b.N; n++ { + b.StopTimer() + scanner := packfile.NewScanner(f.Packfile()) + fs := osfs.New(os.TempDir()) + storage := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + + parser, err := packfile.NewParserWithStorage(scanner, storage) + if err != nil { + b.Error(err) + } + + b.StartTimer() + _, err = parser.Parse() + + b.StopTimer() + if err != nil { + b.Error(err) + } + } +} diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 17da11e..f00562d 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -9,6 +9,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) // See https://github.com/git/git/blob/49fa3dc76179e04b0833542fa52d0f287a4955ac/delta.h @@ -34,18 +35,16 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { defer ioutil.CheckClose(w, &err) - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() + buf := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(buf) _, err = buf.ReadFrom(r) if err != nil { return err } src := buf.Bytes() - dst := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(dst) - dst.Reset() + dst := sync.GetBytesBuffer() + defer sync.PutBytesBuffer(dst) err = patchDelta(dst, src, delta) if err != nil { return err @@ -53,9 +52,9 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { target.SetSize(int64(dst.Len())) - b := byteSlicePool.Get().([]byte) - _, err = io.CopyBuffer(w, dst, b) - byteSlicePool.Put(b) + b := sync.GetByteSlice() + _, err = io.CopyBuffer(w, dst, *b) + sync.PutByteSlice(b) return err } diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 5d9e8fb..730343e 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -3,17 +3,15 @@ package packfile import ( "bufio" "bytes" - "compress/zlib" "fmt" "hash" "hash/crc32" "io" - stdioutil "io/ioutil" - "sync" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/utils/binary" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/go-git/go-git/v5/utils/sync" ) var ( @@ -114,7 +112,7 @@ func (s *Scanner) Header() (version, objects uint32, err error) { return } -// readSignature reads an returns the signature field in the packfile. +// readSignature reads a returns the signature field in the packfile. func (s *Scanner) readSignature() ([]byte, error) { var sig = make([]byte, 4) if _, err := io.ReadFull(s.r, sig); err != nil { @@ -243,7 +241,7 @@ func (s *Scanner) discardObjectIfNeeded() error { } h := s.pendingObject - n, _, err := s.NextObject(stdioutil.Discard) + n, _, err := s.NextObject(io.Discard) if err != nil { return err } @@ -323,14 +321,14 @@ func (s *Scanner) NextObject(w io.Writer) (written int64, crc32 uint32, err erro // ReadObject returns a reader for the object content and an error func (s *Scanner) ReadObject() (io.ReadCloser, error) { s.pendingObject = nil - zr := zlibReaderPool.Get().(io.ReadCloser) + zr, err := sync.GetZlibReader(s.r) - if err := zr.(zlib.Resetter).Reset(s.r, nil); err != nil { + if err != nil { return nil, fmt.Errorf("zlib reset error: %s", err) } - return ioutil.NewReadCloserWithCloser(zr, func() error { - zlibReaderPool.Put(zr) + return ioutil.NewReadCloserWithCloser(zr.Reader, func() error { + sync.PutZlibReader(zr) return nil }), nil } @@ -338,26 +336,20 @@ func (s *Scanner) ReadObject() (io.ReadCloser, error) { // ReadRegularObject reads and write a non-deltified object // from it zlib stream in an object entry in the packfile. func (s *Scanner) copyObject(w io.Writer) (n int64, err error) { - zr := zlibReaderPool.Get().(io.ReadCloser) - defer zlibReaderPool.Put(zr) + zr, err := sync.GetZlibReader(s.r) + defer sync.PutZlibReader(zr) - if err = zr.(zlib.Resetter).Reset(s.r, nil); err != nil { + if err != nil { return 0, fmt.Errorf("zlib reset error: %s", err) } - defer ioutil.CheckClose(zr, &err) - buf := byteSlicePool.Get().([]byte) - n, err = io.CopyBuffer(w, zr, buf) - byteSlicePool.Put(buf) + defer ioutil.CheckClose(zr.Reader, &err) + buf := sync.GetByteSlice() + n, err = io.CopyBuffer(w, zr.Reader, *buf) + sync.PutByteSlice(buf) return } -var byteSlicePool = sync.Pool{ - New: func() interface{} { - return make([]byte, 32*1024) - }, -} - // SeekFromStart sets a new offset from start, returns the old position before // the change. func (s *Scanner) SeekFromStart(offset int64) (previous int64, err error) { @@ -387,9 +379,10 @@ func (s *Scanner) Checksum() (plumbing.Hash, error) { // Close reads the reader until io.EOF func (s *Scanner) Close() error { - buf := byteSlicePool.Get().([]byte) - _, err := io.CopyBuffer(stdioutil.Discard, s.r, buf) - byteSlicePool.Put(buf) + buf := sync.GetByteSlice() + _, err := io.CopyBuffer(io.Discard, s.r, *buf) + sync.PutByteSlice(buf) + return err } @@ -399,13 +392,13 @@ func (s *Scanner) Flush() error { } // scannerReader has the following characteristics: -// - Provides an io.SeekReader impl for bufio.Reader, when the underlying -// reader supports it. -// - Keeps track of the current read position, for when the underlying reader -// isn't an io.SeekReader, but we still want to know the current offset. -// - Writes to the hash writer what it reads, with the aid of a smaller buffer. -// The buffer helps avoid a performance penality for performing small writes -// to the crc32 hash writer. +// - Provides an io.SeekReader impl for bufio.Reader, when the underlying +// reader supports it. +// - Keeps track of the current read position, for when the underlying reader +// isn't an io.SeekReader, but we still want to know the current offset. +// - Writes to the hash writer what it reads, with the aid of a smaller buffer. +// The buffer helps avoid a performance penalty for performing small writes +// to the crc32 hash writer. type scannerReader struct { reader io.Reader crc io.Writer diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 892a27c..9dcc359 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -6,6 +6,7 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/hash" . "gopkg.in/check.v1" ) @@ -71,7 +72,7 @@ func (s *ScannerSuite) testNextObjectHeader(c *C, tag string, n, err := p.Checksum() c.Assert(err, IsNil) - c.Assert(n, HasLen, 20) + c.Assert(n, HasLen, hash.Size) } func (s *ScannerSuite) TestNextObjectHeaderWithOutReadObject(c *C) { |