From 123cdde6f2f6282cb779e03745d384833ac1265b Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Wed, 26 Oct 2022 18:12:39 +0100 Subject: Use Sync.Pool pointers to optimise memory usage Signed-off-by: Paulo Gomes --- plumbing/format/objfile/writer.go | 15 +++++++++++++-- plumbing/format/packfile/parser_test.go | 28 ++++++++++++++++++++++++++++ plumbing/format/packfile/patch_delta.go | 5 +++-- plumbing/format/packfile/scanner.go | 14 +++++++++----- storage/filesystem/object.go | 14 +++++++++++++- worktree.go | 8 +++++--- 6 files changed, 71 insertions(+), 13 deletions(-) diff --git a/plumbing/format/objfile/writer.go b/plumbing/format/objfile/writer.go index 2a96a43..248f81b 100644 --- a/plumbing/format/objfile/writer.go +++ b/plumbing/format/objfile/writer.go @@ -5,6 +5,7 @@ import ( "errors" "io" "strconv" + "sync" "github.com/go-git/go-git/v5/plumbing" ) @@ -18,9 +19,9 @@ var ( // not close the underlying io.Writer. type Writer struct { raw io.Writer - zlib io.WriteCloser hasher plumbing.Hasher multi io.Writer + zlib io.WriteCloser closed bool pending int64 // number of unwritten bytes @@ -31,12 +32,21 @@ type Writer struct { // The returned Writer implements io.WriteCloser. Close should be called when // finished with the Writer. Close will not close the underlying io.Writer. func NewWriter(w io.Writer) *Writer { + zlib := zlibPool.Get().(*zlib.Writer) + zlib.Reset(w) + return &Writer{ raw: w, - zlib: zlib.NewWriter(w), + zlib: zlib, } } +var zlibPool = sync.Pool{ + New: func() interface{} { + return zlib.NewWriter(nil) + }, +} + // WriteHeader writes the type and the size and prepares to accept the object's // contents. If an invalid t is provided, plumbing.ErrInvalidType is returned. If a // negative size is provided, ErrNegativeSize is returned. @@ -100,6 +110,7 @@ func (w *Writer) Hash() plumbing.Hash { // Calling Close does not close the wrapped io.Writer originally passed to // NewWriter. func (w *Writer) Close() error { + defer zlibPool.Put(w.zlib) if err := w.zlib.Close(); err != nil { return err } diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index 09f3f97..651d05f 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -10,8 +10,10 @@ import ( fixtures "github.com/go-git/go-git-fixtures/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/format/packfile" "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage/filesystem" . "gopkg.in/check.v1" ) @@ -248,3 +250,29 @@ func BenchmarkParseBasic(b *testing.B) { } } } + +func BenchmarkParser(b *testing.B) { + f := fixtures.Basic().One() + defer fixtures.Clean() + + b.ResetTimer() + for n := 0; n < b.N; n++ { + b.StopTimer() + scanner := packfile.NewScanner(f.Packfile()) + fs := osfs.New(os.TempDir()) + storage := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + + parser, err := packfile.NewParserWithStorage(scanner, storage) + if err != nil { + b.Error(err) + } + + b.StartTimer() + _, err = parser.Parse() + + b.StopTimer() + if err != nil { + b.Error(err) + } + } +} diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 17da11e..053466d 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -53,9 +53,10 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) { target.SetSize(int64(dst.Len())) - b := byteSlicePool.Get().([]byte) + bufp := byteSlicePool.Get().(*[]byte) + b := *bufp _, err = io.CopyBuffer(w, dst, b) - byteSlicePool.Put(b) + byteSlicePool.Put(bufp) return err } diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 45d480c..b655594 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -346,15 +346,17 @@ func (s *Scanner) copyObject(w io.Writer) (n int64, err error) { } defer ioutil.CheckClose(zr, &err) - buf := byteSlicePool.Get().([]byte) + bufp := byteSlicePool.Get().(*[]byte) + buf := *bufp n, err = io.CopyBuffer(w, zr, buf) - byteSlicePool.Put(buf) + byteSlicePool.Put(bufp) return } var byteSlicePool = sync.Pool{ New: func() interface{} { - return make([]byte, 32*1024) + b := make([]byte, 32*1024) + return &b }, } @@ -387,9 +389,11 @@ func (s *Scanner) Checksum() (plumbing.Hash, error) { // Close reads the reader until io.EOF func (s *Scanner) Close() error { - buf := byteSlicePool.Get().([]byte) + bufp := byteSlicePool.Get().(*[]byte) + buf := *bufp _, err := io.CopyBuffer(stdioutil.Discard, s.r, buf) - byteSlicePool.Put(buf) + byteSlicePool.Put(bufp) + return err } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 5c91bcd..21667fa 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "os" + "sync" "time" "github.com/go-git/go-git/v5/plumbing" @@ -419,10 +420,21 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb s.objectCache.Put(obj) - _, err = io.Copy(w, r) + bufp := copyBufferPool.Get().(*[]byte) + buf := *bufp + _, err = io.CopyBuffer(w, r, buf) + copyBufferPool.Put(bufp) + return obj, err } +var copyBufferPool = sync.Pool{ + New: func() interface{} { + b := make([]byte, 32*1024) + return &b + }, +} + // Get returns the object with the given hash, by searching for it in // the packfile. func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( diff --git a/worktree.go b/worktree.go index c974aed..98116ca 100644 --- a/worktree.go +++ b/worktree.go @@ -534,7 +534,8 @@ func (w *Worktree) checkoutChangeRegularFile(name string, var copyBufferPool = sync.Pool{ New: func() interface{} { - return make([]byte, 32*1024) + b := make([]byte, 32*1024) + return &b }, } @@ -561,9 +562,10 @@ func (w *Worktree) checkoutFile(f *object.File) (err error) { } defer ioutil.CheckClose(to, &err) - buf := copyBufferPool.Get().([]byte) + bufp := copyBufferPool.Get().(*[]byte) + buf := *bufp _, err = io.CopyBuffer(to, from, buf) - copyBufferPool.Put(buf) + copyBufferPool.Put(bufp) return } -- cgit From ffa7e69efb8c4ba8d4e08ec4c65e49e2228fd88b Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Mon, 7 Nov 2022 14:42:00 +0000 Subject: Optimise Reference.String() Decreases allocations and bytes per operation by using string builder with a predefined size. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One additional allocation has been removed by using its own implementation of Strings(). The reason behind this was due to the fact the calls to .String() are more recurrent than .Strings() and the performance impact was worth the code duplication. Benchmark results: cpu: 11th Gen Intel(R) Core(TM) i7-11800H @ 2.30GHz name old time/op new time/op delta ReferenceStringSymbolic-16 140ns ± 4% 40ns ± 9% -71.19% (p=0.008 n=5+5) ReferenceStringHash-16 174ns ±14% 85ns ± 4% -51.13% (p=0.008 n=5+5) ReferenceStringInvalid-16 48.9ns ± 2% 1.5ns ± 3% -96.96% (p=0.008 n=5+5) name old alloc/op new alloc/op delta ReferenceStringSymbolic-16 88.0B ± 0% 32.0B ± 0% -63.64% (p=0.008 n=5+5) ReferenceStringHash-16 176B ± 0% 144B ± 0% -18.18% (p=0.008 n=5+5) ReferenceStringInvalid-16 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta ReferenceStringSymbolic-16 4.00 ± 0% 1.00 ± 0% -75.00% (p=0.008 n=5+5) ReferenceStringHash-16 5.00 ± 0% 3.00 ± 0% -40.00% (p=0.008 n=5+5) ReferenceStringInvalid-16 0.00 0.00 ~ (all equal) Signed-off-by: Paulo Gomes --- plumbing/reference.go | 19 +++++++++++++++++-- plumbing/reference_test.go | 24 +++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/plumbing/reference.go b/plumbing/reference.go index deb5067..eef11e8 100644 --- a/plumbing/reference.go +++ b/plumbing/reference.go @@ -204,6 +204,21 @@ func (r *Reference) Strings() [2]string { } func (r *Reference) String() string { - s := r.Strings() - return fmt.Sprintf("%s %s", s[1], s[0]) + ref := "" + switch r.Type() { + case HashReference: + ref = r.Hash().String() + case SymbolicReference: + ref = symrefPrefix + r.Target().String() + default: + return "" + } + + name := r.Name().String() + var v strings.Builder + v.Grow(len(ref) + len(name) + 1) + v.WriteString(ref) + v.WriteString(" ") + v.WriteString(name) + return v.String() } diff --git a/plumbing/reference_test.go b/plumbing/reference_test.go index b3ccf53..e69076f 100644 --- a/plumbing/reference_test.go +++ b/plumbing/reference_test.go @@ -1,6 +1,10 @@ package plumbing -import . "gopkg.in/check.v1" +import ( + "testing" + + . "gopkg.in/check.v1" +) type ReferenceSuite struct{} @@ -98,3 +102,21 @@ func (s *ReferenceSuite) TestIsTag(c *C) { r := ReferenceName("refs/tags/v3.1.") c.Assert(r.IsTag(), Equals, true) } + +func benchMarkReferenceString(r *Reference, b *testing.B) { + for n := 0; n < b.N; n++ { + r.String() + } +} + +func BenchmarkReferenceStringSymbolic(b *testing.B) { + benchMarkReferenceString(NewSymbolicReference("v3.1.1", "refs/tags/v3.1.1"), b) +} + +func BenchmarkReferenceStringHash(b *testing.B) { + benchMarkReferenceString(NewHashReference("v3.1.1", NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")), b) +} + +func BenchmarkReferenceStringInvalid(b *testing.B) { + benchMarkReferenceString(&Reference{}, b) +} -- cgit