From f4e603f291c7badaa6489d0f9d00e71bd7fc579c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 19 Jan 2018 16:12:11 +0100 Subject: plumbing: packfile, Add a buffer to crc writer. crc update with block smaller than 16 bytes uses a slower version of the function. ReadByte is heavily used by zlib inflate so most of the time crc is update byte by byte. A new Flush method is added to the scanner to flush this crc writer cache. It is only called when the Scanner reader is a teeReader. Signed-off-by: Javi Fontan --- plumbing/format/packfile/scanner.go | 40 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) (limited to 'plumbing/format') diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index d2d776f..8c216f1 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -63,10 +63,7 @@ func NewScanner(r io.Reader) *Scanner { crc := crc32.NewIEEE() return &Scanner{ - r: &teeReader{ - newByteReadSeeker(seeker), - crc, - }, + r: newTeeReader(newByteReadSeeker(seeker), crc), crc: crc, IsSeekable: ok, } @@ -143,6 +140,8 @@ func (s *Scanner) readCount() (uint32, error) { // NextObjectHeader returns the ObjectHeader for the next object in the reader func (s *Scanner) NextObjectHeader() (*ObjectHeader, error) { + defer s.Flush() + if err := s.doPending(); err != nil { return nil, err } @@ -271,6 +270,7 @@ func (s *Scanner) NextObject(w io.Writer) (written int64, crc32 uint32, err erro s.pendingObject = nil written, err = s.copyObject(w) + s.Flush() crc32 = s.crc.Sum32() return } @@ -339,6 +339,16 @@ func (s *Scanner) Close() error { return err } +// Flush finishes writing the buffer to crc hasher in case we are using +// a teeReader. Otherwise it is a no-op. +func (s *Scanner) Flush() error { + tee, ok := s.r.(*teeReader) + if ok { + return tee.Flush() + } + return nil +} + type trackableReader struct { count int64 io.Reader @@ -400,10 +410,21 @@ type reader interface { type teeReader struct { reader - w hash.Hash32 + w hash.Hash32 + bufWriter *bufio.Writer +} + +func newTeeReader(r reader, h hash.Hash32) *teeReader { + return &teeReader{ + reader: r, + w: h, + bufWriter: bufio.NewWriter(h), + } } func (r *teeReader) Read(p []byte) (n int, err error) { + r.Flush() + n, err = r.reader.Read(p) if n > 0 { if n, err := r.w.Write(p[:n]); err != nil { @@ -416,11 +437,12 @@ func (r *teeReader) Read(p []byte) (n int, err error) { func (r *teeReader) ReadByte() (b byte, err error) { b, err = r.reader.ReadByte() if err == nil { - _, err := r.w.Write([]byte{b}) - if err != nil { - return 0, err - } + return b, r.bufWriter.WriteByte(b) } return } + +func (r *teeReader) Flush() (err error) { + return r.bufWriter.Flush() +} -- cgit From 1cb896b25e9c5aef5a8a247f5e4ee0b6854d6f41 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Sun, 21 Jan 2018 21:24:57 +0100 Subject: plumbing: packfile, Add crc check to scanner test. Signed-off-by: Javi Fontan --- plumbing/format/packfile/scanner_test.go | 79 ++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) (limited to 'plumbing/format') diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 000be7f..644d0eb 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -41,14 +41,16 @@ func (s *ScannerSuite) TestNextObjectHeaderWithoutHeader(c *C) { } func (s *ScannerSuite) TestNextObjectHeaderREFDelta(c *C) { - s.testNextObjectHeader(c, "ref-delta", expectedHeadersREF) + s.testNextObjectHeader(c, "ref-delta", expectedHeadersREF, expectedCRCREF) } func (s *ScannerSuite) TestNextObjectHeaderOFSDelta(c *C) { - s.testNextObjectHeader(c, "ofs-delta", expectedHeadersOFS) + s.testNextObjectHeader(c, "ofs-delta", expectedHeadersOFS, expectedCRCOFS) } -func (s *ScannerSuite) testNextObjectHeader(c *C, tag string, expected []ObjectHeader) { +func (s *ScannerSuite) testNextObjectHeader(c *C, tag string, + expected []ObjectHeader, expectedCRC []uint32) { + r := fixtures.Basic().ByTag(tag).One().Packfile() p := NewScanner(r) @@ -61,9 +63,10 @@ func (s *ScannerSuite) testNextObjectHeader(c *C, tag string, expected []ObjectH c.Assert(*h, DeepEquals, expected[i]) buf := bytes.NewBuffer(nil) - n, _, err := p.NextObject(buf) + n, crcFromScanner, err := p.NextObject(buf) c.Assert(err, IsNil) c.Assert(n, Equals, h.Length) + c.Assert(crcFromScanner, Equals, expectedCRC[i]) } n, err := p.Checksum() @@ -149,6 +152,40 @@ var expectedHeadersOFS = []ObjectHeader{ {Type: plumbing.OFSDeltaObject, Offset: 84760, Length: 4, OffsetReference: 84741}, } +var expectedCRCOFS = []uint32{ + 0xaa07ba4b, + 0xf706df58, + 0x12438846, + 0x2905a38c, + 0xd9429436, + 0xbecfde4e, + 0x780e4b3e, + 0xdc18344f, + 0xcf4e4280, + 0x1f08118a, + 0xafded7b8, + 0xcc1428ed, + 0x1631d22f, + 0xbfff5850, + 0xd108e1d8, + 0x8e97ba25, + 0x7316ff70, + 0xdb4fce56, + 0x901cce2c, + 0xec4552b0, + 0x847905bf, + 0x3689459a, + 0xe67af94a, + 0xc2314a2e, + 0xcd987848, + 0x8a853a6d, + 0x70c6518, + 0x4f4108e2, + 0xd6fe09e9, + 0xf07a2804, + 0x1d75d6be, +} + var expectedHeadersREF = []ObjectHeader{ {Type: plumbing.CommitObject, Offset: 12, Length: 254}, {Type: plumbing.REFDeltaObject, Offset: 186, Length: 93, @@ -188,3 +225,37 @@ var expectedHeadersREF = []ObjectHeader{ Reference: plumbing.NewHash("eba74343e2f15d62adedfd8c883ee0262b5c8021")}, {Type: plumbing.TreeObject, Offset: 85485, Length: 73}, } + +var expectedCRCREF = []uint32{ + 0xaa07ba4b, + 0xfb4725a4, + 0x12438846, + 0x2905a38c, + 0xd9429436, + 0xbecfde4e, + 0xdc18344f, + 0x780e4b3e, + 0xcf4e4280, + 0x1f08118a, + 0xafded7b8, + 0xcc1428ed, + 0x1631d22f, + 0x847905bf, + 0x3e20f31d, + 0x3689459a, + 0xd108e1d8, + 0x71143d4a, + 0xe67af94a, + 0x739fb89f, + 0xc2314a2e, + 0x87864926, + 0x415d752f, + 0xf72fb182, + 0x3ffa37d4, + 0xcd987848, + 0x2f20ac8f, + 0xf2f0575, + 0x7d8726e1, + 0x740bf39, + 0x26af4735, +} -- cgit