plumbing: format/packfile, prevent large objects from being read into memory completely (#330)

This PR adds code to prevent large objects from being read into memory
from packfiles or the filesystem.

Objects greater than 1Mb are now no longer directly stored in the cache
or read completely into memory.

This PR differs and improves the previous broken #323 by fixing several
bugs in the reader and transparently wrapping ReaderAt as a Reader.

Signed-off-by: Andrew Thornton <art27@cantab.net>

1 files changed, 51 insertions, 0 deletions

diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go
index 98f53f6..137e485 100644
--- a/plumbing/format/packfile/delta_test.go
+++ b/plumbing/format/packfile/delta_test.go
@@ -1,8 +1,11 @@
 package packfile
 
 import (
+	"bytes"
+	"io/ioutil"
 	"math/rand"
 
+	"github.com/go-git/go-git/v5/plumbing"
 	. "gopkg.in/check.v1"
 )
 
@@ -97,6 +100,32 @@ func (s *DeltaSuite) TestAddDelta(c *C) {
 	}
 }
 
+func (s *DeltaSuite) TestAddDeltaReader(c *C) {
+	for _, t := range s.testCases {
+		baseBuf := genBytes(t.base)
+		baseObj := &plumbing.MemoryObject{}
+		baseObj.Write(baseBuf)
+
+		targetBuf := genBytes(t.target)
+
+		delta := DiffDelta(baseBuf, targetBuf)
+		deltaRC := ioutil.NopCloser(bytes.NewReader(delta))
+
+		c.Log("Executing test case:", t.description)
+
+		resultRC, err := ReaderFromDelta(baseObj, deltaRC)
+		c.Assert(err, IsNil)
+
+		result, err := ioutil.ReadAll(resultRC)
+		c.Assert(err, IsNil)
+
+		err = resultRC.Close()
+		c.Assert(err, IsNil)
+
+		c.Assert(result, DeepEquals, targetBuf)
+	}
+}
+
 func (s *DeltaSuite) TestIncompleteDelta(c *C) {
 	for _, t := range s.testCases {
 		c.Log("Incomplete delta on:", t.description)
@@ -125,3 +154,25 @@ func (s *DeltaSuite) TestMaxCopySizeDelta(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(result, DeepEquals, targetBuf)
 }
+
+func (s *DeltaSuite) TestMaxCopySizeDeltaReader(c *C) {
+	baseBuf := randBytes(maxCopySize)
+	baseObj := &plumbing.MemoryObject{}
+	baseObj.Write(baseBuf)
+
+	targetBuf := baseBuf[0:]
+	targetBuf = append(targetBuf, byte(1))
+
+	delta := DiffDelta(baseBuf, targetBuf)
+	deltaRC := ioutil.NopCloser(bytes.NewReader(delta))
+
+	resultRC, err := ReaderFromDelta(baseObj, deltaRC)
+	c.Assert(err, IsNil)
+
+	result, err := ioutil.ReadAll(resultRC)
+	c.Assert(err, IsNil)
+
+	err = resultRC.Close()
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, targetBuf)
+}