plumbing: format/packfile, prevent large objects from being read into memory completely (#303)v5.4.0

This PR adds code to prevent large objects from being read into memory from packfiles or the filesystem.

Objects greater than 1Mb are now no longer directly stored in the cache
or read completely into memory.

Signed-off-by: Andrew Thornton <art27@cantab.net>

2 files changed, 87 insertions, 1 deletions

diff --git a/storage/filesystem/dotgit/reader.go b/storage/filesystem/dotgit/reader.go
new file mode 100644
index 0000000..a82ac94
--- /dev/null
+++ b/storage/filesystem/dotgit/reader.go
@@ -0,0 +1,79 @@
+package dotgit
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/objfile"
+	"github.com/go-git/go-git/v5/utils/ioutil"
+)
+
+var _ (plumbing.EncodedObject) = &EncodedObject{}
+
+type EncodedObject struct {
+	dir *DotGit
+	h   plumbing.Hash
+	t   plumbing.ObjectType
+	sz  int64
+}
+
+func (e *EncodedObject) Hash() plumbing.Hash {
+	return e.h
+}
+
+func (e *EncodedObject) Reader() (io.ReadCloser, error) {
+	f, err := e.dir.Object(e.h)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, plumbing.ErrObjectNotFound
+		}
+
+		return nil, err
+	}
+	r, err := objfile.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+
+	t, size, err := r.Header()
+	if err != nil {
+		_ = r.Close()
+		return nil, err
+	}
+	if t != e.t {
+		_ = r.Close()
+		return nil, objfile.ErrHeader
+	}
+	if size != e.sz {
+		_ = r.Close()
+		return nil, objfile.ErrHeader
+	}
+	return ioutil.NewReadCloserWithCloser(r, f.Close), nil
+}
+
+func (e *EncodedObject) SetType(plumbing.ObjectType) {}
+
+func (e *EncodedObject) Type() plumbing.ObjectType {
+	return e.t
+}
+
+func (e *EncodedObject) Size() int64 {
+	return e.sz
+}
+
+func (e *EncodedObject) SetSize(int64) {}
+
+func (e *EncodedObject) Writer() (io.WriteCloser, error) {
+	return nil, fmt.Errorf("Not supported")
+}
+
+func NewEncodedObject(dir *DotGit, h plumbing.Hash, t plumbing.ObjectType, size int64) *EncodedObject {
+	return &EncodedObject{
+		dir: dir,
+		h:   h,
+		t:   t,
+		sz:  size,
+	}
+}
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 0c25dad..4862362 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -389,7 +389,6 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
 		return cacheObj, nil
 	}
 
-	obj = s.NewEncodedObject()
 	r, err := objfile.NewReader(f)
 	if err != nil {
 		return nil, err
@@ -402,6 +401,14 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
 		return nil, err
 	}
 
+	if size > packfile.LargeObjectThreshold {
+		obj = dotgit.NewEncodedObject(s.dir, h, t, size)
+		s.objectCache.Put(obj)
+		return obj, nil
+	}
+
+	obj = s.NewEncodedObject()
+
 	obj.SetType(t)
 	obj.SetSize(size)
 	w, err := obj.Writer()