plumbing: format/packfile, prevent large objects from being read into memory completely (#330)

This PR adds code to prevent large objects from being read into memory
from packfiles or the filesystem.

Objects greater than 1Mb are now no longer directly stored in the cache
or read completely into memory.

This PR differs and improves the previous broken #323 by fixing several
bugs in the reader and transparently wrapping ReaderAt as a Reader.

Signed-off-by: Andrew Thornton <art27@cantab.net>

1 files changed, 40 insertions, 0 deletions

diff --git a/utils/ioutil/common.go b/utils/ioutil/common.go
index b52e85a..b0ace4e 100644
--- a/utils/ioutil/common.go
+++ b/utils/ioutil/common.go
@@ -55,6 +55,28 @@ func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser {
 	return &readCloser{Reader: r, closer: c}
 }
 
+type readCloserCloser struct {
+	io.ReadCloser
+	closer func() error
+}
+
+func (r *readCloserCloser) Close() (err error) {
+	defer func() {
+		if err == nil {
+			err = r.closer()
+			return
+		}
+		_ = r.closer()
+	}()
+	return r.ReadCloser.Close()
+}
+
+// NewReadCloserWithCloser creates an `io.ReadCloser` with the given `io.ReaderCloser` and
+// `io.Closer` that ensures that the closer is closed on close
+func NewReadCloserWithCloser(r io.ReadCloser, c func() error) io.ReadCloser {
+	return &readCloserCloser{ReadCloser: r, closer: c}
+}
+
 type writeCloser struct {
 	io.Writer
 	closer io.Closer
@@ -82,6 +104,24 @@ func WriteNopCloser(w io.Writer) io.WriteCloser {
 	return writeNopCloser{w}
 }
 
+type readerAtAsReader struct {
+	io.ReaderAt
+	offset int64
+}
+
+func (r *readerAtAsReader) Read(bs []byte) (int, error) {
+	n, err := r.ReaderAt.ReadAt(bs, r.offset)
+	r.offset += int64(n)
+	return n, err
+}
+
+func NewReaderUsingReaderAt(r io.ReaderAt, offset int64) io.Reader {
+	return &readerAtAsReader{
+		ReaderAt: r,
+		offset:   offset,
+	}
+}
+
 // CheckClose calls Close on the given io.Closer. If the given *error points to
 // nil, it will be assigned the error returned by Close. Otherwise, any error
 // returned by Close will be ignored. CheckClose is usually called with defer.