storage: shallow storage (#180)

* storage: shallow storage

* changes

* changes

11 files changed, 240 insertions, 1 deletions

diff --git a/common.go b/common.go
index 9c642ed..ef1db3b 100644
--- a/common.go
+++ b/common.go
@@ -14,6 +14,7 @@ import (
 type Storer interface {
 	storer.EncodedObjectStorer
 	storer.ReferenceStorer
+	storer.ShallowStorer
 	config.ConfigStorer
 }
 
diff --git a/examples/storage/aerospike/storage.go b/examples/storage/aerospike/storage.go
index 87d4a29..3ce5f5b 100644
--- a/examples/storage/aerospike/storage.go
+++ b/examples/storage/aerospike/storage.go
@@ -248,6 +248,44 @@ func (s *Storage) buildConfigKey() (*driver.Key, error) {
 	return driver.NewKey(s.ns, configSet, fmt.Sprintf("%s|config", s.url))
 }
 
+func (s *Storage) Shallow() ([]plumbing.Hash, error) {
+	key, err := s.buildShallowKey()
+	if err != nil {
+		return nil, err
+	}
+
+	rec, err := s.client.Get(nil, key)
+	if err != nil {
+		return nil, err
+	}
+
+	var h []plumbing.Hash
+	return h, json.Unmarshal(rec.Bins["blob"].([]byte), h)
+}
+
+func (s *Storage) SetShallow(hash []plumbing.Hash) error {
+	key, err := s.buildShallowKey()
+	if err != nil {
+		return err
+	}
+
+	json, err := json.Marshal(hash)
+	if err != nil {
+		return err
+	}
+
+	bins := driver.BinMap{
+		urlField: s.url,
+		"blob":   json,
+	}
+
+	return s.client.Put(nil, key, bins)
+}
+
+func (s *Storage) buildShallowKey() (*driver.Key, error) {
+	return driver.NewKey(s.ns, configSet, fmt.Sprintf("%s|config", s.url))
+}
+
 func createIndexes(c *driver.Client, ns string) error {
 	for _, set := range [...]string{
 		referencesSet,
diff --git a/plumbing/storer/shallow.go b/plumbing/storer/shallow.go
new file mode 100644
index 0000000..4366135
--- /dev/null
+++ b/plumbing/storer/shallow.go
@@ -0,0 +1,10 @@
+package storer
+
+import "gopkg.in/src-d/go-git.v4/plumbing"
+
+// ShallowStorer storage of references to shallow commits by hash, meaning that
+// these commits have missing parents because of a shallow fetch.
+type ShallowStorer interface {
+	SetShallow([]plumbing.Hash) error
+	Shallow() ([]plumbing.Hash, error)
+}
diff --git a/remote.go b/remote.go
index 1d3291b..5d1a8c0 100644
--- a/remote.go
+++ b/remote.go
@@ -129,6 +129,10 @@ func (r *Remote) Fetch(o *FetchOptions) (err error) {
 
 	defer ioutil.CheckClose(reader, &err)
 
+	if err := r.updateShallow(o, reader); err != nil {
+		return err
+	}
+
 	if err = r.updateObjectStorage(
 		r.buildSidebandIfSupported(req.Capabilities, reader),
 	); err != nil {
@@ -295,6 +299,14 @@ func (r *Remote) buildFetchedTags() error {
 	})
 }
 
+func (r *Remote) updateShallow(o *FetchOptions, resp *packp.UploadPackResponse) error {
+	if o.Depth == 0 {
+		return nil
+	}
+
+	return r.s.SetShallow(resp.Shallows)
+}
+
 // Head returns the Reference of the HEAD
 func (r *Remote) Head() *plumbing.Reference {
 	ref, err := storer.ResolveReference(r.refs, plumbing.HEAD)
diff --git a/remote_test.go b/remote_test.go
index 8296f8e..5af2e2c 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -124,6 +124,14 @@ func (s *RemoteSuite) TestFetchDepth(c *C) {
 		r, _ := sto.Reference(exp.Name())
 		c.Assert(exp.String(), Equals, r.String())
 	}
+
+	h, err := sto.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(h, HasLen, 2)
+	c.Assert(h, DeepEquals, []plumbing.Hash{
+		plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"),
+		plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
+	})
 }
 
 func (s *RemoteSuite) TestFetchWithProgress(c *C) {
diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go
index 3a6227e..3405908 100644
--- a/storage/filesystem/internal/dotgit/dotgit.go
+++ b/storage/filesystem/internal/dotgit/dotgit.go
@@ -17,6 +17,7 @@ const (
 	suffix         = ".git"
 	packedRefsPath = "packed-refs"
 	configPath     = "config"
+	shallowPath    = "shallow"
 
 	objectsPath = "objects"
 	packPath    = "pack"
@@ -60,15 +61,35 @@ func New(fs fs.Filesystem) *DotGit {
 	return &DotGit{fs: fs}
 }
 
+// ConfigWriter returns a file pointer for write to the config file
 func (d *DotGit) ConfigWriter() (fs.File, error) {
 	return d.fs.Create(configPath)
 }
 
-// Config returns the path of the config file
+// Config returns a file pointer for read to the config file
 func (d *DotGit) Config() (fs.File, error) {
 	return d.fs.Open(configPath)
 }
 
+// ShallowWriter returns a file pointer for write to the shallow file
+func (d *DotGit) ShallowWriter() (fs.File, error) {
+	return d.fs.Create(shallowPath)
+}
+
+// Shallow returns a file pointer for read to the shallow file
+func (d *DotGit) Shallow() (fs.File, error) {
+	f, err := d.fs.Open(shallowPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+
+		return nil, err
+	}
+
+	return f, nil
+}
+
 // NewObjectPack return a writer for a new packfile, it saves the packfile to
 // disk and also generates and save the index for the given packfile.
 func (d *DotGit) NewObjectPack() (*PackWriter, error) {
diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go
index 3a1f194..b645a85 100644
--- a/storage/filesystem/internal/dotgit/dotgit_test.go
+++ b/storage/filesystem/internal/dotgit/dotgit_test.go
@@ -130,6 +130,61 @@ func (s *SuiteDotGit) TestConfig(c *C) {
 	c.Assert(filepath.Base(file.Filename()), Equals, "config")
 }
 
+func (s *SuiteDotGit) TestConfigWriteAndConfig(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+
+	f, err := dir.ConfigWriter()
+	c.Assert(err, IsNil)
+
+	_, err = f.Write([]byte("foo"))
+	c.Assert(err, IsNil)
+
+	f, err = dir.Config()
+	c.Assert(err, IsNil)
+
+	cnt, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+
+	c.Assert(string(cnt), Equals, "foo")
+}
+
+func (s *SuiteDotGit) TestShallow(c *C) {
+	fs := fixtures.Basic().ByTag(".git").One().DotGit()
+	dir := New(fs)
+
+	file, err := dir.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(file, IsNil)
+}
+
+func (s *SuiteDotGit) TestShallowWriteAndShallow(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+
+	f, err := dir.ShallowWriter()
+	c.Assert(err, IsNil)
+
+	_, err = f.Write([]byte("foo"))
+	c.Assert(err, IsNil)
+
+	f, err = dir.Shallow()
+	c.Assert(err, IsNil)
+
+	cnt, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+
+	c.Assert(string(cnt), Equals, "foo")
+}
+
 func findReference(refs []*plumbing.Reference, name string) *plumbing.Reference {
 	n := plumbing.ReferenceName(name)
 	for _, ref := range refs {
@@ -222,6 +277,18 @@ func (s *SuiteDotGit) TestObjects(c *C) {
 	c.Assert(hashes[2].String(), Equals, "03db8e1fbe133a480f2867aac478fd866686d69e")
 }
 
+func (s *SuiteDotGit) TestObjectsNoFolder(c *C) {
+	tmp, err := ioutil.TempDir("", "dot-git")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(tmp)
+
+	fs := osfs.New(tmp)
+	dir := New(fs)
+	hash, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hash, HasLen, 0)
+}
+
 func (s *SuiteDotGit) TestObject(c *C) {
 	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
 	dir := New(fs)
diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go
new file mode 100644
index 0000000..ec8d20e
--- /dev/null
+++ b/storage/filesystem/shallow.go
@@ -0,0 +1,51 @@
+package filesystem
+
+import (
+	"bufio"
+	"fmt"
+
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+)
+
+// ShallowStorage where the shallow commits are stored, an internal to
+// manipulate the shallow file
+type ShallowStorage struct {
+	dir *dotgit.DotGit
+}
+
+// SetShallow save the shallows in the shallow file in the .git folder as one
+// commit per line represented by 40-byte hexadecimal object terminated by a
+// newline.
+func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
+	f, err := s.dir.ShallowWriter()
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+	for _, h := range commits {
+		if _, err := fmt.Fprintf(f, "%s\n", h); err != err {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Shallow return the shallow commits reading from shallo file from .git
+func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) {
+	f, err := s.dir.Shallow()
+	if err != nil {
+		return nil, err
+	}
+
+	var hash []plumbing.Hash
+
+	scn := bufio.NewScanner(f)
+	for scn.Scan() {
+		hash = append(hash, plumbing.NewHash(scn.Text()))
+	}
+
+	return hash, scn.Err()
+}
diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go
index 7e05cd3..e414428 100644
--- a/storage/filesystem/storage.go
+++ b/storage/filesystem/storage.go
@@ -12,6 +12,7 @@ import (
 type Storage struct {
 	ObjectStorage
 	ReferenceStorage
+	ShallowStorage
 	ConfigStorage
 }
 
@@ -26,6 +27,7 @@ func NewStorage(fs fs.Filesystem) (*Storage, error) {
 	return &Storage{
 		ObjectStorage:    o,
 		ReferenceStorage: ReferenceStorage{dir: dir},
+		ShallowStorage:   ShallowStorage{dir: dir},
 		ConfigStorage:    ConfigStorage{dir: dir},
 	}, nil
 }
diff --git a/storage/memory/storage.go b/storage/memory/storage.go
index a912e94..ecf2efa 100644
--- a/storage/memory/storage.go
+++ b/storage/memory/storage.go
@@ -18,6 +18,7 @@ var ErrUnsupportedObjectType = fmt.Errorf("unsupported object type")
 type Storage struct {
 	ConfigStorage
 	ObjectStorage
+	ShallowStorage
 	ReferenceStorage
 }
 
@@ -26,6 +27,7 @@ func NewStorage() *Storage {
 	return &Storage{
 		ReferenceStorage: make(ReferenceStorage, 0),
 		ConfigStorage:    ConfigStorage{},
+		ShallowStorage:   ShallowStorage{},
 		ObjectStorage: ObjectStorage{
 			Objects: make(map[plumbing.Hash]plumbing.EncodedObject, 0),
 			Commits: make(map[plumbing.Hash]plumbing.EncodedObject, 0),
@@ -195,3 +197,14 @@ func (r ReferenceStorage) IterReferences() (storer.ReferenceIter, error) {
 
 	return storer.NewReferenceSliceIter(refs), nil
 }
+
+type ShallowStorage []plumbing.Hash
+
+func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
+	*s = commits
+	return nil
+}
+
+func (s ShallowStorage) Shallow() ([]plumbing.Hash, error) {
+	return s, nil
+}
diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go
index 9d63ff8..bf0d10d 100644
--- a/storage/test/storage_suite.go
+++ b/storage/test/storage_suite.go
@@ -17,6 +17,7 @@ import (
 type Storer interface {
 	storer.EncodedObjectStorer
 	storer.ReferenceStorer
+	storer.ShallowStorer
 	config.ConfigStorer
 }
 
@@ -263,6 +264,21 @@ func (s *BaseStorageSuite) TestIterReferences(c *C) {
 	c.Assert(err, Equals, io.EOF)
 }
 
+func (s *BaseStorageSuite) TestSetShallowAndShallow(c *C) {
+	expected := []plumbing.Hash{
+		plumbing.NewHash("b66c08ba28aa1f81eb06a1127aa3936ff77e5e2c"),
+		plumbing.NewHash("c3f4688a08fd86f1bf8e055724c84b7a40a09733"),
+		plumbing.NewHash("c78874f116be67ecf54df225a613162b84cc6ebf"),
+	}
+
+	err := s.Storer.SetShallow(expected)
+	c.Assert(err, IsNil)
+
+	result, err := s.Storer.Shallow()
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, expected)
+}
+
 func (s *BaseStorageSuite) TestSetConfigAndConfig(c *C) {
 	expected := config.NewConfig()
 	expected.Core.IsBare = true