Remote.Fetch multiple RefSpec support

10 files changed, 174 insertions, 63 deletions

diff --git a/config/config.go b/config/config.go
index b70cebc..4cf5f7c 100644
--- a/config/config.go
+++ b/config/config.go
@@ -1,9 +1,20 @@
 package config
 
-import "errors"
+import (
+	"errors"
+	"fmt"
+
+	"gopkg.in/src-d/go-git.v3/clients/common"
+)
+
+const (
+	DefaultRefSpec = "+refs/heads/*:refs/remotes/%s/*"
+)
 
 var (
-	ErrRemoteConfigNotFound = errors.New("remote config not found")
+	ErrRemoteConfigNotFound  = errors.New("remote config not found")
+	ErrRemoteConfigEmptyURL  = errors.New("remote config: empty URL")
+	ErrRemoteConfigEmptyName = errors.New("remote config: empty name")
 )
 
 type ConfigStorage interface {
@@ -16,5 +27,26 @@ type ConfigStorage interface {
 type RemoteConfig struct {
 	Name  string
 	URL   string
-	Fetch RefSpec
+	Fetch []RefSpec
+}
+
+// Validate validate the fields and set the default values
+func (c *RemoteConfig) Validate() error {
+	if c.Name == "" {
+		return ErrRemoteConfigEmptyName
+	}
+
+	if c.URL == "" {
+		return ErrRemoteConfigEmptyURL
+	}
+
+	if _, err := common.NewEndpoint(c.URL); err != nil {
+		return err
+	}
+
+	if len(c.Fetch) == 0 {
+		c.Fetch = []RefSpec{RefSpec(fmt.Sprintf(DefaultRefSpec, c.Name))}
+	}
+
+	return nil
 }
diff --git a/config/refspec.go b/config/refspec.go
index e74bf78..dae13be 100644
--- a/config/refspec.go
+++ b/config/refspec.go
@@ -112,3 +112,14 @@ func (s RefSpec) Dst(n core.ReferenceName) core.ReferenceName {
 func (s RefSpec) String() string {
 	return string(s)
 }
+
+// MatchAny returns true if any of the RefSpec match with the given ReferenceName
+func MatchAny(l []RefSpec, n core.ReferenceName) bool {
+	for _, r := range l {
+		if r.Match(n) {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/config/refspec_test.go b/config/refspec_test.go
index b0bb8f5..632f5a4 100644
--- a/config/refspec_test.go
+++ b/config/refspec_test.go
@@ -67,3 +67,13 @@ func (s *RefSpecSuite) TestRefSpecDstBlob(c *C) {
 		"refs/remotes/origin/foo",
 	)
 }
+func (s *RefSpecSuite) TestMatchAny(c *C) {
+	specs := []RefSpec{
+		"refs/heads/bar:refs/remotes/origin/foo",
+		"refs/heads/foo:refs/remotes/origin/bar",
+	}
+
+	c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/foo")), Equals, true)
+	c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/bar")), Equals, true)
+	c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/master")), Equals, false)
+}
diff --git a/core/reference.go b/core/reference.go
index 1325c7e..c33402c 100644
--- a/core/reference.go
+++ b/core/reference.go
@@ -2,6 +2,7 @@ package core
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"strings"
 )
@@ -143,6 +144,11 @@ func (r *Reference) Strings() [2]string {
 	return o
 }
 
+func (r *Reference) String() string {
+	s := r.Strings()
+	return fmt.Sprintf("%s %s", s[1], s[0])
+}
+
 // ReferenceSliceIter implements ReferenceIter. It iterates over a series of
 // references stored in a slice and yields each one in turn when Next() is
 // called.
diff --git a/options.go b/options.go
index 414b7f0..3691a6f 100644
--- a/options.go
+++ b/options.go
@@ -2,7 +2,6 @@ package git
 
 import (
 	"errors"
-	"fmt"
 
 	"gopkg.in/src-d/go-git.v3/clients/common"
 	"gopkg.in/src-d/go-git.v4/config"
@@ -11,9 +10,7 @@ import (
 
 const (
 	// DefaultRemoteName name of the default Remote, just like git command
-	DefaultRemoteName          = "origin"
-	DefaultSingleBranchRefSpec = "+refs/heads/%s:refs/remotes/%s/%[1]s"
-	DefaultRefSpec             = "+refs/heads/*:refs/remotes/%s/*"
+	DefaultRemoteName = "origin"
 )
 
 var (
@@ -54,22 +51,6 @@ func (o *RepositoryCloneOptions) Validate() error {
 	return nil
 }
 
-func (o *RepositoryCloneOptions) refSpec(s core.ReferenceStorage) (config.RefSpec, error) {
-	var spec string
-	if o.SingleBranch {
-		head, err := core.ResolveReference(s, o.ReferenceName)
-		if err != nil {
-			return "", err
-		}
-
-		spec = fmt.Sprintf(DefaultSingleBranchRefSpec, head.Name().Short(), o.RemoteName)
-	} else {
-		spec = fmt.Sprintf(DefaultRefSpec, o.RemoteName)
-	}
-
-	return config.RefSpec(spec), nil
-}
-
 // RepositoryPullOptions describe how a pull should be perform
 type RepositoryPullOptions struct {
 	// Name of the remote to be pulled
@@ -97,14 +78,16 @@ func (o *RepositoryPullOptions) Validate() error {
 
 // RemoteFetchOptions describe how a fetch should be perform
 type RemoteFetchOptions struct {
-	RefSpec config.RefSpec
-	Depth   int
+	RefSpecs []config.RefSpec
+	Depth    int
 }
 
 // Validate validate the fields and set the default values
 func (o *RemoteFetchOptions) Validate() error {
-	if !o.RefSpec.IsValid() {
-		return ErrInvalidRefSpec
+	for _, r := range o.RefSpecs {
+		if !r.IsValid() {
+			return ErrInvalidRefSpec
+		}
 	}
 
 	return nil
diff --git a/remote.go b/remote.go
index b023497..6c020ca 100644
--- a/remote.go
+++ b/remote.go
@@ -1,6 +1,7 @@
 package git
 
 import (
+	"fmt"
 	"io"
 
 	"gopkg.in/src-d/go-git.v4/clients"
@@ -24,6 +25,11 @@ func newRemote(s Storage, c *config.RemoteConfig) *Remote {
 	return &Remote{s: s, c: c}
 }
 
+// Config return the config
+func (r *Remote) Config() *config.RemoteConfig {
+	return r.c
+}
+
 // Connect with the endpoint
 func (r *Remote) Connect() error {
 	if err := r.connectUploadPackService(); err != nil {
@@ -72,7 +78,11 @@ func (r *Remote) Fetch(o *RemoteFetchOptions) (err error) {
 		return err
 	}
 
-	refs, err := r.getWantedReferences(o.RefSpec)
+	if len(o.RefSpecs) == 0 {
+		o.RefSpecs = r.c.Fetch
+	}
+
+	refs, err := r.getWantedReferences(o.RefSpecs)
 	if err != nil {
 		return err
 	}
@@ -88,14 +98,14 @@ func (r *Remote) Fetch(o *RemoteFetchOptions) (err error) {
 	}
 
 	defer checkClose(reader, &err)
-	if err := r.updateObjectStorage(r.s.ObjectStorage(), reader); err != nil {
+	if err := r.updateObjectStorage(reader); err != nil {
 		return err
 	}
 
-	return r.updateLocalReferenceStorage(r.s.ReferenceStorage(), o.RefSpec, refs)
+	return r.updateLocalReferenceStorage(o.RefSpecs, refs)
 }
 
-func (r *Remote) getWantedReferences(spec config.RefSpec) ([]*core.Reference, error) {
+func (r *Remote) getWantedReferences(spec []config.RefSpec) ([]*core.Reference, error) {
 	var refs []*core.Reference
 
 	return refs, r.Refs().ForEach(func(r *core.Reference) error {
@@ -103,7 +113,7 @@ func (r *Remote) getWantedReferences(spec config.RefSpec) ([]*core.Reference, er
 			return nil
 		}
 
-		if spec.Match(r.Name()) {
+		if config.MatchAny(spec, r.Name()) {
 			refs = append(refs, r)
 		}
 
@@ -138,29 +148,29 @@ func (r *Remote) buildRequest(
 	return req, err
 }
 
-func (r *Remote) updateObjectStorage(s core.ObjectStorage, reader io.Reader) error {
+func (r *Remote) updateObjectStorage(reader io.Reader) error {
 	stream := packfile.NewStream(reader)
 
 	d := packfile.NewDecoder(stream)
-	return d.Decode(s)
+	return d.Decode(r.s.ObjectStorage())
 }
 
-func (r *Remote) updateLocalReferenceStorage(
-	local core.ReferenceStorage, spec config.RefSpec, refs []*core.Reference,
-) error {
+func (r *Remote) updateLocalReferenceStorage(specs []config.RefSpec, refs []*core.Reference) error {
 	for _, ref := range refs {
-		if !spec.Match(ref.Name()) {
-			continue
-		}
-
-		if ref.Type() != core.HashReference {
-			continue
-		}
-
-		name := spec.Dst(ref.Name())
-		n := core.NewHashReference(name, ref.Hash())
-		if err := local.Set(n); err != nil {
-			return err
+		for _, spec := range specs {
+			if !spec.Match(ref.Name()) {
+				continue
+			}
+
+			if ref.Type() != core.HashReference {
+				continue
+			}
+
+			name := spec.Dst(ref.Name())
+			n := core.NewHashReference(name, ref.Hash())
+			if err := r.s.ReferenceStorage().Set(n); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -192,3 +202,10 @@ func (r *Remote) Disconnect() error {
 	r.upInfo = nil
 	return r.upSrv.Disconnect()
 }
+
+func (r *Remote) String() string {
+	fetch := r.c.URL
+	push := r.c.URL
+
+	return fmt.Sprintf("%s\t%s (fetch)\n%[1]s\t%s (push)", r.c.Name, fetch, push)
+}
diff --git a/remote_test.go b/remote_test.go
index bc81dad..7bf15e4 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -79,7 +79,7 @@ func (s *RemoteSuite) TestFetch(c *C) {
 	c.Assert(r.Connect(), IsNil)
 
 	err := r.Fetch(&RemoteFetchOptions{
-		RefSpec: DefaultRefSpec,
+		RefSpecs: []config.RefSpec{config.DefaultRefSpec},
 	})
 
 	c.Assert(err, IsNil)
diff --git a/repository.go b/repository.go
index da4aada..64107f1 100644
--- a/repository.go
+++ b/repository.go
@@ -2,6 +2,7 @@ package git
 
 import (
 	"errors"
+	"fmt"
 
 	"gopkg.in/src-d/go-git.v4/config"
 	"gopkg.in/src-d/go-git.v4/core"
@@ -22,8 +23,9 @@ type Repository struct {
 }
 
 // NewMemoryRepository creates a new repository, backed by a memory.Storage
-func NewMemoryRepository() (*Repository, error) {
-	return NewRepository(memory.NewStorage())
+func NewMemoryRepository() *Repository {
+	r, _ := NewRepository(memory.NewStorage())
+	return r
 }
 
 // NewFilesystemRepository creates a new repository, backed by a filesystem.Storage
@@ -73,6 +75,10 @@ func (r *Repository) Remotes() ([]*Remote, error) {
 
 // CreateRemote creates a new remote
 func (r *Repository) CreateRemote(c *config.RemoteConfig) (*Remote, error) {
+	if err := c.Validate(); err != nil {
+		return nil, err
+	}
+
 	remote := newRemote(r.s, c)
 	if err := r.s.ConfigStorage().SetRemote(c); err != nil {
 		return nil, err
@@ -92,11 +98,12 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error {
 		return err
 	}
 
-	remote, err := r.CreateRemote(&config.RemoteConfig{
+	c := &config.RemoteConfig{
 		Name: o.RemoteName,
 		URL:  o.URL,
-	})
+	}
 
+	remote, err := r.CreateRemote(c)
 	if err != nil {
 		return err
 	}
@@ -107,17 +114,11 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error {
 
 	defer remote.Disconnect()
 
-	spec, err := o.refSpec(remote.Info().Refs)
-	if err != nil {
+	if err := r.updateRemoteConfig(remote, o, c); err != nil {
 		return err
 	}
 
-	err = remote.Fetch(&RemoteFetchOptions{
-		RefSpec: spec,
-		Depth:   o.Depth,
-	})
-
-	if err != nil {
+	if err = remote.Fetch(&RemoteFetchOptions{Depth: o.Depth}); err != nil {
 		return err
 	}
 
@@ -129,6 +130,27 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error {
 	return r.createReferences(head)
 }
 
+const refspecSingleBranch = "+refs/heads/%s:refs/remotes/%s/%[1]s"
+
+func (r *Repository) updateRemoteConfig(
+	remote *Remote, o *RepositoryCloneOptions, c *config.RemoteConfig,
+) error {
+	if o.SingleBranch {
+		head, err := core.ResolveReference(remote.Info().Refs, o.ReferenceName)
+		if err != nil {
+			return err
+		}
+
+		c.Fetch = []config.RefSpec{
+			config.RefSpec(fmt.Sprintf(refspecSingleBranch, head.Name().Short(), c.Name)),
+		}
+
+		return r.s.ConfigStorage().SetRemote(c)
+	}
+
+	return nil
+}
+
 func (r *Repository) createReferences(ref *core.Reference) error {
 	if !ref.IsBranch() {
 		// detached HEAD mode
@@ -155,6 +177,12 @@ func (r *Repository) Pull(o *RepositoryPullOptions) error {
 		return err
 	}
 
+	if err = remote.Connect(); err != nil {
+		return err
+	}
+
+	defer remote.Disconnect()
+
 	head, err := remote.Ref(o.ReferenceName, true)
 	if err != nil {
 		return err
diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go
index bf7eda5..cbff1e0 100644
--- a/storage/filesystem/config_test.go
+++ b/storage/filesystem/config_test.go
@@ -18,7 +18,8 @@ func (s *ConfigSuite) TestConfigFileDecode(c *C) {
 
 	c.Assert(config.Remotes, HasLen, 2)
 	c.Assert(config.Remotes["origin"].URL, Equals, "git@github.com:src-d/go-git.git")
-	c.Assert(config.Remotes["origin"].Fetch.String(), Equals, "+refs/heads/*:refs/remotes/origin/*")
+	c.Assert(config.Remotes["origin"].Fetch, HasLen, 1)
+	c.Assert(config.Remotes["origin"].Fetch[0].String(), Equals, "+refs/heads/*:refs/remotes/origin/*")
 }
 
 var configFixture = []byte(`
diff --git a/tree_walker.go b/tree_walker.go
index 5568e1b..1692c2f 100644
--- a/tree_walker.go
+++ b/tree_walker.go
@@ -3,6 +3,8 @@ package git
 import (
 	"io"
 	"path"
+
+	"gopkg.in/src-d/go-git.v4/core"
 )
 
 const (
@@ -91,6 +93,27 @@ func (w *TreeWalker) Next() (name string, entry TreeEntry, obj Object, err error
 	return
 }
 
+func (w *TreeWalker) ForEach(cb func(fullpath string, e TreeEntry) error) error {
+	for {
+		path, e, _, err := w.Next()
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return err
+		}
+
+		if err := cb(path, e); err != nil {
+			if err == core.ErrStop {
+				return nil
+			}
+
+			return err
+		}
+	}
+}
+
 // Tree returns the tree that the tree walker most recently operated on.
 func (w *TreeWalker) Tree() *Tree {
 	current := len(w.stack) - 1