From f42d82364c5d159f65a48e720433ad2bc97f0b7f Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Mon, 22 Aug 2016 00:18:02 +0200 Subject: Remote.Fetch multiple RefSpec support --- config/config.go | 38 ++++++++++++++++++++++-- config/refspec.go | 11 +++++++ config/refspec_test.go | 10 +++++++ core/reference.go | 6 ++++ options.go | 31 +++++--------------- remote.go | 61 +++++++++++++++++++++++++-------------- remote_test.go | 2 +- repository.go | 52 +++++++++++++++++++++++++-------- storage/filesystem/config_test.go | 3 +- tree_walker.go | 23 +++++++++++++++ 10 files changed, 174 insertions(+), 63 deletions(-) diff --git a/config/config.go b/config/config.go index b70cebc..4cf5f7c 100644 --- a/config/config.go +++ b/config/config.go @@ -1,9 +1,20 @@ package config -import "errors" +import ( + "errors" + "fmt" + + "gopkg.in/src-d/go-git.v3/clients/common" +) + +const ( + DefaultRefSpec = "+refs/heads/*:refs/remotes/%s/*" +) var ( - ErrRemoteConfigNotFound = errors.New("remote config not found") + ErrRemoteConfigNotFound = errors.New("remote config not found") + ErrRemoteConfigEmptyURL = errors.New("remote config: empty URL") + ErrRemoteConfigEmptyName = errors.New("remote config: empty name") ) type ConfigStorage interface { @@ -16,5 +27,26 @@ type ConfigStorage interface { type RemoteConfig struct { Name string URL string - Fetch RefSpec + Fetch []RefSpec +} + +// Validate validate the fields and set the default values +func (c *RemoteConfig) Validate() error { + if c.Name == "" { + return ErrRemoteConfigEmptyName + } + + if c.URL == "" { + return ErrRemoteConfigEmptyURL + } + + if _, err := common.NewEndpoint(c.URL); err != nil { + return err + } + + if len(c.Fetch) == 0 { + c.Fetch = []RefSpec{RefSpec(fmt.Sprintf(DefaultRefSpec, c.Name))} + } + + return nil } diff --git a/config/refspec.go b/config/refspec.go index e74bf78..dae13be 100644 --- a/config/refspec.go +++ b/config/refspec.go @@ -112,3 +112,14 @@ func (s RefSpec) Dst(n core.ReferenceName) core.ReferenceName { func (s RefSpec) String() string { return string(s) } + +// MatchAny returns true if any of the RefSpec match with the given ReferenceName +func MatchAny(l []RefSpec, n core.ReferenceName) bool { + for _, r := range l { + if r.Match(n) { + return true + } + } + + return false +} diff --git a/config/refspec_test.go b/config/refspec_test.go index b0bb8f5..632f5a4 100644 --- a/config/refspec_test.go +++ b/config/refspec_test.go @@ -67,3 +67,13 @@ func (s *RefSpecSuite) TestRefSpecDstBlob(c *C) { "refs/remotes/origin/foo", ) } +func (s *RefSpecSuite) TestMatchAny(c *C) { + specs := []RefSpec{ + "refs/heads/bar:refs/remotes/origin/foo", + "refs/heads/foo:refs/remotes/origin/bar", + } + + c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/foo")), Equals, true) + c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/bar")), Equals, true) + c.Assert(MatchAny(specs, core.ReferenceName("refs/heads/master")), Equals, false) +} diff --git a/core/reference.go b/core/reference.go index 1325c7e..c33402c 100644 --- a/core/reference.go +++ b/core/reference.go @@ -2,6 +2,7 @@ package core import ( "errors" + "fmt" "io" "strings" ) @@ -143,6 +144,11 @@ func (r *Reference) Strings() [2]string { return o } +func (r *Reference) String() string { + s := r.Strings() + return fmt.Sprintf("%s %s", s[1], s[0]) +} + // ReferenceSliceIter implements ReferenceIter. It iterates over a series of // references stored in a slice and yields each one in turn when Next() is // called. diff --git a/options.go b/options.go index 414b7f0..3691a6f 100644 --- a/options.go +++ b/options.go @@ -2,7 +2,6 @@ package git import ( "errors" - "fmt" "gopkg.in/src-d/go-git.v3/clients/common" "gopkg.in/src-d/go-git.v4/config" @@ -11,9 +10,7 @@ import ( const ( // DefaultRemoteName name of the default Remote, just like git command - DefaultRemoteName = "origin" - DefaultSingleBranchRefSpec = "+refs/heads/%s:refs/remotes/%s/%[1]s" - DefaultRefSpec = "+refs/heads/*:refs/remotes/%s/*" + DefaultRemoteName = "origin" ) var ( @@ -54,22 +51,6 @@ func (o *RepositoryCloneOptions) Validate() error { return nil } -func (o *RepositoryCloneOptions) refSpec(s core.ReferenceStorage) (config.RefSpec, error) { - var spec string - if o.SingleBranch { - head, err := core.ResolveReference(s, o.ReferenceName) - if err != nil { - return "", err - } - - spec = fmt.Sprintf(DefaultSingleBranchRefSpec, head.Name().Short(), o.RemoteName) - } else { - spec = fmt.Sprintf(DefaultRefSpec, o.RemoteName) - } - - return config.RefSpec(spec), nil -} - // RepositoryPullOptions describe how a pull should be perform type RepositoryPullOptions struct { // Name of the remote to be pulled @@ -97,14 +78,16 @@ func (o *RepositoryPullOptions) Validate() error { // RemoteFetchOptions describe how a fetch should be perform type RemoteFetchOptions struct { - RefSpec config.RefSpec - Depth int + RefSpecs []config.RefSpec + Depth int } // Validate validate the fields and set the default values func (o *RemoteFetchOptions) Validate() error { - if !o.RefSpec.IsValid() { - return ErrInvalidRefSpec + for _, r := range o.RefSpecs { + if !r.IsValid() { + return ErrInvalidRefSpec + } } return nil diff --git a/remote.go b/remote.go index b023497..6c020ca 100644 --- a/remote.go +++ b/remote.go @@ -1,6 +1,7 @@ package git import ( + "fmt" "io" "gopkg.in/src-d/go-git.v4/clients" @@ -24,6 +25,11 @@ func newRemote(s Storage, c *config.RemoteConfig) *Remote { return &Remote{s: s, c: c} } +// Config return the config +func (r *Remote) Config() *config.RemoteConfig { + return r.c +} + // Connect with the endpoint func (r *Remote) Connect() error { if err := r.connectUploadPackService(); err != nil { @@ -72,7 +78,11 @@ func (r *Remote) Fetch(o *RemoteFetchOptions) (err error) { return err } - refs, err := r.getWantedReferences(o.RefSpec) + if len(o.RefSpecs) == 0 { + o.RefSpecs = r.c.Fetch + } + + refs, err := r.getWantedReferences(o.RefSpecs) if err != nil { return err } @@ -88,14 +98,14 @@ func (r *Remote) Fetch(o *RemoteFetchOptions) (err error) { } defer checkClose(reader, &err) - if err := r.updateObjectStorage(r.s.ObjectStorage(), reader); err != nil { + if err := r.updateObjectStorage(reader); err != nil { return err } - return r.updateLocalReferenceStorage(r.s.ReferenceStorage(), o.RefSpec, refs) + return r.updateLocalReferenceStorage(o.RefSpecs, refs) } -func (r *Remote) getWantedReferences(spec config.RefSpec) ([]*core.Reference, error) { +func (r *Remote) getWantedReferences(spec []config.RefSpec) ([]*core.Reference, error) { var refs []*core.Reference return refs, r.Refs().ForEach(func(r *core.Reference) error { @@ -103,7 +113,7 @@ func (r *Remote) getWantedReferences(spec config.RefSpec) ([]*core.Reference, er return nil } - if spec.Match(r.Name()) { + if config.MatchAny(spec, r.Name()) { refs = append(refs, r) } @@ -138,29 +148,29 @@ func (r *Remote) buildRequest( return req, err } -func (r *Remote) updateObjectStorage(s core.ObjectStorage, reader io.Reader) error { +func (r *Remote) updateObjectStorage(reader io.Reader) error { stream := packfile.NewStream(reader) d := packfile.NewDecoder(stream) - return d.Decode(s) + return d.Decode(r.s.ObjectStorage()) } -func (r *Remote) updateLocalReferenceStorage( - local core.ReferenceStorage, spec config.RefSpec, refs []*core.Reference, -) error { +func (r *Remote) updateLocalReferenceStorage(specs []config.RefSpec, refs []*core.Reference) error { for _, ref := range refs { - if !spec.Match(ref.Name()) { - continue - } - - if ref.Type() != core.HashReference { - continue - } - - name := spec.Dst(ref.Name()) - n := core.NewHashReference(name, ref.Hash()) - if err := local.Set(n); err != nil { - return err + for _, spec := range specs { + if !spec.Match(ref.Name()) { + continue + } + + if ref.Type() != core.HashReference { + continue + } + + name := spec.Dst(ref.Name()) + n := core.NewHashReference(name, ref.Hash()) + if err := r.s.ReferenceStorage().Set(n); err != nil { + return err + } } } @@ -192,3 +202,10 @@ func (r *Remote) Disconnect() error { r.upInfo = nil return r.upSrv.Disconnect() } + +func (r *Remote) String() string { + fetch := r.c.URL + push := r.c.URL + + return fmt.Sprintf("%s\t%s (fetch)\n%[1]s\t%s (push)", r.c.Name, fetch, push) +} diff --git a/remote_test.go b/remote_test.go index bc81dad..7bf15e4 100644 --- a/remote_test.go +++ b/remote_test.go @@ -79,7 +79,7 @@ func (s *RemoteSuite) TestFetch(c *C) { c.Assert(r.Connect(), IsNil) err := r.Fetch(&RemoteFetchOptions{ - RefSpec: DefaultRefSpec, + RefSpecs: []config.RefSpec{config.DefaultRefSpec}, }) c.Assert(err, IsNil) diff --git a/repository.go b/repository.go index da4aada..64107f1 100644 --- a/repository.go +++ b/repository.go @@ -2,6 +2,7 @@ package git import ( "errors" + "fmt" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/core" @@ -22,8 +23,9 @@ type Repository struct { } // NewMemoryRepository creates a new repository, backed by a memory.Storage -func NewMemoryRepository() (*Repository, error) { - return NewRepository(memory.NewStorage()) +func NewMemoryRepository() *Repository { + r, _ := NewRepository(memory.NewStorage()) + return r } // NewFilesystemRepository creates a new repository, backed by a filesystem.Storage @@ -73,6 +75,10 @@ func (r *Repository) Remotes() ([]*Remote, error) { // CreateRemote creates a new remote func (r *Repository) CreateRemote(c *config.RemoteConfig) (*Remote, error) { + if err := c.Validate(); err != nil { + return nil, err + } + remote := newRemote(r.s, c) if err := r.s.ConfigStorage().SetRemote(c); err != nil { return nil, err @@ -92,11 +98,12 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error { return err } - remote, err := r.CreateRemote(&config.RemoteConfig{ + c := &config.RemoteConfig{ Name: o.RemoteName, URL: o.URL, - }) + } + remote, err := r.CreateRemote(c) if err != nil { return err } @@ -107,17 +114,11 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error { defer remote.Disconnect() - spec, err := o.refSpec(remote.Info().Refs) - if err != nil { + if err := r.updateRemoteConfig(remote, o, c); err != nil { return err } - err = remote.Fetch(&RemoteFetchOptions{ - RefSpec: spec, - Depth: o.Depth, - }) - - if err != nil { + if err = remote.Fetch(&RemoteFetchOptions{Depth: o.Depth}); err != nil { return err } @@ -129,6 +130,27 @@ func (r *Repository) Clone(o *RepositoryCloneOptions) error { return r.createReferences(head) } +const refspecSingleBranch = "+refs/heads/%s:refs/remotes/%s/%[1]s" + +func (r *Repository) updateRemoteConfig( + remote *Remote, o *RepositoryCloneOptions, c *config.RemoteConfig, +) error { + if o.SingleBranch { + head, err := core.ResolveReference(remote.Info().Refs, o.ReferenceName) + if err != nil { + return err + } + + c.Fetch = []config.RefSpec{ + config.RefSpec(fmt.Sprintf(refspecSingleBranch, head.Name().Short(), c.Name)), + } + + return r.s.ConfigStorage().SetRemote(c) + } + + return nil +} + func (r *Repository) createReferences(ref *core.Reference) error { if !ref.IsBranch() { // detached HEAD mode @@ -155,6 +177,12 @@ func (r *Repository) Pull(o *RepositoryPullOptions) error { return err } + if err = remote.Connect(); err != nil { + return err + } + + defer remote.Disconnect() + head, err := remote.Ref(o.ReferenceName, true) if err != nil { return err diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go index bf7eda5..cbff1e0 100644 --- a/storage/filesystem/config_test.go +++ b/storage/filesystem/config_test.go @@ -18,7 +18,8 @@ func (s *ConfigSuite) TestConfigFileDecode(c *C) { c.Assert(config.Remotes, HasLen, 2) c.Assert(config.Remotes["origin"].URL, Equals, "git@github.com:src-d/go-git.git") - c.Assert(config.Remotes["origin"].Fetch.String(), Equals, "+refs/heads/*:refs/remotes/origin/*") + c.Assert(config.Remotes["origin"].Fetch, HasLen, 1) + c.Assert(config.Remotes["origin"].Fetch[0].String(), Equals, "+refs/heads/*:refs/remotes/origin/*") } var configFixture = []byte(` diff --git a/tree_walker.go b/tree_walker.go index 5568e1b..1692c2f 100644 --- a/tree_walker.go +++ b/tree_walker.go @@ -3,6 +3,8 @@ package git import ( "io" "path" + + "gopkg.in/src-d/go-git.v4/core" ) const ( @@ -91,6 +93,27 @@ func (w *TreeWalker) Next() (name string, entry TreeEntry, obj Object, err error return } +func (w *TreeWalker) ForEach(cb func(fullpath string, e TreeEntry) error) error { + for { + path, e, _, err := w.Next() + if err != nil { + if err == io.EOF { + return nil + } + + return err + } + + if err := cb(path, e); err != nil { + if err == core.ErrStop { + return nil + } + + return err + } + } +} + // Tree returns the tree that the tree walker most recently operated on. func (w *TreeWalker) Tree() *Tree { current := len(w.stack) - 1 -- cgit