diff options
author | Máximo Cuadros <mcuadros@gmail.com> | 2016-12-07 11:16:59 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-12-07 11:16:59 +0100 |
commit | 01ea726be6ed745a21c88fd61fc5e2a62c4ceb88 (patch) | |
tree | 8e0cbfe502afb3187d93e8da95cc1677794c6df5 | |
parent | 5990aeb7198a4961a363eeb422a3faa57c8dc029 (diff) | |
download | go-git-01ea726be6ed745a21c88fd61fc5e2a62c4ceb88.tar.gz |
remote: sideband support (#164)
* remote: sideband support
* changes
-rw-r--r-- | plumbing/protocol/packp/sideband/demux.go | 36 | ||||
-rw-r--r-- | plumbing/protocol/packp/sideband/demux_test.go | 3 | ||||
-rw-r--r-- | plumbing/protocol/packp/ulreq.go | 6 | ||||
-rw-r--r-- | plumbing/protocol/packp/ulreq_test.go | 2 | ||||
-rw-r--r-- | plumbing/transport/common.go | 2 | ||||
-rw-r--r-- | plumbing/transport/http/fetch_pack.go | 47 | ||||
-rw-r--r-- | plumbing/transport/http/fetch_pack_test.go | 4 | ||||
-rw-r--r-- | remote.go | 33 | ||||
-rw-r--r-- | remote_test.go | 49 | ||||
-rw-r--r-- | repository.go | 12 | ||||
-rw-r--r-- | repository_test.go | 21 |
11 files changed, 139 insertions, 76 deletions
diff --git a/plumbing/protocol/packp/sideband/demux.go b/plumbing/protocol/packp/sideband/demux.go index 1669ed4..6470380 100644 --- a/plumbing/protocol/packp/sideband/demux.go +++ b/plumbing/protocol/packp/sideband/demux.go @@ -1,7 +1,6 @@ package sideband import ( - "bytes" "errors" "fmt" "io" @@ -15,21 +14,22 @@ var ErrMaxPackedExceeded = errors.New("max. packed size exceeded") // Progress allows to read the progress information type Progress interface { io.Reader + io.Writer } // Demuxer demultiplex the progress reports and error info interleaved with the // packfile itself. // -// A sideband has three different channels the main one call PackData contains +// A sideband has three different channels the main one, call PackData, contains // the packfile data, the ErrorMessage channel, that contains server errors and // the last one ProgressMessage channel containing information about the ongoing -// task happening in the server (optinal, can be suppressed sending NoProgress +// task happening in the server (optional, can be suppressed sending NoProgress // or Quiet capabilities to the server) // // In order to demultiplex the data stream, method `Read` should be called to // retrieve the PackData channel, the incoming data from the ProgressMessage is -// stored and can be read from `Progress` field, if any message is retrieved -// from the ErrorMessage channel an error is returned and we can assume that the +// written at `Progress` (if any), if any message is retrieved from the +// ErrorMessage channel an error is returned and we can assume that the // connection has been closed. type Demuxer struct { t Type @@ -39,7 +39,7 @@ type Demuxer struct { max int pending []byte - // Progress contains progress information + // Progress is where the progress messages are stored Progress Progress } @@ -51,21 +51,19 @@ func NewDemuxer(t Type, r io.Reader) *Demuxer { } return &Demuxer{ - t: t, - r: r, - max: max, - s: pktline.NewScanner(r), - Progress: bytes.NewBuffer(nil), + t: t, + r: r, + max: max, + s: pktline.NewScanner(r), } } // Read reads up to len(p) bytes from the PackData channel into p, an error can -// be return if an error happends when reading or if a message is sent in the +// be return if an error happens when reading or if a message is sent in the // ErrorMessage channel. // -// If a ProgressMessage is read, it won't be copied to b. Instead of this, it is -// stored and can be read through the reader Progress. If the n value returned -// is zero, err will be nil unless an error reading happens. +// When a ProgressMessage is read, is not copy to b, instead of this is written +// to the Progress func (d *Demuxer) Read(b []byte) (n int, err error) { var read, req int @@ -126,13 +124,17 @@ func (d *Demuxer) nextPackData() ([]byte, error) { case PackData: return content[1:], nil case ProgressMessage: - _, err := d.Progress.(io.Writer).Write(content[1:]) - return nil, err + if d.Progress != nil { + _, err := d.Progress.Write(content[1:]) + return nil, err + } case ErrorMessage: return nil, fmt.Errorf("unexpected error: %s", content[1:]) default: return nil, fmt.Errorf("unknown channel %s", content) } + + return nil, nil } func (d *Demuxer) getPending() (b []byte) { diff --git a/plumbing/protocol/packp/sideband/demux_test.go b/plumbing/protocol/packp/sideband/demux_test.go index 4814d89..ee5f19a 100644 --- a/plumbing/protocol/packp/sideband/demux_test.go +++ b/plumbing/protocol/packp/sideband/demux_test.go @@ -24,6 +24,7 @@ func (s *SidebandSuite) TestDecode(c *C) { buf := bytes.NewBuffer(nil) e := pktline.NewEncoder(buf) e.Encode(PackData.WithPayload(expected[0:8])) + e.Encode(ProgressMessage.WithPayload([]byte{'F', 'O', 'O', '\n'})) e.Encode(PackData.WithPayload(expected[8:16])) e.Encode(PackData.WithPayload(expected[16:26])) @@ -92,6 +93,8 @@ func (s *SidebandSuite) TestDecodeWithProgress(c *C) { content := make([]byte, 26) d := NewDemuxer(Sideband64k, buf) + d.Progress = bytes.NewBuffer(nil) + n, err := io.ReadFull(d, content) c.Assert(err, IsNil) c.Assert(n, Equals, 26) diff --git a/plumbing/protocol/packp/ulreq.go b/plumbing/protocol/packp/ulreq.go index d57c3fc..faf0885 100644 --- a/plumbing/protocol/packp/ulreq.go +++ b/plumbing/protocol/packp/ulreq.go @@ -82,6 +82,12 @@ func NewUploadRequestFromCapabilities(adv *capability.List) *UploadRequest { r.Capabilities.Set(capability.MultiACK) } + if adv.Supports(capability.Sideband64k) { + r.Capabilities.Set(capability.Sideband64k) + } else if adv.Supports(capability.Sideband) { + r.Capabilities.Set(capability.Sideband) + } + if adv.Supports(capability.ThinPack) { r.Capabilities.Set(capability.ThinPack) } diff --git a/plumbing/protocol/packp/ulreq_test.go b/plumbing/protocol/packp/ulreq_test.go index 177cf0e..f000b91 100644 --- a/plumbing/protocol/packp/ulreq_test.go +++ b/plumbing/protocol/packp/ulreq_test.go @@ -29,7 +29,7 @@ func (s *UlReqSuite) TestNewUploadRequestFromCapabilities(c *C) { r := NewUploadRequestFromCapabilities(cap) c.Assert(r.Capabilities.String(), Equals, - "multi_ack_detailed thin-pack ofs-delta agent=go-git/4.x", + "multi_ack_detailed side-band-64k thin-pack ofs-delta agent=go-git/4.x", ) } diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index 73b7d27..a20fa14 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -131,8 +131,6 @@ func transformSCPLikeIfNeeded(endpoint string) string { var UnsupportedCapabilities = []capability.Capability{ capability.MultiACK, capability.MultiACKDetailed, - capability.Sideband, - capability.Sideband64k, capability.ThinPack, } diff --git a/plumbing/transport/http/fetch_pack.go b/plumbing/transport/http/fetch_pack.go index 20cdb55..1548110 100644 --- a/plumbing/transport/http/fetch_pack.go +++ b/plumbing/transport/http/fetch_pack.go @@ -5,7 +5,7 @@ import ( "fmt" "io" "net/http" - "strings" + "strconv" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/pktline" @@ -117,7 +117,7 @@ func (s *fetchPackSession) Close() error { return nil } -func (s *fetchPackSession) doRequest(method, url string, content *strings.Reader) (*http.Response, error) { +func (s *fetchPackSession) doRequest(method, url string, content *bytes.Buffer) (*http.Response, error) { var body io.Reader if content != nil { body = content @@ -144,44 +144,33 @@ func (s *fetchPackSession) doRequest(method, url string, content *strings.Reader return res, nil } -func (s *fetchPackSession) applyHeadersToRequest(req *http.Request, content *strings.Reader) { +// it requires a bytes.Buffer, because we need to know the length +func (s *fetchPackSession) applyHeadersToRequest(req *http.Request, content *bytes.Buffer) { req.Header.Add("User-Agent", "git/1.0") - req.Header.Add("Host", "github.com") + req.Header.Add("Host", s.endpoint.Host) if content == nil { req.Header.Add("Accept", "*/*") - } else { - req.Header.Add("Accept", "application/x-git-upload-pack-result") - req.Header.Add("Content-Type", "application/x-git-upload-pack-request") - req.Header.Add("Content-Length", string(content.Len())) + return } + + req.Header.Add("Accept", "application/x-git-upload-pack-result") + req.Header.Add("Content-Type", "application/x-git-upload-pack-request") + req.Header.Add("Content-Length", strconv.Itoa(content.Len())) } -func uploadPackRequestToReader(r *packp.UploadPackRequest) (*strings.Reader, error) { - var buf bytes.Buffer - e := pktline.NewEncoder(&buf) +func uploadPackRequestToReader(req *packp.UploadPackRequest) (*bytes.Buffer, error) { + buf := bytes.NewBuffer(nil) + e := pktline.NewEncoder(buf) - for _, want := range r.Wants { - _ = e.Encodef("want %s\n", want) + if err := req.UploadRequest.Encode(buf); err != nil { + return nil, fmt.Errorf("sending upload-req message: %s", err) } - for _, have := range r.Haves { - _ = e.Encodef("have %s\n", have) + if err := req.UploadHaves.Encode(buf); err != nil { + return nil, fmt.Errorf("sending haves message: %s", err) } - if r.Depth != nil { - depth, ok := r.Depth.(packp.DepthCommits) - if !ok { - return nil, fmt.Errorf("only commit depth is supported") - } - - if depth != 0 { - _ = e.Encodef("deepen %d\n", depth) - } - } - - _ = e.Flush() _ = e.EncodeString("done\n") - - return strings.NewReader(buf.String()), nil + return buf, nil } diff --git a/plumbing/transport/http/fetch_pack_test.go b/plumbing/transport/http/fetch_pack_test.go index 920b623..6c40e60 100644 --- a/plumbing/transport/http/fetch_pack_test.go +++ b/plumbing/transport/http/fetch_pack_test.go @@ -51,8 +51,8 @@ func (s *FetchPackSuite) TestuploadPackRequestToReader(c *C) { c.Assert(err, IsNil) b, _ := ioutil.ReadAll(sr) c.Assert(string(b), Equals, - "0032want d82f291cde9987322c8a0c81a325e1ba6159684c\n"+ - "0032want 2b41ef280fdb67a9b250678686a0c3e03b0a9989\n"+ + "0032want 2b41ef280fdb67a9b250678686a0c3e03b0a9989\n"+ + "0032want d82f291cde9987322c8a0c81a325e1ba6159684c\n0000"+ "0032have 6ecf0ef2c2dffb796033e5a02219af86ec6584e5\n0000"+ "0009done\n", ) @@ -10,6 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/client" @@ -22,6 +23,7 @@ var NoErrAlreadyUpToDate = errors.New("already up-to-date") type Remote struct { c *config.RemoteConfig s Storer + p sideband.Progress // cache fields, there during the connection is open endpoint transport.Endpoint @@ -31,8 +33,8 @@ type Remote struct { refs memory.ReferenceStorage } -func newRemote(s Storer, c *config.RemoteConfig) *Remote { - return &Remote{s: s, c: c} +func newRemote(s Storer, p sideband.Progress, c *config.RemoteConfig) *Remote { + return &Remote{s: s, p: p, c: c} } // Config return the config @@ -125,7 +127,10 @@ func (r *Remote) Fetch(o *FetchOptions) (err error) { } defer checkClose(reader, &err) - if err := r.updateObjectStorage(reader); err != nil { + + if err = r.updateObjectStorage( + r.buildSidebandIfSupported(req.Capabilities, reader), + ); err != nil { return err } @@ -178,6 +183,10 @@ func (r *Remote) buildRequest( req.Capabilities.Set(capability.Shallow) } + if r.p == nil && r.advRefs.Capabilities.Supports(capability.NoProgress) { + req.Capabilities.Set(capability.NoProgress) + } + for _, ref := range refs { req.Wants = append(req.Wants, ref.Hash()) } @@ -221,6 +230,24 @@ func (r *Remote) updateObjectStorage(reader io.Reader) error { return err } +func (r *Remote) buildSidebandIfSupported(l *capability.List, reader io.Reader) io.Reader { + var t sideband.Type + + switch { + case l.Supports(capability.Sideband): + t = sideband.Sideband + case l.Supports(capability.Sideband64k): + t = sideband.Sideband64k + default: + return reader + } + + d := sideband.NewDemuxer(t, reader) + d.Progress = r.p + + return d +} + func (r *Remote) updateLocalReferenceStorage(specs []config.RefSpec, refs []*plumbing.Reference) error { for _, spec := range specs { for _, ref := range refs { diff --git a/remote_test.go b/remote_test.go index 30f68ad..acf646f 100644 --- a/remote_test.go +++ b/remote_test.go @@ -1,6 +1,7 @@ package git import ( + "bytes" "crypto/tls" "fmt" "io" @@ -30,21 +31,21 @@ var _ = Suite(&RemoteSuite{}) func (s *RemoteSuite) TestConnect(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) err := r.Connect() c.Assert(err, IsNil) } func (s *RemoteSuite) TestnewRemoteInvalidEndpoint(c *C) { - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: "qux"}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: "qux"}) err := r.Connect() c.Assert(err, NotNil) } func (s *RemoteSuite) TestnewRemoteInvalidSchemaEndpoint(c *C) { - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: "qux://foo"}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: "qux://foo"}) err := r.Connect() c.Assert(err, NotNil) @@ -52,7 +53,7 @@ func (s *RemoteSuite) TestnewRemoteInvalidSchemaEndpoint(c *C) { func (s *RemoteSuite) TestInfo(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.AdvertisedReferences(), IsNil) c.Assert(r.Connect(), IsNil) c.Assert(r.AdvertisedReferences(), NotNil) @@ -61,14 +62,14 @@ func (s *RemoteSuite) TestInfo(c *C) { func (s *RemoteSuite) TestDefaultBranch(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) c.Assert(r.Head().Name(), Equals, plumbing.ReferenceName("refs/heads/master")) } func (s *RemoteSuite) TestCapabilities(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) c.Assert(r.Capabilities().Get(capability.Agent), HasLen, 1) } @@ -76,7 +77,7 @@ func (s *RemoteSuite) TestCapabilities(c *C) { func (s *RemoteSuite) TestFetch(c *C) { url := s.GetBasicLocalRepositoryURL() sto := memory.NewStorage() - r := newRemote(sto, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(sto, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") @@ -101,7 +102,7 @@ func (s *RemoteSuite) TestFetch(c *C) { func (s *RemoteSuite) TestFetchDepth(c *C) { url := s.GetBasicLocalRepositoryURL() sto := memory.NewStorage() - r := newRemote(sto, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(sto, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") @@ -124,6 +125,25 @@ func (s *RemoteSuite) TestFetchDepth(c *C) { } } +func (s *RemoteSuite) TestFetchWithProgress(c *C) { + url := s.GetBasicLocalRepositoryURL() + sto := memory.NewStorage() + buf := bytes.NewBuffer(nil) + + r := newRemote(sto, buf, &config.RemoteConfig{Name: "foo", URL: url}) + c.Assert(r.Connect(), IsNil) + + refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") + err := r.Fetch(&FetchOptions{ + RefSpecs: []config.RefSpec{refspec}, + }) + + c.Assert(err, IsNil) + c.Assert(sto.Objects, HasLen, 31) + + c.Assert(buf.Len(), Not(Equals), 0) +} + type mockPackfileWriter struct { Storer PackfileWriterCalled bool @@ -135,7 +155,6 @@ func (m *mockPackfileWriter) PackfileWriter() (io.WriteCloser, error) { } func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { - dir, err := ioutil.TempDir("", "fetch") c.Assert(err, IsNil) @@ -147,7 +166,7 @@ func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { mock := &mockPackfileWriter{Storer: fss} url := s.GetBasicLocalRepositoryURL() - r := newRemote(mock, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(mock, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") @@ -173,7 +192,7 @@ func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { func (s *RemoteSuite) TestFetchNoErrAlreadyUpToDate(c *C) { url := s.GetBasicLocalRepositoryURL() sto := memory.NewStorage() - r := newRemote(sto, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(sto, nil, &config.RemoteConfig{Name: "foo", URL: url}) c.Assert(r.Connect(), IsNil) refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") @@ -189,7 +208,7 @@ func (s *RemoteSuite) TestFetchNoErrAlreadyUpToDate(c *C) { func (s *RemoteSuite) TestHead(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) err := r.Connect() c.Assert(err, IsNil) @@ -198,7 +217,7 @@ func (s *RemoteSuite) TestHead(c *C) { func (s *RemoteSuite) TestRef(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) err := r.Connect() c.Assert(err, IsNil) @@ -213,7 +232,7 @@ func (s *RemoteSuite) TestRef(c *C) { func (s *RemoteSuite) TestRefs(c *C) { url := s.GetBasicLocalRepositoryURL() - r := newRemote(nil, &config.RemoteConfig{Name: "foo", URL: url}) + r := newRemote(nil, nil, &config.RemoteConfig{Name: "foo", URL: url}) err := r.Connect() c.Assert(err, IsNil) @@ -223,7 +242,7 @@ func (s *RemoteSuite) TestRefs(c *C) { } func (s *RemoteSuite) TestString(c *C) { - r := newRemote(nil, &config.RemoteConfig{ + r := newRemote(nil, nil, &config.RemoteConfig{ Name: "foo", URL: "https://github.com/git-fixtures/basic.git", }) diff --git a/repository.go b/repository.go index a1a5cae..4486ff4 100644 --- a/repository.go +++ b/repository.go @@ -6,6 +6,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -24,6 +25,11 @@ var ( type Repository struct { r map[string]*Remote s Storer + + // Progress is where the human readable information sent by the server is + // stored, if nil nothing is stored and the capability (if supported) + // no-progress, is sent to the server to avoid send this information + Progress sideband.Progress } // NewMemoryRepository creates a new repository, backed by a memory.Storage @@ -64,7 +70,7 @@ func (r *Repository) Remote(name string) (*Remote, error) { return nil, ErrRemoteNotFound } - return newRemote(r.s, c), nil + return newRemote(r.s, r.Progress, c), nil } // Remotes return all the remotes @@ -78,7 +84,7 @@ func (r *Repository) Remotes() ([]*Remote, error) { var i int for _, c := range cfg.Remotes { - remotes[i] = newRemote(r.s, c) + remotes[i] = newRemote(r.s, r.Progress, c) i++ } @@ -91,7 +97,7 @@ func (r *Repository) CreateRemote(c *config.RemoteConfig) (*Remote, error) { return nil, err } - remote := newRemote(r.s, c) + remote := newRemote(r.s, r.Progress, c) cfg, err := r.s.Config() if err != nil { diff --git a/repository_test.go b/repository_test.go index adc722c..0b6a603 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1,7 +1,9 @@ package git import ( + "bytes" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -11,10 +13,6 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/memory" - "os" - - "bytes" - . "gopkg.in/check.v1" ) @@ -45,6 +43,21 @@ func (s *RepositorySuite) TestCreateRemoteAndRemote(c *C) { c.Assert(alt.Config().Name, Equals, "foo") } +func (s *RepositorySuite) TestRemoteWithProgress(c *C) { + buf := bytes.NewBuffer(nil) + + r := NewMemoryRepository() + r.Progress = buf + + remote, err := r.CreateRemote(&config.RemoteConfig{ + Name: "foo", + URL: "http://foo/foo.git", + }) + + c.Assert(err, IsNil) + c.Assert(remote.p, Equals, buf) +} + func (s *RepositorySuite) TestCreateRemoteInvalid(c *C) { r := NewMemoryRepository() remote, err := r.CreateRemote(&config.RemoteConfig{}) |