From df9748cfb51db9c406e3df063badbd8c78ee819d Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Mon, 12 Dec 2016 15:50:15 +0100 Subject: transport: new git protocol (#175) --- plumbing/protocol/packp/advrefs_decode.go | 8 +- plumbing/protocol/packp/advrefs_decode_test.go | 2 +- plumbing/protocol/packp/common.go | 24 +++++ plumbing/protocol/packp/ulreq_decode.go | 8 +- plumbing/transport/client/client.go | 2 + plumbing/transport/git/common.go | 118 +++++++++++++++++++++++++ plumbing/transport/git/common_test.go | 9 ++ plumbing/transport/git/fetch_pack_test.go | 35 ++++++++ plumbing/transport/internal/common/common.go | 91 ++++++++++++------- utils/ioutil/common.go | 12 +++ 10 files changed, 272 insertions(+), 37 deletions(-) create mode 100644 plumbing/transport/git/common.go create mode 100644 plumbing/transport/git/common_test.go create mode 100644 plumbing/transport/git/fetch_pack_test.go diff --git a/plumbing/protocol/packp/advrefs_decode.go b/plumbing/protocol/packp/advrefs_decode.go index fddd69b..e0a449e 100644 --- a/plumbing/protocol/packp/advrefs_decode.go +++ b/plumbing/protocol/packp/advrefs_decode.go @@ -55,8 +55,12 @@ type decoderStateFn func(*advRefsDecoder) decoderStateFn // fills out the parser stiky error func (d *advRefsDecoder) error(format string, a ...interface{}) { - d.err = fmt.Errorf("pkt-line %d: %s", d.nLine, - fmt.Sprintf(format, a...)) + msg := fmt.Sprintf( + "pkt-line %d: %s", d.nLine, + fmt.Sprintf(format, a...), + ) + + d.err = NewErrUnexpectedData(msg, d.line) } // Reads a new pkt-line from the scanner, makes its payload available as diff --git a/plumbing/protocol/packp/advrefs_decode_test.go b/plumbing/protocol/packp/advrefs_decode_test.go index 2cc2568..e9a01f8 100644 --- a/plumbing/protocol/packp/advrefs_decode_test.go +++ b/plumbing/protocol/packp/advrefs_decode_test.go @@ -46,7 +46,7 @@ func (s *AdvRefsDecodeSuite) TestShortForHash(c *C) { pktline.FlushString, } r := toPktLines(c, payloads) - s.testDecoderErrorMatches(c, r, ".*too short") + s.testDecoderErrorMatches(c, r, ".*too short.*") } func (s *AdvRefsDecodeSuite) testDecoderErrorMatches(c *C, input io.Reader, pattern string) { diff --git a/plumbing/protocol/packp/common.go b/plumbing/protocol/packp/common.go index 93dfaed..ab07ac8 100644 --- a/plumbing/protocol/packp/common.go +++ b/plumbing/protocol/packp/common.go @@ -1,5 +1,9 @@ package packp +import ( + "fmt" +) + type stateFn func() stateFn const ( @@ -44,3 +48,23 @@ var ( func isFlush(payload []byte) bool { return len(payload) == 0 } + +// ErrUnexpectedData represents an unexpected data decoding a message +type ErrUnexpectedData struct { + Msg string + Data []byte +} + +// NewErrUnexpectedData returns a new ErrUnexpectedData containing the data and +// the message given +func NewErrUnexpectedData(msg string, data []byte) error { + return &ErrUnexpectedData{Msg: msg, Data: data} +} + +func (err *ErrUnexpectedData) Error() string { + if len(err.Data) == 0 { + return err.Msg + } + + return fmt.Sprintf("%s (%s)", err.Msg, err.Data) +} diff --git a/plumbing/protocol/packp/ulreq_decode.go b/plumbing/protocol/packp/ulreq_decode.go index 541a077..bcd642d 100644 --- a/plumbing/protocol/packp/ulreq_decode.go +++ b/plumbing/protocol/packp/ulreq_decode.go @@ -45,8 +45,12 @@ func (d *ulReqDecoder) Decode(v *UploadRequest) error { // fills out the parser stiky error func (d *ulReqDecoder) error(format string, a ...interface{}) { - d.err = fmt.Errorf("pkt-line %d: %s", d.nLine, - fmt.Sprintf(format, a...)) + msg := fmt.Sprintf( + "pkt-line %d: %s", d.nLine, + fmt.Sprintf(format, a...), + ) + + d.err = NewErrUnexpectedData(msg, d.line) } // Reads a new pkt-line from the scanner, makes its payload available as diff --git a/plumbing/transport/client/client.go b/plumbing/transport/client/client.go index 770b7dc..095c51d 100644 --- a/plumbing/transport/client/client.go +++ b/plumbing/transport/client/client.go @@ -5,6 +5,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/file" + "gopkg.in/src-d/go-git.v4/plumbing/transport/git" "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" ) @@ -14,6 +15,7 @@ var Protocols = map[string]transport.Client{ "http": http.DefaultClient, "https": http.DefaultClient, "ssh": ssh.DefaultClient, + "git": git.DefaultClient, "file": file.DefaultClient, } diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go new file mode 100644 index 0000000..0fa7f5d --- /dev/null +++ b/plumbing/transport/git/common.go @@ -0,0 +1,118 @@ +package git + +import ( + "errors" + "fmt" + "io" + "net" + "strings" + "time" + + "gopkg.in/src-d/go-git.v2/formats/pktline" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +var ( + errAlreadyConnected = errors.New("tcp connection already connected") +) + +// DefaultClient is the default git client. +var DefaultClient = common.NewClient(&runner{}) + +type runner struct{} + +// Command returns a new Command for the given cmd in the given Endpoint +func (r *runner) Command(cmd string, ep transport.Endpoint) (common.Command, error) { + c := &command{command: cmd, endpoint: ep} + if err := c.connect(); err != nil { + return nil, err + } + + return c, nil +} + +type command struct { + conn net.Conn + connected bool + command string + endpoint transport.Endpoint +} + +// SetAuth cannot be called since git protocol doesn't support authentication +func (c *command) SetAuth(auth transport.AuthMethod) error { + return transport.ErrInvalidAuthMethod +} + +// Start executes the command sending the required message to the TCP connection +func (c *command) Start() error { + cmd := endpointToCommand(c.command, c.endpoint) + line, err := pktline.EncodeFromString(cmd) + if err != nil { + return err + } + + _, err = c.conn.Write([]byte(line)) + return err +} + +func (c *command) connect() error { + if c.connected { + return errAlreadyConnected + } + + var err error + c.conn, err = net.DialTimeout("tcp", c.getHostWithPort(), time.Second) + if err != nil { + return err + } + + c.connected = true + return nil +} + +func (c *command) getHostWithPort() string { + host := c.endpoint.Host + if strings.Index(c.endpoint.Host, ":") == -1 { + host += ":9418" + } + + return host +} + +// StderrPipe git protocol doesn't have any dedicated error channel +func (c *command) StderrPipe() (io.Reader, error) { + return nil, nil +} + +// StdinPipe return the underlying connection as WriteCloser, wrapped to prevent +// call to the Close function from the connection, a command execution in git +// protocol can't be closed or killed +func (c *command) StdinPipe() (io.WriteCloser, error) { + return ioutil.WriteNopCloser(c.conn), nil +} + +// StdoutPipe return the underlying connection as Reader +func (c *command) StdoutPipe() (io.Reader, error) { + return c.conn, nil +} + +func endpointToCommand(cmd string, ep transport.Endpoint) string { + return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, ep.Host, 0) +} + +// Wait no-op function, required by the interface +func (c *command) Wait() error { + return nil +} + +// Close closes the TCP connection and connection. +func (c *command) Close() error { + if !c.connected { + return nil + } + + c.connected = false + return c.conn.Close() +} diff --git a/plumbing/transport/git/common_test.go b/plumbing/transport/git/common_test.go new file mode 100644 index 0000000..3f25ad9 --- /dev/null +++ b/plumbing/transport/git/common_test.go @@ -0,0 +1,9 @@ +package git + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } diff --git a/plumbing/transport/git/fetch_pack_test.go b/plumbing/transport/git/fetch_pack_test.go new file mode 100644 index 0000000..dc40240 --- /dev/null +++ b/plumbing/transport/git/fetch_pack_test.go @@ -0,0 +1,35 @@ +package git + +import ( + "gopkg.in/src-d/go-git.v4/fixtures" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/test" + + . "gopkg.in/check.v1" +) + +type FetchPackSuite struct { + test.FetchPackSuite + fixtures.Suite +} + +var _ = Suite(&FetchPackSuite{}) + +func (s *FetchPackSuite) SetUpSuite(c *C) { + s.Suite.SetUpSuite(c) + + s.FetchPackSuite.Client = DefaultClient + + ep, err := transport.NewEndpoint("git://github.com/git-fixtures/basic.git") + c.Assert(err, IsNil) + s.FetchPackSuite.Endpoint = ep + + ep, err = transport.NewEndpoint("git://github.com/git-fixtures/empty.git") + c.Assert(err, IsNil) + s.FetchPackSuite.EmptyEndpoint = ep + + ep, err = transport.NewEndpoint("git://github.com/git-fixtures/non-existent.git") + c.Assert(err, IsNil) + s.FetchPackSuite.NonExistentEndpoint = ep + +} diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index f0e1691..f6d1249 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -132,22 +132,30 @@ func (c *client) newSession(s string, ep transport.Endpoint) (*session, error) { return nil, err } + return &session{ + Stdin: stdin, + Stdout: stdout, + Command: cmd, + errLines: c.listenErrors(stderr), + isReceivePack: s == transport.ReceivePackServiceName, + }, nil +} + +func (c *client) listenErrors(r io.Reader) chan string { + if r == nil { + return nil + } + errLines := make(chan string, errLinesBuffer) go func() { - s := bufio.NewScanner(stderr) + s := bufio.NewScanner(r) for s.Scan() { line := string(s.Bytes()) errLines <- line } }() - return &session{ - Stdin: stdin, - Stdout: stdout, - Command: cmd, - errLines: errLines, - isReceivePack: s == transport.ReceivePackServiceName, - }, nil + return errLines } // SetAuth delegates to the command's SetAuth. @@ -163,38 +171,52 @@ func (s *session) AdvertisedReferences() (*packp.AdvRefs, error) { ar := packp.NewAdvRefs() if err := ar.Decode(s.Stdout); err != nil { - // If repository is not found, we get empty stdout and server - // writes an error to stderr. - if err == packp.ErrEmptyInput { - if err := s.checkNotFoundError(); err != nil { - return nil, err - } - - return nil, io.ErrUnexpectedEOF + if err := s.handleAdvRefDecodeError(err); err != nil { + return nil, err } + } - // For empty (but existing) repositories, we get empty - // advertised-references message. But valid. That is, it - // includes at least a flush. - if err == packp.ErrEmptyAdvRefs { - // Empty repositories are valid for git-receive-pack. - if s.isReceivePack { - return ar, nil - } + transport.FilterUnsupportedCapabilities(ar.Capabilities) + s.advRefs = ar + return ar, nil +} + +func (s *session) handleAdvRefDecodeError(err error) error { + // If repository is not found, we get empty stdout and server writes an + // error to stderr. + if err == packp.ErrEmptyInput { + if err := s.checkNotFoundError(); err != nil { + return err + } + + return io.ErrUnexpectedEOF + } - if err := s.finish(); err != nil { - return nil, err - } + // For empty (but existing) repositories, we get empty advertised-references + // message. But valid. That is, it includes at least a flush. + if err == packp.ErrEmptyAdvRefs { + // Empty repositories are valid for git-receive-pack. + if s.isReceivePack { + return nil + } - return nil, transport.ErrEmptyRemoteRepository + if err := s.finish(); err != nil { + return err } - return nil, err + return transport.ErrEmptyRemoteRepository } - transport.FilterUnsupportedCapabilities(ar.Capabilities) - s.advRefs = ar - return ar, nil + // Some server sends the errors as normal content (git protocol), so when + // we try to decode it fails, we need to check the content of it, to detect + // not found errors + if uerr, ok := err.(*packp.ErrUnexpectedData); ok { + if isRepoNotFoundError(string(uerr.Data)) { + return transport.ErrRepositoryNotFound + } + } + + return err } // FetchPack performs a request to the server to fetch a packfile. A reader is @@ -317,6 +339,7 @@ var ( githubRepoNotFoundErr = "ERROR: Repository not found." bitbucketRepoNotFoundErr = "conq: repository does not exist." localRepoNotFoundErr = "does not appear to be a git repository" + gitProtocolNotFoundErr = "ERR \n Repository not found." ) func isRepoNotFoundError(s string) bool { @@ -332,6 +355,10 @@ func isRepoNotFoundError(s string) bool { return true } + if strings.HasPrefix(s, gitProtocolNotFoundErr) { + return true + } + return false } diff --git a/utils/ioutil/common.go b/utils/ioutil/common.go index 13db692..f5b78df 100644 --- a/utils/ioutil/common.go +++ b/utils/ioutil/common.go @@ -50,3 +50,15 @@ func (r *readCloser) Close() error { func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser { return &readCloser{Reader: r, closer: c} } + +type writeNopCloser struct { + io.Writer +} + +func (writeNopCloser) Close() error { return nil } + +// WriteNopCloser returns a WriteCloser with a no-op Close method wrapping +// the provided Writer w. +func WriteNopCloser(w io.Writer) io.WriteCloser { + return writeNopCloser{w} +} -- cgit