new plumbing package (#118)

* plumbing: now core was renamed to core, and formats and clients moved inside

4 files changed, 712 insertions, 0 deletions

diff --git a/plumbing/client/ssh/auth_method.go b/plumbing/client/ssh/auth_method.go
new file mode 100644
index 0000000..587f59a
--- /dev/null
+++ b/plumbing/client/ssh/auth_method.go
@@ -0,0 +1,159 @@
+package ssh
+
+import (
+	"fmt"
+	"net"
+	"os"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+	"gopkg.in/src-d/go-git.v4/plumbing/client/common"
+)
+
+// AuthMethod is the interface all auth methods for the ssh client
+// must implement. The clientConfig method returns the ssh client
+// configuration needed to establish an ssh connection.
+type AuthMethod interface {
+	common.AuthMethod
+	clientConfig() *ssh.ClientConfig
+}
+
+// The names of the AuthMethod implementations. To be returned by the
+// Name() method. Most git servers only allow PublicKeysName and
+// PublicKeysCallbackName.
+const (
+	KeyboardInteractiveName = "ssh-keyboard-interactive"
+	PasswordName            = "ssh-password"
+	PasswordCallbackName    = "ssh-password-callback"
+	PublicKeysName          = "ssh-public-keys"
+	PublicKeysCallbackName  = "ssh-public-key-callback"
+)
+
+// KeyboardInteractive implements AuthMethod by using a
+// prompt/response sequence controlled by the server.
+type KeyboardInteractive struct {
+	User      string
+	Challenge ssh.KeyboardInteractiveChallenge
+}
+
+func (a *KeyboardInteractive) Name() string {
+	return KeyboardInteractiveName
+}
+
+func (a *KeyboardInteractive) String() string {
+	return fmt.Sprintf("user: %s, name: %s", a.User, a.Name())
+}
+
+func (a *KeyboardInteractive) clientConfig() *ssh.ClientConfig {
+	return &ssh.ClientConfig{
+		User: a.User,
+		Auth: []ssh.AuthMethod{ssh.KeyboardInteractiveChallenge(a.Challenge)},
+	}
+}
+
+// Password implements AuthMethod by using the given password.
+type Password struct {
+	User string
+	Pass string
+}
+
+func (a *Password) Name() string {
+	return PasswordName
+}
+
+func (a *Password) String() string {
+	return fmt.Sprintf("user: %s, name: %s", a.User, a.Name())
+}
+
+func (a *Password) clientConfig() *ssh.ClientConfig {
+	return &ssh.ClientConfig{
+		User: a.User,
+		Auth: []ssh.AuthMethod{ssh.Password(a.Pass)},
+	}
+}
+
+// PasswordCallback implements AuthMethod by using a callback
+// to fetch the password.
+type PasswordCallback struct {
+	User     string
+	Callback func() (pass string, err error)
+}
+
+func (a *PasswordCallback) Name() string {
+	return PasswordCallbackName
+}
+
+func (a *PasswordCallback) String() string {
+	return fmt.Sprintf("user: %s, name: %s", a.User, a.Name())
+}
+
+func (a *PasswordCallback) clientConfig() *ssh.ClientConfig {
+	return &ssh.ClientConfig{
+		User: a.User,
+		Auth: []ssh.AuthMethod{ssh.PasswordCallback(a.Callback)},
+	}
+}
+
+// PublicKeys implements AuthMethod by using the given
+// key pairs.
+type PublicKeys struct {
+	User   string
+	Signer ssh.Signer
+}
+
+func (a *PublicKeys) Name() string {
+	return PublicKeysName
+}
+
+func (a *PublicKeys) String() string {
+	return fmt.Sprintf("user: %s, name: %s", a.User, a.Name())
+}
+
+func (a *PublicKeys) clientConfig() *ssh.ClientConfig {
+	return &ssh.ClientConfig{
+		User: a.User,
+		Auth: []ssh.AuthMethod{ssh.PublicKeys(a.Signer)},
+	}
+}
+
+// PublicKeysCallback implements AuthMethod by asking a
+// ssh.agent.Agent to act as a signer.
+type PublicKeysCallback struct {
+	User     string
+	Callback func() (signers []ssh.Signer, err error)
+}
+
+func (a *PublicKeysCallback) Name() string {
+	return PublicKeysCallbackName
+}
+
+func (a *PublicKeysCallback) String() string {
+	return fmt.Sprintf("user: %s, name: %s", a.User, a.Name())
+}
+
+func (a *PublicKeysCallback) clientConfig() *ssh.ClientConfig {
+	return &ssh.ClientConfig{
+		User: a.User,
+		Auth: []ssh.AuthMethod{ssh.PublicKeysCallback(a.Callback)},
+	}
+}
+
+const DefaultSSHUsername = "git"
+
+// Opens a pipe with the ssh agent and uses the pipe
+// as the implementer of the public key callback function.
+func NewSSHAgentAuth(user string) (*PublicKeysCallback, error) {
+	if user == "" {
+		user = DefaultSSHUsername
+	}
+
+	pipe, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &PublicKeysCallback{
+		User:     user,
+		Callback: agent.NewClient(pipe).Signers,
+	}, nil
+}
diff --git a/plumbing/client/ssh/auth_method_test.go b/plumbing/client/ssh/auth_method_test.go
new file mode 100644
index 0000000..a87c950
--- /dev/null
+++ b/plumbing/client/ssh/auth_method_test.go
@@ -0,0 +1,94 @@
+package ssh
+
+import (
+	"fmt"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type SuiteCommon struct{}
+
+var _ = Suite(&SuiteCommon{})
+
+func (s *SuiteCommon) TestKeyboardInteractiveName(c *C) {
+	a := &KeyboardInteractive{
+		User:      "test",
+		Challenge: nil,
+	}
+	c.Assert(a.Name(), Equals, KeyboardInteractiveName)
+}
+
+func (s *SuiteCommon) TestKeyboardInteractiveString(c *C) {
+	a := &KeyboardInteractive{
+		User:      "test",
+		Challenge: nil,
+	}
+	c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", KeyboardInteractiveName))
+}
+
+func (s *SuiteCommon) TestPasswordName(c *C) {
+	a := &Password{
+		User: "test",
+		Pass: "",
+	}
+	c.Assert(a.Name(), Equals, PasswordName)
+}
+
+func (s *SuiteCommon) TestPasswordString(c *C) {
+	a := &Password{
+		User: "test",
+		Pass: "",
+	}
+	c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", PasswordName))
+}
+
+func (s *SuiteCommon) TestPasswordCallbackName(c *C) {
+	a := &PasswordCallback{
+		User:     "test",
+		Callback: nil,
+	}
+	c.Assert(a.Name(), Equals, PasswordCallbackName)
+}
+
+func (s *SuiteCommon) TestPasswordCallbackString(c *C) {
+	a := &PasswordCallback{
+		User:     "test",
+		Callback: nil,
+	}
+	c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", PasswordCallbackName))
+}
+
+func (s *SuiteCommon) TestPublicKeysName(c *C) {
+	a := &PublicKeys{
+		User:   "test",
+		Signer: nil,
+	}
+	c.Assert(a.Name(), Equals, PublicKeysName)
+}
+
+func (s *SuiteCommon) TestPublicKeysString(c *C) {
+	a := &PublicKeys{
+		User:   "test",
+		Signer: nil,
+	}
+	c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", PublicKeysName))
+}
+
+func (s *SuiteCommon) TestPublicKeysCallbackName(c *C) {
+	a := &PublicKeysCallback{
+		User:     "test",
+		Callback: nil,
+	}
+	c.Assert(a.Name(), Equals, PublicKeysCallbackName)
+}
+
+func (s *SuiteCommon) TestPublicKeysCallbackString(c *C) {
+	a := &PublicKeysCallback{
+		User:     "test",
+		Callback: nil,
+	}
+	c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", PublicKeysCallbackName))
+}
diff --git a/plumbing/client/ssh/git_upload_pack.go b/plumbing/client/ssh/git_upload_pack.go
new file mode 100644
index 0000000..e2b73fd
--- /dev/null
+++ b/plumbing/client/ssh/git_upload_pack.go
@@ -0,0 +1,315 @@
+// Package ssh implements a ssh client for go-git.
+package ssh
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"gopkg.in/src-d/go-git.v4/plumbing/client/common"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/advrefs"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/pktline"
+	"gopkg.in/src-d/go-git.v4/plumbing/format/packp/ulreq"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// New errors introduced by this package.
+var (
+	ErrInvalidAuthMethod      = errors.New("invalid ssh auth method")
+	ErrAuthRequired           = errors.New("cannot connect: auth required")
+	ErrNotConnected           = errors.New("not connected")
+	ErrAlreadyConnected       = errors.New("already connected")
+	ErrUploadPackAnswerFormat = errors.New("git-upload-pack bad answer format")
+	ErrUnsupportedVCS         = errors.New("only git is supported")
+	ErrUnsupportedRepo        = errors.New("only github.com is supported")
+
+	nak = []byte("NAK")
+	eol = []byte("\n")
+)
+
+// GitUploadPackService holds the service information.
+// The zero value is safe to use.
+type GitUploadPackService struct {
+	connected bool
+	endpoint  common.Endpoint
+	client    *ssh.Client
+	auth      AuthMethod
+}
+
+// NewGitUploadPackService initialises a GitUploadPackService,
+func NewGitUploadPackService(endpoint common.Endpoint) common.GitUploadPackService {
+	return &GitUploadPackService{endpoint: endpoint}
+}
+
+// Connect connects to the SSH server, unless a AuthMethod was set with SetAuth
+// method, by default uses an auth method based on PublicKeysCallback, it
+// connects to a SSH agent, using the address stored in the SSH_AUTH_SOCK
+// environment var
+func (s *GitUploadPackService) Connect() error {
+	if s.connected {
+		return ErrAlreadyConnected
+	}
+
+	if err := s.setAuthFromEndpoint(); err != nil {
+		return err
+	}
+
+	var err error
+	s.client, err = ssh.Dial("tcp", s.getHostWithPort(), s.auth.clientConfig())
+	if err != nil {
+		return err
+	}
+
+	s.connected = true
+	return nil
+}
+
+func (s *GitUploadPackService) getHostWithPort() string {
+	host := s.endpoint.Host
+	if strings.Index(s.endpoint.Host, ":") == -1 {
+		host += ":22"
+	}
+
+	return host
+}
+
+func (s *GitUploadPackService) setAuthFromEndpoint() error {
+	var u string
+	if info := s.endpoint.User; info != nil {
+		u = info.Username()
+	}
+
+	var err error
+	s.auth, err = NewSSHAgentAuth(u)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SetAuth sets the AuthMethod
+func (s *GitUploadPackService) SetAuth(auth common.AuthMethod) error {
+	var ok bool
+	s.auth, ok = auth.(AuthMethod)
+	if !ok {
+		return ErrInvalidAuthMethod
+	}
+
+	return nil
+}
+
+// Info returns the GitUploadPackInfo of the repository. The client must be
+// connected with the repository (using the ConnectWithAuth() method) before
+// using this method.
+func (s *GitUploadPackService) Info() (i *common.GitUploadPackInfo, err error) {
+	if !s.connected {
+		return nil, ErrNotConnected
+	}
+
+	session, err := s.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		// the session can be closed by the other endpoint,
+		// therefore we must ignore a close error.
+		_ = session.Close()
+	}()
+
+	out, err := session.Output(s.getCommand())
+	if err != nil {
+		return nil, err
+	}
+
+	i = common.NewGitUploadPackInfo()
+	return i, i.Decode(bytes.NewReader(out))
+}
+
+// Disconnect the SSH client.
+func (s *GitUploadPackService) Disconnect() (err error) {
+	if !s.connected {
+		return ErrNotConnected
+	}
+	s.connected = false
+	return s.client.Close()
+}
+
+// Fetch returns a packfile for a given upload request.  It opens a new
+// SSH session on a connected GitUploadPackService, sends the given
+// upload request to the server and returns a reader for the received
+// packfile.  Closing the returned reader will close the SSH session.
+func (s *GitUploadPackService) Fetch(req *common.GitUploadPackRequest) (rc io.ReadCloser, err error) {
+	if !s.connected {
+		return nil, ErrNotConnected
+	}
+
+	session, i, o, done, err := openSSHSession(s.client, s.getCommand())
+	if err != nil {
+		return nil, fmt.Errorf("cannot open SSH session: %s", err)
+	}
+
+	if err := talkPackProtocol(i, o, req); err != nil {
+		return nil, err
+	}
+
+	return &fetchSession{
+		Reader:  o,
+		session: session,
+		done:    done,
+	}, nil
+}
+
+func openSSHSession(c *ssh.Client, cmd string) (
+	*ssh.Session, io.WriteCloser, io.Reader, <-chan error, error) {
+
+	session, err := c.NewSession()
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("cannot open SSH session: %s", err)
+	}
+
+	i, err := session.StdinPipe()
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("cannot pipe remote stdin: %s", err)
+	}
+
+	o, err := session.StdoutPipe()
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("cannot pipe remote stdout: %s", err)
+	}
+
+	done := make(chan error)
+	go func() {
+		done <- session.Run(cmd)
+	}()
+
+	return session, i, o, done, nil
+}
+
+// TODO support multi_ack mode
+// TODO support multi_ack_detailed mode
+// TODO support acks for common objects
+// TODO build a proper state machine for all these processing options
+func talkPackProtocol(w io.WriteCloser, r io.Reader,
+	req *common.GitUploadPackRequest) error {
+
+	if err := skipAdvRef(r); err != nil {
+		return fmt.Errorf("skipping advertised-refs: %s", err)
+	}
+
+	if err := sendUlReq(w, req); err != nil {
+		return fmt.Errorf("sending upload-req message: %s", err)
+	}
+
+	if err := sendHaves(w, req); err != nil {
+		return fmt.Errorf("sending haves message: %s", err)
+	}
+
+	if err := sendDone(w); err != nil {
+		return fmt.Errorf("sending done message: %s", err)
+	}
+
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("closing input: %s", err)
+	}
+
+	if err := readNAK(r); err != nil {
+		return fmt.Errorf("reading NAK: %s", err)
+	}
+
+	return nil
+}
+
+func skipAdvRef(r io.Reader) error {
+	d := advrefs.NewDecoder(r)
+	ar := advrefs.New()
+
+	return d.Decode(ar)
+}
+
+func sendUlReq(w io.Writer, req *common.GitUploadPackRequest) error {
+	ur := ulreq.New()
+	ur.Wants = req.Wants
+	ur.Depth = ulreq.DepthCommits(req.Depth)
+	e := ulreq.NewEncoder(w)
+
+	return e.Encode(ur)
+}
+
+func sendHaves(w io.Writer, req *common.GitUploadPackRequest) error {
+	e := pktline.NewEncoder(w)
+	for _, have := range req.Haves {
+		if err := e.Encodef("have %s\n", have); err != nil {
+			return fmt.Errorf("sending haves for %q: err ", have, err)
+		}
+	}
+
+	if len(req.Haves) != 0 {
+		if err := e.Flush(); err != nil {
+			return fmt.Errorf("sending flush-pkt after haves: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func sendDone(w io.Writer) error {
+	e := pktline.NewEncoder(w)
+
+	return e.Encodef("done\n")
+}
+
+func readNAK(r io.Reader) error {
+	s := pktline.NewScanner(r)
+	if !s.Scan() {
+		return s.Err()
+	}
+
+	b := s.Bytes()
+	b = bytes.TrimSuffix(b, eol)
+	if !bytes.Equal(b, nak) {
+		return fmt.Errorf("expecting NAK, found %q instead", string(b))
+	}
+
+	return nil
+}
+
+type fetchSession struct {
+	io.Reader
+	session *ssh.Session
+	done    <-chan error
+}
+
+// Close closes the session and collects the output state of the remote
+// SSH command.
+//
+// If both the remote command and the closing of the session completes
+// susccessfully it returns nil.
+//
+// If the remote command completes unsuccessfully or is interrupted by a
+// signal, it returns the corresponding *ExitError.
+//
+// Otherwise, if clossing the SSH session fails it returns the close
+// error.  Closing the session when the other has already close it is
+// not cosidered an error.
+func (f *fetchSession) Close() (err error) {
+	if err := <-f.done; err != nil {
+		return err
+	}
+
+	if err := f.session.Close(); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+func (s *GitUploadPackService) getCommand() string {
+	directory := s.endpoint.Path
+	directory = directory[1:]
+
+	return fmt.Sprintf("git-upload-pack '%s'", directory)
+}
diff --git a/plumbing/client/ssh/git_upload_pack_test.go b/plumbing/client/ssh/git_upload_pack_test.go
new file mode 100644
index 0000000..4d5b2b1
--- /dev/null
+++ b/plumbing/client/ssh/git_upload_pack_test.go
@@ -0,0 +1,144 @@
+package ssh
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/client/common"
+)
+
+type RemoteSuite struct {
+	Endpoint common.Endpoint
+}
+
+var _ = Suite(&RemoteSuite{})
+
+func (s *RemoteSuite) SetUpSuite(c *C) {
+	var err error
+	s.Endpoint, err = common.NewEndpoint("git@github.com:git-fixtures/basic.git")
+	c.Assert(err, IsNil)
+
+	if os.Getenv("SSH_AUTH_SOCK") == "" {
+		c.Skip("SSH_AUTH_SOCK is not set")
+	}
+}
+
+// A mock implementation of client.common.AuthMethod
+// to test non ssh auth method detection.
+type mockAuth struct{}
+
+func (*mockAuth) Name() string   { return "" }
+func (*mockAuth) String() string { return "" }
+
+func (s *RemoteSuite) TestSetAuthWrongType(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.SetAuth(&mockAuth{}), Equals, ErrInvalidAuthMethod)
+}
+
+func (s *RemoteSuite) TestAlreadyConnected(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	defer func() {
+		c.Assert(r.Disconnect(), IsNil)
+	}()
+
+	c.Assert(r.Connect(), Equals, ErrAlreadyConnected)
+}
+
+func (s *RemoteSuite) TestDisconnect(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	c.Assert(r.Disconnect(), IsNil)
+}
+
+func (s *RemoteSuite) TestDisconnectedWhenNonConnected(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Disconnect(), Equals, ErrNotConnected)
+}
+
+func (s *RemoteSuite) TestAlreadyDisconnected(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	c.Assert(r.Disconnect(), IsNil)
+	c.Assert(r.Disconnect(), Equals, ErrNotConnected)
+}
+
+func (s *RemoteSuite) TestServeralConnections(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	c.Assert(r.Disconnect(), IsNil)
+
+	c.Assert(r.Connect(), IsNil)
+	c.Assert(r.Disconnect(), IsNil)
+
+	c.Assert(r.Connect(), IsNil)
+	c.Assert(r.Disconnect(), IsNil)
+}
+
+func (s *RemoteSuite) TestInfoNotConnected(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	_, err := r.Info()
+	c.Assert(err, Equals, ErrNotConnected)
+}
+
+func (s *RemoteSuite) TestDefaultBranch(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	defer func() { c.Assert(r.Disconnect(), IsNil) }()
+
+	info, err := r.Info()
+	c.Assert(err, IsNil)
+	c.Assert(info.Capabilities.SymbolicReference("HEAD"), Equals, "refs/heads/master")
+}
+
+func (s *RemoteSuite) TestCapabilities(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	defer func() { c.Assert(r.Disconnect(), IsNil) }()
+
+	info, err := r.Info()
+	c.Assert(err, IsNil)
+	c.Assert(info.Capabilities.Get("agent").Values, HasLen, 1)
+}
+
+func (s *RemoteSuite) TestFetchNotConnected(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	pr := &common.GitUploadPackRequest{}
+	pr.Want(plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))
+	_, err := r.Fetch(pr)
+	c.Assert(err, Equals, ErrNotConnected)
+}
+
+func (s *RemoteSuite) TestFetch(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	defer func() { c.Assert(r.Disconnect(), IsNil) }()
+
+	req := &common.GitUploadPackRequest{}
+	req.Want(plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))
+	req.Want(plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"))
+	reader, err := r.Fetch(req)
+	c.Assert(err, IsNil)
+	defer func() { c.Assert(reader.Close(), IsNil) }()
+
+	b, err := ioutil.ReadAll(reader)
+	c.Assert(err, IsNil)
+	c.Check(len(b), Equals, 85585)
+}
+
+func (s *RemoteSuite) TestFetchError(c *C) {
+	r := NewGitUploadPackService(s.Endpoint)
+	c.Assert(r.Connect(), IsNil)
+	defer func() { c.Assert(r.Disconnect(), IsNil) }()
+
+	req := &common.GitUploadPackRequest{}
+	req.Want(plumbing.NewHash("1111111111111111111111111111111111111111"))
+
+	reader, err := r.Fetch(req)
+	c.Assert(err, IsNil)
+
+	err = reader.Close()
+	c.Assert(err, Not(IsNil))
+}