diff options
Diffstat (limited to 'plumbing/transport')
29 files changed, 1036 insertions, 113 deletions
diff --git a/plumbing/transport/client/client.go b/plumbing/transport/client/client.go index 20c3d05..1948c23 100644 --- a/plumbing/transport/client/client.go +++ b/plumbing/transport/client/client.go @@ -3,10 +3,7 @@ package client import ( - "crypto/tls" - "crypto/x509" "fmt" - gohttp "net/http" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/file" @@ -24,14 +21,6 @@ var Protocols = map[string]transport.Transport{ "file": file.DefaultClient, } -var insecureClient = http.NewClient(&gohttp.Client{ - Transport: &gohttp.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, -}) - // InstallProtocol adds or modifies an existing protocol. func InstallProtocol(scheme string, c transport.Transport) { if c == nil { @@ -50,27 +39,6 @@ func NewClient(endpoint *transport.Endpoint) (transport.Transport, error) { } func getTransport(endpoint *transport.Endpoint) (transport.Transport, error) { - if endpoint.Protocol == "https" { - if endpoint.InsecureSkipTLS { - return insecureClient, nil - } - - if len(endpoint.CaBundle) != 0 { - rootCAs, _ := x509.SystemCertPool() - if rootCAs == nil { - rootCAs = x509.NewCertPool() - } - rootCAs.AppendCertsFromPEM(endpoint.CaBundle) - return http.NewClient(&gohttp.Client{ - Transport: &gohttp.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: rootCAs, - }, - }, - }), nil - } - } - f, ok := Protocols[endpoint.Protocol] if !ok { return nil, fmt.Errorf("unsupported scheme %q", endpoint.Protocol) diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index a9ee2ca..c6a054a 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -112,10 +112,41 @@ type Endpoint struct { Port int // Path is the repository path. Path string - // InsecureSkipTLS skips ssl verify if protocal is https + // InsecureSkipTLS skips ssl verify if protocol is https InsecureSkipTLS bool // CaBundle specify additional ca bundle with system cert pool CaBundle []byte + // Proxy provides info required for connecting to a proxy. + Proxy ProxyOptions +} + +type ProxyOptions struct { + URL string + Username string + Password string +} + +func (o *ProxyOptions) Validate() error { + if o.URL != "" { + _, err := url.Parse(o.URL) + return err + } + return nil +} + +func (o *ProxyOptions) FullURL() (*url.URL, error) { + proxyURL, err := url.Parse(o.URL) + if err != nil { + return nil, err + } + if o.Username != "" { + if o.Password != "" { + proxyURL.User = url.UserPassword(o.Username, o.Password) + } else { + proxyURL.User = url.User(o.Username) + } + } + return proxyURL, nil } var defaultPorts = map[string]int{ @@ -196,11 +227,17 @@ func parseURL(endpoint string) (*Endpoint, error) { pass, _ = u.User.Password() } + host := u.Hostname() + if strings.Contains(host, ":") { + // IPv6 address + host = "[" + host + "]" + } + return &Endpoint{ Protocol: u.Scheme, User: user, Password: pass, - Host: u.Hostname(), + Host: host, Port: getPort(u), Path: getPath(u), }, nil diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index 0c5a01a..d9f12ab 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -95,16 +95,28 @@ func (s *SuiteCommon) TestNewEndpointSCPLike(c *C) { c.Assert(e.String(), Equals, "ssh://git@github.com/user/repository.git") } -func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { +func (s *SuiteCommon) TestNewEndpointSCPLikeWithNumericPath(c *C) { e, err := NewEndpoint("git@github.com:9999/user/repository.git") c.Assert(err, IsNil) c.Assert(e.Protocol, Equals, "ssh") c.Assert(e.User, Equals, "git") c.Assert(e.Password, Equals, "") c.Assert(e.Host, Equals, "github.com") - c.Assert(e.Port, Equals, 9999) - c.Assert(e.Path, Equals, "user/repository.git") - c.Assert(e.String(), Equals, "ssh://git@github.com:9999/user/repository.git") + c.Assert(e.Port, Equals, 22) + c.Assert(e.Path, Equals, "9999/user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com/9999/user/repository.git") +} + +func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { + e, err := NewEndpoint("git@github.com:8080:9999/user/repository.git") + c.Assert(err, IsNil) + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 8080) + c.Assert(e.Path, Equals, "9999/user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com:8080/9999/user/repository.git") } func (s *SuiteCommon) TestNewEndpointFileAbs(c *C) { @@ -186,3 +198,15 @@ func (s *SuiteCommon) TestFilterUnsupportedCapabilities(c *C) { FilterUnsupportedCapabilities(l) c.Assert(l.Supports(capability.MultiACK), Equals, false) } + +func (s *SuiteCommon) TestNewEndpointIPv6(c *C) { + // see issue https://github.com/go-git/go-git/issues/740 + // + // IPv6 host names are not being properly handled, which results in unhelpful + // error messages depending on the format used. + // + e, err := NewEndpoint("http://[::1]:8080/foo.git") + c.Assert(err, IsNil) + c.Assert(e.Host, Equals, "[::1]") + c.Assert(e.String(), Equals, "http://[::1]:8080/foo.git") +} diff --git a/plumbing/transport/file/common_test.go b/plumbing/transport/file/common_test.go index 4d6612b..7e033a8 100644 --- a/plumbing/transport/file/common_test.go +++ b/plumbing/transport/file/common_test.go @@ -1,7 +1,6 @@ package file import ( - "io/ioutil" "os" "os/exec" "path/filepath" @@ -25,7 +24,7 @@ func (s *CommonSuite) SetUpSuite(c *C) { } var err error - s.tmpDir, err = ioutil.TempDir("", "") + s.tmpDir, err = os.MkdirTemp("", "") c.Assert(err, IsNil) s.ReceivePackBin = filepath.Join(s.tmpDir, "git-receive-pack") s.UploadPackBin = filepath.Join(s.tmpDir, "git-upload-pack") diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go index 306aae2..92fc0be 100644 --- a/plumbing/transport/git/common.go +++ b/plumbing/transport/git/common.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net" + "strconv" "github.com/go-git/go-git/v5/plumbing/format/pktline" "github.com/go-git/go-git/v5/plumbing/transport" @@ -69,7 +70,7 @@ func (c *command) getHostWithPort() string { port = DefaultPort } - return fmt.Sprintf("%s:%d", host, port) + return net.JoinHostPort(host, strconv.Itoa(port)) } // StderrPipe git protocol doesn't have any dedicated error channel @@ -77,14 +78,14 @@ func (c *command) StderrPipe() (io.Reader, error) { return nil, nil } -// StdinPipe return the underlying connection as WriteCloser, wrapped to prevent +// StdinPipe returns the underlying connection as WriteCloser, wrapped to prevent // call to the Close function from the connection, a command execution in git // protocol can't be closed or killed func (c *command) StdinPipe() (io.WriteCloser, error) { return ioutil.WriteNopCloser(c.conn), nil } -// StdoutPipe return the underlying connection as Reader +// StdoutPipe returns the underlying connection as Reader func (c *command) StdoutPipe() (io.Reader, error) { return c.conn, nil } @@ -92,7 +93,7 @@ func (c *command) StdoutPipe() (io.Reader, error) { func endpointToCommand(cmd string, ep *transport.Endpoint) string { host := ep.Host if ep.Port != DefaultPort { - host = fmt.Sprintf("%s:%d", ep.Host, ep.Port) + host = net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port)) } return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, host, 0) diff --git a/plumbing/transport/git/common_test.go b/plumbing/transport/git/common_test.go index 3391aaf..7389919 100644 --- a/plumbing/transport/git/common_test.go +++ b/plumbing/transport/git/common_test.go @@ -2,7 +2,6 @@ package git import ( "fmt" - "io/ioutil" "net" "os" "os/exec" @@ -37,7 +36,7 @@ func (s *BaseSuite) SetUpTest(c *C) { s.port, err = freePort() c.Assert(err, IsNil) - s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-protocol-%d", s.port)) + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-protocol-%d", s.port)) c.Assert(err, IsNil) } diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index d57c0fe..a7cdc1e 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -4,16 +4,22 @@ package http import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "fmt" "net" "net/http" + "net/url" + "reflect" "strconv" "strings" + "sync" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/protocol/packp" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/utils/ioutil" + "github.com/golang/groupcache/lru" ) // it requires a bytes.Buffer, because we need to know the length @@ -67,6 +73,17 @@ func advertisedReferences(ctx context.Context, s *session, serviceName string) ( return nil, err } + // Git 2.41+ returns a zero-id plus capabilities when an empty + // repository is being cloned. This skips the existing logic within + // advrefs_decode.decodeFirstHash, which expects a flush-pkt instead. + // + // This logic aligns with plumbing/transport/internal/common/common.go. + if ar.IsEmpty() && + // Empty repositories are valid for git-receive-pack. + transport.ReceivePackServiceName != serviceName { + return nil, transport.ErrEmptyRemoteRepository + } + transport.FilterUnsupportedCapabilities(ar.Capabilities) s.advRefs = ar @@ -74,40 +91,83 @@ func advertisedReferences(ctx context.Context, s *session, serviceName string) ( } type client struct { - c *http.Client + c *http.Client + transports *lru.Cache + m sync.RWMutex } -// DefaultClient is the default HTTP client, which uses `http.DefaultClient`. -var DefaultClient = NewClient(nil) +// ClientOptions holds user configurable options for the client. +type ClientOptions struct { + // CacheMaxEntries is the max no. of entries that the transport objects + // cache will hold at any given point of time. It must be a positive integer. + // Calling `client.addTransport()` after the cache has reached the specified + // size, will result in the least recently used transport getting deleted + // before the provided transport is added to the cache. + CacheMaxEntries int +} + +var ( + // defaultTransportCacheSize is the default capacity of the transport objects cache. + // Its value is 0 because transport caching is turned off by default and is an + // opt-in feature. + defaultTransportCacheSize = 0 + + // DefaultClient is the default HTTP client, which uses a net/http client configured + // with http.DefaultTransport. + DefaultClient = NewClient(nil) +) // NewClient creates a new client with a custom net/http client. // See `InstallProtocol` to install and override default http client. -// Unless a properly initialized client is given, it will fall back into -// `http.DefaultClient`. +// If the net/http client is nil or empty, it will use a net/http client configured +// with http.DefaultTransport. // // Note that for HTTP client cannot distinguish between private repositories and // unexistent repositories on GitHub. So it returns `ErrAuthorizationRequired` // for both. func NewClient(c *http.Client) transport.Transport { if c == nil { - return &client{http.DefaultClient} + c = &http.Client{ + Transport: http.DefaultTransport, + } } + return NewClientWithOptions(c, &ClientOptions{ + CacheMaxEntries: defaultTransportCacheSize, + }) +} - return &client{ +// NewClientWithOptions returns a new client configured with the provided net/http client +// and other custom options specific to the client. +// If the net/http client is nil or empty, it will use a net/http client configured +// with http.DefaultTransport. +func NewClientWithOptions(c *http.Client, opts *ClientOptions) transport.Transport { + if c == nil { + c = &http.Client{ + Transport: http.DefaultTransport, + } + } + cl := &client{ c: c, } + + if opts != nil { + if opts.CacheMaxEntries > 0 { + cl.transports = lru.New(opts.CacheMaxEntries) + } + } + return cl } func (c *client) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.UploadPackSession, error) { - return newUploadPackSession(c.c, ep, auth) + return newUploadPackSession(c, ep, auth) } func (c *client) NewReceivePackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.ReceivePackSession, error) { - return newReceivePackSession(c.c, ep, auth) + return newReceivePackSession(c, ep, auth) } type session struct { @@ -117,10 +177,106 @@ type session struct { advRefs *packp.AdvRefs } -func newSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { +func transportWithInsecureTLS(transport *http.Transport) { + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = true +} + +func transportWithCABundle(transport *http.Transport, caBundle []byte) error { + rootCAs, err := x509.SystemCertPool() + if err != nil { + return err + } + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + rootCAs.AppendCertsFromPEM(caBundle) + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = rootCAs + return nil +} + +func transportWithProxy(transport *http.Transport, proxyURL *url.URL) { + transport.Proxy = http.ProxyURL(proxyURL) +} + +func configureTransport(transport *http.Transport, ep *transport.Endpoint) error { + if len(ep.CaBundle) > 0 { + if err := transportWithCABundle(transport, ep.CaBundle); err != nil { + return err + } + } + if ep.InsecureSkipTLS { + transportWithInsecureTLS(transport) + } + + if ep.Proxy.URL != "" { + proxyURL, err := ep.Proxy.FullURL() + if err != nil { + return err + } + transportWithProxy(transport, proxyURL) + } + return nil +} + +func newSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { + var httpClient *http.Client + + // We need to configure the http transport if there are transport specific + // options present in the endpoint. + if len(ep.CaBundle) > 0 || ep.InsecureSkipTLS || ep.Proxy.URL != "" { + var transport *http.Transport + // if the client wasn't configured to have a cache for transports then just configure + // the transport and use it directly, otherwise try to use the cache. + if c.transports == nil { + tr, ok := c.c.Transport.(*http.Transport) + if !ok { + return nil, fmt.Errorf("expected underlying client transport to be of type: %s; got: %s", + reflect.TypeOf(transport), reflect.TypeOf(c.c.Transport)) + } + + transport = tr.Clone() + configureTransport(transport, ep) + } else { + transportOpts := transportOptions{ + caBundle: string(ep.CaBundle), + insecureSkipTLS: ep.InsecureSkipTLS, + } + if ep.Proxy.URL != "" { + proxyURL, err := ep.Proxy.FullURL() + if err != nil { + return nil, err + } + transportOpts.proxyURL = *proxyURL + } + var found bool + transport, found = c.fetchTransport(transportOpts) + + if !found { + transport = c.c.Transport.(*http.Transport).Clone() + configureTransport(transport, ep) + c.addTransport(transportOpts, transport) + } + } + + httpClient = &http.Client{ + Transport: transport, + CheckRedirect: c.c.CheckRedirect, + Jar: c.c.Jar, + Timeout: c.c.Timeout, + } + } else { + httpClient = c.c + } + s := &session{ auth: basicAuthFromEndpoint(ep), - client: c, + client: httpClient, endpoint: ep, } if auth != nil { diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index 4122e62..1517228 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -3,7 +3,6 @@ package http import ( "crypto/tls" "fmt" - "io/ioutil" "log" "net" "net/http" @@ -91,6 +90,60 @@ func (s *ClientSuite) TestNewHTTPError40x(c *C) { "unexpected client error.*") } +func (s *ClientSuite) Test_newSession(c *C) { + cl := NewClientWithOptions(nil, &ClientOptions{ + CacheMaxEntries: 2, + }).(*client) + + insecureEP := s.Endpoint + insecureEP.InsecureSkipTLS = true + session, err := newSession(cl, insecureEP, nil) + c.Assert(err, IsNil) + + sessionTransport := session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) + t, ok := cl.fetchTransport(transportOptions{ + insecureSkipTLS: true, + }) + // transport should be cached. + c.Assert(ok, Equals, true) + // cached transport should be the one that's used. + c.Assert(sessionTransport, Equals, t) + + caEndpoint := insecureEP + caEndpoint.CaBundle = []byte("this is the way") + session, err = newSession(cl, caEndpoint, nil) + c.Assert(err, IsNil) + + sessionTransport = session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) + c.Assert(sessionTransport.TLSClientConfig.RootCAs, NotNil) + t, ok = cl.fetchTransport(transportOptions{ + insecureSkipTLS: true, + caBundle: "this is the way", + }) + // transport should be cached. + c.Assert(ok, Equals, true) + // cached transport should be the one that's used. + c.Assert(sessionTransport, Equals, t) + + session, err = newSession(cl, caEndpoint, nil) + c.Assert(err, IsNil) + sessionTransport = session.client.Transport.(*http.Transport) + // transport that's going to be used should be cached already. + c.Assert(sessionTransport, Equals, t) + // no new transport got cached. + c.Assert(cl.transports.Len(), Equals, 2) + + // if the cache does not exist, the transport should still be correctly configured. + cl.transports = nil + session, err = newSession(cl, insecureEP, nil) + c.Assert(err, IsNil) + + sessionTransport = session.client.Transport.(*http.Transport) + c.Assert(sessionTransport.TLSClientConfig.InsecureSkipVerify, Equals, true) +} + func (s *ClientSuite) testNewHTTPError(c *C, code int, msg string) { req, _ := http.NewRequest("GET", "foo", nil) res := &http.Response{ @@ -168,7 +221,7 @@ func (s *BaseSuite) SetUpTest(c *C) { l, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - base, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-http-%d", s.port)) + base, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-http-%d", s.port)) c.Assert(err, IsNil) s.port = l.Addr().(*net.TCPAddr).Port diff --git a/plumbing/transport/http/internal/test/proxy_test.go b/plumbing/transport/http/internal/test/proxy_test.go new file mode 100644 index 0000000..6ae2943 --- /dev/null +++ b/plumbing/transport/http/internal/test/proxy_test.go @@ -0,0 +1,72 @@ +package test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + nethttp "net/http" + "os" + "sync/atomic" + "testing" + + "github.com/elazarl/goproxy" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + + . "gopkg.in/check.v1" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type ProxySuite struct{} + +var _ = Suite(&ProxySuite{}) + +var proxiedRequests int32 + +// This test tests proxy support via an env var, i.e. `HTTPS_PROXY`. +// Its located in a separate package because golang caches the value +// of proxy env vars leading to misleading/unexpected test results. +func (s *ProxySuite) TestAdvertisedReferences(c *C) { + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + SetupHTTPSProxy(proxy, &proxiedRequests) + httpsListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpsListener.Close() + httpProxyAddr := fmt.Sprintf("localhost:%d", httpsListener.Addr().(*net.TCPAddr).Port) + + proxyServer := nethttp.Server{ + Addr: httpProxyAddr, + Handler: proxy, + // Due to how golang manages http/2 when provided with custom TLS config, + // servers and clients running in the same process leads to issues. + // Ref: https://github.com/golang/go/issues/21336 + TLSConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + }, + } + go proxyServer.ServeTLS(httpsListener, "../../testdata/certs/server.crt", "../../testdata/certs/server.key") + defer proxyServer.Close() + os.Setenv("HTTPS_PROXY", fmt.Sprintf("https://user:pass@%s", httpProxyAddr)) + defer os.Unsetenv("HTTPS_PROXY") + + endpoint, err := transport.NewEndpoint("https://github.com/git-fixtures/basic.git") + c.Assert(err, IsNil) + endpoint.InsecureSkipTLS = true + + client := http.DefaultClient + session, err := client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + info, err := session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} diff --git a/plumbing/transport/http/internal/test/test_utils.go b/plumbing/transport/http/internal/test/test_utils.go new file mode 100644 index 0000000..6665fb3 --- /dev/null +++ b/plumbing/transport/http/internal/test/test_utils.go @@ -0,0 +1,43 @@ +package test + +import ( + "encoding/base64" + "strings" + "sync/atomic" + + "github.com/elazarl/goproxy" +) + +func SetupHTTPSProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) { + var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + if strings.Contains(host, "github.com") { + user, pass, _ := ParseBasicAuth(ctx.Req.Header.Get("Proxy-Authorization")) + if user != "user" || pass != "pass" { + return goproxy.RejectConnect, host + } + atomic.AddInt32(proxiedRequests, 1) + return goproxy.OkConnect, host + } + // Reject if it isn't our request. + return goproxy.RejectConnect, host + } + proxy.OnRequest().HandleConnect(proxyHandler) +} + +// adapted from https://github.com/golang/go/blob/2ef70d9d0f98832c8103a7968b195e560a8bb262/src/net/http/request.go#L959 +func ParseBasicAuth(auth string) (username, password string, ok bool) { + const prefix = "Basic " + if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) { + return "", "", false + } + c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) + if err != nil { + return "", "", false + } + cs := string(c) + username, password, ok = strings.Cut(cs, ":") + if !ok { + return "", "", false + } + return username, password, true +} diff --git a/plumbing/transport/http/proxy_test.go b/plumbing/transport/http/proxy_test.go new file mode 100644 index 0000000..f3024da --- /dev/null +++ b/plumbing/transport/http/proxy_test.go @@ -0,0 +1,119 @@ +package http + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "strings" + "sync/atomic" + + "github.com/elazarl/goproxy" + fixtures "github.com/go-git/go-git-fixtures/v4" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http/internal/test" + + . "gopkg.in/check.v1" +) + +type ProxySuite struct { + u UploadPackSuite + fixtures.Suite +} + +var _ = Suite(&ProxySuite{}) + +var proxiedRequests int32 + +func (s *ProxySuite) TestAdvertisedReferences(c *C) { + s.u.SetUpTest(c) + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + setupHTTPProxy(proxy, &proxiedRequests) + httpListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpListener.Close() + + httpProxyAddr := fmt.Sprintf("http://localhost:%d", httpListener.Addr().(*net.TCPAddr).Port) + proxyServer := http.Server{ + Addr: httpProxyAddr, + Handler: proxy, + } + go proxyServer.Serve(httpListener) + defer proxyServer.Close() + + endpoint := s.u.prepareRepository(c, fixtures.Basic().One(), "basic.git") + endpoint.Proxy = transport.ProxyOptions{ + URL: httpProxyAddr, + Username: "user", + Password: "pass", + } + + s.u.Client = NewClient(nil) + session, err := s.u.Client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + info, err := session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) + + atomic.StoreInt32(&proxiedRequests, 0) + test.SetupHTTPSProxy(proxy, &proxiedRequests) + httpsListener, err := net.Listen("tcp", ":0") + c.Assert(err, IsNil) + defer httpsListener.Close() + httpsProxyAddr := fmt.Sprintf("https://localhost:%d", httpsListener.Addr().(*net.TCPAddr).Port) + + tlsProxyServer := http.Server{ + Addr: httpsProxyAddr, + Handler: proxy, + // Due to how golang manages http/2 when provided with custom TLS config, + // servers and clients running in the same process leads to issues. + // Ref: https://github.com/golang/go/issues/21336 + TLSConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + }, + } + go tlsProxyServer.ServeTLS(httpsListener, "testdata/certs/server.crt", "testdata/certs/server.key") + defer tlsProxyServer.Close() + + endpoint, err = transport.NewEndpoint("https://github.com/git-fixtures/basic.git") + c.Assert(err, IsNil) + endpoint.Proxy = transport.ProxyOptions{ + URL: httpsProxyAddr, + Username: "user", + Password: "pass", + } + endpoint.InsecureSkipTLS = true + + session, err = s.u.Client.NewUploadPackSession(endpoint, nil) + c.Assert(err, IsNil) + + info, err = session.AdvertisedReferencesContext(ctx) + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed = atomic.LoadInt32(&proxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +func setupHTTPProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) { + // The request is being forwarded to the local test git server in this handler. + var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if strings.Contains(req.Host, "localhost") { + user, pass, _ := test.ParseBasicAuth(req.Header.Get("Proxy-Authorization")) + if user != "user" || pass != "pass" { + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusUnauthorized, "") + } + atomic.AddInt32(proxiedRequests, 1) + return req, nil + } + // Reject if it isn't our request. + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "") + } + proxy.OnRequest().Do(proxyHandler) +} diff --git a/plumbing/transport/http/receive_pack.go b/plumbing/transport/http/receive_pack.go index 4d14ff2..4387ecf 100644 --- a/plumbing/transport/http/receive_pack.go +++ b/plumbing/transport/http/receive_pack.go @@ -19,7 +19,7 @@ type rpSession struct { *session } -func newReceivePackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { +func newReceivePackSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { s, err := newSession(c, ep, auth) return &rpSession{s}, err } diff --git a/plumbing/transport/http/testdata/certs/server.crt b/plumbing/transport/http/testdata/certs/server.crt new file mode 100644 index 0000000..9bdec2c --- /dev/null +++ b/plumbing/transport/http/testdata/certs/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUWcuzUyG3EfGsXVUH0BAmnCJyNHswDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMDMwNzA3MTgwNloXDTI0MDMwNjA3MTgwNlowWTELMAkGA1UEBhMCQVUxEzAR +BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 +IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAvyKX6vJXt1u+WBfBNJByFDAb7msdsk6SiPFlX5uyilaWmlRxvLo1 +GZMjjuQbs4wU6BAoZcgiELnsC9GSyxgrhk7NIW3ud/QD7s8ZxETxFLb0ur6tJj7/ +ETEcU/AKSl1FpeJbGHqGipYp5+0GU0zPDxRYqC2N3+fcGZPQbBwxb1f+MrBjWutb +3eNYTLdPH3W7RUqbunC1KZRJ8XOcU5XZ4qEaMkZYdz1QItxwPnpPuSZs53ga3TDF +zclpQcT8OH2JNwSI6bwlwFJ0Es06manw7XHmgd8anhix9FdsQYaTOW4kqh1iKQ/P +jPG50bdTUEqlOsaa+9av/qf+90npzt3xqQIDAQABo1MwUTAdBgNVHQ4EFgQUqTqb +q+jiJVgwftQS+YLcQWnvTuAwHwYDVR0jBBgwFoAUqTqbq+jiJVgwftQS+YLcQWnv +TuAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAVUaFSikxyCy1 +4P/ZZgeuR7vEJ5vWBxKPw/jFNZUFWy2Ag32w1BhrDwoYoc1Awg76QF2TqBQAhFNm +ek9aE+L83P/R2UhE9+LHnzwdMXt9HYOI1grONk2z3lMI1y4FCJBxHfGyC/XMoNgZ +qP7UdLgLGTIMN3O1Fww416Hn8BHzxN4o5ZEHJZ6QPMuN8OLk9oVu3yQIq/QcmSDH +GT2RiwT5IJWMUKK1UrV+y3/T9FwW2qqu+LX+coxjk7HgDWy3y66V9ahLBt8kONcr +qK0zutoQ5WPSmvnD2Nr0LVLGXEd7hbQNO7bgjO2YOBtnagUQJt72i/OmvZv8Mfnp +Bu6Qgl5hDw== +-----END CERTIFICATE----- diff --git a/plumbing/transport/http/testdata/certs/server.key b/plumbing/transport/http/testdata/certs/server.key new file mode 100644 index 0000000..9a0cd8f --- /dev/null +++ b/plumbing/transport/http/testdata/certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC/Ipfq8le3W75Y +F8E0kHIUMBvuax2yTpKI8WVfm7KKVpaaVHG8ujUZkyOO5BuzjBToEChlyCIQuewL +0ZLLGCuGTs0hbe539APuzxnERPEUtvS6vq0mPv8RMRxT8ApKXUWl4lsYeoaKlinn +7QZTTM8PFFioLY3f59wZk9BsHDFvV/4ysGNa61vd41hMt08fdbtFSpu6cLUplEnx +c5xTldnioRoyRlh3PVAi3HA+ek+5JmzneBrdMMXNyWlBxPw4fYk3BIjpvCXAUnQS +zTqZqfDtceaB3xqeGLH0V2xBhpM5biSqHWIpD8+M8bnRt1NQSqU6xpr71q/+p/73 +SenO3fGpAgMBAAECggEAQUjenQhzv5Rdmpdajcq8vHqGP9Rki0/dK1tQpex3elsD +C+nGA5GSq46feaIeeCBjz7QdKE7Im+/1WUAXJLm3vCNUW5PB/UTixwIEKg7mTY4E +X3jbiZHA661boKv/x9C+BmAff2fyZonN/ILwQymcG+l2MtOEfzMh8baUXSjwFbhg +B08u4iXjee0y9I0CGMYWfasHLOIuhACCFKtqnvdQp8B82g8eSPhme5IjfPP8KZVr +00n6z8m00HVk6/yYJ8pVZ82j3T+wH6IqvlvaC320sbto8YXV6i8GWHaJumzU4/9s +IRm4459E+NmNcLNY/TCu89zsfrgNirN+qFfvJIOTxQKBgQDtME8s4UP0MhGuJ2lD +1HD64fAxMC6Xp/QSzY91Yn79UNssUUV+IwjuUnLIz3U8DBs/QETLm7CkNtI7h5m5 +dBdeBBzCRGxhe8WqRfvceu4s0zr08ZkDaKLjFsBSnBsXZhKAAuRqBjnGAoAiKgVa +WpEAug00ThhQjipSY9tO9NSBawKBgQDOSz+8m2HJFktEdSctKIB9DesqlAg7YCyy +dHzywP0/r7wEvsCN7xPgCT5g8JBkRaFvLLKgw7gMKAUx8V2iwizEoDCAs/pbTWji +uZwPC8lWtbkpBMQIaP4Wap+GyFQJKv1/qZduwpkwkj+ok+m3WwIW55VFGyLn3XGG +VcLZm83aOwKBgQDXXI/nXjqHVZb8HEjWD+Ttx4yB/Q+xIAzbrc3edap8c5guKzUA +DOulCTOz5bq65PsweTh970V6NVS6PKt12lUFRpKeSeZmtS2LJ7RCQ1RTWxAjK+MV +V0LfEt9ZouhuXH3bwcSICFMY2VhirOjjW2xhzo0Cuw4UxqDi4kxU6rSxNQKBgQCI +sn5KmV/jot0/QK40E0mJFEcHkM4foiwcGGqPZWiq4eUh89CefJTb+OQX0nCrsSQ3 +ChRXyTlU/NPsczcL2cVWiZt6PUihZZsh2cJaigHhbkuCrcDEneX4rrCE3IwrAwy1 +oohRAawG7nI2X8UYFbs9uDlGcKPhpvBKBtw13DM87wKBgE8fOiFoTph//6piU7dV +pN33UfhPcAFwsIzxAH6Ljo6BYx2hfPRCxI2g0wchk6ydbDecLgMwVgugdJZ6+tRf +P+YV3wEwPcWOvWby3+EmJh0cXUTl6ZMA+eR4pvCi6kf2xJf9dRmEeNNhOuzn9Y0J +cT9yhBFG4iejKP0iTwET1JKY +-----END PRIVATE KEY----- diff --git a/plumbing/transport/http/transport.go b/plumbing/transport/http/transport.go new file mode 100644 index 0000000..052f3c8 --- /dev/null +++ b/plumbing/transport/http/transport.go @@ -0,0 +1,40 @@ +package http + +import ( + "net/http" + "net/url" +) + +// transportOptions contains transport specific configuration. +type transportOptions struct { + insecureSkipTLS bool + // []byte is not comparable. + caBundle string + proxyURL url.URL +} + +func (c *client) addTransport(opts transportOptions, transport *http.Transport) { + c.m.Lock() + c.transports.Add(opts, transport) + c.m.Unlock() +} + +func (c *client) removeTransport(opts transportOptions) { + c.m.Lock() + c.transports.Remove(opts) + c.m.Unlock() +} + +func (c *client) fetchTransport(opts transportOptions) (*http.Transport, bool) { + c.m.RLock() + t, ok := c.transports.Get(opts) + c.m.RUnlock() + if !ok { + return nil, false + } + transport, ok := t.(*http.Transport) + if !ok { + return nil, false + } + return transport, true +} diff --git a/plumbing/transport/http/upload_pack.go b/plumbing/transport/http/upload_pack.go index e735b3d..4f85145 100644 --- a/plumbing/transport/http/upload_pack.go +++ b/plumbing/transport/http/upload_pack.go @@ -19,7 +19,7 @@ type upSession struct { *session } -func newUploadPackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { +func newUploadPackSession(c *client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { s, err := newSession(c, ep, auth) return &upSession{s}, err } diff --git a/plumbing/transport/http/upload_pack_test.go b/plumbing/transport/http/upload_pack_test.go index c088ecc..abb7adf 100644 --- a/plumbing/transport/http/upload_pack_test.go +++ b/plumbing/transport/http/upload_pack_test.go @@ -3,7 +3,7 @@ package http import ( "context" "fmt" - "io/ioutil" + "io" "net/url" "os" "path/filepath" @@ -49,7 +49,7 @@ func (s *UploadPackSuite) TestuploadPackRequestToReader(c *C) { sr, err := uploadPackRequestToReader(r) c.Assert(err, IsNil) - b, _ := ioutil.ReadAll(sr) + b, _ := io.ReadAll(sr) c.Assert(string(b), Equals, "0032want 2b41ef280fdb67a9b250678686a0c3e03b0a9989\n"+ "0032want d82f291cde9987322c8a0c81a325e1ba6159684c\n0000"+ diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index d0e9a29..5fdf425 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - stdioutil "io/ioutil" "strings" "time" @@ -156,7 +155,7 @@ func (c *client) listenFirstError(r io.Reader) chan string { close(errLine) } - _, _ = io.Copy(stdioutil.Discard, r) + _, _ = io.Copy(io.Discard, r) }() return errLine @@ -233,7 +232,7 @@ func (s *session) handleAdvRefDecodeError(err error) error { // UploadPack performs a request to the server to fetch a packfile. A reader is // returned with the packfile content. The reader must be closed after reading. func (s *session) UploadPack(ctx context.Context, req *packp.UploadPackRequest) (*packp.UploadPackResponse, error) { - if req.IsEmpty() && len(req.Shallows) == 0 { + if req.IsEmpty() { return nil, transport.ErrEmptyUploadPackRequest } @@ -374,7 +373,7 @@ func (s *session) checkNotFoundError() error { case <-t.C: return ErrTimeoutExceeded case line, ok := <-s.firstErrLine: - if !ok { + if !ok || len(line) == 0 { return nil } diff --git a/plumbing/transport/internal/common/common_test.go b/plumbing/transport/internal/common/common_test.go index c60ef3b..affa787 100644 --- a/plumbing/transport/internal/common/common_test.go +++ b/plumbing/transport/internal/common/common_test.go @@ -76,3 +76,17 @@ func (s *CommonSuite) TestIsRepoNotFoundErrorForGogsAccessDenied(c *C) { c.Assert(isRepoNotFound, Equals, true) } + +func (s *CommonSuite) TestCheckNotFoundError(c *C) { + firstErrLine := make(chan string, 1) + + session := session{ + firstErrLine: firstErrLine, + } + + firstErrLine <- "" + + err := session.checkNotFoundError() + + c.Assert(err, IsNil) +} diff --git a/plumbing/transport/server/server.go b/plumbing/transport/server/server.go index 8ab70fe..11fa0c8 100644 --- a/plumbing/transport/server/server.go +++ b/plumbing/transport/server/server.go @@ -189,7 +189,7 @@ func (s *upSession) objectsToUpload(req *packp.UploadPackRequest) ([]plumbing.Ha } func (*upSession) setSupportedCapabilities(c *capability.List) error { - if err := c.Set(capability.Agent, capability.DefaultAgent); err != nil { + if err := c.Set(capability.Agent, capability.DefaultAgent()); err != nil { return err } @@ -355,7 +355,7 @@ func (s *rpSession) reportStatus() *packp.ReportStatus { } func (*rpSession) setSupportedCapabilities(c *capability.List) error { - if err := c.Set(capability.Agent, capability.DefaultAgent); err != nil { + if err := c.Set(capability.Agent, capability.DefaultAgent()); err != nil { return err } diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index 3514669..ac4e358 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -3,17 +3,15 @@ package ssh import ( "errors" "fmt" - "io/ioutil" "os" "os/user" "path/filepath" "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/mitchellh/go-homedir" + "github.com/skeema/knownhosts" sshagent "github.com/xanzy/ssh-agent" "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/knownhosts" ) const DefaultUsername = "git" @@ -135,7 +133,7 @@ func NewPublicKeys(user string, pemBytes []byte, password string) (*PublicKeys, // encoded private key. An encryption password should be given if the pemBytes // contains a password encrypted PEM block otherwise password should be empty. func NewPublicKeysFromFile(user, pemFile, password string) (*PublicKeys, error) { - bytes, err := ioutil.ReadFile(pemFile) + bytes, err := os.ReadFile(pemFile) if err != nil { return nil, err } @@ -224,12 +222,19 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // // If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: -// /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file +// +// /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // // If SSH_KNOWN_HOSTS is not set the following file locations will be used: -// ~/.ssh/known_hosts -// /etc/ssh/ssh_known_hosts +// +// ~/.ssh/known_hosts +// /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { + kh, err := newKnownHosts(files...) + return ssh.HostKeyCallback(kh), err +} + +func newKnownHosts(files ...string) (knownhosts.HostKeyCallback, error) { var err error if len(files) == 0 { @@ -251,7 +256,7 @@ func getDefaultKnownHostsFiles() ([]string, error) { return files, nil } - homeDirPath, err := homedir.Dir() + homeDirPath, err := os.UserHomeDir() if err != nil { return nil, err } diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index 46e7913..1531603 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -4,12 +4,14 @@ package ssh import ( "context" "fmt" + "net" "reflect" "strconv" "strings" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/internal/common" + "github.com/skeema/knownhosts" "github.com/kevinburke/ssh_config" "golang.org/x/crypto/ssh" @@ -121,10 +123,24 @@ func (c *command) connect() error { if err != nil { return err } + hostWithPort := c.getHostWithPort() + if config.HostKeyCallback == nil { + kh, err := newKnownHosts() + if err != nil { + return err + } + config.HostKeyCallback = kh.HostKeyCallback() + config.HostKeyAlgorithms = kh.HostKeyAlgorithms(hostWithPort) + } else if len(config.HostKeyAlgorithms) == 0 { + // Set the HostKeyAlgorithms based on HostKeyCallback. + // For background see https://github.com/go-git/go-git/issues/411 as well as + // https://github.com/golang/go/issues/29286 for root cause. + config.HostKeyAlgorithms = knownhosts.HostKeyAlgorithms(config.HostKeyCallback, hostWithPort) + } overrideConfig(c.config, config) - c.client, err = dial("tcp", c.getHostWithPort(), config) + c.client, err = dial("tcp", hostWithPort, c.endpoint.Proxy, config) if err != nil { return err } @@ -139,7 +155,7 @@ func (c *command) connect() error { return nil } -func dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { +func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.ClientConfig) (*ssh.Client, error) { var ( ctx = context.Background() cancel context.CancelFunc @@ -151,10 +167,33 @@ func dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { } defer cancel() - conn, err := proxy.Dial(ctx, network, addr) + var conn net.Conn + var err error + + if proxyOpts.URL != "" { + proxyUrl, err := proxyOpts.FullURL() + if err != nil { + return nil, err + } + dialer, err := proxy.FromURL(proxyUrl, proxy.Direct) + if err != nil { + return nil, err + } + + // Try to use a ContextDialer, but fall back to a Dialer if that goes south. + ctxDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, fmt.Errorf("expected ssh proxy dialer to be of type %s; got %s", + reflect.TypeOf(ctxDialer), reflect.TypeOf(dialer)) + } + conn, err = ctxDialer.DialContext(ctx, "tcp", addr) + } else { + conn, err = proxy.Dial(ctx, network, addr) + } if err != nil { return nil, err } + c, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { return nil, err @@ -173,7 +212,7 @@ func (c *command) getHostWithPort() string { port = DefaultPort } - return fmt.Sprintf("%s:%d", host, port) + return net.JoinHostPort(host, strconv.Itoa(port)) } func (c *command) doGetHostWithPortFromSSHConfig() (addr string, found bool) { @@ -201,7 +240,7 @@ func (c *command) doGetHostWithPortFromSSHConfig() (addr string, found bool) { } } - addr = fmt.Sprintf("%s:%d", host, port) + addr = net.JoinHostPort(host, strconv.Itoa(port)) return } diff --git a/plumbing/transport/ssh/common_test.go b/plumbing/transport/ssh/common_test.go index 6d634d5..496e82d 100644 --- a/plumbing/transport/ssh/common_test.go +++ b/plumbing/transport/ssh/common_test.go @@ -5,23 +5,25 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/gliderlabs/ssh" "github.com/kevinburke/ssh_config" - "golang.org/x/crypto/ssh" + stdssh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/testdata" . "gopkg.in/check.v1" ) func Test(t *testing.T) { TestingT(t) } func (s *SuiteCommon) TestOverrideConfig(c *C) { - config := &ssh.ClientConfig{ + config := &stdssh.ClientConfig{ User: "foo", - Auth: []ssh.AuthMethod{ - ssh.Password("yourpassword"), + Auth: []stdssh.AuthMethod{ + stdssh.Password("yourpassword"), }, - HostKeyCallback: ssh.FixedHostKey(nil), + HostKeyCallback: stdssh.FixedHostKey(nil), } - target := &ssh.ClientConfig{} + target := &stdssh.ClientConfig{} overrideConfig(config, target) c.Assert(target.User, Equals, "foo") @@ -30,11 +32,11 @@ func (s *SuiteCommon) TestOverrideConfig(c *C) { } func (s *SuiteCommon) TestOverrideConfigKeep(c *C) { - config := &ssh.ClientConfig{ + config := &stdssh.ClientConfig{ User: "foo", } - target := &ssh.ClientConfig{ + target := &stdssh.ClientConfig{ User: "bar", } @@ -93,12 +95,69 @@ func (s *SuiteCommon) TestDefaultSSHConfigWildcard(c *C) { c.Assert(cmd.getHostWithPort(), Equals, "github.com:22") } +func (s *SuiteCommon) TestIgnoreHostKeyCallback(c *C) { + uploadPack := &UploadPackSuite{ + opts: []ssh.Option{ + ssh.HostKeyPEM(testdata.PEMBytes["ed25519"]), + }, + } + uploadPack.SetUpSuite(c) + // Use the default client, which does not have a host key callback + uploadPack.Client = DefaultClient + auth, err := NewPublicKeys("foo", testdata.PEMBytes["rsa"], "") + c.Assert(err, IsNil) + c.Assert(auth, NotNil) + auth.HostKeyCallback = stdssh.InsecureIgnoreHostKey() + ep := uploadPack.newEndpoint(c, "bar.git") + ps, err := uploadPack.Client.NewUploadPackSession(ep, auth) + c.Assert(err, IsNil) + c.Assert(ps, NotNil) +} + +func (s *SuiteCommon) TestFixedHostKeyCallback(c *C) { + hostKey, err := stdssh.ParsePrivateKey(testdata.PEMBytes["ed25519"]) + c.Assert(err, IsNil) + uploadPack := &UploadPackSuite{ + opts: []ssh.Option{ + ssh.HostKeyPEM(testdata.PEMBytes["ed25519"]), + }, + } + uploadPack.SetUpSuite(c) + // Use the default client, which does not have a host key callback + uploadPack.Client = DefaultClient + auth, err := NewPublicKeys("foo", testdata.PEMBytes["rsa"], "") + c.Assert(err, IsNil) + c.Assert(auth, NotNil) + auth.HostKeyCallback = stdssh.FixedHostKey(hostKey.PublicKey()) + ep := uploadPack.newEndpoint(c, "bar.git") + ps, err := uploadPack.Client.NewUploadPackSession(ep, auth) + c.Assert(err, IsNil) + c.Assert(ps, NotNil) +} + +func (s *SuiteCommon) TestFailHostKeyCallback(c *C) { + uploadPack := &UploadPackSuite{ + opts: []ssh.Option{ + ssh.HostKeyPEM(testdata.PEMBytes["ed25519"]), + }, + } + uploadPack.SetUpSuite(c) + // Use the default client, which does not have a host key callback + uploadPack.Client = DefaultClient + auth, err := NewPublicKeys("foo", testdata.PEMBytes["rsa"], "") + c.Assert(err, IsNil) + c.Assert(auth, NotNil) + ep := uploadPack.newEndpoint(c, "bar.git") + _, err = uploadPack.Client.NewUploadPackSession(ep, auth) + c.Assert(err, NotNil) +} + func (s *SuiteCommon) TestIssue70(c *C) { uploadPack := &UploadPackSuite{} uploadPack.SetUpSuite(c) - config := &ssh.ClientConfig{ - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + config := &stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), } r := &runner{ config: config, diff --git a/plumbing/transport/ssh/internal/test/proxy_test.go b/plumbing/transport/ssh/internal/test/proxy_test.go new file mode 100644 index 0000000..8e775f8 --- /dev/null +++ b/plumbing/transport/ssh/internal/test/proxy_test.go @@ -0,0 +1,112 @@ +package test + +import ( + "context" + "fmt" + "log" + "net" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/armon/go-socks5" + "github.com/gliderlabs/ssh" + "github.com/go-git/go-git/v5/plumbing/transport" + ggssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + + fixtures "github.com/go-git/go-git-fixtures/v4" + stdssh "golang.org/x/crypto/ssh" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type ProxyEnvSuite struct { + fixtures.Suite + port int + base string +} + +var _ = Suite(&ProxyEnvSuite{}) + +var socksProxiedRequests int32 + +// This test tests proxy support via an env var, i.e. `ALL_PROXY`. +// Its located in a separate package because golang caches the value +// of proxy env vars leading to misleading/unexpected test results. +func (s *ProxyEnvSuite) TestCommand(c *C) { + socksListener, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) + + socksServer, err := socks5.New(&socks5.Config{ + Rules: TestProxyRule{}, + }) + c.Assert(err, IsNil) + go func() { + socksServer.Serve(socksListener) + }() + socksProxyAddr := fmt.Sprintf("socks5://localhost:%d", socksListener.Addr().(*net.TCPAddr).Port) + os.Setenv("ALL_PROXY", socksProxyAddr) + defer os.Unsetenv("ALL_PROXY") + + sshListener, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) + sshServer := &ssh.Server{Handler: HandlerSSH} + go func() { + log.Fatal(sshServer.Serve(sshListener)) + }() + + s.port = sshListener.Addr().(*net.TCPAddr).Port + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) + c.Assert(err, IsNil) + + ggssh.DefaultAuthBuilder = func(user string) (ggssh.AuthMethod, error) { + return &ggssh.Password{User: user}, nil + } + + ep := s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + c.Assert(err, IsNil) + + client := ggssh.NewClient(&stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + }) + r, err := client.NewUploadPackSession(ep, nil) + c.Assert(err, IsNil) + defer func() { c.Assert(r.Close(), IsNil) }() + + info, err := r.AdvertisedReferences() + c.Assert(err, IsNil) + c.Assert(info, NotNil) + proxyUsed := atomic.LoadInt32(&socksProxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +func (s *ProxyEnvSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) + c.Assert(err, IsNil) + + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) + c.Assert(err, IsNil) + + return s.newEndpoint(c, name) +} + +func (s *ProxyEnvSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf( + "ssh://git@localhost:%d/%s/%s", s.port, filepath.ToSlash(s.base), name, + )) + + c.Assert(err, IsNil) + return ep +} + +type TestProxyRule struct{} + +func (dr TestProxyRule) Allow(ctx context.Context, req *socks5.Request) (context.Context, bool) { + atomic.AddInt32(&socksProxiedRequests, 1) + return ctx, true +} diff --git a/plumbing/transport/ssh/internal/test/test_utils.go b/plumbing/transport/ssh/internal/test/test_utils.go new file mode 100644 index 0000000..c3797b1 --- /dev/null +++ b/plumbing/transport/ssh/internal/test/test_utils.go @@ -0,0 +1,83 @@ +package test + +import ( + "fmt" + "io" + "os/exec" + "runtime" + "strings" + "sync" + + "github.com/gliderlabs/ssh" +) + +func HandlerSSH(s ssh.Session) { + cmd, stdin, stderr, stdout, err := buildCommand(s.Command()) + if err != nil { + fmt.Println(err) + return + } + + if err := cmd.Start(); err != nil { + fmt.Println(err) + return + } + + go func() { + defer stdin.Close() + io.Copy(stdin, s) + }() + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + io.Copy(s.Stderr(), stderr) + }() + + go func() { + defer wg.Done() + io.Copy(s, stdout) + }() + + wg.Wait() + + if err := cmd.Wait(); err != nil { + return + } + +} + +func buildCommand(c []string) (cmd *exec.Cmd, stdin io.WriteCloser, stderr, stdout io.ReadCloser, err error) { + if len(c) != 2 { + err = fmt.Errorf("invalid command") + return + } + + // fix for Windows environments + var path string + if runtime.GOOS == "windows" { + path = strings.Replace(c[1], "/C:/", "C:/", 1) + } else { + path = c[1] + } + + cmd = exec.Command(c[0], path) + stdout, err = cmd.StdoutPipe() + if err != nil { + return + } + + stdin, err = cmd.StdinPipe() + if err != nil { + return + } + + stderr, err = cmd.StderrPipe() + if err != nil { + return + } + + return +} diff --git a/plumbing/transport/ssh/proxy_test.go b/plumbing/transport/ssh/proxy_test.go index 3caf1ff..2ba98e8 100644 --- a/plumbing/transport/ssh/proxy_test.go +++ b/plumbing/transport/ssh/proxy_test.go @@ -1,36 +1,87 @@ package ssh import ( + "context" "fmt" "log" "net" "os" + "sync/atomic" "github.com/armon/go-socks5" + "github.com/gliderlabs/ssh" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/ssh/internal/test" + + fixtures "github.com/go-git/go-git-fixtures/v4" + stdssh "golang.org/x/crypto/ssh" . "gopkg.in/check.v1" ) type ProxySuite struct { - UploadPackSuite + u UploadPackSuite + fixtures.Suite } var _ = Suite(&ProxySuite{}) -func (s *ProxySuite) SetUpSuite(c *C) { - s.UploadPackSuite.SetUpSuite(c) +var socksProxiedRequests int32 - l, err := net.Listen("tcp", "localhost:0") +func (s *ProxySuite) TestCommand(c *C) { + socksListener, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - server, err := socks5.New(&socks5.Config{}) + socksServer, err := socks5.New(&socks5.Config{ + AuthMethods: []socks5.Authenticator{socks5.UserPassAuthenticator{ + Credentials: socks5.StaticCredentials{ + "user": "pass", + }, + }}, + Rules: TestProxyRule{}, + }) c.Assert(err, IsNil) + go func() { + socksServer.Serve(socksListener) + }() + socksProxyAddr := fmt.Sprintf("socks5://localhost:%d", socksListener.Addr().(*net.TCPAddr).Port) - port := l.Addr().(*net.TCPAddr).Port - - err = os.Setenv("ALL_PROXY", fmt.Sprintf("socks5://localhost:%d", port)) + sshListener, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) - + sshServer := &ssh.Server{Handler: test.HandlerSSH} go func() { - log.Fatal(server.Serve(l)) + log.Fatal(sshServer.Serve(sshListener)) }() + + s.u.port = sshListener.Addr().(*net.TCPAddr).Port + s.u.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.u.port)) + c.Assert(err, IsNil) + + DefaultAuthBuilder = func(user string) (AuthMethod, error) { + return &Password{User: user}, nil + } + + ep := s.u.prepareRepository(c, fixtures.Basic().One(), "basic.git") + c.Assert(err, IsNil) + ep.Proxy = transport.ProxyOptions{ + URL: socksProxyAddr, + Username: "user", + Password: "pass", + } + + runner := runner{ + config: &stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + }, + } + _, err = runner.Command(transport.UploadPackServiceName, ep, nil) + c.Assert(err, IsNil) + proxyUsed := atomic.LoadInt32(&socksProxiedRequests) > 0 + c.Assert(proxyUsed, Equals, true) +} + +type TestProxyRule struct{} + +func (dr TestProxyRule) Allow(ctx context.Context, req *socks5.Request) (context.Context, bool) { + atomic.AddInt32(&socksProxiedRequests, 1) + return ctx, true } diff --git a/plumbing/transport/ssh/upload_pack_test.go b/plumbing/transport/ssh/upload_pack_test.go index e65e04a..67af566 100644 --- a/plumbing/transport/ssh/upload_pack_test.go +++ b/plumbing/transport/ssh/upload_pack_test.go @@ -3,7 +3,6 @@ package ssh import ( "fmt" "io" - "io/ioutil" "log" "net" "os" @@ -14,6 +13,7 @@ import ( "sync" "github.com/go-git/go-git/v5/plumbing/transport" + testutils "github.com/go-git/go-git/v5/plumbing/transport/ssh/internal/test" "github.com/go-git/go-git/v5/plumbing/transport/test" "github.com/gliderlabs/ssh" @@ -25,6 +25,7 @@ import ( type UploadPackSuite struct { test.UploadPackSuite fixtures.Suite + opts []ssh.Option port int base string @@ -41,7 +42,7 @@ func (s *UploadPackSuite) SetUpSuite(c *C) { c.Assert(err, IsNil) s.port = l.Addr().(*net.TCPAddr).Port - s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) + s.base, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) c.Assert(err, IsNil) DefaultAuthBuilder = func(user string) (AuthMethod, error) { @@ -56,7 +57,10 @@ func (s *UploadPackSuite) SetUpSuite(c *C) { s.UploadPackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") s.UploadPackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") - server := &ssh.Server{Handler: handlerSSH} + server := &ssh.Server{Handler: testutils.HandlerSSH} + for _, opt := range s.opts { + opt(server) + } go func() { log.Fatal(server.Serve(l)) }() diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index 018d38e..9414fba 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -1,13 +1,11 @@ // Package test implements common test suite for different transport // implementations. -// package test import ( "bytes" "context" "io" - "io/ioutil" "os" "path/filepath" @@ -235,7 +233,7 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, if rootPath != "" && err == nil && stat.IsDir() { objectPath := filepath.Join(rootPath, "objects/pack") - files, err := ioutil.ReadDir(objectPath) + files, err := os.ReadDir(objectPath) c.Assert(err, IsNil) for _, file := range files { @@ -371,5 +369,5 @@ func (s *ReceivePackSuite) emptyPackfile() io.ReadCloser { panic(err) } - return ioutil.NopCloser(&buf) + return io.NopCloser(&buf) } diff --git a/plumbing/transport/test/upload_pack.go b/plumbing/transport/test/upload_pack.go index 3ee029d..f7842eb 100644 --- a/plumbing/transport/test/upload_pack.go +++ b/plumbing/transport/test/upload_pack.go @@ -1,13 +1,11 @@ // Package test implements common test suite for different transport // implementations. -// package test import ( "bytes" "context" "io" - "io/ioutil" "time" "github.com/go-git/go-git/v5/plumbing" @@ -154,7 +152,7 @@ func (s *UploadPackSuite) TestUploadPackWithContextOnRead(c *C) { cancel() - _, err = io.Copy(ioutil.Discard, reader) + _, err = io.Copy(io.Discard, reader) c.Assert(err, NotNil) err = reader.Close() @@ -255,7 +253,7 @@ func (s *UploadPackSuite) TestFetchError(c *C) { } func (s *UploadPackSuite) checkObjectNumber(c *C, r io.Reader, n int) { - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) c.Assert(err, IsNil) buf := bytes.NewBuffer(b) storage := memory.NewStorage() |