diff options
134 files changed, 2770 insertions, 1069 deletions
diff --git a/.travis.yml b/.travis.yml index c81b17e..ee975e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.8 - - tip + - 1.8.x + - 1.9.x go_import_path: gopkg.in/src-d/go-git.v4 @@ -11,11 +11,6 @@ env: - GIT_VERSION=v1.9.3 - GIT_VERSION=v2.11.0 -matrix: - fast_finish: true - allow_failures: - - go: tip - cache: directories: - $HOME/.git-dist @@ -28,15 +23,6 @@ before_install: - git config --global user.email "travis@example.com" - git config --global user.name "Travis CI" - # we only decrypt the SSH key when we aren't in a pull request - - > - if [ "$TRAVIS_PULL_REQUEST" = "false" ] ; then \ - bash .travis/install_key.sh; \ - export SSH_TEST_PRIVATE_KEY=$HOME/.travis/deploy.pem; \ - else \ - export SSH_AUTH_SOCK=""; \ - fi - install: - go get -v -t ./... @@ -48,4 +34,4 @@ script: - go vet ./... after_success: - - bash <(curl -s https://codecov.io/bash)
\ No newline at end of file + - bash <(curl -s https://codecov.io/bash) diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index fa998bf..4464ed8 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -75,8 +75,8 @@ is supported by go-git. | worktree | ✖ | | annotate | (see blame) | | **gpg** | -| git-verify-commit | ✖ | -| git-verify-tag | ✖ | +| git-verify-commit | ✔ | +| git-verify-tag | ✔ | | **plumbing commands** | | cat-file | ✔ | | check-ignore | | @@ -119,7 +119,7 @@ table of git with go-git. Contribute ---------- -If you are interested on contributing to go-git, open an [issue](https://github.com/src-d/go-git/issues) explaining which missing functionality you want to work in, and we will guide you through the implementation. +If you are interested in contributing to go-git, open an [issue](https://github.com/src-d/go-git/issues) explaining which missing functionality you want to work on, and we will guide you through the implementation. License ------- diff --git a/_examples/commit/main.go b/_examples/commit/main.go index aeb1d4f..556cb9c 100644 --- a/_examples/commit/main.go +++ b/_examples/commit/main.go @@ -12,13 +12,13 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/object" ) -// Basic example of how to commit changes to the current branch to an existant +// Basic example of how to commit changes to the current branch to an existent // repository. func main() { CheckArgs("<directory>") directory := os.Args[1] - // Opens an already existant repository. + // Opens an already existent repository. r, err := git.PlainOpen(directory) CheckIfError(err) diff --git a/_examples/context/main.go b/_examples/context/main.go index 72885ff..f873055 100644 --- a/_examples/context/main.go +++ b/_examples/context/main.go @@ -9,7 +9,7 @@ import ( . "gopkg.in/src-d/go-git.v4/_examples" ) -// Gracefull cancellation example of a basic git operation such as Clone. +// Graceful cancellation example of a basic git operation such as Clone. func main() { CheckArgs("<url>", "<directory>") url := os.Args[1] diff --git a/_examples/storage/README.md b/_examples/storage/README.md index b7207ee..fc72e6f 100644 --- a/_examples/storage/README.md +++ b/_examples/storage/README.md @@ -8,7 +8,7 @@ ### and what this means ... *git* has as very well defined storage system, the `.git` directory, present on any repository. This is the place where `git` stores al the [`objects`](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), [`references`](https://git-scm.com/book/es/v2/Git-Internals-Git-References) and [`configuration`](https://git-scm.com/docs/git-config#_configuration_file). This information is stored in plain files. -Our original **go-git** version was designed to work in memory, some time after we added support to read the `.git`, and now we have added support for fully customized [storages](https://godoc.org/github.com/src-d/go-git#Storer). +Our original **go-git** version was designed to work in memory, some time after we added support to read the `.git`, and now we have added support for fully customized [storages](https://godoc.org/gopkg.in/src-d/go-git.v4/storage#Storer). This means that the internal database of any repository can be saved and accessed on any support, databases, distributed filesystems, etc. This functionality is pretty similar to the [libgit2 backends](http://blog.deveo.com/your-git-repository-in-a-database-pluggable-backends-in-libgit2/) @@ -147,10 +147,7 @@ func (b *blame) fillRevs() error { var err error b.revs, err = references(b.fRev, b.path) - if err != nil { - return err - } - return nil + return err } // build graph of a file from its revision history @@ -244,7 +241,7 @@ func (b *blame) GoString() string { lines := strings.Split(contents, "\n") // max line number length - mlnl := len(fmt.Sprintf("%s", strconv.Itoa(len(lines)))) + mlnl := len(strconv.Itoa(len(lines))) // max author length mal := b.maxAuthorLength() format := fmt.Sprintf("%%s (%%-%ds %%%dd) %%s\n", diff --git a/blame_test.go b/blame_test.go index 8bef4d0..5374610 100644 --- a/blame_test.go +++ b/blame_test.go @@ -1,10 +1,10 @@ package git import ( - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type BlameSuite struct { diff --git a/common_test.go b/common_test.go index a7cd755..f8f4e61 100644 --- a/common_test.go +++ b/common_test.go @@ -3,17 +3,17 @@ package git import ( "testing" - billy "gopkg.in/src-d/go-billy.v3" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" - "github.com/src-d/go-git-fixtures" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/util" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/util" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -30,7 +30,7 @@ func (s *BaseSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.buildBasicRepository(c) - s.cache = make(map[string]*Repository, 0) + s.cache = make(map[string]*Repository) } func (s *BaseSuite) TearDownSuite(c *C) { diff --git a/config/config.go b/config/config.go index 475045e..fc4cd28 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sort" + "strconv" format "gopkg.in/src-d/go-git.v4/plumbing/format/config" ) @@ -40,6 +41,14 @@ type Config struct { // Worktree is the path to the root of the working tree. Worktree string } + + Pack struct { + // Window controls the size of the sliding window for delta + // compression. The default is 10. A value of 0 turns off + // delta compression entirely. + Window uint + } + // Remotes list of repository remotes, the key of the map is the name // of the remote, should equal to RemoteConfig.Name. Remotes map[string]*RemoteConfig @@ -56,8 +65,8 @@ type Config struct { // NewConfig returns a new empty Config. func NewConfig() *Config { return &Config{ - Remotes: make(map[string]*RemoteConfig, 0), - Submodules: make(map[string]*Submodule, 0), + Remotes: make(map[string]*RemoteConfig), + Submodules: make(map[string]*Submodule), Raw: format.New(), } } @@ -81,10 +90,14 @@ const ( remoteSection = "remote" submoduleSection = "submodule" coreSection = "core" + packSection = "pack" fetchKey = "fetch" urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + windowKey = "window" + + defaultPackWindow = uint(10) ) // Unmarshal parses a git-config file and stores it. @@ -98,6 +111,9 @@ func (c *Config) Unmarshal(b []byte) error { } c.unmarshalCore() + if err := c.unmarshalPack(); err != nil { + return err + } c.unmarshalSubmodules() return c.unmarshalRemotes() } @@ -111,6 +127,21 @@ func (c *Config) unmarshalCore() { c.Core.Worktree = s.Options.Get(worktreeKey) } +func (c *Config) unmarshalPack() error { + s := c.Raw.Section(packSection) + window := s.Options.Get(windowKey) + if window == "" { + c.Pack.Window = defaultPackWindow + } else { + winUint, err := strconv.ParseUint(window, 10, 32) + if err != nil { + return err + } + c.Pack.Window = uint(winUint) + } + return nil +} + func (c *Config) unmarshalRemotes() error { s := c.Raw.Section(remoteSection) for _, sub := range s.Subsections { @@ -138,6 +169,7 @@ func (c *Config) unmarshalSubmodules() { // Marshal returns Config encoded as a git-config file. func (c *Config) Marshal() ([]byte, error) { c.marshalCore() + c.marshalPack() c.marshalRemotes() c.marshalSubmodules() @@ -158,6 +190,13 @@ func (c *Config) marshalCore() { } } +func (c *Config) marshalPack() { + s := c.Raw.Section(packSection) + if c.Pack.Window != defaultPackWindow { + s.SetOption(windowKey, fmt.Sprintf("%d", c.Pack.Window)) + } +} + func (c *Config) marshalRemotes() { s := c.Raw.Section(remoteSection) newSubsections := make(format.Subsections, 0, len(c.Remotes)) @@ -251,13 +290,8 @@ func (c *RemoteConfig) unmarshal(s *format.Subsection) error { fetch = append(fetch, rs) } - var urls []string - for _, f := range c.raw.Options.GetAll(urlKey) { - urls = append(urls, f) - } - c.Name = c.raw.Name - c.URLs = urls + c.URLs = append([]string(nil), c.raw.Options.GetAll(urlKey)...) c.Fetch = fetch return nil diff --git a/config/config_test.go b/config/config_test.go index c27ee26..019cee6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -10,6 +10,8 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { input := []byte(`[core] bare = true worktree = foo +[pack] + window = 20 [remote "origin"] url = git@github.com:mcuadros/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* @@ -33,6 +35,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) c.Assert(cfg.Core.Worktree, Equals, "foo") + c.Assert(cfg.Pack.Window, Equals, uint(20)) c.Assert(cfg.Remotes, HasLen, 2) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") c.Assert(cfg.Remotes["origin"].URLs, DeepEquals, []string{"git@github.com:mcuadros/go-git.git"}) @@ -51,6 +54,8 @@ func (s *ConfigSuite) TestMarshall(c *C) { output := []byte(`[core] bare = true worktree = bar +[pack] + window = 20 [remote "alt"] url = git@github.com:mcuadros/go-git.git url = git@github.com:src-d/go-git.git @@ -65,6 +70,7 @@ func (s *ConfigSuite) TestMarshall(c *C) { cfg := NewConfig() cfg.Core.IsBare = true cfg.Core.Worktree = "bar" + cfg.Pack.Window = 20 cfg.Remotes["origin"] = &RemoteConfig{ Name: "origin", URLs: []string{"git@github.com:mcuadros/go-git.git"}, @@ -92,6 +98,8 @@ func (s *ConfigSuite) TestUnmarshallMarshall(c *C) { bare = true worktree = foo custom = ignored +[pack] + window = 20 [remote "origin"] url = git@github.com:mcuadros/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* diff --git a/config/modules.go b/config/modules.go index 24ef304..b208984 100644 --- a/config/modules.go +++ b/config/modules.go @@ -24,7 +24,7 @@ type Modules struct { // NewModules returns a new empty Modules func NewModules() *Modules { return &Modules{ - Submodules: make(map[string]*Submodule, 0), + Submodules: make(map[string]*Submodule), raw: format.New(), } } diff --git a/config/refspec.go b/config/refspec.go index 7e4106a..af7e732 100644 --- a/config/refspec.go +++ b/config/refspec.go @@ -51,20 +51,12 @@ func (s RefSpec) Validate() error { // IsForceUpdate returns if update is allowed in non fast-forward merges. func (s RefSpec) IsForceUpdate() bool { - if s[0] == refSpecForce[0] { - return true - } - - return false + return s[0] == refSpecForce[0] } // IsDelete returns true if the refspec indicates a delete (empty src). func (s RefSpec) IsDelete() bool { - if s[0] == refSpecSeparator[0] { - return true - } - - return false + return s[0] == refSpecSeparator[0] } // Src return the src side. @@ -87,7 +79,7 @@ func (s RefSpec) Match(n plumbing.ReferenceName) bool { // IsWildcard returns true if the RefSpec contains a wildcard. func (s RefSpec) IsWildcard() bool { - return strings.Index(string(s), refSpecWildcard) != -1 + return strings.Contains(string(s), refSpecWildcard) } func (s RefSpec) matchExact(n plumbing.ReferenceName) bool { diff --git a/example_test.go b/example_test.go index 1b369ba..e9d8e8b 100644 --- a/example_test.go +++ b/example_test.go @@ -13,7 +13,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/memory" - "gopkg.in/src-d/go-billy.v3/memfs" + "gopkg.in/src-d/go-billy.v4/memfs" ) func ExampleClone() { diff --git a/internal/revision/parser.go b/internal/revision/parser.go index b45a6d8..d2c509e 100644 --- a/internal/revision/parser.go +++ b/internal/revision/parser.go @@ -254,7 +254,7 @@ func (p *Parser) parseAt() (Revisioner, error) { var lit, nextLit string var err error - tok, lit, err = p.scan() + tok, _, err = p.scan() if err != nil { return nil, err @@ -95,6 +95,9 @@ type PullOptions struct { // stored, if nil nothing is stored and the capability (if supported) // no-progress, is sent to the server to avoid send this information. Progress sideband.Progress + // Force allows the pull to update a local branch even when the remote + // branch does not descend from it. + Force bool } // Validate validates the fields and sets the default values. @@ -142,6 +145,9 @@ type FetchOptions struct { // Tags describe how the tags will be fetched from the remote repository, // by default is TagFollowing. Tags TagMode + // Force allows the fetch to update a local branch even when the remote + // branch does not descend from it. + Force bool } // Validate validates the fields and sets the default values. @@ -348,3 +354,9 @@ func (o *CommitOptions) Validate(r *Repository) error { return nil } + +// ListOptions describes how a remote list should be performed. +type ListOptions struct { + // Auth credentials, if required, to use with the remote repository. + Auth transport.AuthMethod +} diff --git a/plumbing/format/config/encoder.go b/plumbing/format/config/encoder.go index 6d17a5a..4eac896 100644 --- a/plumbing/format/config/encoder.go +++ b/plumbing/format/config/encoder.go @@ -53,17 +53,13 @@ func (e *Encoder) encodeSubsection(sectionName string, s *Subsection) error { return err } - if err := e.encodeOptions(s.Options); err != nil { - return err - } - - return nil + return e.encodeOptions(s.Options) } func (e *Encoder) encodeOptions(opts Options) error { for _, o := range opts { pattern := "\t%s = %s\n" - if strings.Index(o.Value, "\\") != -1 { + if strings.Contains(o.Value, "\\") { pattern = "\t%s = %q\n" } diff --git a/plumbing/format/gitignore/dir.go b/plumbing/format/gitignore/dir.go index c3bfc53..41dd624 100644 --- a/plumbing/format/gitignore/dir.go +++ b/plumbing/format/gitignore/dir.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) const ( diff --git a/plumbing/format/gitignore/dir_test.go b/plumbing/format/gitignore/dir_test.go index d28a714..b8a5453 100644 --- a/plumbing/format/gitignore/dir_test.go +++ b/plumbing/format/gitignore/dir_test.go @@ -3,10 +3,9 @@ package gitignore import ( "os" - "gopkg.in/src-d/go-billy.v3" - "gopkg.in/src-d/go-billy.v3/memfs" - . "gopkg.in/check.v1" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/memfs" ) type MatcherSuite struct { diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go index c7decb2..20d6859 100644 --- a/plumbing/format/idxfile/decoder_test.go +++ b/plumbing/format/idxfile/decoder_test.go @@ -6,12 +6,12 @@ import ( "fmt" "testing" - "github.com/src-d/go-git-fixtures" . "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } diff --git a/plumbing/format/idxfile/encoder_test.go b/plumbing/format/idxfile/encoder_test.go index d566b0d..e5b96b7 100644 --- a/plumbing/format/idxfile/encoder_test.go +++ b/plumbing/format/idxfile/encoder_test.go @@ -4,11 +4,11 @@ import ( "bytes" "io/ioutil" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func (s *IdxfileSuite) TestEncode(c *C) { diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index 5bf6a52..1a58128 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -200,11 +200,8 @@ func (d *Decoder) padEntry(idx *Index, e *Entry, read int) error { entrySize := read + len(e.Name) padLen := 8 - entrySize%8 - if _, err := io.CopyN(ioutil.Discard, d.r, int64(padLen)); err != nil { - return err - } - - return nil + _, err := io.CopyN(ioutil.Discard, d.r, int64(padLen)) + return err } func (d *Decoder) readExtensions(idx *Index) error { @@ -288,7 +285,7 @@ func (d *Decoder) readChecksum(expected []byte, alreadyRead [4]byte) error { return err } - if bytes.Compare(h[:], expected) != 0 { + if !bytes.Equal(h[:], expected) { return ErrInvalidChecksum } @@ -407,7 +404,7 @@ func (d *resolveUndoDecoder) Decode(ru *ResolveUndo) error { func (d *resolveUndoDecoder) readEntry() (*ResolveUndoEntry, error) { e := &ResolveUndoEntry{ - Stages: make(map[Stage]plumbing.Hash, 0), + Stages: make(map[Stage]plumbing.Hash), } path, err := binary.ReadUntil(d.r, '\x00') diff --git a/plumbing/format/index/decoder_test.go b/plumbing/format/index/decoder_test.go index c3fa590..8940bfb 100644 --- a/plumbing/format/index/decoder_test.go +++ b/plumbing/format/index/decoder_test.go @@ -6,7 +6,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/format/index/encoder_test.go b/plumbing/format/index/encoder_test.go index bc5df0f..78cbbba 100644 --- a/plumbing/format/index/encoder_test.go +++ b/plumbing/format/index/encoder_test.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/google/go-cmp/cmp" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -46,7 +47,7 @@ func (s *IndexSuite) TestEncode(c *C) { err = d.Decode(output) c.Assert(err, IsNil) - c.Assert(idx, DeepEquals, output) + c.Assert(cmp.Equal(idx, output), Equals, true) c.Assert(output.Entries[0].Name, Equals, strings.Repeat(" ", 20)) c.Assert(output.Entries[1].Name, Equals, "bar") diff --git a/plumbing/format/objfile/reader.go b/plumbing/format/objfile/reader.go index e7e119c..c4467e4 100644 --- a/plumbing/format/objfile/reader.go +++ b/plumbing/format/objfile/reader.go @@ -110,9 +110,5 @@ func (r *Reader) Hash() plumbing.Hash { // Close releases any resources consumed by the Reader. Calling Close does not // close the wrapped io.Reader originally passed to NewReader. func (r *Reader) Close() error { - if err := r.zlib.Close(); err != nil { - return err - } - - return nil + return r.zlib.Close() } diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index 3d475b2..ad72ea0 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -105,7 +105,7 @@ func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, o: o, idx: NewIndex(0), - offsetToType: make(map[int64]plumbing.ObjectType, 0), + offsetToType: make(map[int64]plumbing.ObjectType), decoderType: t, }, nil } @@ -207,12 +207,16 @@ func (d *Decoder) decodeObjectsWithObjectStorerTx(count int) error { // constructor, if the object decoded is not equals to the specified one, nil will // be returned func (d *Decoder) DecodeObject() (plumbing.EncodedObject, error) { + return d.doDecodeObject(d.decoderType) +} + +func (d *Decoder) doDecodeObject(t plumbing.ObjectType) (plumbing.EncodedObject, error) { h, err := d.s.NextObjectHeader() if err != nil { return nil, err } - if d.decoderType == plumbing.AnyObject { + if t == plumbing.AnyObject { return d.decodeByHeader(h) } @@ -279,6 +283,7 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error obj := d.newObject() obj.SetSize(h.Length) obj.SetType(h.Type) + var crc uint32 var err error switch h.Type { @@ -315,7 +320,8 @@ func (d *Decoder) newObject() plumbing.EncodedObject { // returned is added into a internal index. This is intended to be able to regenerate // objects from deltas (offset deltas or reference deltas) without an package index // (.idx file). If Decode wasn't called previously objects offset should provided -// using the SetOffsets method. +// using the SetOffsets method. It decodes the object regardless of the Decoder +// type. func (d *Decoder) DecodeObjectAt(offset int64) (plumbing.EncodedObject, error) { if !d.s.IsSeekable { return nil, ErrNonSeekable @@ -333,7 +339,7 @@ func (d *Decoder) DecodeObjectAt(offset int64) (plumbing.EncodedObject, error) { } }() - return d.DecodeObject() + return d.doDecodeObject(plumbing.AnyObject) } func (d *Decoder) fillRegularObjectContent(obj plumbing.EncodedObject) (uint32, error) { diff --git a/plumbing/format/packfile/decoder_test.go b/plumbing/format/packfile/decoder_test.go index ecf7c81..1a1a74a 100644 --- a/plumbing/format/packfile/decoder_test.go +++ b/plumbing/format/packfile/decoder_test.go @@ -3,9 +3,6 @@ package packfile_test import ( "io" - "gopkg.in/src-d/go-billy.v3/memfs" - - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" @@ -14,6 +11,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ReaderSuite struct { @@ -293,7 +292,7 @@ func (s *ReaderSuite) TestDecodeCRCs(c *C) { c.Assert(int(sum), Equals, 78022211966) } -func (s *ReaderSuite) TestReadObjectAt(c *C) { +func (s *ReaderSuite) TestDecodeObjectAt(c *C) { f := fixtures.Basic().One() scanner := packfile.NewScanner(f.Packfile()) d, err := packfile.NewDecoder(scanner, nil) @@ -311,6 +310,25 @@ func (s *ReaderSuite) TestReadObjectAt(c *C) { c.Assert(obj.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") } +func (s *ReaderSuite) TestDecodeObjectAtForType(c *C) { + f := fixtures.Basic().One() + scanner := packfile.NewScanner(f.Packfile()) + d, err := packfile.NewDecoderForType(scanner, nil, plumbing.TreeObject) + c.Assert(err, IsNil) + + // when the packfile is ref-delta based, the offsets are required + if f.Is("ref-delta") { + d.SetIndex(getIndexFromIdxFile(f.Idx())) + } + + // the objects at reference 186, is a delta, so should be recall, + // without being read before. + obj, err := d.DecodeObjectAt(186) + c.Assert(err, IsNil) + c.Assert(obj.Type(), Equals, plumbing.CommitObject) + c.Assert(obj.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") +} + func (s *ReaderSuite) TestIndex(c *C) { f := fixtures.Basic().One() scanner := packfile.NewScanner(f.Packfile()) diff --git a/plumbing/format/packfile/delta_index.go b/plumbing/format/packfile/delta_index.go new file mode 100644 index 0000000..07a6112 --- /dev/null +++ b/plumbing/format/packfile/delta_index.go @@ -0,0 +1,297 @@ +package packfile + +const blksz = 16 +const maxChainLength = 64 + +// deltaIndex is a modified version of JGit's DeltaIndex adapted to our current +// design. +type deltaIndex struct { + table []int + entries []int + mask int +} + +func (idx *deltaIndex) init(buf []byte) { + scanner := newDeltaIndexScanner(buf, len(buf)) + idx.mask = scanner.mask + idx.table = scanner.table + idx.entries = make([]int, countEntries(scanner)+1) + idx.copyEntries(scanner) +} + +// findMatch returns the offset of src where the block starting at tgtOffset +// is and the length of the match. A length of 0 means there was no match. A +// length of -1 means the src length is lower than the blksz and whatever +// other positive length is the length of the match in bytes. +func (idx *deltaIndex) findMatch(src, tgt []byte, tgtOffset int) (srcOffset, l int) { + if len(tgt) < tgtOffset+s { + return 0, len(tgt) - tgtOffset + } + + if len(src) < blksz { + return 0, -1 + } + + if len(tgt) >= tgtOffset+s && len(src) >= blksz { + h := hashBlock(tgt, tgtOffset) + tIdx := h & idx.mask + eIdx := idx.table[tIdx] + if eIdx != 0 { + srcOffset = idx.entries[eIdx] + } else { + return + } + + l = matchLength(src, tgt, tgtOffset, srcOffset) + } + + return +} + +func matchLength(src, tgt []byte, otgt, osrc int) (l int) { + lensrc := len(src) + lentgt := len(tgt) + for (osrc < lensrc && otgt < lentgt) && src[osrc] == tgt[otgt] { + l++ + osrc++ + otgt++ + } + return +} + +func countEntries(scan *deltaIndexScanner) (cnt int) { + // Figure out exactly how many entries we need. As we do the + // enumeration truncate any delta chains longer than what we + // are willing to scan during encode. This keeps the encode + // logic linear in the size of the input rather than quadratic. + for i := 0; i < len(scan.table); i++ { + h := scan.table[i] + if h == 0 { + continue + } + + size := 0 + for { + size++ + if size == maxChainLength { + scan.next[h] = 0 + break + } + h = scan.next[h] + + if h == 0 { + break + } + } + cnt += size + } + + return +} + +func (idx *deltaIndex) copyEntries(scanner *deltaIndexScanner) { + // Rebuild the entries list from the scanner, positioning all + // blocks in the same hash chain next to each other. We can + // then later discard the next list, along with the scanner. + // + next := 1 + for i := 0; i < len(idx.table); i++ { + h := idx.table[i] + if h == 0 { + continue + } + + idx.table[i] = next + for { + idx.entries[next] = scanner.entries[h] + next++ + h = scanner.next[h] + + if h == 0 { + break + } + } + } +} + +type deltaIndexScanner struct { + table []int + entries []int + next []int + mask int + count int +} + +func newDeltaIndexScanner(buf []byte, size int) *deltaIndexScanner { + size -= size % blksz + worstCaseBlockCnt := size / blksz + if worstCaseBlockCnt < 1 { + return new(deltaIndexScanner) + } + + tableSize := tableSize(worstCaseBlockCnt) + scanner := &deltaIndexScanner{ + table: make([]int, tableSize), + mask: tableSize - 1, + entries: make([]int, worstCaseBlockCnt+1), + next: make([]int, worstCaseBlockCnt+1), + } + + scanner.scan(buf, size) + return scanner +} + +// slightly modified version of JGit's DeltaIndexScanner. We store the offset on the entries +// instead of the entries and the key, so we avoid operations to retrieve the offset later, as +// we don't use the key. +// See: https://github.com/eclipse/jgit/blob/005e5feb4ecd08c4e4d141a38b9e7942accb3212/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaIndexScanner.java +func (s *deltaIndexScanner) scan(buf []byte, end int) { + lastHash := 0 + ptr := end - blksz + + for { + key := hashBlock(buf, ptr) + tIdx := key & s.mask + head := s.table[tIdx] + if head != 0 && lastHash == key { + s.entries[head] = ptr + } else { + s.count++ + eIdx := s.count + s.entries[eIdx] = ptr + s.next[eIdx] = head + s.table[tIdx] = eIdx + } + + lastHash = key + ptr -= blksz + + if 0 > ptr { + break + } + } +} + +func tableSize(worstCaseBlockCnt int) int { + shift := 32 - leadingZeros(uint32(worstCaseBlockCnt)) + sz := 1 << uint(shift-1) + if sz < worstCaseBlockCnt { + sz <<= 1 + } + return sz +} + +// use https://golang.org/pkg/math/bits/#LeadingZeros32 in the future +func leadingZeros(x uint32) (n int) { + if x >= 1<<16 { + x >>= 16 + n = 16 + } + if x >= 1<<8 { + x >>= 8 + n += 8 + } + n += int(len8tab[x]) + return 32 - n +} + +var len8tab = [256]uint8{ + 0x00, 0x01, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, +} + +func hashBlock(raw []byte, ptr int) int { + // The first 4 steps collapse out into a 4 byte big-endian decode, + // with a larger right shift as we combined shift lefts together. + // + hash := ((uint32(raw[ptr]) & 0xff) << 24) | + ((uint32(raw[ptr+1]) & 0xff) << 16) | + ((uint32(raw[ptr+2]) & 0xff) << 8) | + (uint32(raw[ptr+3]) & 0xff) + hash ^= T[hash>>31] + + hash = ((hash << 8) | (uint32(raw[ptr+4]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+5]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+6]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+7]) & 0xff)) ^ T[hash>>23] + + hash = ((hash << 8) | (uint32(raw[ptr+8]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+9]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+10]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+11]) & 0xff)) ^ T[hash>>23] + + hash = ((hash << 8) | (uint32(raw[ptr+12]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+13]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+14]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+15]) & 0xff)) ^ T[hash>>23] + + return int(hash) +} + +var T = []uint32{0x00000000, 0xd4c6b32d, 0x7d4bd577, + 0xa98d665a, 0x2e5119c3, 0xfa97aaee, 0x531accb4, 0x87dc7f99, + 0x5ca23386, 0x886480ab, 0x21e9e6f1, 0xf52f55dc, 0x72f32a45, + 0xa6359968, 0x0fb8ff32, 0xdb7e4c1f, 0x6d82d421, 0xb944670c, + 0x10c90156, 0xc40fb27b, 0x43d3cde2, 0x97157ecf, 0x3e981895, + 0xea5eabb8, 0x3120e7a7, 0xe5e6548a, 0x4c6b32d0, 0x98ad81fd, + 0x1f71fe64, 0xcbb74d49, 0x623a2b13, 0xb6fc983e, 0x0fc31b6f, + 0xdb05a842, 0x7288ce18, 0xa64e7d35, 0x219202ac, 0xf554b181, + 0x5cd9d7db, 0x881f64f6, 0x536128e9, 0x87a79bc4, 0x2e2afd9e, + 0xfaec4eb3, 0x7d30312a, 0xa9f68207, 0x007be45d, 0xd4bd5770, + 0x6241cf4e, 0xb6877c63, 0x1f0a1a39, 0xcbcca914, 0x4c10d68d, + 0x98d665a0, 0x315b03fa, 0xe59db0d7, 0x3ee3fcc8, 0xea254fe5, + 0x43a829bf, 0x976e9a92, 0x10b2e50b, 0xc4745626, 0x6df9307c, + 0xb93f8351, 0x1f8636de, 0xcb4085f3, 0x62cde3a9, 0xb60b5084, + 0x31d72f1d, 0xe5119c30, 0x4c9cfa6a, 0x985a4947, 0x43240558, + 0x97e2b675, 0x3e6fd02f, 0xeaa96302, 0x6d751c9b, 0xb9b3afb6, + 0x103ec9ec, 0xc4f87ac1, 0x7204e2ff, 0xa6c251d2, 0x0f4f3788, + 0xdb8984a5, 0x5c55fb3c, 0x88934811, 0x211e2e4b, 0xf5d89d66, + 0x2ea6d179, 0xfa606254, 0x53ed040e, 0x872bb723, 0x00f7c8ba, + 0xd4317b97, 0x7dbc1dcd, 0xa97aaee0, 0x10452db1, 0xc4839e9c, + 0x6d0ef8c6, 0xb9c84beb, 0x3e143472, 0xead2875f, 0x435fe105, + 0x97995228, 0x4ce71e37, 0x9821ad1a, 0x31accb40, 0xe56a786d, + 0x62b607f4, 0xb670b4d9, 0x1ffdd283, 0xcb3b61ae, 0x7dc7f990, + 0xa9014abd, 0x008c2ce7, 0xd44a9fca, 0x5396e053, 0x8750537e, + 0x2edd3524, 0xfa1b8609, 0x2165ca16, 0xf5a3793b, 0x5c2e1f61, + 0x88e8ac4c, 0x0f34d3d5, 0xdbf260f8, 0x727f06a2, 0xa6b9b58f, + 0x3f0c6dbc, 0xebcade91, 0x4247b8cb, 0x96810be6, 0x115d747f, + 0xc59bc752, 0x6c16a108, 0xb8d01225, 0x63ae5e3a, 0xb768ed17, + 0x1ee58b4d, 0xca233860, 0x4dff47f9, 0x9939f4d4, 0x30b4928e, + 0xe47221a3, 0x528eb99d, 0x86480ab0, 0x2fc56cea, 0xfb03dfc7, + 0x7cdfa05e, 0xa8191373, 0x01947529, 0xd552c604, 0x0e2c8a1b, + 0xdaea3936, 0x73675f6c, 0xa7a1ec41, 0x207d93d8, 0xf4bb20f5, + 0x5d3646af, 0x89f0f582, 0x30cf76d3, 0xe409c5fe, 0x4d84a3a4, + 0x99421089, 0x1e9e6f10, 0xca58dc3d, 0x63d5ba67, 0xb713094a, + 0x6c6d4555, 0xb8abf678, 0x11269022, 0xc5e0230f, 0x423c5c96, + 0x96faefbb, 0x3f7789e1, 0xebb13acc, 0x5d4da2f2, 0x898b11df, + 0x20067785, 0xf4c0c4a8, 0x731cbb31, 0xa7da081c, 0x0e576e46, + 0xda91dd6b, 0x01ef9174, 0xd5292259, 0x7ca44403, 0xa862f72e, + 0x2fbe88b7, 0xfb783b9a, 0x52f55dc0, 0x8633eeed, 0x208a5b62, + 0xf44ce84f, 0x5dc18e15, 0x89073d38, 0x0edb42a1, 0xda1df18c, + 0x739097d6, 0xa75624fb, 0x7c2868e4, 0xa8eedbc9, 0x0163bd93, + 0xd5a50ebe, 0x52797127, 0x86bfc20a, 0x2f32a450, 0xfbf4177d, + 0x4d088f43, 0x99ce3c6e, 0x30435a34, 0xe485e919, 0x63599680, + 0xb79f25ad, 0x1e1243f7, 0xcad4f0da, 0x11aabcc5, 0xc56c0fe8, + 0x6ce169b2, 0xb827da9f, 0x3ffba506, 0xeb3d162b, 0x42b07071, + 0x9676c35c, 0x2f49400d, 0xfb8ff320, 0x5202957a, 0x86c42657, + 0x011859ce, 0xd5deeae3, 0x7c538cb9, 0xa8953f94, 0x73eb738b, + 0xa72dc0a6, 0x0ea0a6fc, 0xda6615d1, 0x5dba6a48, 0x897cd965, + 0x20f1bf3f, 0xf4370c12, 0x42cb942c, 0x960d2701, 0x3f80415b, + 0xeb46f276, 0x6c9a8def, 0xb85c3ec2, 0x11d15898, 0xc517ebb5, + 0x1e69a7aa, 0xcaaf1487, 0x632272dd, 0xb7e4c1f0, 0x3038be69, + 0xe4fe0d44, 0x4d736b1e, 0x99b5d833, +} diff --git a/plumbing/format/packfile/delta_selector.go b/plumbing/format/packfile/delta_selector.go index cc0ae0f..51adcdf 100644 --- a/plumbing/format/packfile/delta_selector.go +++ b/plumbing/format/packfile/delta_selector.go @@ -2,15 +2,13 @@ package packfile import ( "sort" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" ) const ( - // How far back in the sorted list to search for deltas. 10 is - // the default in command line git. - deltaWindowSize = 10 // deltas based on deltas, how many steps we can do. // 50 is the default value used in JGit maxDepth = int64(50) @@ -30,27 +28,75 @@ func newDeltaSelector(s storer.EncodedObjectStorer) *deltaSelector { return &deltaSelector{s} } -// ObjectsToPack creates a list of ObjectToPack from the hashes provided, -// creating deltas if it's suitable, using an specific internal logic -func (dw *deltaSelector) ObjectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, error) { - otp, err := dw.objectsToPack(hashes) +// ObjectsToPack creates a list of ObjectToPack from the hashes +// provided, creating deltas if it's suitable, using an specific +// internal logic. `packWindow` specifies the size of the sliding +// window used to compare objects for delta compression; 0 turns off +// delta compression entirely. +func (dw *deltaSelector) ObjectsToPack( + hashes []plumbing.Hash, + packWindow uint, +) ([]*ObjectToPack, error) { + otp, err := dw.objectsToPack(hashes, packWindow) if err != nil { return nil, err } + if packWindow == 0 { + return otp, nil + } + dw.sort(otp) - if err := dw.walk(otp); err != nil { + var objectGroups [][]*ObjectToPack + var prev *ObjectToPack + i := -1 + for _, obj := range otp { + if prev == nil || prev.Type() != obj.Type() { + objectGroups = append(objectGroups, []*ObjectToPack{obj}) + i++ + prev = obj + } else { + objectGroups[i] = append(objectGroups[i], obj) + } + } + + var wg sync.WaitGroup + var once sync.Once + for _, objs := range objectGroups { + objs := objs + wg.Add(1) + go func() { + if walkErr := dw.walk(objs, packWindow); walkErr != nil { + once.Do(func() { + err = walkErr + }) + } + wg.Done() + }() + } + wg.Wait() + + if err != nil { return nil, err } return otp, nil } -func (dw *deltaSelector) objectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, error) { +func (dw *deltaSelector) objectsToPack( + hashes []plumbing.Hash, + packWindow uint, +) ([]*ObjectToPack, error) { var objectsToPack []*ObjectToPack for _, h := range hashes { - o, err := dw.encodedDeltaObject(h) + var o plumbing.EncodedObject + var err error + if packWindow == 0 { + o, err = dw.encodedObject(h) + } else { + o, err = dw.encodedDeltaObject(h) + } if err != nil { return nil, err } @@ -63,6 +109,10 @@ func (dw *deltaSelector) objectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, objectsToPack = append(objectsToPack, otp) } + if packWindow == 0 { + return objectsToPack, nil + } + if err := dw.fixAndBreakChains(objectsToPack); err != nil { return nil, err } @@ -171,8 +221,18 @@ func (dw *deltaSelector) sort(objectsToPack []*ObjectToPack) { sort.Sort(byTypeAndSize(objectsToPack)) } -func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { +func (dw *deltaSelector) walk( + objectsToPack []*ObjectToPack, + packWindow uint, +) error { + indexMap := make(map[plumbing.Hash]*deltaIndex) for i := 0; i < len(objectsToPack); i++ { + // Clean up the index map for anything outside our pack + // window, to save memory. + if i > int(packWindow) { + delete(indexMap, objectsToPack[i-int(packWindow)].Hash()) + } + target := objectsToPack[i] // If we already have a delta, we don't try to find a new one for this @@ -187,7 +247,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { continue } - for j := i - 1; j >= 0 && i-j < deltaWindowSize; j-- { + for j := i - 1; j >= 0 && i-j < int(packWindow); j-- { base := objectsToPack[j] // Objects must use only the same type as their delta base. // Since objectsToPack is sorted by type and size, once we find @@ -196,7 +256,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { break } - if err := dw.tryToDeltify(base, target); err != nil { + if err := dw.tryToDeltify(indexMap, base, target); err != nil { return err } } @@ -205,7 +265,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { return nil } -func (dw *deltaSelector) tryToDeltify(base, target *ObjectToPack) error { +func (dw *deltaSelector) tryToDeltify(indexMap map[plumbing.Hash]*deltaIndex, base, target *ObjectToPack) error { // If the sizes are radically different, this is a bad pairing. if target.Size() < base.Size()>>4 { return nil @@ -238,8 +298,12 @@ func (dw *deltaSelector) tryToDeltify(base, target *ObjectToPack) error { return err } + if _, ok := indexMap[base.Hash()]; !ok { + indexMap[base.Hash()] = new(deltaIndex) + } + // Now we can generate the delta using originals - delta, err := GetDelta(base.Original, target.Original) + delta, err := getDelta(indexMap[base.Hash()], base.Original, target.Original) if err != nil { return err } diff --git a/plumbing/format/packfile/delta_selector_test.go b/plumbing/format/packfile/delta_selector_test.go index ca4a96b..7d7fd0c 100644 --- a/plumbing/format/packfile/delta_selector_test.go +++ b/plumbing/format/packfile/delta_selector_test.go @@ -146,7 +146,8 @@ func (s *DeltaSelectorSuite) createTestObjects() { func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Different type hashes := []plumbing.Hash{s.hashes["base"], s.hashes["treeType"]} - otp, err := s.ds.ObjectsToPack(hashes) + deltaWindowSize := uint(10) + otp, err := s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["base"]]) @@ -154,7 +155,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Size radically different hashes = []plumbing.Hash{s.hashes["bigBase"], s.hashes["target"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["bigBase"]]) @@ -162,7 +163,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Delta Size Limit with no best delta yet hashes = []plumbing.Hash{s.hashes["smallBase"], s.hashes["smallTarget"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["smallBase"]]) @@ -170,7 +171,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // It will create the delta hashes = []plumbing.Hash{s.hashes["base"], s.hashes["target"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["target"]]) @@ -185,7 +186,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { s.hashes["o2"], s.hashes["o3"], } - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 3) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["o1"]]) @@ -201,20 +202,32 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // a delta. hashes = make([]plumbing.Hash, 0, deltaWindowSize+2) hashes = append(hashes, s.hashes["base"]) - for i := 0; i < deltaWindowSize; i++ { + for i := uint(0); i < deltaWindowSize; i++ { hashes = append(hashes, s.hashes["smallTarget"]) } hashes = append(hashes, s.hashes["target"]) // Don't sort so we can easily check the sliding window without // creating a bunch of new objects. - otp, err = s.ds.objectsToPack(hashes) + otp, err = s.ds.objectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) - err = s.ds.walk(otp) + err = s.ds.walk(otp, deltaWindowSize) c.Assert(err, IsNil) - c.Assert(len(otp), Equals, deltaWindowSize+2) + c.Assert(len(otp), Equals, int(deltaWindowSize)+2) targetIdx := len(otp) - 1 c.Assert(otp[targetIdx].IsDelta(), Equals, false) + + // Check that no deltas are created, and the objects are unsorted, + // if compression is off. + hashes = []plumbing.Hash{s.hashes["base"], s.hashes["target"]} + otp, err = s.ds.ObjectsToPack(hashes, 0) + c.Assert(err, IsNil) + c.Assert(len(otp), Equals, 2) + c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["base"]]) + c.Assert(otp[0].IsDelta(), Equals, false) + c.Assert(otp[1].Original, Equals, s.store.Objects[s.hashes["target"]]) + c.Assert(otp[1].IsDelta(), Equals, false) + c.Assert(otp[1].Depth, Equals, 0) } func (s *DeltaSelectorSuite) TestMaxDepth(c *C) { diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 7e9f822..4d56dc1 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -2,8 +2,6 @@ package packfile import ( "bytes" - "hash/adler32" - "io/ioutil" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -26,26 +24,40 @@ const ( // To generate target again, you will need the obtained object and "base" one. // Error will be returned if base or target object cannot be read. func GetDelta(base, target plumbing.EncodedObject) (plumbing.EncodedObject, error) { + return getDelta(new(deltaIndex), base, target) +} + +func getDelta(index *deltaIndex, base, target plumbing.EncodedObject) (plumbing.EncodedObject, error) { br, err := base.Reader() if err != nil { return nil, err } + defer br.Close() tr, err := target.Reader() if err != nil { return nil, err } + defer tr.Close() - bb, err := ioutil.ReadAll(br) + bb := bufPool.Get().(*bytes.Buffer) + bb.Reset() + defer bufPool.Put(bb) + + _, err = bb.ReadFrom(br) if err != nil { return nil, err } - tb, err := ioutil.ReadAll(tr) + tb := bufPool.Get().(*bytes.Buffer) + tb.Reset() + defer bufPool.Put(tb) + + _, err = tb.ReadFrom(tr) if err != nil { return nil, err } - db := DiffDelta(bb, tb) + db := diffDelta(index, bb.Bytes(), tb.Bytes()) delta := &plumbing.MemoryObject{} _, err = delta.Write(db) if err != nil { @@ -59,21 +71,41 @@ func GetDelta(base, target plumbing.EncodedObject) (plumbing.EncodedObject, erro } // DiffDelta returns the delta that transforms src into tgt. -func DiffDelta(src []byte, tgt []byte) []byte { +func DiffDelta(src, tgt []byte) []byte { + return diffDelta(new(deltaIndex), src, tgt) +} + +func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { buf := bufPool.Get().(*bytes.Buffer) buf.Reset() buf.Write(deltaEncodeSize(len(src))) buf.Write(deltaEncodeSize(len(tgt))) - sindex := initMatch(src) + if len(index.entries) == 0 { + index.init(src) + } ibuf := bufPool.Get().(*bytes.Buffer) ibuf.Reset() for i := 0; i < len(tgt); i++ { - offset, l := findMatch(src, tgt, sindex, i) + offset, l := index.findMatch(src, tgt, i) - if l < s { + if l == 0 { + // couldn't find a match, just write the current byte and continue ibuf.WriteByte(tgt[i]) + } else if l < 0 { + // src is less than blksz, copy the rest of the target to avoid + // calls to findMatch + for ; i < len(tgt); i++ { + ibuf.WriteByte(tgt[i]) + } + } else if l < s { + // remaining target is less than blksz, copy what's left of it + // and avoid calls to findMatch + for j := i; j < i+l; j++ { + ibuf.WriteByte(tgt[j]) + } + i += l - 1 } else { encodeInsertOperation(ibuf, buf) @@ -126,52 +158,6 @@ func encodeInsertOperation(ibuf, buf *bytes.Buffer) { ibuf.Reset() } -func initMatch(src []byte) map[uint32]int { - i := 0 - index := make(map[uint32]int) - for { - if i+s > len(src) { - break - } - - ch := adler32.Checksum(src[i : i+s]) - index[ch] = i - i += s - } - - return index -} - -func findMatch(src, tgt []byte, sindex map[uint32]int, tgtOffset int) (srcOffset, l int) { - if len(tgt) >= tgtOffset+s { - ch := adler32.Checksum(tgt[tgtOffset : tgtOffset+s]) - var ok bool - srcOffset, ok = sindex[ch] - if !ok { - return - } - - l = matchLength(src, tgt, tgtOffset, srcOffset) - } - - return -} - -func matchLength(src, tgt []byte, otgt, osrc int) int { - l := 0 - for { - if (osrc >= len(src) || otgt >= len(tgt)) || src[osrc] != tgt[otgt] { - break - } - - l++ - osrc++ - otgt++ - } - - return l -} - func deltaEncodeSize(size int) []byte { var ret []byte c := size & 0x7f diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index 1426559..7ee6546 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -14,10 +14,10 @@ import ( // Encoder gets the data from the storage and write it into the writer in PACK // format type Encoder struct { - selector *deltaSelector - w *offsetWriter - zw *zlib.Writer - hasher plumbing.Hasher + selector *deltaSelector + w *offsetWriter + zw *zlib.Writer + hasher plumbing.Hasher // offsets is a map of object hashes to corresponding offsets in the packfile. // It is used to determine offset of the base of a delta when a OFS_DELTA is // used. @@ -45,10 +45,15 @@ func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *E } } -// Encode creates a packfile containing all the objects referenced in hashes -// and writes it to the writer in the Encoder. -func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { - objects, err := e.selector.ObjectsToPack(hashes) +// Encode creates a packfile containing all the objects referenced in +// hashes and writes it to the writer in the Encoder. `packWindow` +// specifies the size of the sliding window used to compare objects +// for delta compression; 0 turns off delta compression entirely. +func (e *Encoder) Encode( + hashes []plumbing.Hash, + packWindow uint, +) (plumbing.Hash, error) { + objects, err := e.selector.ObjectsToPack(hashes, packWindow) if err != nil { return plumbing.ZeroHash, err } @@ -137,7 +142,7 @@ func (e *Encoder) writeOfsDeltaHeader(deltaOffset int64, base plumbing.Hash) err // for OFS_DELTA, offset of the base is interpreted as negative offset // relative to the type-byte of the header of the ofs-delta entry. - relativeOffset := deltaOffset-baseOffset + relativeOffset := deltaOffset - baseOffset if relativeOffset <= 0 { return fmt.Errorf("bad offset for OFS_DELTA entry: %d", relativeOffset) } diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index d92e2c4..8011596 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -10,7 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) @@ -27,12 +27,23 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { fixs.Test(c, func(f *fixtures.Fixture) { storage, err := filesystem.NewStorage(f.DotGit()) c.Assert(err, IsNil) - s.testEncodeDecode(c, storage) + s.testEncodeDecode(c, storage, 10) }) } -func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer) { +func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { + fixs := fixtures.Basic().ByTag("packfile").ByTag(".git") + fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). + ByTag("packfile").ByTag(".git").One()) + fixs.Test(c, func(f *fixtures.Fixture) { + storage, err := filesystem.NewStorage(f.DotGit()) + c.Assert(err, IsNil) + s.testEncodeDecode(c, storage, 0) + }) +} + +func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer, packWindow uint) { objIter, err := storage.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -57,7 +68,7 @@ func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer) { buf := bytes.NewBuffer(nil) enc := NewEncoder(buf, storage, false) - _, err = enc.Encode(hashes) + _, err = enc.Encode(hashes, packWindow) c.Assert(err, IsNil) scanner := NewScanner(buf) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index b5b0c42..f40517d 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -3,11 +3,11 @@ package packfile import ( "bytes" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type EncoderSuite struct { @@ -26,7 +26,7 @@ func (s *EncoderSuite) SetUpTest(c *C) { } func (s *EncoderSuite) TestCorrectPackHeader(c *C) { - hash, err := s.enc.Encode([]plumbing.Hash{}) + hash, err := s.enc.Encode([]plumbing.Hash{}, 10) c.Assert(err, IsNil) hb := [20]byte(hash) @@ -47,7 +47,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) // PACK + VERSION(2) + OBJECT NUMBER(1) @@ -74,13 +74,13 @@ func (s *EncoderSuite) TestMaxObjectSize(c *C) { o.SetType(plumbing.CommitObject) _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) c.Assert(hash.IsZero(), Not(Equals), true) } func (s *EncoderSuite) TestHashNotFound(c *C) { - h, err := s.enc.Encode([]plumbing.Hash{plumbing.NewHash("BAD")}) + h, err := s.enc.Encode([]plumbing.Hash{plumbing.NewHash("BAD")}, 10) c.Assert(h, Equals, plumbing.ZeroHash) c.Assert(err, NotNil) c.Assert(err, Equals, plumbing.ErrObjectNotFound) diff --git a/plumbing/format/packfile/object_pack.go b/plumbing/format/packfile/object_pack.go index 14337d1..e22e783 100644 --- a/plumbing/format/packfile/object_pack.go +++ b/plumbing/format/packfile/object_pack.go @@ -84,11 +84,7 @@ func (o *ObjectToPack) Size() int64 { } func (o *ObjectToPack) IsDelta() bool { - if o.Base != nil { - return true - } - - return false + return o.Base != nil } func (o *ObjectToPack) SetDelta(base *ObjectToPack, delta plumbing.EncodedObject) { diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 976cabc..c604851 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -38,11 +38,8 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { target.SetSize(int64(len(dst))) - if _, err := w.Write(dst); err != nil { - return err - } - - return nil + _, err = w.Write(dst) + return err } var ( diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 1ca8b6e..ab87642 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -6,7 +6,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/format/pktline/encoder.go b/plumbing/format/pktline/encoder.go index 797b813..eae85cc 100644 --- a/plumbing/format/pktline/encoder.go +++ b/plumbing/format/pktline/encoder.go @@ -63,21 +63,15 @@ func (e *Encoder) encodeLine(p []byte) error { } if bytes.Equal(p, Flush) { - if err := e.Flush(); err != nil { - return err - } - return nil + return e.Flush() } n := len(p) + 4 if _, err := e.w.Write(asciiHex16(n)); err != nil { return err } - if _, err := e.w.Write(p); err != nil { - return err - } - - return nil + _, err := e.w.Write(p) + return err } // Returns the hexadecimal ascii representation of the 16 less diff --git a/plumbing/object/change_adaptor.go b/plumbing/object/change_adaptor.go index 49b6545..491c399 100644 --- a/plumbing/object/change_adaptor.go +++ b/plumbing/object/change_adaptor.go @@ -8,7 +8,7 @@ import ( "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" ) -// The folowing functions transform changes types form the merkletrie +// The following functions transform changes types form the merkletrie // package to changes types from this package. func newChange(c merkletrie.Change) (*Change, error) { diff --git a/plumbing/object/change_adaptor_test.go b/plumbing/object/change_adaptor_test.go index 317c0d6..dd2921d 100644 --- a/plumbing/object/change_adaptor_test.go +++ b/plumbing/object/change_adaptor_test.go @@ -10,7 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/utils/merkletrie" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/object/change_test.go b/plumbing/object/change_test.go index ded7ff2..7036fa3 100644 --- a/plumbing/object/change_test.go +++ b/plumbing/object/change_test.go @@ -10,8 +10,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/utils/merkletrie" - fixtures "github.com/src-d/go-git-fixtures" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ChangeSuite struct { diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index eee015b..a317714 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -3,15 +3,23 @@ package object import ( "bufio" "bytes" + "errors" "fmt" "io" "strings" + "golang.org/x/crypto/openpgp" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) +const ( + beginpgp string = "-----BEGIN PGP SIGNATURE-----" + endpgp string = "-----END PGP SIGNATURE-----" +) + // Hash represents the hash of an object type Hash plumbing.Hash @@ -19,7 +27,7 @@ type Hash plumbing.Hash // at a certain point in time. It contains meta-information about that point // in time, such as a timestamp, the author of the changes since the last // commit, a pointer to the previous commit(s), etc. -// http://schacon.github.io/gitbook/1_the_git_object_model.html +// http://shafiulazam.com/gitbook/1_the_git_object_model.html type Commit struct { // Hash of the commit object. Hash plumbing.Hash @@ -28,6 +36,8 @@ type Commit struct { // Committer is the one performing the commit, might be different from // Author. Committer Signature + // PGPSignature is the PGP signature of the commit. + PGPSignature string // Message is the commit message, contains arbitrary text. Message string // TreeHash is the hash of the root tree of the commit. @@ -91,6 +101,17 @@ func (c *Commit) NumParents() int { return len(c.ParentHashes) } +var ErrParentNotFound = errors.New("commit parent not found") + +// Parent returns the ith parent of a commit. +func (c *Commit) Parent(i int) (*Commit, error) { + if len(c.ParentHashes) == 0 || i > len(c.ParentHashes)-1 { + return nil, ErrParentNotFound + } + + return GetCommit(c.s, c.ParentHashes[i]) +} + // File returns the file with the specified "path" in the commit and a // nil error if the file exists. If the file does not exist, it returns // a nil file and the ErrFileNotFound error. @@ -145,12 +166,33 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { r := bufio.NewReader(reader) var message bool + var pgpsig bool for { line, err := r.ReadBytes('\n') if err != nil && err != io.EOF { return err } + if pgpsig { + // Check if it's the end of a PGP signature. + if bytes.Contains(line, []byte(endpgp)) { + c.PGPSignature += endpgp + "\n" + pgpsig = false + } else { + // Trim the left padding. + line = bytes.TrimLeft(line, " ") + c.PGPSignature += string(line) + } + continue + } + + // Check if it's the beginning of a PGP signature. + if bytes.Contains(line, []byte(beginpgp)) { + c.PGPSignature += beginpgp + "\n" + pgpsig = true + continue + } + if !message { line = bytes.TrimSpace(line) if len(line) == 0 { @@ -181,6 +223,10 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { // Encode transforms a Commit into a plumbing.EncodedObject. func (b *Commit) Encode(o plumbing.EncodedObject) error { + return b.encode(o, true) +} + +func (b *Commit) encode(o plumbing.EncodedObject, includeSig bool) error { o.SetType(plumbing.CommitObject) w, err := o.Writer() if err != nil { @@ -215,6 +261,21 @@ func (b *Commit) Encode(o plumbing.EncodedObject) error { return err } + if b.PGPSignature != "" && includeSig { + if _, err = fmt.Fprint(w, "pgpsig"); err != nil { + return err + } + + // Split all the signature lines and write with a left padding and + // newline at the end. + lines := strings.Split(b.PGPSignature, "\n") + for _, line := range lines { + if _, err = fmt.Fprintf(w, " %s\n", line); err != nil { + return err + } + } + } + if _, err = fmt.Fprintf(w, "\n\n%s", b.Message); err != nil { return err } @@ -222,6 +283,32 @@ func (b *Commit) Encode(o plumbing.EncodedObject) error { return err } +// Stats shows the status of commit. +func (c *Commit) Stats() (FileStats, error) { + // Get the previous commit. + ci := c.Parents() + parentCommit, err := ci.Next() + if err != nil { + if err == io.EOF { + emptyNoder := treeNoder{} + parentCommit = &Commit{ + Hash: emptyNoder.hash, + // TreeHash: emptyNoder.parent.Hash, + s: c.s, + } + } else { + return nil, err + } + } + + patch, err := parentCommit.Patch(c) + if err != nil { + return nil, err + } + + return getFileStatsFromFilePatches(patch.FilePatches()), nil +} + func (c *Commit) String() string { return fmt.Sprintf( "%s %s\nAuthor: %s\nDate: %s\n\n%s\n", @@ -230,6 +317,31 @@ func (c *Commit) String() string { ) } +// Verify performs PGP verification of the commit with a provided armored +// keyring and returns openpgp.Entity associated with verifying key on success. +func (c *Commit) Verify(armoredKeyRing string) (*openpgp.Entity, error) { + keyRingReader := strings.NewReader(armoredKeyRing) + keyring, err := openpgp.ReadArmoredKeyRing(keyRingReader) + if err != nil { + return nil, err + } + + // Extract signature. + signature := strings.NewReader(c.PGPSignature) + + encoded := &plumbing.MemoryObject{} + // Encode commit components, excluding signature and get a reader object. + if err := c.encode(encoded, false); err != nil { + return nil, err + } + er, err := encoded.Reader() + if err != nil { + return nil, err + } + + return openpgp.CheckArmoredDetachedSignature(keyring, er, signature) +} + func indent(t string) string { var output []string for _, line := range strings.Split(t, "\n") { diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index e89302d..191b14d 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -6,10 +6,10 @@ import ( "strings" "time" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -67,6 +67,18 @@ func (s *SuiteCommit) TestParents(c *C) { i.Close() } +func (s *SuiteCommit) TestParent(c *C) { + commit, err := s.Commit.Parent(1) + c.Assert(err, IsNil) + c.Assert(commit.Hash.String(), Equals, "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69") +} + +func (s *SuiteCommit) TestParentNotFound(c *C) { + commit, err := s.Commit.Parent(42) + c.Assert(err, Equals, ErrParentNotFound) + c.Assert(commit, IsNil) +} + func (s *SuiteCommit) TestPatch(c *C) { from := s.commit(c, plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294")) to := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) @@ -182,6 +194,7 @@ func (s *SuiteCommit) TestStringMultiLine(c *C) { f := fixtures.ByURL("https://github.com/src-d/go-git.git").One() sto, err := filesystem.NewStorage(f.DotGit()) + c.Assert(err, IsNil) o, err := sto.EncodedObject(plumbing.CommitObject, hash) c.Assert(err, IsNil) @@ -232,3 +245,121 @@ func (s *SuiteCommit) TestLongCommitMessageSerialization(c *C) { c.Assert(err, IsNil) c.Assert(decoded.Message, Equals, longMessage) } + +func (s *SuiteCommit) TestPGPSignatureSerialization(c *C) { + encoded := &plumbing.MemoryObject{} + decoded := &Commit{} + commit := *s.Commit + + pgpsignature := `-----BEGIN PGP SIGNATURE----- + +iQEcBAABAgAGBQJTZbQlAAoJEF0+sviABDDrZbQH/09PfE51KPVPlanr6q1v4/Ut +LQxfojUWiLQdg2ESJItkcuweYg+kc3HCyFejeDIBw9dpXt00rY26p05qrpnG+85b +hM1/PswpPLuBSr+oCIDj5GMC2r2iEKsfv2fJbNW8iWAXVLoWZRF8B0MfqX/YTMbm +ecorc4iXzQu7tupRihslbNkfvfciMnSDeSvzCpWAHl7h8Wj6hhqePmLm9lAYqnKp +8S5B/1SSQuEAjRZgI4IexpZoeKGVDptPHxLLS38fozsyi0QyDyzEgJxcJQVMXxVi +RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= +=EFTF +-----END PGP SIGNATURE----- +` + commit.PGPSignature = pgpsignature + + err := commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, pgpsignature) +} + +func (s *SuiteCommit) TestStat(c *C) { + aCommit := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) + fileStats, err := aCommit.Stats() + c.Assert(err, IsNil) + + c.Assert(fileStats[0].Name, Equals, "vendor/foo.go") + c.Assert(fileStats[0].Addition, Equals, 7) + c.Assert(fileStats[0].Deletion, Equals, 0) + c.Assert(fileStats[0].String(), Equals, " vendor/foo.go | 7 +++++++\n") + + // Stats for another commit. + aCommit = s.commit(c, plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294")) + fileStats, err = aCommit.Stats() + c.Assert(err, IsNil) + + c.Assert(fileStats[0].Name, Equals, "go/example.go") + c.Assert(fileStats[0].Addition, Equals, 142) + c.Assert(fileStats[0].Deletion, Equals, 0) + c.Assert(fileStats[0].String(), Equals, " go/example.go | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + + c.Assert(fileStats[1].Name, Equals, "php/crappy.php") + c.Assert(fileStats[1].Addition, Equals, 259) + c.Assert(fileStats[1].Deletion, Equals, 0) + c.Assert(fileStats[1].String(), Equals, " php/crappy.php | 259 ++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func (s *SuiteCommit) TestVerify(c *C) { + ts := time.Unix(1511197315, 0) + loc, _ := time.LoadLocation("Asia/Kolkata") + commit := &Commit{ + Hash: plumbing.NewHash("8a9cea36fe052711fbc42b86e1f99a4fa0065deb"), + Author: Signature{Name: "Sunny", Email: "me@darkowlzz.space", When: ts.In(loc)}, + Committer: Signature{Name: "Sunny", Email: "me@darkowlzz.space", When: ts.In(loc)}, + Message: `status: simplify template command selection +`, + TreeHash: plumbing.NewHash("6572ba6df4f1fb323c8aaa24ce07bca0648b161e"), + ParentHashes: []plumbing.Hash{plumbing.NewHash("ede5f57ea1280a0065beec96d3e1a3453d010dbd")}, + PGPSignature: ` +-----BEGIN PGP SIGNATURE----- + +iQFHBAABCAAxFiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAloTCrsTHG1lQGRhcmtv +d2x6ei5zcGFjZQAKCRBDIt4ypybJTul5CADmVxB4kqlqRZ9fAcSU5LKva3GRXx0+ +leX6vbzoyQztSWYgl7zALh4kB3a3t2C9EnnM6uehlgaORNigyMArCSY1ivWVviCT +BvldSVi8f8OvnqwbWX0I/5a8KmItthDf5WqZRFjhcRlY1AK5Bo2hUGVRq71euf8F +rE6wNhDoyBCEpftXuXbq8duD7D6qJ7QiOS4m5+ej1UCssS2WQ60yta7q57odduHY ++txqTKI8MQUpBgoTqh+V4lOkwQQxLiz7hIQ/ZYLUcnp6fan7/kY/G7YoLt9pOG1Y +vLzAWdidLH2P+EUOqlNMuVScHYWD1FZB0/L5LJ8no5pTowQd2Z+Nggxl +=0uC8 +-----END PGP SIGNATURE----- +`, + } + + armoredKeyRing := ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFmtHgABCADnfThM7q8D4pgUub9jMppSpgFh3ev84g3Csc3yQUlszEOVgXmu +YiSWP1oAiWFQ8ahCydh3LT8TnEB2QvoRNiExUI5XlXFwVfKW3cpDu8gdhtufs90Q +NvpaHOgTqRf/texGEKwXi6fvS47fpyaQ9BKNdN52LeaaHzDDZkVsAFmroE+7MMvj +P4Mq8qDn2WcWnX9zheQKYrX6Cs48Tx80eehHor4f/XnuaP8DLmPQx7URdJ0Igckh +N+i91Qv2ujin8zxUwhkfus66EZS9lQ4qR9iVHs4WHOs3j7whsejd4VhajonilVHj +uqTtqHmpN/4njbIKb8q8uQkS26VQYoSYm2UvABEBAAG0GlN1bm55IDxtZUBkYXJr +b3dsenouc3BhY2U+iQFUBBMBCAA+FiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAlmt +HgACGwMFCQPCZwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQQyLeMqcmyU7V +nAf+J5BYu26B2i+iwctOzDRFcPwCLka9cBwe5wcDvoF2qL8QRo8NPWBBH4zWHa/k +BthtGo1b89a53I2hnTwTQ0NOtAUNV+Vvu6nOHJd9Segsx3E1nM43bd2bUfGJ1eeO +jDOlOvtP4ozuV6Ej+0Ln2ouMOc87yAwbAzTfQ9axU6CKUbqy0/t2dW1jdKntGH+t +VPeFxJHL2gXjP89skCSPYA7yKqqyJRPFvC+7rde1OLdCmZi4VwghUiNbh3s1+xM3 +gfr2ahsRDTN2SQzwuHu4y1EgZgPtuWfRxzHqduoRoSgfOfFr9H9Il3UMHf2Etleu +rif40YZJhge6STwsIycGh4wOiLkBDQRZrR4AAQgArpUvPdGC/W9X4AuZXrXEShvx +TqM4K2Jk9n0j+ABx87k9fm48qgtae7+TayMbb0i7kcbgnjltKbauTbyRbju/EJvN +CdIw76IPpjy6jUM37wG2QGLFo6Ku3x8/ZpNGGOZ8KMU258/EBqDlJQ/4g4kJ8D+m +9yOH0r6/Xpe/jOY2V8Jo9pdFTm+8eAsSyZF0Cl7drz603Pymq1IS2wrwQbdxQA/w +B75pQ5es7X34Ac7/9UZCwCPmZDAldnjHyw5dZgZe8XLrG84BIfbG0Hj8PjrFdF1D +Czt9bk+PbYAnLORW2oX1oedxVrNFo5UrbWgBSjA1ppbGFjwSDHFlyjuEuxqyFwAR +AQABiQE8BBgBCAAmFiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAlmtHgACGwwFCQPC +ZwAACgkQQyLeMqcmyU7ZBggArzc8UUVSjde987Vqnu/S5Cv8Qhz+UB7gAFyTW2iF +VYvB86r30H/NnfjvjCVkBE6FHCNHoxWVyDWmuxKviB7nkReHuwqniQHPgdJDcTKC +tBboeX2IYBLJbEvEJuz5NSvnvFuYkIpZHqySFaqdl/qu9XcmoPL5AmIzIFOeiNty +qT0ldkf3ru6yQQDDqBDpkfz4AzkpFnLYL59z6IbJDK2Hz7aKeSEeVOGiZLCjIZZV +uISZThYqh5zUkvF346OHLDqfDdgQ4RZriqd/DTtRJPlz2uL0QcEIjJuYCkG0UWgl +sYyf9RfOnw/KUFAQbdtvLx3ikODQC+D3KBtuKI9ISHQfgw== +=FPev +-----END PGP PUBLIC KEY BLOCK----- +` + + e, err := commit.Verify(armoredKeyRing) + c.Assert(err, IsNil) + + _, ok := e.Identities["Sunny <me@darkowlzz.space>"] + c.Assert(ok, Equals, true) +} diff --git a/plumbing/object/commit_walker.go b/plumbing/object/commit_walker.go index 797c17a..40ad258 100644 --- a/plumbing/object/commit_walker.go +++ b/plumbing/object/commit_walker.go @@ -8,9 +8,10 @@ import ( ) type commitPreIterator struct { - seen map[plumbing.Hash]bool - stack []CommitIter - start *Commit + seenExternal map[plumbing.Hash]bool + seen map[plumbing.Hash]bool + stack []CommitIter + start *Commit } // NewCommitPreorderIter returns a CommitIter that walks the commit history, @@ -20,16 +21,21 @@ type commitPreIterator struct { // and will return the error. Other errors might be returned if the history // cannot be traversed (e.g. missing objects). Ignore allows to skip some // commits from being iterated. -func NewCommitPreorderIter(c *Commit, ignore []plumbing.Hash) CommitIter { +func NewCommitPreorderIter( + c *Commit, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitIter { seen := make(map[plumbing.Hash]bool) for _, h := range ignore { seen[h] = true } return &commitPreIterator{ - seen: seen, - stack: make([]CommitIter, 0), - start: c, + seenExternal: seenExternal, + seen: seen, + stack: make([]CommitIter, 0), + start: c, } } @@ -57,7 +63,7 @@ func (w *commitPreIterator) Next() (*Commit, error) { } } - if w.seen[c.Hash] { + if w.seen[c.Hash] || w.seenExternal[c.Hash] { continue } diff --git a/plumbing/object/commit_walker_test.go b/plumbing/object/commit_walker_test.go index 48b504d..a27104e 100644 --- a/plumbing/object/commit_walker_test.go +++ b/plumbing/object/commit_walker_test.go @@ -16,7 +16,7 @@ func (s *CommitWalkerSuite) TestCommitPreIterator(c *C) { commit := s.commit(c, s.Fixture.Head) var commits []*Commit - NewCommitPreorderIter(commit, nil).ForEach(func(c *Commit) error { + NewCommitPreorderIter(commit, nil, nil).ForEach(func(c *Commit) error { commits = append(commits, c) return nil }) @@ -42,7 +42,7 @@ func (s *CommitWalkerSuite) TestCommitPreIteratorWithIgnore(c *C) { commit := s.commit(c, s.Fixture.Head) var commits []*Commit - NewCommitPreorderIter(commit, []plumbing.Hash{ + NewCommitPreorderIter(commit, nil, []plumbing.Hash{ plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), }).ForEach(func(c *Commit) error { commits = append(commits, c) @@ -60,6 +60,30 @@ func (s *CommitWalkerSuite) TestCommitPreIteratorWithIgnore(c *C) { } } +func (s *CommitWalkerSuite) TestCommitPreIteratorWithSeenExternal(c *C) { + commit := s.commit(c, s.Fixture.Head) + + var commits []*Commit + seenExternal := map[plumbing.Hash]bool{ + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"): true, + } + NewCommitPreorderIter(commit, seenExternal, nil). + ForEach(func(c *Commit) error { + commits = append(commits, c) + return nil + }) + + c.Assert(commits, HasLen, 2) + + expected := []string{ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + "918c48b83bd081e863dbe1b80f8998f058cd8294", + } + for i, commit := range commits { + c.Assert(commit.Hash.String(), Equals, expected[i]) + } +} + func (s *CommitWalkerSuite) TestCommitPostIterator(c *C) { commit := s.commit(c, s.Fixture.Head) diff --git a/plumbing/object/difftree_test.go b/plumbing/object/difftree_test.go index eb68d4d..c9344b8 100644 --- a/plumbing/object/difftree_test.go +++ b/plumbing/object/difftree_test.go @@ -11,7 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/merkletrie" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/object/file_test.go b/plumbing/object/file_test.go index 8c8634d..2288697 100644 --- a/plumbing/object/file_test.go +++ b/plumbing/object/file_test.go @@ -8,7 +8,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/object/object_test.go b/plumbing/object/object_test.go index 6d9028f..2ac5d12 100644 --- a/plumbing/object/object_test.go +++ b/plumbing/object/object_test.go @@ -11,7 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/plumbing/object/patch.go b/plumbing/object/patch.go index d413114..a920631 100644 --- a/plumbing/object/patch.go +++ b/plumbing/object/patch.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" "io" + "math" + "strings" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" @@ -105,6 +107,10 @@ func (p *Patch) Encode(w io.Writer) error { return ue.Encode(p) } +func (p *Patch) Stats() FileStats { + return getFileStatsFromFilePatches(p.FilePatches()) +} + func (p *Patch) String() string { buf := bytes.NewBuffer(nil) err := p.Encode(buf) @@ -185,3 +191,112 @@ func (t *textChunk) Content() string { func (t *textChunk) Type() fdiff.Operation { return t.op } + +// FileStat stores the status of changes in content of a file. +type FileStat struct { + Name string + Addition int + Deletion int +} + +func (fs FileStat) String() string { + return printStat([]FileStat{fs}) +} + +// FileStats is a collection of FileStat. +type FileStats []FileStat + +func (fileStats FileStats) String() string { + return printStat(fileStats) +} + +func printStat(fileStats []FileStat) string { + padLength := float64(len(" ")) + newlineLength := float64(len("\n")) + separatorLength := float64(len("|")) + // Soft line length limit. The text length calculation below excludes + // length of the change number. Adding that would take it closer to 80, + // but probably not more than 80, until it's a huge number. + lineLength := 72.0 + + // Get the longest filename and longest total change. + var longestLength float64 + var longestTotalChange float64 + for _, fs := range fileStats { + if int(longestLength) < len(fs.Name) { + longestLength = float64(len(fs.Name)) + } + totalChange := fs.Addition + fs.Deletion + if int(longestTotalChange) < totalChange { + longestTotalChange = float64(totalChange) + } + } + + // Parts of the output: + // <pad><filename><pad>|<pad><changeNumber><pad><+++/---><newline> + // example: " main.go | 10 +++++++--- " + + // <pad><filename><pad> + leftTextLength := padLength + longestLength + padLength + + // <pad><number><pad><+++++/-----><newline> + // Excluding number length here. + rightTextLength := padLength + padLength + newlineLength + + totalTextArea := leftTextLength + separatorLength + rightTextLength + heightOfHistogram := lineLength - totalTextArea + + // Scale the histogram. + var scaleFactor float64 + if longestTotalChange > heightOfHistogram { + // Scale down to heightOfHistogram. + scaleFactor = float64(longestTotalChange / heightOfHistogram) + } else { + scaleFactor = 1.0 + } + + finalOutput := "" + for _, fs := range fileStats { + addn := float64(fs.Addition) + deln := float64(fs.Deletion) + adds := strings.Repeat("+", int(math.Floor(addn/scaleFactor))) + dels := strings.Repeat("-", int(math.Floor(deln/scaleFactor))) + finalOutput += fmt.Sprintf(" %s | %d %s%s\n", fs.Name, (fs.Addition + fs.Deletion), adds, dels) + } + + return finalOutput +} + +func getFileStatsFromFilePatches(filePatches []fdiff.FilePatch) FileStats { + var fileStats FileStats + + for _, fp := range filePatches { + cs := FileStat{} + from, to := fp.Files() + if from == nil { + // New File is created. + cs.Name = to.Path() + } else if to == nil { + // File is deleted. + cs.Name = from.Path() + } else if from.Path() != to.Path() { + // File is renamed. Not supported. + // cs.Name = fmt.Sprintf("%s => %s", from.Path(), to.Path()) + } else { + cs.Name = from.Path() + } + + for _, chunk := range fp.Chunks() { + switch chunk.Type() { + case fdiff.Add: + cs.Addition += strings.Count(chunk.Content(), "\n") + case fdiff.Delete: + cs.Deletion += strings.Count(chunk.Content(), "\n") + } + } + + fileStats = append(fileStats, cs) + } + + return fileStats +} diff --git a/plumbing/object/tag.go b/plumbing/object/tag.go index 7b091d0..19e55cf 100644 --- a/plumbing/object/tag.go +++ b/plumbing/object/tag.go @@ -6,6 +6,9 @@ import ( "fmt" "io" stdioutil "io/ioutil" + "strings" + + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -30,6 +33,8 @@ type Tag struct { Tagger Signature // Message is an arbitrary text message. Message string + // PGPSignature is the PGP signature of the tag. + PGPSignature string // TargetType is the object type of the target. TargetType plumbing.ObjectType // Target is the hash of the target object. @@ -124,13 +129,46 @@ func (t *Tag) Decode(o plumbing.EncodedObject) (err error) { if err != nil { return err } - t.Message = string(data) + + var pgpsig bool + // Check if data contains PGP signature. + if bytes.Contains(data, []byte(beginpgp)) { + // Split the lines at newline. + messageAndSig := bytes.Split(data, []byte("\n")) + + for _, l := range messageAndSig { + if pgpsig { + if bytes.Contains(l, []byte(endpgp)) { + t.PGPSignature += endpgp + "\n" + pgpsig = false + } else { + t.PGPSignature += string(l) + "\n" + } + continue + } + + // Check if it's the beginning of a PGP signature. + if bytes.Contains(l, []byte(beginpgp)) { + t.PGPSignature += beginpgp + "\n" + pgpsig = true + continue + } + + t.Message += string(l) + "\n" + } + } else { + t.Message = string(data) + } return nil } // Encode transforms a Tag into a plumbing.EncodedObject. func (t *Tag) Encode(o plumbing.EncodedObject) error { + return t.encode(o, true) +} + +func (t *Tag) encode(o plumbing.EncodedObject, includeSig bool) error { o.SetType(plumbing.TagObject) w, err := o.Writer() if err != nil { @@ -156,6 +194,16 @@ func (t *Tag) Encode(o plumbing.EncodedObject) error { return err } + if t.PGPSignature != "" && includeSig { + // Split all the signature lines and write with a newline at the end. + lines := strings.Split(t.PGPSignature, "\n") + for _, line := range lines { + if _, err = fmt.Fprintf(w, "%s\n", line); err != nil { + return err + } + } + } + return err } @@ -225,6 +273,31 @@ func (t *Tag) String() string { ) } +// Verify performs PGP verification of the tag with a provided armored +// keyring and returns openpgp.Entity associated with verifying key on success. +func (t *Tag) Verify(armoredKeyRing string) (*openpgp.Entity, error) { + keyRingReader := strings.NewReader(armoredKeyRing) + keyring, err := openpgp.ReadArmoredKeyRing(keyRingReader) + if err != nil { + return nil, err + } + + // Extract signature. + signature := strings.NewReader(t.PGPSignature) + + encoded := &plumbing.MemoryObject{} + // Encode tag components, excluding signature and get a reader object. + if err := t.encode(encoded, false); err != nil { + return nil, err + } + er, err := encoded.Reader() + if err != nil { + return nil, err + } + + return openpgp.CheckArmoredDetachedSignature(keyring, er, signature) +} + // TagIter provides an iterator for a set of tags. type TagIter struct { storer.EncodedObjectIter @@ -252,7 +325,7 @@ func (iter *TagIter) Next() (*Tag, error) { } // ForEach call the cb function for each tag contained on this iter until -// an error happends or the end of the iter is reached. If ErrStop is sent +// an error happens or the end of the iter is reached. If ErrStop is sent // the iteration is stop but no error is returned. The iterator is closed. func (iter *TagIter) ForEach(cb func(*Tag) error) error { return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error { diff --git a/plumbing/object/tag_test.go b/plumbing/object/tag_test.go index 9f2d28c..9900093 100644 --- a/plumbing/object/tag_test.go +++ b/plumbing/object/tag_test.go @@ -6,12 +6,12 @@ import ( "strings" "time" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type TagSuite struct { @@ -285,3 +285,94 @@ func (s *TagSuite) TestLongTagNameSerialization(c *C) { c.Assert(err, IsNil) c.Assert(decoded.Name, Equals, longName) } + +func (s *TagSuite) TestPGPSignatureSerialization(c *C) { + encoded := &plumbing.MemoryObject{} + decoded := &Tag{} + tag := s.tag(c, plumbing.NewHash("b742a2a9fa0afcfa9a6fad080980fbc26b007c69")) + + pgpsignature := `-----BEGIN PGP SIGNATURE----- + +iQEcBAABAgAGBQJTZbQlAAoJEF0+sviABDDrZbQH/09PfE51KPVPlanr6q1v4/Ut +LQxfojUWiLQdg2ESJItkcuweYg+kc3HCyFejeDIBw9dpXt00rY26p05qrpnG+85b +hM1/PswpPLuBSr+oCIDj5GMC2r2iEKsfv2fJbNW8iWAXVLoWZRF8B0MfqX/YTMbm +ecorc4iXzQu7tupRihslbNkfvfciMnSDeSvzCpWAHl7h8Wj6hhqePmLm9lAYqnKp +8S5B/1SSQuEAjRZgI4IexpZoeKGVDptPHxLLS38fozsyi0QyDyzEgJxcJQVMXxVi +RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= +=EFTF +-----END PGP SIGNATURE----- +` + tag.PGPSignature = pgpsignature + + err := tag.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, pgpsignature) +} + +func (s *TagSuite) TestVerify(c *C) { + ts := time.Unix(1511524851, 0) + loc, _ := time.LoadLocation("Asia/Kolkata") + tag := &Tag{ + Name: "v0.2", + Tagger: Signature{Name: "Sunny", Email: "me@darkowlzz.space", When: ts.In(loc)}, + Message: `This is a signed tag +`, + TargetType: plumbing.CommitObject, + Target: plumbing.NewHash("064f92fe00e70e6b64cb358a65039daa4b6ae8d2"), + PGPSignature: ` +-----BEGIN PGP SIGNATURE----- + +iQFHBAABCAAxFiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAloYCg8THG1lQGRhcmtv +d2x6ei5zcGFjZQAKCRBDIt4ypybJTs0cCACjQZe2610t3gfbUPbgQiWDL9uvlCeb +sNSeTC6hLAFSvHTMqLr/6RpiLlfQXyATD7TZUH0DUSLsERLheG82OgVxkOTzPCpy +GL6iGKeZ4eZ1KiV+SBPjqizC9ShhGooPUw9oUSVdj4jsaHDdDHtY63Pjl0KvJmms +OVi9SSxjeMbmaC81C8r0ZuOLTXJh/JRKh2BsehdcnK3736BK+16YRD7ugXLpkQ5d +nsCFVbuYYoLMoJL5NmEun0pbUrpY+MI8VPK0f9HV5NeaC4NksC+ke/xYMT+P2lRL +CN+9zcCIU+mXr2fCl1xOQcnQzwOElObDxpDcPcxVn0X+AhmPc+uj0mqD +=l75D +-----END PGP SIGNATURE----- +`, + } + + armoredKeyRing := ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFmtHgABCADnfThM7q8D4pgUub9jMppSpgFh3ev84g3Csc3yQUlszEOVgXmu +YiSWP1oAiWFQ8ahCydh3LT8TnEB2QvoRNiExUI5XlXFwVfKW3cpDu8gdhtufs90Q +NvpaHOgTqRf/texGEKwXi6fvS47fpyaQ9BKNdN52LeaaHzDDZkVsAFmroE+7MMvj +P4Mq8qDn2WcWnX9zheQKYrX6Cs48Tx80eehHor4f/XnuaP8DLmPQx7URdJ0Igckh +N+i91Qv2ujin8zxUwhkfus66EZS9lQ4qR9iVHs4WHOs3j7whsejd4VhajonilVHj +uqTtqHmpN/4njbIKb8q8uQkS26VQYoSYm2UvABEBAAG0GlN1bm55IDxtZUBkYXJr +b3dsenouc3BhY2U+iQFUBBMBCAA+FiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAlmt +HgACGwMFCQPCZwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQQyLeMqcmyU7V +nAf+J5BYu26B2i+iwctOzDRFcPwCLka9cBwe5wcDvoF2qL8QRo8NPWBBH4zWHa/k +BthtGo1b89a53I2hnTwTQ0NOtAUNV+Vvu6nOHJd9Segsx3E1nM43bd2bUfGJ1eeO +jDOlOvtP4ozuV6Ej+0Ln2ouMOc87yAwbAzTfQ9axU6CKUbqy0/t2dW1jdKntGH+t +VPeFxJHL2gXjP89skCSPYA7yKqqyJRPFvC+7rde1OLdCmZi4VwghUiNbh3s1+xM3 +gfr2ahsRDTN2SQzwuHu4y1EgZgPtuWfRxzHqduoRoSgfOfFr9H9Il3UMHf2Etleu +rif40YZJhge6STwsIycGh4wOiLkBDQRZrR4AAQgArpUvPdGC/W9X4AuZXrXEShvx +TqM4K2Jk9n0j+ABx87k9fm48qgtae7+TayMbb0i7kcbgnjltKbauTbyRbju/EJvN +CdIw76IPpjy6jUM37wG2QGLFo6Ku3x8/ZpNGGOZ8KMU258/EBqDlJQ/4g4kJ8D+m +9yOH0r6/Xpe/jOY2V8Jo9pdFTm+8eAsSyZF0Cl7drz603Pymq1IS2wrwQbdxQA/w +B75pQ5es7X34Ac7/9UZCwCPmZDAldnjHyw5dZgZe8XLrG84BIfbG0Hj8PjrFdF1D +Czt9bk+PbYAnLORW2oX1oedxVrNFo5UrbWgBSjA1ppbGFjwSDHFlyjuEuxqyFwAR +AQABiQE8BBgBCAAmFiEEoRt6IzxHaZkkUslhQyLeMqcmyU4FAlmtHgACGwwFCQPC +ZwAACgkQQyLeMqcmyU7ZBggArzc8UUVSjde987Vqnu/S5Cv8Qhz+UB7gAFyTW2iF +VYvB86r30H/NnfjvjCVkBE6FHCNHoxWVyDWmuxKviB7nkReHuwqniQHPgdJDcTKC +tBboeX2IYBLJbEvEJuz5NSvnvFuYkIpZHqySFaqdl/qu9XcmoPL5AmIzIFOeiNty +qT0ldkf3ru6yQQDDqBDpkfz4AzkpFnLYL59z6IbJDK2Hz7aKeSEeVOGiZLCjIZZV +uISZThYqh5zUkvF346OHLDqfDdgQ4RZriqd/DTtRJPlz2uL0QcEIjJuYCkG0UWgl +sYyf9RfOnw/KUFAQbdtvLx3ikODQC+D3KBtuKI9ISHQfgw== +=FPev +-----END PGP PUBLIC KEY BLOCK----- +` + + e, err := tag.Verify(armoredKeyRing) + c.Assert(err, IsNil) + + _, ok := e.Identities["Sunny <me@darkowlzz.space>"] + c.Assert(ok, Equals, true) +} diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index 44ac720..2fcd979 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -136,9 +136,9 @@ func (t *Tree) dir(baseName string) (*Tree, error) { } tree := &Tree{s: t.s} - tree.Decode(obj) + err = tree.Decode(obj) - return tree, nil + return tree, err } var errEntryNotFound = errors.New("entry not found") diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index 796d979..3a687dd 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -1,6 +1,7 @@ package object import ( + "errors" "io" "gopkg.in/src-d/go-git.v4/plumbing" @@ -8,8 +9,8 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" - fixtures "github.com/src-d/go-git-fixtures" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type TreeSuite struct { @@ -113,6 +114,42 @@ func (s *TreeSuite) TestFindEntry(c *C) { c.Assert(e.Name, Equals, "foo.go") } +// Overrides returned plumbing.EncodedObject for given hash. +// Otherwise, delegates to actual storer to get real object +type fakeStorer struct { + storer.EncodedObjectStorer + hash plumbing.Hash + fake fakeEncodedObject +} + +func (fs fakeStorer) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { + if fs.hash == h { + return fs.fake, nil + } + return fs.EncodedObjectStorer.EncodedObject(t, h) +} + +// Overrides reader of plumbing.EncodedObject to simulate read error +type fakeEncodedObject struct{ plumbing.EncodedObject } + +func (fe fakeEncodedObject) Reader() (io.ReadCloser, error) { + return nil, errors.New("Simulate encoded object can't be read") +} + +func (s *TreeSuite) TestDir(c *C) { + vendor, err := s.Tree.dir("vendor") + c.Assert(err, IsNil) + + t, err := GetTree(s.Tree.s, s.Tree.ID()) + c.Assert(err, IsNil) + o, err := t.s.EncodedObject(plumbing.AnyObject, vendor.ID()) + c.Assert(err, IsNil) + + t.s = fakeStorer{t.s, vendor.ID(), fakeEncodedObject{o}} + _, err = t.dir("vendor") + c.Assert(err, NotNil) +} + // This plumbing.EncodedObject implementation has a reader that only returns 6 // bytes at a time, this should simulate the conditions when a read // returns less bytes than asked, for example when reading a hash which diff --git a/plumbing/protocol/packp/advrefs_decode.go b/plumbing/protocol/packp/advrefs_decode.go index e0a449e..1b4c62c 100644 --- a/plumbing/protocol/packp/advrefs_decode.go +++ b/plumbing/protocol/packp/advrefs_decode.go @@ -169,7 +169,7 @@ func decodeSkipNoRefs(p *advRefsDecoder) decoderStateFn { return decodeCaps } -// decode the refname, expectes SP refname NULL +// decode the refname, expects SP refname NULL func decodeFirstRef(l *advRefsDecoder) decoderStateFn { if len(l.line) < 3 { l.error("line too short after hash") diff --git a/plumbing/protocol/packp/advrefs_encode.go b/plumbing/protocol/packp/advrefs_encode.go index cb93d46..c23e3fe 100644 --- a/plumbing/protocol/packp/advrefs_encode.go +++ b/plumbing/protocol/packp/advrefs_encode.go @@ -133,7 +133,7 @@ func encodeRefs(e *advRefsEncoder) encoderStateFn { continue } - hash, _ := e.data.References[r] + hash := e.data.References[r] if e.err = e.pe.Encodef("%s %s\n", hash.String(), r); e.err != nil { return nil } diff --git a/plumbing/protocol/packp/capability/capability.go b/plumbing/protocol/packp/capability/capability.go index 96d93f6..a129781 100644 --- a/plumbing/protocol/packp/capability/capability.go +++ b/plumbing/protocol/packp/capability/capability.go @@ -234,7 +234,7 @@ const ( const DefaultAgent = "go-git/4.x" -var valid = map[Capability]bool{ +var known = map[Capability]bool{ MultiACK: true, MultiACKDetailed: true, NoDone: true, ThinPack: true, Sideband: true, Sideband64k: true, OFSDelta: true, Agent: true, Shallow: true, DeepenSince: true, DeepenNot: true, DeepenRelative: true, diff --git a/plumbing/protocol/packp/capability/list.go b/plumbing/protocol/packp/capability/list.go index 3904a4e..26a79b6 100644 --- a/plumbing/protocol/packp/capability/list.go +++ b/plumbing/protocol/packp/capability/list.go @@ -108,7 +108,7 @@ func (l *List) Add(c Capability, values ...string) error { return nil } - if !multipleArgument[c] && len(l.m[c].Values) > 0 { + if known[c] && !multipleArgument[c] && len(l.m[c].Values) > 0 { return ErrMultipleArguments } @@ -116,7 +116,19 @@ func (l *List) Add(c Capability, values ...string) error { return nil } +func (l *List) validateNoEmptyArgs(values []string) error { + for _, v := range values { + if v == "" { + return ErrEmtpyArgument + } + } + return nil +} + func (l *List) validate(c Capability, values []string) error { + if !known[c] { + return l.validateNoEmptyArgs(values) + } if requiresArgument[c] && len(values) == 0 { return ErrArgumentsRequired } @@ -128,14 +140,7 @@ func (l *List) validate(c Capability, values []string) error { if !multipleArgument[c] && len(values) > 1 { return ErrMultipleArguments } - - for _, v := range values { - if v == "" { - return ErrEmtpyArgument - } - } - - return nil + return l.validateNoEmptyArgs(values) } // Supports returns true if capability is present diff --git a/plumbing/protocol/packp/capability/list_test.go b/plumbing/protocol/packp/capability/list_test.go index 9665e89..82dd63f 100644 --- a/plumbing/protocol/packp/capability/list_test.go +++ b/plumbing/protocol/packp/capability/list_test.go @@ -65,6 +65,26 @@ func (s *SuiteCapabilities) TestDecodeWithUnknownCapability(c *check.C) { c.Assert(cap.Supports(Capability("foo")), check.Equals, true) } +func (s *SuiteCapabilities) TestDecodeWithUnknownCapabilityWithArgument(c *check.C) { + cap := NewList() + err := cap.Decode([]byte("oldref=HEAD:refs/heads/v2 thin-pack")) + c.Assert(err, check.IsNil) + + c.Assert(cap.m, check.HasLen, 2) + c.Assert(cap.Get("oldref"), check.DeepEquals, []string{"HEAD:refs/heads/v2"}) + c.Assert(cap.Get(ThinPack), check.IsNil) +} + +func (s *SuiteCapabilities) TestDecodeWithUnknownCapabilityWithMultipleArgument(c *check.C) { + cap := NewList() + err := cap.Decode([]byte("foo=HEAD:refs/heads/v2 foo=HEAD:refs/heads/v1 thin-pack")) + c.Assert(err, check.IsNil) + + c.Assert(cap.m, check.HasLen, 2) + c.Assert(cap.Get("foo"), check.DeepEquals, []string{"HEAD:refs/heads/v2", "HEAD:refs/heads/v1"}) + c.Assert(cap.Get(ThinPack), check.IsNil) +} + func (s *SuiteCapabilities) TestString(c *check.C) { cap := NewList() cap.Set(Agent, "bar") @@ -153,7 +173,7 @@ func (s *SuiteCapabilities) TestAddErrArgumentsNotAllowed(c *check.C) { c.Assert(err, check.Equals, ErrArguments) } -func (s *SuiteCapabilities) TestAddErrArgumendts(c *check.C) { +func (s *SuiteCapabilities) TestAddErrArguments(c *check.C) { cap := NewList() err := cap.Add(SymRef, "") c.Assert(err, check.Equals, ErrEmtpyArgument) diff --git a/plumbing/protocol/packp/shallowupd.go b/plumbing/protocol/packp/shallowupd.go index 40f58e8..fce4e3b 100644 --- a/plumbing/protocol/packp/shallowupd.go +++ b/plumbing/protocol/packp/shallowupd.go @@ -32,7 +32,7 @@ func (r *ShallowUpdate) Decode(reader io.Reader) error { err = r.decodeShallowLine(line) case bytes.HasPrefix(line, unshallow): err = r.decodeUnshallowLine(line) - case bytes.Compare(line, pktline.Flush) == 0: + case bytes.Equal(line, pktline.Flush): return nil } diff --git a/plumbing/protocol/packp/srvresp.go b/plumbing/protocol/packp/srvresp.go index b214341..6a91991 100644 --- a/plumbing/protocol/packp/srvresp.go +++ b/plumbing/protocol/packp/srvresp.go @@ -35,8 +35,8 @@ func (r *ServerResponse) Decode(reader *bufio.Reader, isMultiACK bool) error { return err } - // we need to detect when the end of a response header and the begining - // of a packfile header happend, some requests to the git daemon + // we need to detect when the end of a response header and the beginning + // of a packfile header happened, some requests to the git daemon // produces a duplicate ACK header even when multi_ack is not supported. stop, err := r.stopReading(reader) if err != nil { @@ -77,7 +77,7 @@ func (r *ServerResponse) stopReading(reader *bufio.Reader) (bool, error) { func (r *ServerResponse) isValidCommand(b []byte) bool { commands := [][]byte{ack, nak} for _, c := range commands { - if bytes.Compare(b, c) == 0 { + if bytes.Equal(b, c) { return true } } @@ -90,11 +90,11 @@ func (r *ServerResponse) decodeLine(line []byte) error { return fmt.Errorf("unexpected flush") } - if bytes.Compare(line[0:3], ack) == 0 { + if bytes.Equal(line[0:3], ack) { return r.decodeACKLine(line) } - if bytes.Compare(line[0:3], nak) == 0 { + if bytes.Equal(line[0:3], nak) { return nil } diff --git a/plumbing/protocol/packp/ulreq.go b/plumbing/protocol/packp/ulreq.go index 7832007..74109d8 100644 --- a/plumbing/protocol/packp/ulreq.go +++ b/plumbing/protocol/packp/ulreq.go @@ -28,7 +28,7 @@ type Depth interface { // DepthCommits values stores the maximum number of requested commits in // the packfile. Zero means infinite. A negative value will have -// undefined consecuences. +// undefined consequences. type DepthCommits int func (d DepthCommits) isDepth() {} diff --git a/plumbing/protocol/packp/ulreq_encode.go b/plumbing/protocol/packp/ulreq_encode.go index 4a26e74..89a5986 100644 --- a/plumbing/protocol/packp/ulreq_encode.go +++ b/plumbing/protocol/packp/ulreq_encode.go @@ -70,7 +70,7 @@ func (e *ulReqEncoder) encodeFirstWant() stateFn { func (e *ulReqEncoder) encodeAditionalWants() stateFn { last := e.data.Wants[0] for _, w := range e.data.Wants[1:] { - if bytes.Compare(last[:], w[:]) == 0 { + if bytes.Equal(last[:], w[:]) { continue } @@ -90,7 +90,7 @@ func (e *ulReqEncoder) encodeShallows() stateFn { var last plumbing.Hash for _, s := range e.data.Shallows { - if bytes.Compare(last[:], s[:]) == 0 { + if bytes.Equal(last[:], s[:]) { continue } diff --git a/plumbing/protocol/packp/uppackreq.go b/plumbing/protocol/packp/uppackreq.go index 4bb22d0..1144139 100644 --- a/plumbing/protocol/packp/uppackreq.go +++ b/plumbing/protocol/packp/uppackreq.go @@ -77,7 +77,7 @@ func (u *UploadHaves) Encode(w io.Writer, flush bool) error { var last plumbing.Hash for _, have := range u.Haves { - if bytes.Compare(last[:], have[:]) == 0 { + if bytes.Equal(last[:], have[:]) { continue } diff --git a/plumbing/protocol/packp/uppackresp_test.go b/plumbing/protocol/packp/uppackresp_test.go index 789444d..0d96ce7 100644 --- a/plumbing/protocol/packp/uppackresp_test.go +++ b/plumbing/protocol/packp/uppackresp_test.go @@ -92,7 +92,7 @@ func (s *UploadPackResponseSuite) TestEncodeNAK(c *C) { c.Assert(res.Encode(b), IsNil) expected := "0008NAK\n[PACK]" - c.Assert(string(b.Bytes()), Equals, expected) + c.Assert(b.String(), Equals, expected) } func (s *UploadPackResponseSuite) TestEncodeDepth(c *C) { @@ -107,7 +107,7 @@ func (s *UploadPackResponseSuite) TestEncodeDepth(c *C) { c.Assert(res.Encode(b), IsNil) expected := "00000008NAK\nPACK" - c.Assert(string(b.Bytes()), Equals, expected) + c.Assert(b.String(), Equals, expected) } func (s *UploadPackResponseSuite) TestEncodeMultiACK(c *C) { diff --git a/plumbing/revlist/revlist.go b/plumbing/revlist/revlist.go index 5b2ff99..0a9d1e8 100644 --- a/plumbing/revlist/revlist.go +++ b/plumbing/revlist/revlist.go @@ -35,9 +35,9 @@ func objects( ignore []plumbing.Hash, allowMissingObjects bool, ) ([]plumbing.Hash, error) { - seen := hashListToSet(ignore) result := make(map[plumbing.Hash]bool) + visited := make(map[plumbing.Hash]bool) walkerFunc := func(h plumbing.Hash) { if !seen[h] { @@ -47,7 +47,7 @@ func objects( } for _, h := range objects { - if err := processObject(s, h, seen, ignore, walkerFunc); err != nil { + if err := processObject(s, h, seen, visited, ignore, walkerFunc); err != nil { if allowMissingObjects && err == plumbing.ErrObjectNotFound { continue } @@ -64,6 +64,7 @@ func processObject( s storer.EncodedObjectStorer, h plumbing.Hash, seen map[plumbing.Hash]bool, + visited map[plumbing.Hash]bool, ignore []plumbing.Hash, walkerFunc func(h plumbing.Hash), ) error { @@ -83,12 +84,12 @@ func processObject( switch do := do.(type) { case *object.Commit: - return reachableObjects(do, seen, ignore, walkerFunc) + return reachableObjects(do, seen, visited, ignore, walkerFunc) case *object.Tree: return iterateCommitTrees(seen, do, walkerFunc) case *object.Tag: walkerFunc(do.Hash) - return processObject(s, do.Target, seen, ignore, walkerFunc) + return processObject(s, do.Target, seen, visited, ignore, walkerFunc) case *object.Blob: walkerFunc(do.Hash) default: @@ -106,13 +107,36 @@ func processObject( func reachableObjects( commit *object.Commit, seen map[plumbing.Hash]bool, + visited map[plumbing.Hash]bool, ignore []plumbing.Hash, - cb func(h plumbing.Hash)) error { + cb func(h plumbing.Hash), +) error { + i := object.NewCommitPreorderIter(commit, seen, ignore) + pending := make(map[plumbing.Hash]bool) + addPendingParents(pending, visited, commit) + + for { + commit, err := i.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + if pending[commit.Hash] { + delete(pending, commit.Hash) + } + + addPendingParents(pending, visited, commit) + + if visited[commit.Hash] && len(pending) == 0 { + break + } - i := object.NewCommitPreorderIter(commit, ignore) - return i.ForEach(func(commit *object.Commit) error { if seen[commit.Hash] { - return nil + continue } cb(commit.Hash) @@ -122,15 +146,28 @@ func reachableObjects( return err } - return iterateCommitTrees(seen, tree, cb) - }) + if err := iterateCommitTrees(seen, tree, cb); err != nil { + return err + } + } + + return nil +} + +func addPendingParents(pending, visited map[plumbing.Hash]bool, commit *object.Commit) { + for _, p := range commit.ParentHashes { + if !visited[p] { + pending[p] = true + } + } } // iterateCommitTrees iterate all reachable trees from the given commit func iterateCommitTrees( seen map[plumbing.Hash]bool, tree *object.Tree, - cb func(h plumbing.Hash)) error { + cb func(h plumbing.Hash), +) error { if seen[tree.Hash] { return nil } diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go index dd1e8c1..e6419f4 100644 --- a/plumbing/revlist/revlist_test.go +++ b/plumbing/revlist/revlist_test.go @@ -8,7 +8,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) @@ -217,3 +217,60 @@ func (s *RevListSuite) TestRevListObjectsNewBranch(c *C) { } c.Assert(len(remoteHist), Equals, len(revList)) } + +// This tests will ensure that a5b8b09 and b8e471f will be visited even if +// 35e8510 has already been visited and will not stop iterating until they +// have been as well. +// +// * af2d6a6 some json +// * 1669dce Merge branch 'master' +// |\ +// | * a5b8b09 Merge pull request #1 +// | |\ +// | | * b8e471f Creating changelog +// | |/ +// * | 35e8510 binary file +// |/ +// * b029517 Initial commit +func (s *RevListSuite) TestReachableObjectsNoRevisit(c *C) { + obj, err := s.Storer.EncodedObject(plumbing.CommitObject, plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a")) + c.Assert(err, IsNil) + + do, err := object.DecodeObject(s.Storer, obj) + c.Assert(err, IsNil) + + commit, ok := do.(*object.Commit) + c.Assert(ok, Equals, true) + + var visited []plumbing.Hash + err = reachableObjects( + commit, + map[plumbing.Hash]bool{ + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"): true, + }, + map[plumbing.Hash]bool{ + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"): true, + }, + nil, + func(h plumbing.Hash) { + obj, err := s.Storer.EncodedObject(plumbing.AnyObject, h) + c.Assert(err, IsNil) + + do, err := object.DecodeObject(s.Storer, obj) + c.Assert(err, IsNil) + + if _, ok := do.(*object.Commit); ok { + visited = append(visited, h) + } + }, + ) + c.Assert(err, IsNil) + + c.Assert(visited, DeepEquals, []plumbing.Hash{ + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), + plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea"), + plumbing.NewHash("a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"), + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"), + }) +} diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index 3f41468..e793211 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -123,7 +123,7 @@ func (iter *EncodedObjectLookupIter) Next() (plumbing.EncodedObject, error) { } // ForEach call the cb function for each object contained on this iter until -// an error happends or the end of the iter is reached. If ErrStop is sent +// an error happens or the end of the iter is reached. If ErrStop is sent // the iteration is stop but no error is returned. The iterator is closed. func (iter *EncodedObjectLookupIter) ForEach(cb func(plumbing.EncodedObject) error) error { return ForEachIterator(iter, cb) @@ -168,7 +168,7 @@ func (iter *EncodedObjectSliceIter) Next() (plumbing.EncodedObject, error) { } // ForEach call the cb function for each object contained on this iter until -// an error happends or the end of the iter is reached. If ErrStop is sent +// an error happens or the end of the iter is reached. If ErrStop is sent // the iteration is stop but no error is returned. The iterator is closed. func (iter *EncodedObjectSliceIter) ForEach(cb func(plumbing.EncodedObject) error) error { return ForEachIterator(iter, cb) @@ -213,7 +213,7 @@ func (iter *MultiEncodedObjectIter) Next() (plumbing.EncodedObject, error) { } // ForEach call the cb function for each object contained on this iter until -// an error happends or the end of the iter is reached. If ErrStop is sent +// an error happens or the end of the iter is reached. If ErrStop is sent // the iteration is stop but no error is returned. The iterator is closed. func (iter *MultiEncodedObjectIter) ForEach(cb func(plumbing.EncodedObject) error) error { return ForEachIterator(iter, cb) diff --git a/plumbing/storer/reference.go b/plumbing/storer/reference.go index 988c784..ae80a39 100644 --- a/plumbing/storer/reference.go +++ b/plumbing/storer/reference.go @@ -16,6 +16,11 @@ var ErrMaxResolveRecursion = errors.New("max. recursion level reached") // ReferenceStorer is a generic storage of references. type ReferenceStorer interface { SetReference(*plumbing.Reference) error + // CheckAndSetReference sets the reference `new`, but if `old` is + // not `nil`, it first checks that the current stored value for + // `old.Name()` matches the given reference value in `old`. If + // not, it returns an error and doesn't update `new`. + CheckAndSetReference(new, old *plumbing.Reference) error Reference(plumbing.ReferenceName) (*plumbing.Reference, error) IterReferences() (ReferenceIter, error) RemoveReference(plumbing.ReferenceName) error @@ -121,7 +126,7 @@ func (iter *ReferenceSliceIter) Next() (*plumbing.Reference, error) { } // ForEach call the cb function for each reference contained on this iter until -// an error happends or the end of the iter is reached. If ErrStop is sent +// an error happens or the end of the iter is reached. If ErrStop is sent // the iteration is stop but no error is returned. The iterator is closed. func (iter *ReferenceSliceIter) ForEach(cb func(*plumbing.Reference) error) error { defer iter.Close() diff --git a/plumbing/storer/reference_test.go b/plumbing/storer/reference_test.go index 5738eef..490ec95 100644 --- a/plumbing/storer/reference_test.go +++ b/plumbing/storer/reference_test.go @@ -97,11 +97,7 @@ func (s *ReferenceSuite) TestReferenceFilteredIterNext(c *C) { } i := NewReferenceFilteredIter(func(r *plumbing.Reference) bool { - if r.Name() == "bar" { - return true - } - - return false + return r.Name() == "bar" }, NewReferenceSliceIter(slice)) foo, err := i.Next() c.Assert(err, IsNil) @@ -120,11 +116,7 @@ func (s *ReferenceSuite) TestReferenceFilteredIterForEach(c *C) { } i := NewReferenceFilteredIter(func(r *plumbing.Reference) bool { - if r.Name() == "bar" { - return true - } - - return false + return r.Name() == "bar" }, NewReferenceSliceIter(slice)) var count int i.ForEach(func(r *plumbing.Reference) error { @@ -143,11 +135,7 @@ func (s *ReferenceSuite) TestReferenceFilteredIterError(c *C) { } i := NewReferenceFilteredIter(func(r *plumbing.Reference) bool { - if r.Name() == "bar" { - return true - } - - return false + return r.Name() == "bar" }, NewReferenceSliceIter(slice)) var count int exampleErr := errors.New("SOME ERROR") @@ -172,11 +160,7 @@ func (s *ReferenceSuite) TestReferenceFilteredIterForEachStop(c *C) { } i := NewReferenceFilteredIter(func(r *plumbing.Reference) bool { - if r.Name() == "bar" { - return true - } - - return false + return r.Name() == "bar" }, NewReferenceSliceIter(slice)) var count int diff --git a/plumbing/transport/client/client.go b/plumbing/transport/client/client.go index 76c1469..90635a5 100644 --- a/plumbing/transport/client/client.go +++ b/plumbing/transport/client/client.go @@ -34,14 +34,14 @@ func InstallProtocol(scheme string, c transport.Transport) { // NewClient returns the appropriate client among of the set of known protocols: // http://, https://, ssh:// and file://. // See `InstallProtocol` to add or modify protocols. -func NewClient(endpoint transport.Endpoint) (transport.Transport, error) { - f, ok := Protocols[endpoint.Protocol()] +func NewClient(endpoint *transport.Endpoint) (transport.Transport, error) { + f, ok := Protocols[endpoint.Protocol] if !ok { - return nil, fmt.Errorf("unsupported scheme %q", endpoint.Protocol()) + return nil, fmt.Errorf("unsupported scheme %q", endpoint.Protocol) } if f == nil { - return nil, fmt.Errorf("malformed client for scheme %q, client is defined as nil", endpoint.Protocol()) + return nil, fmt.Errorf("malformed client for scheme %q, client is defined as nil", endpoint.Protocol) } return f, nil diff --git a/plumbing/transport/client/client_test.go b/plumbing/transport/client/client_test.go index 2b686b6..65cf574 100644 --- a/plumbing/transport/client/client_test.go +++ b/plumbing/transport/client/client_test.go @@ -59,12 +59,12 @@ type dummyClient struct { *http.Client } -func (*dummyClient) NewUploadPackSession(transport.Endpoint, transport.AuthMethod) ( +func (*dummyClient) NewUploadPackSession(*transport.Endpoint, transport.AuthMethod) ( transport.UploadPackSession, error) { return nil, nil } -func (*dummyClient) NewReceivePackSession(transport.Endpoint, transport.AuthMethod) ( +func (*dummyClient) NewReceivePackSession(*transport.Endpoint, transport.AuthMethod) ( transport.ReceivePackSession, error) { return nil, nil } diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index 2088500..cc9682f 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -13,6 +13,7 @@ package transport import ( + "bytes" "context" "errors" "fmt" @@ -20,6 +21,7 @@ import ( "net/url" "regexp" "strconv" + "strings" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" @@ -45,9 +47,9 @@ const ( // It is implemented both by the client and the server, making this a RPC. type Transport interface { // NewUploadPackSession starts a git-upload-pack session for an endpoint. - NewUploadPackSession(Endpoint, AuthMethod) (UploadPackSession, error) + NewUploadPackSession(*Endpoint, AuthMethod) (UploadPackSession, error) // NewReceivePackSession starts a git-receive-pack session for an endpoint. - NewReceivePackSession(Endpoint, AuthMethod) (ReceivePackSession, error) + NewReceivePackSession(*Endpoint, AuthMethod) (ReceivePackSession, error) } type Session interface { @@ -91,26 +93,71 @@ type ReceivePackSession interface { } // Endpoint represents a Git URL in any supported protocol. -type Endpoint interface { - // Protocol returns the protocol (e.g. git, https, file). It should never - // return the empty string. - Protocol() string - // User returns the user or an empty string if none is given. - User() string - // Password returns the password or an empty string if none is given. - Password() string - // Host returns the host or an empty string if none is given. - Host() string - // Port returns the port or 0 if there is no port or a default should be - // used. - Port() int - // Path returns the repository path. - Path() string - // String returns a string representation of the Git URL. - String() string +type Endpoint struct { + // Protocol is the protocol of the endpoint (e.g. git, https, file). + Protocol string + // User is the user. + User string + // Password is the password. + Password string + // Host is the host. + Host string + // Port is the port to connect, if 0 the default port for the given protocol + // wil be used. + Port int + // Path is the repository path. + Path string } -func NewEndpoint(endpoint string) (Endpoint, error) { +var defaultPorts = map[string]int{ + "http": 80, + "https": 443, + "git": 9418, + "ssh": 22, +} + +// String returns a string representation of the Git URL. +func (u *Endpoint) String() string { + var buf bytes.Buffer + if u.Protocol != "" { + buf.WriteString(u.Protocol) + buf.WriteByte(':') + } + + if u.Protocol != "" || u.Host != "" || u.User != "" || u.Password != "" { + buf.WriteString("//") + + if u.User != "" || u.Password != "" { + buf.WriteString(u.User) + if u.Password != "" { + buf.WriteByte(':') + buf.WriteString(u.Password) + } + + buf.WriteByte('@') + } + + if u.Host != "" { + buf.WriteString(u.Host) + + if u.Port != 0 { + port, ok := defaultPorts[strings.ToLower(u.Protocol)] + if !ok || ok && port != u.Port { + fmt.Fprintf(&buf, ":%d", u.Port) + } + } + } + } + + if u.Path != "" && u.Path[0] != '/' && u.Host != "" { + buf.WriteByte('/') + } + + buf.WriteString(u.Path) + return buf.String() +} + +func NewEndpoint(endpoint string) (*Endpoint, error) { if e, ok := parseSCPLike(endpoint); ok { return e, nil } @@ -119,9 +166,13 @@ func NewEndpoint(endpoint string) (Endpoint, error) { return e, nil } + return parseURL(endpoint) +} + +func parseURL(endpoint string) (*Endpoint, error) { u, err := url.Parse(endpoint) if err != nil { - return nil, plumbing.NewPermanentError(err) + return nil, err } if !u.IsAbs() { @@ -130,40 +181,29 @@ func NewEndpoint(endpoint string) (Endpoint, error) { )) } - return urlEndpoint{u}, nil -} - -type urlEndpoint struct { - *url.URL -} - -func (e urlEndpoint) Protocol() string { return e.URL.Scheme } -func (e urlEndpoint) Host() string { return e.URL.Hostname() } - -func (e urlEndpoint) User() string { - if e.URL.User == nil { - return "" + var user, pass string + if u.User != nil { + user = u.User.Username() + pass, _ = u.User.Password() } - return e.URL.User.Username() + return &Endpoint{ + Protocol: u.Scheme, + User: user, + Password: pass, + Host: u.Hostname(), + Port: getPort(u), + Path: getPath(u), + }, nil } -func (e urlEndpoint) Password() string { - if e.URL.User == nil { - return "" - } - - p, _ := e.URL.User.Password() - return p -} - -func (e urlEndpoint) Port() int { - p := e.URL.Port() +func getPort(u *url.URL) int { + p := u.Port() if p == "" { return 0 } - i, err := strconv.Atoi(e.URL.Port()) + i, err := strconv.Atoi(p) if err != nil { return 0 } @@ -171,78 +211,55 @@ func (e urlEndpoint) Port() int { return i } -func (e urlEndpoint) Path() string { - var res string = e.URL.Path - if e.URL.RawQuery != "" { - res += "?" + e.URL.RawQuery +func getPath(u *url.URL) string { + var res string = u.Path + if u.RawQuery != "" { + res += "?" + u.RawQuery } - if e.URL.Fragment != "" { - res += "#" + e.URL.Fragment + if u.Fragment != "" { + res += "#" + u.Fragment } return res } -type scpEndpoint struct { - user string - host string - path string -} - -func (e *scpEndpoint) Protocol() string { return "ssh" } -func (e *scpEndpoint) User() string { return e.user } -func (e *scpEndpoint) Password() string { return "" } -func (e *scpEndpoint) Host() string { return e.host } -func (e *scpEndpoint) Port() int { return 22 } -func (e *scpEndpoint) Path() string { return e.path } - -func (e *scpEndpoint) String() string { - var user string - if e.user != "" { - user = fmt.Sprintf("%s@", e.user) - } - - return fmt.Sprintf("%s%s:%s", user, e.host, e.path) -} - -type fileEndpoint struct { - path string -} - -func (e *fileEndpoint) Protocol() string { return "file" } -func (e *fileEndpoint) User() string { return "" } -func (e *fileEndpoint) Password() string { return "" } -func (e *fileEndpoint) Host() string { return "" } -func (e *fileEndpoint) Port() int { return 0 } -func (e *fileEndpoint) Path() string { return e.path } -func (e *fileEndpoint) String() string { return e.path } - var ( isSchemeRegExp = regexp.MustCompile(`^[^:]+://`) - scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?P<path>[^\\].*)$`) + scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?:(?P<port>[0-9]{1,5})/)?(?P<path>[^\\].*)$`) ) -func parseSCPLike(endpoint string) (Endpoint, bool) { +func parseSCPLike(endpoint string) (*Endpoint, bool) { if isSchemeRegExp.MatchString(endpoint) || !scpLikeUrlRegExp.MatchString(endpoint) { return nil, false } m := scpLikeUrlRegExp.FindStringSubmatch(endpoint) - return &scpEndpoint{ - user: m[1], - host: m[2], - path: m[3], + + port, err := strconv.Atoi(m[3]) + if err != nil { + port = 22 + } + + return &Endpoint{ + Protocol: "ssh", + User: m[1], + Host: m[2], + Port: port, + Path: m[4], }, true } -func parseFile(endpoint string) (Endpoint, bool) { +func parseFile(endpoint string) (*Endpoint, bool) { if isSchemeRegExp.MatchString(endpoint) { return nil, false } path := endpoint - return &fileEndpoint{path}, true + return &Endpoint{ + Protocol: "file", + Path: path, + }, true } // UnsupportedCapabilities are the capabilities not supported by any client diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index ec617bd..4203ce9 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -17,108 +17,139 @@ var _ = Suite(&SuiteCommon{}) func (s *SuiteCommon) TestNewEndpointHTTP(c *C) { e, err := NewEndpoint("http://git:pass@github.com/user/repository.git?foo#bar") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "http") - c.Assert(e.User(), Equals, "git") - c.Assert(e.Password(), Equals, "pass") - c.Assert(e.Host(), Equals, "github.com") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "/user/repository.git?foo#bar") + c.Assert(e.Protocol, Equals, "http") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "pass") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "/user/repository.git?foo#bar") c.Assert(e.String(), Equals, "http://git:pass@github.com/user/repository.git?foo#bar") } +func (s *SuiteCommon) TestNewEndpointPorts(c *C) { + e, err := NewEndpoint("http://git:pass@github.com:8080/user/repository.git?foo#bar") + c.Assert(err, IsNil) + c.Assert(e.String(), Equals, "http://git:pass@github.com:8080/user/repository.git?foo#bar") + + e, err = NewEndpoint("https://git:pass@github.com:443/user/repository.git?foo#bar") + c.Assert(err, IsNil) + c.Assert(e.String(), Equals, "https://git:pass@github.com/user/repository.git?foo#bar") + + e, err = NewEndpoint("ssh://git:pass@github.com:22/user/repository.git?foo#bar") + c.Assert(err, IsNil) + c.Assert(e.String(), Equals, "ssh://git:pass@github.com/user/repository.git?foo#bar") + + e, err = NewEndpoint("git://github.com:9418/user/repository.git?foo#bar") + c.Assert(err, IsNil) + c.Assert(e.String(), Equals, "git://github.com/user/repository.git?foo#bar") + +} + func (s *SuiteCommon) TestNewEndpointSSH(c *C) { e, err := NewEndpoint("ssh://git@github.com/user/repository.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "ssh") - c.Assert(e.User(), Equals, "git") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "github.com") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "/user/repository.git") + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "/user/repository.git") c.Assert(e.String(), Equals, "ssh://git@github.com/user/repository.git") } func (s *SuiteCommon) TestNewEndpointSSHNoUser(c *C) { e, err := NewEndpoint("ssh://github.com/user/repository.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "ssh") - c.Assert(e.User(), Equals, "") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "github.com") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "/user/repository.git") + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "/user/repository.git") c.Assert(e.String(), Equals, "ssh://github.com/user/repository.git") } func (s *SuiteCommon) TestNewEndpointSSHWithPort(c *C) { e, err := NewEndpoint("ssh://git@github.com:777/user/repository.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "ssh") - c.Assert(e.User(), Equals, "git") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "github.com") - c.Assert(e.Port(), Equals, 777) - c.Assert(e.Path(), Equals, "/user/repository.git") + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 777) + c.Assert(e.Path, Equals, "/user/repository.git") c.Assert(e.String(), Equals, "ssh://git@github.com:777/user/repository.git") } func (s *SuiteCommon) TestNewEndpointSCPLike(c *C) { e, err := NewEndpoint("git@github.com:user/repository.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "ssh") - c.Assert(e.User(), Equals, "git") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "github.com") - c.Assert(e.Port(), Equals, 22) - c.Assert(e.Path(), Equals, "user/repository.git") - c.Assert(e.String(), Equals, "git@github.com:user/repository.git") + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 22) + c.Assert(e.Path, Equals, "user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com/user/repository.git") +} + +func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { + e, err := NewEndpoint("git@github.com:9999/user/repository.git") + c.Assert(err, IsNil) + c.Assert(e.Protocol, Equals, "ssh") + c.Assert(e.User, Equals, "git") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Port, Equals, 9999) + c.Assert(e.Path, Equals, "user/repository.git") + c.Assert(e.String(), Equals, "ssh://git@github.com:9999/user/repository.git") } func (s *SuiteCommon) TestNewEndpointFileAbs(c *C) { e, err := NewEndpoint("/foo.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "file") - c.Assert(e.User(), Equals, "") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "/foo.git") - c.Assert(e.String(), Equals, "/foo.git") + c.Assert(e.Protocol, Equals, "file") + c.Assert(e.User, Equals, "") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "/foo.git") + c.Assert(e.String(), Equals, "file:///foo.git") } func (s *SuiteCommon) TestNewEndpointFileRel(c *C) { e, err := NewEndpoint("foo.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "file") - c.Assert(e.User(), Equals, "") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "foo.git") - c.Assert(e.String(), Equals, "foo.git") + c.Assert(e.Protocol, Equals, "file") + c.Assert(e.User, Equals, "") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "foo.git") + c.Assert(e.String(), Equals, "file://foo.git") } func (s *SuiteCommon) TestNewEndpointFileWindows(c *C) { e, err := NewEndpoint("C:\\foo.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "file") - c.Assert(e.User(), Equals, "") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "C:\\foo.git") - c.Assert(e.String(), Equals, "C:\\foo.git") + c.Assert(e.Protocol, Equals, "file") + c.Assert(e.User, Equals, "") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "C:\\foo.git") + c.Assert(e.String(), Equals, "file://C:\\foo.git") } func (s *SuiteCommon) TestNewEndpointFileURL(c *C) { e, err := NewEndpoint("file:///foo.git") c.Assert(err, IsNil) - c.Assert(e.Protocol(), Equals, "file") - c.Assert(e.User(), Equals, "") - c.Assert(e.Password(), Equals, "") - c.Assert(e.Host(), Equals, "") - c.Assert(e.Port(), Equals, 0) - c.Assert(e.Path(), Equals, "/foo.git") + c.Assert(e.Protocol, Equals, "file") + c.Assert(e.User, Equals, "") + c.Assert(e.Password, Equals, "") + c.Assert(e.Host, Equals, "") + c.Assert(e.Port, Equals, 0) + c.Assert(e.Path, Equals, "/foo.git") c.Assert(e.String(), Equals, "file:///foo.git") } diff --git a/plumbing/transport/file/client.go b/plumbing/transport/file/client.go index d229fdd..e799ee1 100644 --- a/plumbing/transport/file/client.go +++ b/plumbing/transport/file/client.go @@ -73,7 +73,7 @@ func prefixExecPath(cmd string) (string, error) { return cmd, nil } -func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthMethod, +func (r *runner) Command(cmd string, ep *transport.Endpoint, auth transport.AuthMethod, ) (common.Command, error) { switch cmd { @@ -95,7 +95,7 @@ func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthM } } - return &command{cmd: exec.Command(cmd, ep.Path())}, nil + return &command{cmd: exec.Command(cmd, ep.Path)}, nil } type command struct { diff --git a/plumbing/transport/file/client_test.go b/plumbing/transport/file/client_test.go index 864cddc..25ea278 100644 --- a/plumbing/transport/file/client_test.go +++ b/plumbing/transport/file/client_test.go @@ -41,7 +41,7 @@ repositoryformatversion = 0 filemode = true bare = true` -func prepareRepo(c *C, path string) transport.Endpoint { +func prepareRepo(c *C, path string) *transport.Endpoint { ep, err := transport.NewEndpoint(path) c.Assert(err, IsNil) diff --git a/plumbing/transport/file/common_test.go b/plumbing/transport/file/common_test.go index 4f3ae8f..99866d7 100644 --- a/plumbing/transport/file/common_test.go +++ b/plumbing/transport/file/common_test.go @@ -6,9 +6,8 @@ import ( "os/exec" "path/filepath" - "github.com/src-d/go-git-fixtures" - . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type CommonSuite struct { diff --git a/plumbing/transport/file/receive_pack_test.go b/plumbing/transport/file/receive_pack_test.go index a7dc399..3e7b140 100644 --- a/plumbing/transport/file/receive_pack_test.go +++ b/plumbing/transport/file/receive_pack_test.go @@ -3,10 +3,10 @@ package file import ( "os" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ReceivePackSuite struct { diff --git a/plumbing/transport/file/server_test.go b/plumbing/transport/file/server_test.go index 080beef..1793c0f 100644 --- a/plumbing/transport/file/server_test.go +++ b/plumbing/transport/file/server_test.go @@ -4,9 +4,8 @@ import ( "os" "os/exec" - "github.com/src-d/go-git-fixtures" - . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ServerSuite struct { diff --git a/plumbing/transport/file/upload_pack_test.go b/plumbing/transport/file/upload_pack_test.go index 9a922d1..0b9b562 100644 --- a/plumbing/transport/file/upload_pack_test.go +++ b/plumbing/transport/file/upload_pack_test.go @@ -4,11 +4,11 @@ import ( "os" "path/filepath" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type UploadPackSuite struct { diff --git a/plumbing/transport/git/common.go b/plumbing/transport/git/common.go index fcd02f8..78aaa3b 100644 --- a/plumbing/transport/git/common.go +++ b/plumbing/transport/git/common.go @@ -20,7 +20,7 @@ const DefaultPort = 9418 type runner struct{} // Command returns a new Command for the given cmd in the given Endpoint -func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthMethod) (common.Command, error) { +func (r *runner) Command(cmd string, ep *transport.Endpoint, auth transport.AuthMethod) (common.Command, error) { // auth not allowed since git protocol doesn't support authentication if auth != nil { return nil, transport.ErrInvalidAuthMethod @@ -36,7 +36,7 @@ type command struct { conn net.Conn connected bool command string - endpoint transport.Endpoint + endpoint *transport.Endpoint } // Start executes the command sending the required message to the TCP connection @@ -63,8 +63,8 @@ func (c *command) connect() error { } func (c *command) getHostWithPort() string { - host := c.endpoint.Host() - port := c.endpoint.Port() + host := c.endpoint.Host + port := c.endpoint.Port if port <= 0 { port = DefaultPort } @@ -89,13 +89,13 @@ func (c *command) StdoutPipe() (io.Reader, error) { return c.conn, nil } -func endpointToCommand(cmd string, ep transport.Endpoint) string { - host := ep.Host() - if ep.Port() != DefaultPort { - host = fmt.Sprintf("%s:%d", ep.Host(), ep.Port()) +func endpointToCommand(cmd string, ep *transport.Endpoint) string { + host := ep.Host + if ep.Port != DefaultPort { + host = fmt.Sprintf("%s:%d", ep.Host, ep.Port) } - return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path(), 0, host, 0) + return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, host, 0) } // Close closes the TCP connection and connection. diff --git a/plumbing/transport/git/common_test.go b/plumbing/transport/git/common_test.go index 3f25ad9..61097e7 100644 --- a/plumbing/transport/git/common_test.go +++ b/plumbing/transport/git/common_test.go @@ -1,9 +1,108 @@ package git import ( + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" "testing" + "time" + + "gopkg.in/src-d/go-git.v4/plumbing/transport" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } + +type BaseSuite struct { + fixtures.Suite + + base string + port int + daemon *exec.Cmd +} + +func (s *BaseSuite) SetUpTest(c *C) { + if runtime.GOOS == "windows" { + c.Skip(`git for windows has issues with write operations through git:// protocol. + See https://github.com/git-for-windows/git/issues/907`) + } + + var err error + s.port, err = freePort() + c.Assert(err, IsNil) + + s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-protocol-%d", s.port)) + c.Assert(err, IsNil) +} + +func (s *BaseSuite) StartDaemon(c *C) { + s.daemon = exec.Command( + "git", + "daemon", + fmt.Sprintf("--base-path=%s", s.base), + "--export-all", + "--enable=receive-pack", + "--reuseaddr", + fmt.Sprintf("--port=%d", s.port), + // Unless max-connections is limited to 1, a git-receive-pack + // might not be seen by a subsequent operation. + "--max-connections=1", + ) + + // Environment must be inherited in order to acknowledge GIT_EXEC_PATH if set. + s.daemon.Env = os.Environ() + + err := s.daemon.Start() + c.Assert(err, IsNil) + + // Connections might be refused if we start sending request too early. + time.Sleep(time.Millisecond * 500) +} + +func (s *BaseSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf("git://localhost:%d/%s", s.port, name)) + c.Assert(err, IsNil) + + return ep +} + +func (s *BaseSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) + c.Assert(err, IsNil) + + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) + c.Assert(err, IsNil) + + return s.newEndpoint(c, name) +} + +func (s *BaseSuite) TearDownTest(c *C) { + _ = s.daemon.Process.Signal(os.Kill) + _ = s.daemon.Wait() + + err := os.RemoveAll(s.base) + c.Assert(err, IsNil) +} + +func freePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + + return l.Addr().(*net.TCPAddr).Port, l.Close() +} diff --git a/plumbing/transport/git/receive_pack_test.go b/plumbing/transport/git/receive_pack_test.go index 7b0fa46..fa10735 100644 --- a/plumbing/transport/git/receive_pack_test.go +++ b/plumbing/transport/git/receive_pack_test.go @@ -1,148 +1,26 @@ package git import ( - "fmt" - "io" - "io/ioutil" - "net" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/src-d/go-git-fixtures" - "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ReceivePackSuite struct { test.ReceivePackSuite - fixtures.Suite - - base string - daemon *exec.Cmd + BaseSuite } var _ = Suite(&ReceivePackSuite{}) func (s *ReceivePackSuite) SetUpTest(c *C) { - if runtime.GOOS == "windows" { - c.Skip(`git for windows has issues with write operations through git:// protocol. - See https://github.com/git-for-windows/git/issues/907`) - } + s.BaseSuite.SetUpTest(c) s.ReceivePackSuite.Client = DefaultClient + s.ReceivePackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.ReceivePackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.ReceivePackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") - port, err := freePort() - c.Assert(err, IsNil) - - base, err := ioutil.TempDir(os.TempDir(), "go-git-daemon-test") - c.Assert(err, IsNil) - s.base = base - - host := fmt.Sprintf("localhost_%d", port) - interpolatedBase := filepath.Join(base, host) - err = os.MkdirAll(interpolatedBase, 0755) - c.Assert(err, IsNil) - - dotgit := fixtures.Basic().One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "basic.git")) - c.Assert(err, IsNil) - - ep, err := transport.NewEndpoint(fmt.Sprintf("git://localhost:%d/basic.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.Endpoint = ep - - dotgit = fixtures.ByTag("empty").One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "empty.git")) - c.Assert(err, IsNil) - - ep, err = transport.NewEndpoint(fmt.Sprintf("git://localhost:%d/empty.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.EmptyEndpoint = ep - - ep, err = transport.NewEndpoint(fmt.Sprintf("git://localhost:%d/non-existent.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.NonExistentEndpoint = ep - - s.daemon = exec.Command( - "git", - "daemon", - fmt.Sprintf("--base-path=%s", base), - "--export-all", - "--enable=receive-pack", - "--reuseaddr", - fmt.Sprintf("--port=%d", port), - // Use interpolated paths to validate that clients are specifying - // host and port properly. - // Note that some git versions (e.g. v2.11.0) had a bug that prevented - // the use of repository paths containing colons (:), so we use - // underscore (_) instead of colon in the interpolation. - // See https://github.com/git/git/commit/fe050334074c5132d01e1df2c1b9a82c9b8d394c - fmt.Sprintf("--interpolated-path=%s/%%H_%%P%%D", base), - // Unless max-connections is limited to 1, a git-receive-pack - // might not be seen by a subsequent operation. - "--max-connections=1", - // Whitelist required for interpolated paths. - fmt.Sprintf("%s/%s", interpolatedBase, "basic.git"), - fmt.Sprintf("%s/%s", interpolatedBase, "empty.git"), - ) - - // Environment must be inherited in order to acknowledge GIT_EXEC_PATH if set. - s.daemon.Env = os.Environ() - - err = s.daemon.Start() - c.Assert(err, IsNil) - - // Connections might be refused if we start sending request too early. - time.Sleep(time.Millisecond * 500) -} - -func (s *ReceivePackSuite) TearDownTest(c *C) { - err := s.daemon.Process.Signal(os.Kill) - c.Assert(err, IsNil) - - _ = s.daemon.Wait() - - err = os.RemoveAll(s.base) - c.Assert(err, IsNil) -} - -func freePort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - return 0, err - } - - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, err - } - - return l.Addr().(*net.TCPAddr).Port, l.Close() -} - -const bareConfig = `[core] -repositoryformatversion = 0 -filemode = true -bare = true` - -func prepareRepo(c *C, path string) { - // git-receive-pack refuses to update refs/heads/master on non-bare repo - // so we ensure bare repo config. - config := filepath.Join(path, "config") - if _, err := os.Stat(config); err == nil { - f, err := os.OpenFile(config, os.O_TRUNC|os.O_WRONLY, 0) - c.Assert(err, IsNil) - content := strings.NewReader(bareConfig) - _, err = io.Copy(f, content) - c.Assert(err, IsNil) - c.Assert(f.Close(), IsNil) - } + s.StartDaemon(c) } diff --git a/plumbing/transport/git/upload_pack_test.go b/plumbing/transport/git/upload_pack_test.go index d367a7f..7058564 100644 --- a/plumbing/transport/git/upload_pack_test.go +++ b/plumbing/transport/git/upload_pack_test.go @@ -1,35 +1,26 @@ package git import ( - "github.com/src-d/go-git-fixtures" - "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type UploadPackSuite struct { test.UploadPackSuite - fixtures.Suite + BaseSuite } var _ = Suite(&UploadPackSuite{}) func (s *UploadPackSuite) SetUpSuite(c *C) { - s.Suite.SetUpSuite(c) + s.BaseSuite.SetUpTest(c) s.UploadPackSuite.Client = DefaultClient + s.UploadPackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.UploadPackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.UploadPackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") - ep, err := transport.NewEndpoint("git://github.com/git-fixtures/basic.git") - c.Assert(err, IsNil) - s.UploadPackSuite.Endpoint = ep - - ep, err = transport.NewEndpoint("git://github.com/git-fixtures/empty.git") - c.Assert(err, IsNil) - s.UploadPackSuite.EmptyEndpoint = ep - - ep, err = transport.NewEndpoint("git://github.com/git-fixtures/non-existent.git") - c.Assert(err, IsNil) - s.UploadPackSuite.NonExistentEndpoint = ep - + s.StartDaemon(c) } diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index 6b40d42..edf1c6c 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -10,6 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/utils/ioutil" ) // it requires a bytes.Buffer, because we need to know the length @@ -39,14 +40,15 @@ func advertisedReferences(s *session, serviceName string) (*packp.AdvRefs, error } s.applyAuthToRequest(req) - applyHeadersToRequest(req, nil, s.endpoint.Host(), serviceName) + applyHeadersToRequest(req, nil, s.endpoint.Host, serviceName) res, err := s.client.Do(req) if err != nil { return nil, err } + defer ioutil.CheckClose(res.Body, &err) + if err := NewErr(res); err != nil { - _ = res.Body.Close() return nil, err } @@ -90,13 +92,13 @@ func NewClient(c *http.Client) transport.Transport { } } -func (c *client) NewUploadPackSession(ep transport.Endpoint, auth transport.AuthMethod) ( +func (c *client) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.UploadPackSession, error) { return newUploadPackSession(c.c, ep, auth) } -func (c *client) NewReceivePackSession(ep transport.Endpoint, auth transport.AuthMethod) ( +func (c *client) NewReceivePackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.ReceivePackSession, error) { return newReceivePackSession(c.c, ep, auth) @@ -105,11 +107,11 @@ func (c *client) NewReceivePackSession(ep transport.Endpoint, auth transport.Aut type session struct { auth AuthMethod client *http.Client - endpoint transport.Endpoint + endpoint *transport.Endpoint advRefs *packp.AdvRefs } -func newSession(c *http.Client, ep transport.Endpoint, auth transport.AuthMethod) (*session, error) { +func newSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { s := &session{ auth: basicAuthFromEndpoint(ep), client: c, @@ -145,23 +147,18 @@ type AuthMethod interface { setAuth(r *http.Request) } -func basicAuthFromEndpoint(ep transport.Endpoint) *BasicAuth { - u := ep.User() +func basicAuthFromEndpoint(ep *transport.Endpoint) *BasicAuth { + u := ep.User if u == "" { return nil } - return NewBasicAuth(u, ep.Password()) + return &BasicAuth{u, ep.Password} } // BasicAuth represent a HTTP basic auth type BasicAuth struct { - username, password string -} - -// NewBasicAuth returns a basicAuth base on the given user and password -func NewBasicAuth(username, password string) *BasicAuth { - return &BasicAuth{username, password} + Username, Password string } func (a *BasicAuth) setAuth(r *http.Request) { @@ -169,7 +166,7 @@ func (a *BasicAuth) setAuth(r *http.Request) { return } - r.SetBasicAuth(a.username, a.password) + r.SetBasicAuth(a.Username, a.Password) } // Name is name of the auth @@ -179,11 +176,11 @@ func (a *BasicAuth) Name() string { func (a *BasicAuth) String() string { masked := "*******" - if a.password == "" { + if a.Password == "" { masked = "<empty>" } - return fmt.Sprintf("%s - %s:%s", a.Name(), a.username, masked) + return fmt.Sprintf("%s - %s:%s", a.Name(), a.Username, masked) } // Err is a dedicated error to return errors based on status code diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index d1f36d3..8d57996 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -2,18 +2,28 @@ package http import ( "crypto/tls" + "fmt" + "io/ioutil" + "log" + "net" "net/http" + "net/http/cgi" + "os" + "os/exec" + "path/filepath" + "strings" "testing" "gopkg.in/src-d/go-git.v4/plumbing/transport" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } type ClientSuite struct { - Endpoint transport.Endpoint + Endpoint *transport.Endpoint EmptyAuth transport.AuthMethod } @@ -38,7 +48,7 @@ func (s *UploadPackSuite) TestNewClient(c *C) { } func (s *ClientSuite) TestNewBasicAuth(c *C) { - a := NewBasicAuth("foo", "qux") + a := &BasicAuth{"foo", "qux"} c.Assert(a.Name(), Equals, "http-basic-auth") c.Assert(a.String(), Equals, "http-basic-auth - foo:*******") @@ -95,3 +105,64 @@ func (s *ClientSuite) TestSetAuthWrongType(c *C) { _, err := DefaultClient.NewUploadPackSession(s.Endpoint, &mockAuth{}) c.Assert(err, Equals, transport.ErrInvalidAuthMethod) } + +type BaseSuite struct { + fixtures.Suite + + base string + host string + port int +} + +func (s *BaseSuite) SetUpTest(c *C) { + l, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) + + base, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-http-%d", s.port)) + c.Assert(err, IsNil) + + s.port = l.Addr().(*net.TCPAddr).Port + s.base = filepath.Join(base, s.host) + + err = os.MkdirAll(s.base, 0755) + c.Assert(err, IsNil) + + cmd := exec.Command("git", "--exec-path") + out, err := cmd.CombinedOutput() + c.Assert(err, IsNil) + + server := &http.Server{ + Handler: &cgi.Handler{ + Path: filepath.Join(strings.Trim(string(out), "\n"), "git-http-backend"), + Env: []string{"GIT_HTTP_EXPORT_ALL=true", fmt.Sprintf("GIT_PROJECT_ROOT=%s", s.base)}, + }, + } + go func() { + log.Fatal(server.Serve(l)) + }() +} + +func (s *BaseSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) + c.Assert(err, IsNil) + + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) + c.Assert(err, IsNil) + + return s.newEndpoint(c, name) +} + +func (s *BaseSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/%s", s.port, name)) + c.Assert(err, IsNil) + + return ep +} + +func (s *BaseSuite) TearDownTest(c *C) { + err := os.RemoveAll(s.base) + c.Assert(err, IsNil) +} diff --git a/plumbing/transport/http/receive_pack.go b/plumbing/transport/http/receive_pack.go index d2dfeb7..e5cae28 100644 --- a/plumbing/transport/http/receive_pack.go +++ b/plumbing/transport/http/receive_pack.go @@ -19,7 +19,7 @@ type rpSession struct { *session } -func newReceivePackSession(c *http.Client, ep transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { +func newReceivePackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { s, err := newSession(c, ep, auth) return &rpSession{s}, err } @@ -89,7 +89,7 @@ func (s *rpSession) doRequest( return nil, plumbing.NewPermanentError(err) } - applyHeadersToRequest(req, content, s.endpoint.Host(), transport.ReceivePackServiceName) + applyHeadersToRequest(req, content, s.endpoint.Host, transport.ReceivePackServiceName) s.applyAuthToRequest(req) res, err := s.client.Do(req.WithContext(ctx)) diff --git a/plumbing/transport/http/receive_pack_test.go b/plumbing/transport/http/receive_pack_test.go index d870e5d..737d792 100644 --- a/plumbing/transport/http/receive_pack_test.go +++ b/plumbing/transport/http/receive_pack_test.go @@ -1,122 +1,24 @@ package http import ( - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "net/http/cgi" - "os" - "os/exec" - "path/filepath" - "strings" - - "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) type ReceivePackSuite struct { test.ReceivePackSuite - fixtures.Suite - - base string + BaseSuite } var _ = Suite(&ReceivePackSuite{}) func (s *ReceivePackSuite) SetUpTest(c *C) { - s.ReceivePackSuite.Client = DefaultClient - - port, err := freePort() - c.Assert(err, IsNil) - - base, err := ioutil.TempDir(os.TempDir(), "go-git-http-backend-test") - c.Assert(err, IsNil) - s.base = base - - host := fmt.Sprintf("localhost_%d", port) - interpolatedBase := filepath.Join(base, host) - err = os.MkdirAll(interpolatedBase, 0755) - c.Assert(err, IsNil) - - dotgit := fixtures.Basic().One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "basic.git")) - c.Assert(err, IsNil) - - ep, err := transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/basic.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.Endpoint = ep - - dotgit = fixtures.ByTag("empty").One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "empty.git")) - c.Assert(err, IsNil) - - ep, err = transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/empty.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.EmptyEndpoint = ep - - ep, err = transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/non-existent.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.NonExistentEndpoint = ep - - cmd := exec.Command("git", "--exec-path") - out, err := cmd.CombinedOutput() - c.Assert(err, IsNil) - p := filepath.Join(strings.Trim(string(out), "\n"), "git-http-backend") + s.BaseSuite.SetUpTest(c) - h := &cgi.Handler{ - Path: p, - Env: []string{"GIT_HTTP_EXPORT_ALL=true", fmt.Sprintf("GIT_PROJECT_ROOT=%s", interpolatedBase)}, - } - - go func() { - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), h)) - }() -} - -func (s *ReceivePackSuite) TearDownTest(c *C) { - err := os.RemoveAll(s.base) - c.Assert(err, IsNil) -} - -func freePort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - return 0, err - } - - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, err - } - - return l.Addr().(*net.TCPAddr).Port, l.Close() -} - -const bareConfig = `[core] -repositoryformatversion = 0 -filemode = true -bare = true -[http] -receivepack = true` - -func prepareRepo(c *C, path string) { - // git-receive-pack refuses to update refs/heads/master on non-bare repo - // so we ensure bare repo config. - config := filepath.Join(path, "config") - if _, err := os.Stat(config); err == nil { - f, err := os.OpenFile(config, os.O_TRUNC|os.O_WRONLY, 0) - c.Assert(err, IsNil) - content := strings.NewReader(bareConfig) - _, err = io.Copy(f, content) - c.Assert(err, IsNil) - c.Assert(f.Close(), IsNil) - } + s.ReceivePackSuite.Client = DefaultClient + s.ReceivePackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.ReceivePackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.ReceivePackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") } diff --git a/plumbing/transport/http/upload_pack.go b/plumbing/transport/http/upload_pack.go index c5ac325..85a57a5 100644 --- a/plumbing/transport/http/upload_pack.go +++ b/plumbing/transport/http/upload_pack.go @@ -19,9 +19,8 @@ type upSession struct { *session } -func newUploadPackSession(c *http.Client, ep transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { +func newUploadPackSession(c *http.Client, ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { s, err := newSession(c, ep, auth) - return &upSession{s}, err } @@ -88,7 +87,7 @@ func (s *upSession) doRequest( return nil, plumbing.NewPermanentError(err) } - applyHeadersToRequest(req, content, s.endpoint.Host(), transport.UploadPackServiceName) + applyHeadersToRequest(req, content, s.endpoint.Host, transport.UploadPackServiceName) s.applyAuthToRequest(req) res, err := s.client.Do(req.WithContext(ctx)) diff --git a/plumbing/transport/http/upload_pack_test.go b/plumbing/transport/http/upload_pack_test.go index 57d5f46..fbd28c7 100644 --- a/plumbing/transport/http/upload_pack_test.go +++ b/plumbing/transport/http/upload_pack_test.go @@ -1,7 +1,10 @@ package http import ( + "fmt" "io/ioutil" + "os" + "path/filepath" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" @@ -9,28 +12,22 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type UploadPackSuite struct { test.UploadPackSuite + BaseSuite } var _ = Suite(&UploadPackSuite{}) func (s *UploadPackSuite) SetUpSuite(c *C) { + s.BaseSuite.SetUpTest(c) s.UploadPackSuite.Client = DefaultClient - - ep, err := transport.NewEndpoint("https://github.com/git-fixtures/basic.git") - c.Assert(err, IsNil) - s.UploadPackSuite.Endpoint = ep - - ep, err = transport.NewEndpoint("https://github.com/git-fixtures/empty.git") - c.Assert(err, IsNil) - s.UploadPackSuite.EmptyEndpoint = ep - - ep, err = transport.NewEndpoint("https://github.com/git-fixtures/non-existent.git") - c.Assert(err, IsNil) - s.UploadPackSuite.NonExistentEndpoint = ep + s.UploadPackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.UploadPackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.UploadPackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") } // Overwritten, different behaviour for HTTP. @@ -38,7 +35,7 @@ func (s *UploadPackSuite) TestAdvertisedReferencesNotExists(c *C) { r, err := s.Client.NewUploadPackSession(s.NonExistentEndpoint, s.EmptyAuth) c.Assert(err, IsNil) info, err := r.AdvertisedReferences() - c.Assert(err, Equals, transport.ErrAuthenticationRequired) + c.Assert(err, Equals, transport.ErrRepositoryNotFound) c.Assert(info, IsNil) } @@ -58,3 +55,23 @@ func (s *UploadPackSuite) TestuploadPackRequestToReader(c *C) { "0009done\n", ) } + +func (s *UploadPackSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) + c.Assert(err, IsNil) + + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) + c.Assert(err, IsNil) + + return s.newEndpoint(c, name) +} + +func (s *UploadPackSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/%s", s.port, name)) + c.Assert(err, IsNil) + + return ep +} diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index 598c6b1..8ec1ea5 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -39,7 +39,7 @@ type Commander interface { // error should be returned if the endpoint is not supported or the // command cannot be created (e.g. binary does not exist, connection // cannot be established). - Command(cmd string, ep transport.Endpoint, auth transport.AuthMethod) (Command, error) + Command(cmd string, ep *transport.Endpoint, auth transport.AuthMethod) (Command, error) } // Command is used for a single command execution. @@ -83,14 +83,14 @@ func NewClient(runner Commander) transport.Transport { } // NewUploadPackSession creates a new UploadPackSession. -func (c *client) NewUploadPackSession(ep transport.Endpoint, auth transport.AuthMethod) ( +func (c *client) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.UploadPackSession, error) { return c.newSession(transport.UploadPackServiceName, ep, auth) } // NewReceivePackSession creates a new ReceivePackSession. -func (c *client) NewReceivePackSession(ep transport.Endpoint, auth transport.AuthMethod) ( +func (c *client) NewReceivePackSession(ep *transport.Endpoint, auth transport.AuthMethod) ( transport.ReceivePackSession, error) { return c.newSession(transport.ReceivePackServiceName, ep, auth) @@ -108,7 +108,7 @@ type session struct { firstErrLine chan string } -func (c *client) newSession(s string, ep transport.Endpoint, auth transport.AuthMethod) (*session, error) { +func (c *client) newSession(s string, ep *transport.Endpoint, auth transport.AuthMethod) (*session, error) { cmd, err := c.cmdr.Command(s, ep, auth) if err != nil { return nil, err diff --git a/plumbing/transport/server/loader.go b/plumbing/transport/server/loader.go index 028ead4..c83752c 100644 --- a/plumbing/transport/server/loader.go +++ b/plumbing/transport/server/loader.go @@ -5,8 +5,8 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" - "gopkg.in/src-d/go-billy.v3" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/osfs" ) // DefaultLoader is a filesystem loader ignoring host and resolving paths to /. @@ -17,7 +17,7 @@ type Loader interface { // Load loads a storer.Storer given a transport.Endpoint. // Returns transport.ErrRepositoryNotFound if the repository does not // exist. - Load(ep transport.Endpoint) (storer.Storer, error) + Load(ep *transport.Endpoint) (storer.Storer, error) } type fsLoader struct { @@ -33,8 +33,8 @@ func NewFilesystemLoader(base billy.Filesystem) Loader { // Load looks up the endpoint's path in the base file system and returns a // storer for it. Returns transport.ErrRepositoryNotFound if a repository does // not exist in the given path. -func (l *fsLoader) Load(ep transport.Endpoint) (storer.Storer, error) { - fs, err := l.base.Chroot(ep.Path()) +func (l *fsLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { + fs, err := l.base.Chroot(ep.Path) if err != nil { return nil, err } @@ -53,7 +53,7 @@ type MapLoader map[string]storer.Storer // Load returns a storer.Storer for given a transport.Endpoint by looking it up // in the map. Returns transport.ErrRepositoryNotFound if the endpoint does not // exist. -func (l MapLoader) Load(ep transport.Endpoint) (storer.Storer, error) { +func (l MapLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { s, ok := l[ep.String()] if !ok { return nil, transport.ErrRepositoryNotFound diff --git a/plumbing/transport/server/loader_test.go b/plumbing/transport/server/loader_test.go index 38fabe3..f35511d 100644 --- a/plumbing/transport/server/loader_test.go +++ b/plumbing/transport/server/loader_test.go @@ -26,7 +26,7 @@ func (s *LoaderSuite) SetUpSuite(c *C) { c.Assert(exec.Command("git", "init", "--bare", s.RepoPath).Run(), IsNil) } -func (s *LoaderSuite) endpoint(c *C, url string) transport.Endpoint { +func (s *LoaderSuite) endpoint(c *C, url string) *transport.Endpoint { ep, err := transport.NewEndpoint(url) c.Assert(err, IsNil) return ep diff --git a/plumbing/transport/server/receive_pack_test.go b/plumbing/transport/server/receive_pack_test.go index 54c2fba..39fa979 100644 --- a/plumbing/transport/server/receive_pack_test.go +++ b/plumbing/transport/server/receive_pack_test.go @@ -2,14 +2,12 @@ package server_test import ( "gopkg.in/src-d/go-git.v4/plumbing/transport" - "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" ) type ReceivePackSuite struct { BaseSuite - test.ReceivePackSuite } var _ = Suite(&ReceivePackSuite{}) @@ -20,7 +18,7 @@ func (s *ReceivePackSuite) SetUpSuite(c *C) { } func (s *ReceivePackSuite) SetUpTest(c *C) { - s.prepareRepositories(c, &s.Endpoint, &s.EmptyEndpoint, &s.NonExistentEndpoint) + s.prepareRepositories(c) } func (s *ReceivePackSuite) TearDownTest(c *C) { diff --git a/plumbing/transport/server/server.go b/plumbing/transport/server/server.go index be36de5..2357bd6 100644 --- a/plumbing/transport/server/server.go +++ b/plumbing/transport/server/server.go @@ -43,7 +43,7 @@ func NewClient(loader Loader) transport.Transport { } } -func (s *server) NewUploadPackSession(ep transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { +func (s *server) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) { sto, err := s.loader.Load(ep) if err != nil { return nil, err @@ -52,7 +52,7 @@ func (s *server) NewUploadPackSession(ep transport.Endpoint, auth transport.Auth return s.handler.NewUploadPackSession(sto) } -func (s *server) NewReceivePackSession(ep transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { +func (s *server) NewReceivePackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.ReceivePackSession, error) { sto, err := s.loader.Load(ep) if err != nil { return nil, err @@ -165,7 +165,8 @@ func (s *upSession) UploadPack(ctx context.Context, req *packp.UploadPackRequest pr, pw := io.Pipe() e := packfile.NewEncoder(pw, s.storer, false) go func() { - _, err := e.Encode(objs) + // TODO: plumb through a pack window. + _, err := e.Encode(objs, 10) pw.CloseWithError(err) }() diff --git a/plumbing/transport/server/server_test.go b/plumbing/transport/server/server_test.go index 7912768..33d74d1 100644 --- a/plumbing/transport/server/server_test.go +++ b/plumbing/transport/server/server_test.go @@ -3,20 +3,23 @@ package server_test import ( "testing" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/client" "gopkg.in/src-d/go-git.v4/plumbing/transport/server" + "gopkg.in/src-d/go-git.v4/plumbing/transport/test" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } type BaseSuite struct { fixtures.Suite + test.ReceivePackSuite + loader server.MapLoader client transport.Transport clientBackup transport.Transport @@ -44,27 +47,19 @@ func (s *BaseSuite) TearDownSuite(c *C) { } } -func (s *BaseSuite) prepareRepositories(c *C, basic *transport.Endpoint, - empty *transport.Endpoint, nonExistent *transport.Endpoint) { +func (s *BaseSuite) prepareRepositories(c *C) { + var err error - f := fixtures.Basic().One() - fs := f.DotGit() - path := fs.Root() - ep, err := transport.NewEndpoint(path) + fs := fixtures.Basic().One().DotGit() + s.Endpoint, err = transport.NewEndpoint(fs.Root()) c.Assert(err, IsNil) - *basic = ep - sto, err := filesystem.NewStorage(fs) + s.loader[s.Endpoint.String()], err = filesystem.NewStorage(fs) c.Assert(err, IsNil) - s.loader[ep.String()] = sto - path = "/empty.git" - ep, err = transport.NewEndpoint(path) + s.EmptyEndpoint, err = transport.NewEndpoint("/empty.git") c.Assert(err, IsNil) - *empty = ep - s.loader[ep.String()] = memory.NewStorage() + s.loader[s.EmptyEndpoint.String()] = memory.NewStorage() - path = "/non-existent.git" - ep, err = transport.NewEndpoint(path) + s.NonExistentEndpoint, err = transport.NewEndpoint("/non-existent.git") c.Assert(err, IsNil) - *nonExistent = ep } diff --git a/plumbing/transport/server/upload_pack_test.go b/plumbing/transport/server/upload_pack_test.go index 99473d3..f252a75 100644 --- a/plumbing/transport/server/upload_pack_test.go +++ b/plumbing/transport/server/upload_pack_test.go @@ -2,34 +2,23 @@ package server_test import ( "gopkg.in/src-d/go-git.v4/plumbing/transport" - "gopkg.in/src-d/go-git.v4/plumbing/transport/test" . "gopkg.in/check.v1" ) type UploadPackSuite struct { BaseSuite - test.UploadPackSuite } var _ = Suite(&UploadPackSuite{}) func (s *UploadPackSuite) SetUpSuite(c *C) { s.BaseSuite.SetUpSuite(c) - s.UploadPackSuite.Client = s.client + s.Client = s.client } func (s *UploadPackSuite) SetUpTest(c *C) { - s.prepareRepositories(c, &s.Endpoint, &s.EmptyEndpoint, &s.NonExistentEndpoint) -} - -// Overwritten, it's not an error in server-side. -func (s *UploadPackSuite) TestAdvertisedReferencesEmpty(c *C) { - r, err := s.Client.NewUploadPackSession(s.EmptyEndpoint, s.EmptyAuth) - c.Assert(err, IsNil) - ar, err := r.AdvertisedReferences() - c.Assert(err, IsNil) - c.Assert(len(ar.References), Equals, 0) + s.prepareRepositories(c) } // Overwritten, server returns error earlier. @@ -57,5 +46,5 @@ func (s *ClientLikeUploadPackSuite) SetUpSuite(c *C) { } func (s *ClientLikeUploadPackSuite) TestAdvertisedReferencesEmpty(c *C) { - s.UploadPackSuite.UploadPackSuite.TestAdvertisedReferencesEmpty(c) + s.UploadPackSuite.TestAdvertisedReferencesEmpty(c) } diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index baae181..a092b29 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -25,8 +25,9 @@ const DefaultUsername = "git" // configuration needed to establish an ssh connection. type AuthMethod interface { transport.AuthMethod - clientConfig() *ssh.ClientConfig - hostKeyCallback() (ssh.HostKeyCallback, error) + // ClientConfig should return a valid ssh.ClientConfig to be used to create + // a connection to the SSH server. + ClientConfig() (*ssh.ClientConfig, error) } // The names of the AuthMethod implementations. To be returned by the @@ -45,7 +46,7 @@ const ( type KeyboardInteractive struct { User string Challenge ssh.KeyboardInteractiveChallenge - baseAuthMethod + HostKeyCallbackHelper } func (a *KeyboardInteractive) Name() string { @@ -56,18 +57,20 @@ func (a *KeyboardInteractive) String() string { return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) } -func (a *KeyboardInteractive) clientConfig() *ssh.ClientConfig { - return &ssh.ClientConfig{ +func (a *KeyboardInteractive) ClientConfig() (*ssh.ClientConfig, error) { + return a.SetHostKeyCallback(&ssh.ClientConfig{ User: a.User, - Auth: []ssh.AuthMethod{ssh.KeyboardInteractiveChallenge(a.Challenge)}, - } + Auth: []ssh.AuthMethod{ + ssh.KeyboardInteractiveChallenge(a.Challenge), + }, + }) } // Password implements AuthMethod by using the given password. type Password struct { - User string - Pass string - baseAuthMethod + User string + Password string + HostKeyCallbackHelper } func (a *Password) Name() string { @@ -78,11 +81,11 @@ func (a *Password) String() string { return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) } -func (a *Password) clientConfig() *ssh.ClientConfig { - return &ssh.ClientConfig{ +func (a *Password) ClientConfig() (*ssh.ClientConfig, error) { + return a.SetHostKeyCallback(&ssh.ClientConfig{ User: a.User, - Auth: []ssh.AuthMethod{ssh.Password(a.Pass)}, - } + Auth: []ssh.AuthMethod{ssh.Password(a.Password)}, + }) } // PasswordCallback implements AuthMethod by using a callback @@ -90,7 +93,7 @@ func (a *Password) clientConfig() *ssh.ClientConfig { type PasswordCallback struct { User string Callback func() (pass string, err error) - baseAuthMethod + HostKeyCallbackHelper } func (a *PasswordCallback) Name() string { @@ -101,25 +104,25 @@ func (a *PasswordCallback) String() string { return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) } -func (a *PasswordCallback) clientConfig() *ssh.ClientConfig { - return &ssh.ClientConfig{ +func (a *PasswordCallback) ClientConfig() (*ssh.ClientConfig, error) { + return a.SetHostKeyCallback(&ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PasswordCallback(a.Callback)}, - } + }) } // PublicKeys implements AuthMethod by using the given key pairs. type PublicKeys struct { User string Signer ssh.Signer - baseAuthMethod + HostKeyCallbackHelper } // NewPublicKeys returns a PublicKeys from a PEM encoded private key. An // encryption password should be given if the pemBytes contains a password // encrypted PEM block otherwise password should be empty. It supports RSA // (PKCS#1), DSA (OpenSSL), and ECDSA private keys. -func NewPublicKeys(user string, pemBytes []byte, password string) (AuthMethod, error) { +func NewPublicKeys(user string, pemBytes []byte, password string) (*PublicKeys, error) { block, _ := pem.Decode(pemBytes) if x509.IsEncryptedPEMBlock(block) { key, err := x509.DecryptPEMBlock(block, []byte(password)) @@ -142,7 +145,7 @@ func NewPublicKeys(user string, pemBytes []byte, password string) (AuthMethod, e // NewPublicKeysFromFile returns a PublicKeys from a file containing a PEM // encoded private key. An encryption password should be given if the pemBytes // contains a password encrypted PEM block otherwise password should be empty. -func NewPublicKeysFromFile(user, pemFile, password string) (AuthMethod, error) { +func NewPublicKeysFromFile(user, pemFile, password string) (*PublicKeys, error) { bytes, err := ioutil.ReadFile(pemFile) if err != nil { return nil, err @@ -159,11 +162,11 @@ func (a *PublicKeys) String() string { return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) } -func (a *PublicKeys) clientConfig() *ssh.ClientConfig { - return &ssh.ClientConfig{ +func (a *PublicKeys) ClientConfig() (*ssh.ClientConfig, error) { + return a.SetHostKeyCallback(&ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PublicKeys(a.Signer)}, - } + }) } func username() (string, error) { @@ -173,9 +176,11 @@ func username() (string, error) { } else { username = os.Getenv("USER") } + if username == "" { return "", errors.New("failed to get username") } + return username, nil } @@ -184,13 +189,13 @@ func username() (string, error) { type PublicKeysCallback struct { User string Callback func() (signers []ssh.Signer, err error) - baseAuthMethod + HostKeyCallbackHelper } // NewSSHAgentAuth returns a PublicKeysCallback based on a SSH agent, it opens // a pipe with the SSH agent and uses the pipe as the implementer of the public // key callback function. -func NewSSHAgentAuth(u string) (AuthMethod, error) { +func NewSSHAgentAuth(u string) (*PublicKeysCallback, error) { var err error if u == "" { u, err = username() @@ -218,11 +223,11 @@ func (a *PublicKeysCallback) String() string { return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) } -func (a *PublicKeysCallback) clientConfig() *ssh.ClientConfig { - return &ssh.ClientConfig{ +func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { + return a.SetHostKeyCallback(&ssh.ClientConfig{ User: a.User, Auth: []ssh.AuthMethod{ssh.PublicKeysCallback(a.Callback)}, - } + }) } // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a @@ -287,17 +292,26 @@ func filterKnownHostsFiles(files ...string) ([]string, error) { return out, nil } -type baseAuthMethod struct { +// HostKeyCallbackHelper is a helper that provides common functionality to +// configure HostKeyCallback into a ssh.ClientConfig. +type HostKeyCallbackHelper struct { // HostKeyCallback is the function type used for verifying server keys. - // If nil default callback will be create using NewKnownHostsHostKeyCallback + // If nil default callback will be create using NewKnownHostsCallback // without argument. HostKeyCallback ssh.HostKeyCallback } -func (m *baseAuthMethod) hostKeyCallback() (ssh.HostKeyCallback, error) { +// SetHostKeyCallback sets the field HostKeyCallback in the given cfg. If +// HostKeyCallback is empty a default callback is created using +// NewKnownHostsCallback. +func (m *HostKeyCallbackHelper) SetHostKeyCallback(cfg *ssh.ClientConfig) (*ssh.ClientConfig, error) { + var err error if m.HostKeyCallback == nil { - return NewKnownHostsCallback() + if m.HostKeyCallback, err = NewKnownHostsCallback(); err != nil { + return cfg, err + } } - return m.HostKeyCallback, nil + cfg.HostKeyCallback = m.HostKeyCallback + return cfg, nil } diff --git a/plumbing/transport/ssh/auth_method_test.go b/plumbing/transport/ssh/auth_method_test.go index 2ee5100..1e77ca0 100644 --- a/plumbing/transport/ssh/auth_method_test.go +++ b/plumbing/transport/ssh/auth_method_test.go @@ -32,16 +32,16 @@ func (s *SuiteCommon) TestKeyboardInteractiveString(c *C) { func (s *SuiteCommon) TestPasswordName(c *C) { a := &Password{ - User: "test", - Pass: "", + User: "test", + Password: "", } c.Assert(a.Name(), Equals, PasswordName) } func (s *SuiteCommon) TestPasswordString(c *C) { a := &Password{ - User: "test", - Pass: "", + User: "test", + Password: "", } c.Assert(a.String(), Equals, fmt.Sprintf("user: test, name: %s", PasswordName)) } diff --git a/plumbing/transport/ssh/common.go b/plumbing/transport/ssh/common.go index af79dfb..f5bc9a7 100644 --- a/plumbing/transport/ssh/common.go +++ b/plumbing/transport/ssh/common.go @@ -31,7 +31,7 @@ type runner struct { config *ssh.ClientConfig } -func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthMethod) (common.Command, error) { +func (r *runner) Command(cmd string, ep *transport.Endpoint, auth transport.AuthMethod) (common.Command, error) { c := &command{command: cmd, endpoint: ep, config: r.config} if auth != nil { c.setAuth(auth) @@ -47,7 +47,7 @@ type command struct { *ssh.Session connected bool command string - endpoint transport.Endpoint + endpoint *transport.Endpoint client *ssh.Client auth AuthMethod config *ssh.ClientConfig @@ -98,8 +98,7 @@ func (c *command) connect() error { } var err error - config := c.auth.clientConfig() - config.HostKeyCallback, err = c.auth.hostKeyCallback() + config, err := c.auth.ClientConfig() if err != nil { return err } @@ -122,8 +121,8 @@ func (c *command) connect() error { } func (c *command) getHostWithPort() string { - host := c.endpoint.Host() - port := c.endpoint.Port() + host := c.endpoint.Host + port := c.endpoint.Port if port <= 0 { port = DefaultPort } @@ -133,12 +132,12 @@ func (c *command) getHostWithPort() string { func (c *command) setAuthFromEndpoint() error { var err error - c.auth, err = DefaultAuthBuilder(c.endpoint.User()) + c.auth, err = DefaultAuthBuilder(c.endpoint.User) return err } -func endpointToCommand(cmd string, ep transport.Endpoint) string { - return fmt.Sprintf("%s '%s'", cmd, ep.Path()) +func endpointToCommand(cmd string, ep *transport.Endpoint) string { + return fmt.Sprintf("%s '%s'", cmd, ep.Path) } func overrideConfig(overrides *ssh.ClientConfig, c *ssh.ClientConfig) { @@ -154,14 +153,8 @@ func overrideConfig(overrides *ssh.ClientConfig, c *ssh.ClientConfig) { f := t.Field(i) vcf := vc.FieldByName(f.Name) vof := vo.FieldByName(f.Name) - if isZeroValue(vcf) { - vcf.Set(vof) - } + vcf.Set(vof) } *c = vc.Interface().(ssh.ClientConfig) } - -func isZeroValue(v reflect.Value) bool { - return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) -} diff --git a/plumbing/transport/ssh/common_test.go b/plumbing/transport/ssh/common_test.go index 1b07eee..5315e28 100644 --- a/plumbing/transport/ssh/common_test.go +++ b/plumbing/transport/ssh/common_test.go @@ -37,5 +37,5 @@ func (s *SuiteCommon) TestOverrideConfigKeep(c *C) { } overrideConfig(config, target) - c.Assert(target.User, Equals, "bar") + c.Assert(target.User, Equals, "foo") } diff --git a/plumbing/transport/ssh/upload_pack_test.go b/plumbing/transport/ssh/upload_pack_test.go index cb9baa5..56d1601 100644 --- a/plumbing/transport/ssh/upload_pack_test.go +++ b/plumbing/transport/ssh/upload_pack_test.go @@ -1,47 +1,139 @@ package ssh import ( + "fmt" + "io" + "io/ioutil" + "log" + "net" "os" + "os/exec" + "path/filepath" + "strings" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" + "github.com/gliderlabs/ssh" + "gopkg.in/src-d/go-git-fixtures.v3" + stdssh "golang.org/x/crypto/ssh" . "gopkg.in/check.v1" ) type UploadPackSuite struct { test.UploadPackSuite + fixtures.Suite + + port int + base string } var _ = Suite(&UploadPackSuite{}) func (s *UploadPackSuite) SetUpSuite(c *C) { - s.setAuthBuilder(c) - s.UploadPackSuite.Client = DefaultClient + s.Suite.SetUpSuite(c) + + l, err := net.Listen("tcp", "localhost:0") + c.Assert(err, IsNil) - ep, err := transport.NewEndpoint("git@github.com:git-fixtures/basic.git") + s.port = l.Addr().(*net.TCPAddr).Port + s.base, err = ioutil.TempDir(os.TempDir(), fmt.Sprintf("go-git-ssh-%d", s.port)) c.Assert(err, IsNil) - s.UploadPackSuite.Endpoint = ep - ep, err = transport.NewEndpoint("git@github.com:git-fixtures/empty.git") + DefaultAuthBuilder = func(user string) (AuthMethod, error) { + return &Password{User: user}, nil + } + + s.UploadPackSuite.Client = NewClient(&stdssh.ClientConfig{ + HostKeyCallback: stdssh.InsecureIgnoreHostKey(), + }) + + s.UploadPackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.UploadPackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.UploadPackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") + + server := &ssh.Server{Handler: handlerSSH} + go func() { + log.Fatal(server.Serve(l)) + }() +} + +func (s *UploadPackSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) *transport.Endpoint { + fs := f.DotGit() + + err := fixtures.EnsureIsBare(fs) c.Assert(err, IsNil) - s.UploadPackSuite.EmptyEndpoint = ep - ep, err = transport.NewEndpoint("git@github.com:git-fixtures/non-existent.git") + path := filepath.Join(s.base, name) + err = os.Rename(fs.Root(), path) c.Assert(err, IsNil) - s.UploadPackSuite.NonExistentEndpoint = ep + + return s.newEndpoint(c, name) +} + +func (s *UploadPackSuite) newEndpoint(c *C, name string) *transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf( + "ssh://git@localhost:%d/%s/%s", s.port, filepath.ToSlash(s.base), name, + )) + + c.Assert(err, IsNil) + return ep } -func (s *UploadPackSuite) setAuthBuilder(c *C) { - privateKey := os.Getenv("SSH_TEST_PRIVATE_KEY") - if privateKey != "" { - DefaultAuthBuilder = func(user string) (AuthMethod, error) { - return NewPublicKeysFromFile(user, privateKey, "") - } +func handlerSSH(s ssh.Session) { + cmd, stdin, stderr, stdout, err := buildCommand(s.Command()) + if err != nil { + fmt.Println(err) + return } - if privateKey == "" && os.Getenv("SSH_AUTH_SOCK") == "" { - c.Skip("SSH_AUTH_SOCK or SSH_TEST_PRIVATE_KEY are required") + if err := cmd.Start(); err != nil { + fmt.Println(err) return } + + go func() { + defer stdin.Close() + io.Copy(stdin, s) + }() + + go func() { + defer stderr.Close() + io.Copy(s.Stderr(), stderr) + }() + + defer stdout.Close() + io.Copy(s, stdout) + + if err := cmd.Wait(); err != nil { + return + } +} + +func buildCommand(c []string) (cmd *exec.Cmd, stdin io.WriteCloser, stderr, stdout io.ReadCloser, err error) { + if len(c) != 2 { + err = fmt.Errorf("invalid command") + return + } + + // fix for Windows environments + path := strings.Replace(c[1], "/C:/", "C:/", 1) + + cmd = exec.Command(c[0], path) + stdout, err = cmd.StdoutPipe() + if err != nil { + return + } + + stdin, err = cmd.StdinPipe() + if err != nil { + return + } + + stderr, err = cmd.StderrPipe() + if err != nil { + return + } + + return } diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index d29d9ca..0f3352c 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -9,7 +9,6 @@ import ( "io" "io/ioutil" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" @@ -18,12 +17,13 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ReceivePackSuite struct { - Endpoint transport.Endpoint - EmptyEndpoint transport.Endpoint - NonExistentEndpoint transport.Endpoint + Endpoint *transport.Endpoint + EmptyEndpoint *transport.Endpoint + NonExistentEndpoint *transport.Endpoint EmptyAuth transport.AuthMethod Client transport.Transport } @@ -213,7 +213,7 @@ func (s *ReceivePackSuite) TestSendPackOnNonEmptyWithReportStatusWithError(c *C) s.checkRemoteHead(c, endpoint, fixture.Head) } -func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep transport.Endpoint, +func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, req *packp.ReferenceUpdateRequest, fixture *fixtures.Fixture, callAdvertisedReferences bool) (*packp.ReportStatus, error) { url := "" @@ -245,7 +245,7 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep transport.Endpoint, return r.ReceivePack(context.Background(), req) } -func (s *ReceivePackSuite) receivePack(c *C, ep transport.Endpoint, +func (s *ReceivePackSuite) receivePack(c *C, ep *transport.Endpoint, req *packp.ReferenceUpdateRequest, fixture *fixtures.Fixture, callAdvertisedReferences bool) { @@ -269,11 +269,11 @@ func (s *ReceivePackSuite) receivePack(c *C, ep transport.Endpoint, } } -func (s *ReceivePackSuite) checkRemoteHead(c *C, ep transport.Endpoint, head plumbing.Hash) { +func (s *ReceivePackSuite) checkRemoteHead(c *C, ep *transport.Endpoint, head plumbing.Hash) { s.checkRemoteReference(c, ep, "refs/heads/master", head) } -func (s *ReceivePackSuite) checkRemoteReference(c *C, ep transport.Endpoint, +func (s *ReceivePackSuite) checkRemoteReference(c *C, ep *transport.Endpoint, refName string, head plumbing.Hash) { r, err := s.Client.NewUploadPackSession(ep, s.EmptyAuth) @@ -348,7 +348,7 @@ func (s *ReceivePackSuite) testSendPackDeleteReference(c *C) { func (s *ReceivePackSuite) emptyPackfile() io.ReadCloser { var buf bytes.Buffer e := packfile.NewEncoder(&buf, memory.NewStorage(), false) - _, err := e.Encode(nil) + _, err := e.Encode(nil, 10) if err != nil { panic(err) } diff --git a/plumbing/transport/test/upload_pack.go b/plumbing/transport/test/upload_pack.go index b3acc4f..70e4e56 100644 --- a/plumbing/transport/test/upload_pack.go +++ b/plumbing/transport/test/upload_pack.go @@ -21,9 +21,9 @@ import ( ) type UploadPackSuite struct { - Endpoint transport.Endpoint - EmptyEndpoint transport.Endpoint - NonExistentEndpoint transport.Endpoint + Endpoint *transport.Endpoint + EmptyEndpoint *transport.Endpoint + NonExistentEndpoint *transport.Endpoint EmptyAuth transport.AuthMethod Client transport.Transport } diff --git a/references.go b/references.go index ff2c1d3..a1872a5 100644 --- a/references.go +++ b/references.go @@ -26,7 +26,7 @@ import ( // to fix this). func references(c *object.Commit, path string) ([]*object.Commit, error) { var result []*object.Commit - seen := make(map[plumbing.Hash]struct{}, 0) + seen := make(map[plumbing.Hash]struct{}) if err := walkGraph(&result, &seen, c, path); err != nil { return nil, err } diff --git a/references_test.go b/references_test.go index 3697949..cefc7a2 100644 --- a/references_test.go +++ b/references_test.go @@ -4,12 +4,12 @@ import ( "bytes" "fmt" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ReferencesSuite struct { @@ -25,6 +25,15 @@ import ( var ( NoErrAlreadyUpToDate = errors.New("already up-to-date") ErrDeleteRefNotSupported = errors.New("server does not support delete-refs") + ErrForceNeeded = errors.New("some refs were not updated") +) + +const ( + // This describes the maximum number of commits to walk when + // computing the haves to send to a server, for each ref in the + // repo containing this remote, when not using the multi-ack + // protocol. Setting this to 0 means there is no limit. + maxHavesToVisitPerRef = 100 ) // Remote represents a connection to a remote repository. @@ -107,7 +116,12 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { return ErrDeleteRefNotSupported } - req, err := r.newReferenceUpdateRequest(o, remoteRefs, ar) + localRefs, err := r.references() + if err != nil { + return err + } + + req, err := r.newReferenceUpdateRequest(o, localRefs, remoteRefs, ar) if err != nil { return err } @@ -156,7 +170,12 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { return r.updateRemoteReferenceStorage(req, rs) } -func (r *Remote) newReferenceUpdateRequest(o *PushOptions, remoteRefs storer.ReferenceStorer, ar *packp.AdvRefs) (*packp.ReferenceUpdateRequest, error) { +func (r *Remote) newReferenceUpdateRequest( + o *PushOptions, + localRefs []*plumbing.Reference, + remoteRefs storer.ReferenceStorer, + ar *packp.AdvRefs, +) (*packp.ReferenceUpdateRequest, error) { req := packp.NewReferenceUpdateRequestFromCapabilities(ar.Capabilities) if o.Progress != nil { @@ -168,7 +187,7 @@ func (r *Remote) newReferenceUpdateRequest(o *PushOptions, remoteRefs storer.Ref } } - if err := r.addReferencesToUpdate(o.RefSpecs, remoteRefs, req); err != nil { + if err := r.addReferencesToUpdate(o.RefSpecs, localRefs, remoteRefs, req); err != nil { return nil, err } @@ -262,6 +281,11 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt return nil, err } + localRefs, err := r.references() + if err != nil { + return nil, err + } + refs, err := calculateRefs(o.RefSpecs, remoteRefs, o.Tags) if err != nil { return nil, err @@ -269,7 +293,7 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt req.Wants, err = getWants(r.s, refs) if len(req.Wants) > 0 { - req.Haves, err = getHaves(r.s) + req.Haves, err = getHaves(localRefs, remoteRefs, r.s) if err != nil { return nil, err } @@ -279,7 +303,7 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt } } - updated, err := r.updateLocalReferenceStorage(o.RefSpecs, refs, remoteRefs, o.Tags) + updated, err := r.updateLocalReferenceStorage(o.RefSpecs, refs, remoteRefs, o.Tags, o.Force) if err != nil { return nil, err } @@ -309,7 +333,7 @@ func newSendPackSession(url string, auth transport.AuthMethod) (transport.Receiv return c.NewReceivePackSession(ep, auth) } -func newClient(url string) (transport.Transport, transport.Endpoint, error) { +func newClient(url string) (transport.Transport, *transport.Endpoint, error) { ep, err := transport.NewEndpoint(url) if err != nil { return nil, nil, err @@ -346,17 +370,18 @@ func (r *Remote) fetchPack(ctx context.Context, o *FetchOptions, s transport.Upl return err } -func (r *Remote) addReferencesToUpdate(refspecs []config.RefSpec, +func (r *Remote) addReferencesToUpdate( + refspecs []config.RefSpec, + localRefs []*plumbing.Reference, remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest) error { - for _, rs := range refspecs { if rs.IsDelete() { if err := r.deleteReferences(rs, remoteRefs, req); err != nil { return err } } else { - if err := r.addOrUpdateReferences(rs, remoteRefs, req); err != nil { + if err := r.addOrUpdateReferences(rs, localRefs, remoteRefs, req); err != nil { return err } } @@ -365,18 +390,20 @@ func (r *Remote) addReferencesToUpdate(refspecs []config.RefSpec, return nil } -func (r *Remote) addOrUpdateReferences(rs config.RefSpec, - remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest) error { - iter, err := r.s.IterReferences() - if err != nil { - return err +func (r *Remote) addOrUpdateReferences( + rs config.RefSpec, + localRefs []*plumbing.Reference, + remoteRefs storer.ReferenceStorer, + req *packp.ReferenceUpdateRequest, +) error { + for _, ref := range localRefs { + err := r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) + if err != nil { + return err + } } - return iter.ForEach(func(ref *plumbing.Reference) error { - return r.addReferenceIfRefSpecMatches( - rs, remoteRefs, ref, req, - ) - }) + return nil } func (r *Remote) deleteReferences(rs config.RefSpec, @@ -449,30 +476,124 @@ func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, return nil } -func getHaves(localRefs storer.ReferenceStorer) ([]plumbing.Hash, error) { - iter, err := localRefs.IterReferences() +func (r *Remote) references() ([]*plumbing.Reference, error) { + var localRefs []*plumbing.Reference + iter, err := r.s.IterReferences() if err != nil { return nil, err } - haves := map[plumbing.Hash]bool{} - err = iter.ForEach(func(ref *plumbing.Reference) error { - if haves[ref.Hash()] == true { - return nil + for { + ref, err := iter.Next() + if err == io.EOF { + break } + if err != nil { + return nil, err + } + + localRefs = append(localRefs, ref) + } + + return localRefs, nil +} + +func getRemoteRefsFromStorer(remoteRefStorer storer.ReferenceStorer) ( + map[plumbing.Hash]bool, error) { + remoteRefs := map[plumbing.Hash]bool{} + iter, err := remoteRefStorer.IterReferences() + if err != nil { + return nil, err + } + err = iter.ForEach(func(ref *plumbing.Reference) error { if ref.Type() != plumbing.HashReference { return nil } + remoteRefs[ref.Hash()] = true + return nil + }) + if err != nil { + return nil, err + } + return remoteRefs, nil +} + +// getHavesFromRef populates the given `haves` map with the given +// reference, and up to `maxHavesToVisitPerRef` ancestor commits. +func getHavesFromRef( + ref *plumbing.Reference, + remoteRefs map[plumbing.Hash]bool, + s storage.Storer, + haves map[plumbing.Hash]bool, +) error { + h := ref.Hash() + if haves[h] { + return nil + } + // No need to load the commit if we know the remote already + // has this hash. + if remoteRefs[h] { + haves[h] = true + return nil + } + + commit, err := object.GetCommit(s, h) + if err != nil { + // Ignore the error if this isn't a commit. haves[ref.Hash()] = true return nil + } + + // Until go-git supports proper commit negotiation during an + // upload pack request, include up to `maxHavesToVisitPerRef` + // commits from the history of each ref. + walker := object.NewCommitPreorderIter(commit, haves, nil) + toVisit := maxHavesToVisitPerRef + return walker.ForEach(func(c *object.Commit) error { + haves[c.Hash] = true + toVisit-- + // If toVisit starts out at 0 (indicating there is no + // max), then it will be negative here and we won't stop + // early. + if toVisit == 0 || remoteRefs[c.Hash] { + return storer.ErrStop + } + return nil }) +} + +func getHaves( + localRefs []*plumbing.Reference, + remoteRefStorer storer.ReferenceStorer, + s storage.Storer, +) ([]plumbing.Hash, error) { + haves := map[plumbing.Hash]bool{} + // Build a map of all the remote references, to avoid loading too + // many parent commits for references we know don't need to be + // transferred. + remoteRefs, err := getRemoteRefsFromStorer(remoteRefStorer) if err != nil { return nil, err } + for _, ref := range localRefs { + if haves[ref.Hash()] { + continue + } + + if ref.Type() != plumbing.HashReference { + continue + } + + err = getHavesFromRef(ref, remoteRefs, s, haves) + if err != nil { + return nil, err + } + } + var result []plumbing.Hash for h := range haves { result = append(result, h) @@ -497,7 +618,7 @@ func calculateRefs( return nil, err } - refs := make(memory.ReferenceStorage, 0) + refs := make(memory.ReferenceStorage) return refs, iter.ForEach(func(ref *plumbing.Reference) error { if !config.MatchAny(spec, ref.Name()) { return nil @@ -584,8 +705,8 @@ func isFastForward(s storer.EncodedObjectStorer, old, new plumbing.Hash) (bool, } found := false - iter := object.NewCommitPreorderIter(c, nil) - return found, iter.ForEach(func(c *object.Commit) error { + iter := object.NewCommitPreorderIter(c, nil, nil) + err = iter.ForEach(func(c *object.Commit) error { if c.Hash != old { return nil } @@ -593,6 +714,7 @@ func isFastForward(s storer.EncodedObjectStorer, old, new plumbing.Hash) (bool, found = true return storer.ErrStop }) + return found, err } func (r *Remote) newUploadPackRequest(o *FetchOptions, @@ -617,6 +739,7 @@ func (r *Remote) newUploadPackRequest(o *FetchOptions, for _, s := range o.RefSpecs { if !s.IsWildcard() { isWildcard = false + break } } @@ -651,8 +774,11 @@ func (r *Remote) updateLocalReferenceStorage( specs []config.RefSpec, fetchedRefs, remoteRefs memory.ReferenceStorage, tagMode TagMode, + force bool, ) (updated bool, err error) { isWildcard := true + forceNeeded := false + for _, spec := range specs { if !spec.IsWildcard() { isWildcard = false @@ -667,9 +793,25 @@ func (r *Remote) updateLocalReferenceStorage( continue } - new := plumbing.NewHashReference(spec.Dst(ref.Name()), ref.Hash()) + localName := spec.Dst(ref.Name()) + old, _ := storer.ResolveReference(r.s, localName) + new := plumbing.NewHashReference(localName, ref.Hash()) - refUpdated, err := updateReferenceStorerIfNeeded(r.s, new) + // If the ref exists locally as a branch and force is not specified, + // only update if the new ref is an ancestor of the old + if old != nil && old.Name().IsBranch() && !force && !spec.IsForceUpdate() { + ff, err := isFastForward(r.s, old.Hash(), new.Hash()) + if err != nil { + return updated, err + } + + if !ff { + forceNeeded = true + continue + } + } + + refUpdated, err := checkAndUpdateReferenceStorerIfNeeded(r.s, new, old) if err != nil { return updated, err } @@ -697,6 +839,10 @@ func (r *Remote) updateLocalReferenceStorage( updated = true } + if err == nil && forceNeeded { + err = ErrForceNeeded + } + return } @@ -728,6 +874,39 @@ func (r *Remote) buildFetchedTags(refs memory.ReferenceStorage) (updated bool, e return } +// List the references on the remote repository. +func (r *Remote) List(o *ListOptions) ([]*plumbing.Reference, error) { + s, err := newUploadPackSession(r.c.URLs[0], o.Auth) + if err != nil { + return nil, err + } + + defer ioutil.CheckClose(s, &err) + + ar, err := s.AdvertisedReferences() + if err != nil { + return nil, err + } + + allRefs, err := ar.AllReferences() + if err != nil { + return nil, err + } + + refs, err := allRefs.IterReferences() + if err != nil { + return nil, err + } + + var resultRefs []*plumbing.Reference + refs.ForEach(func(ref *plumbing.Reference) error { + resultRefs = append(resultRefs, ref) + return nil + }) + + return resultRefs, nil +} + func objectsToPush(commands []*packp.Command) ([]plumbing.Hash, error) { var objects []plumbing.Hash for _, cmd := range commands { @@ -766,17 +945,21 @@ func referencesToHashes(refs storer.ReferenceStorer) ([]plumbing.Hash, error) { func pushHashes( ctx context.Context, sess transport.ReceivePackSession, - sto storer.EncodedObjectStorer, + s storage.Storer, req *packp.ReferenceUpdateRequest, hs []plumbing.Hash, ) (*packp.ReportStatus, error) { rd, wr := io.Pipe() req.Packfile = rd + config, err := s.Config() + if err != nil { + return nil, err + } done := make(chan error) go func() { - e := packfile.NewEncoder(wr, sto, false) - if _, err := e.Encode(hs); err != nil { + e := packfile.NewEncoder(wr, s, false) + if _, err := e.Encode(hs, config.Pack.Window); err != nil { done <- wr.CloseWithError(err) return } diff --git a/remote_test.go b/remote_test.go index 10f6708..e586e7a 100644 --- a/remote_test.go +++ b/remote_test.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -16,7 +15,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) type RemoteSuite struct { @@ -176,6 +176,7 @@ func (s *RemoteSuite) testFetch(c *C, r *Remote, o *FetchOptions, expected []*pl var refs int l, err := r.s.IterReferences() + c.Assert(err, IsNil) l.ForEach(func(r *plumbing.Reference) error { refs++; return nil }) c.Assert(refs, Equals, len(expected)) @@ -307,6 +308,65 @@ func (s *RemoteSuite) doTestFetchNoErrAlreadyUpToDate(c *C, url string) { c.Assert(err, Equals, NoErrAlreadyUpToDate) } +func (s *RemoteSuite) testFetchFastForward(c *C, sto storage.Storer) { + r := newRemote(sto, &config.RemoteConfig{ + URLs: []string{s.GetBasicLocalRepositoryURL()}, + }) + + s.testFetch(c, r, &FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/heads/master:refs/heads/master"), + }, + }, []*plumbing.Reference{ + plumbing.NewReferenceFromStrings("refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + }) + + // First make sure that we error correctly when a force is required. + err := r.Fetch(&FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("refs/heads/branch:refs/heads/master"), + }, + }) + c.Assert(err, Equals, ErrForceNeeded) + + // And that forcing it fixes the problem. + err = r.Fetch(&FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/heads/branch:refs/heads/master"), + }, + }) + c.Assert(err, IsNil) + + // Now test that a fast-forward, non-force fetch works. + r.s.SetReference(plumbing.NewReferenceFromStrings( + "refs/heads/master", "918c48b83bd081e863dbe1b80f8998f058cd8294", + )) + s.testFetch(c, r, &FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("refs/heads/master:refs/heads/master"), + }, + }, []*plumbing.Reference{ + plumbing.NewReferenceFromStrings("refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + }) +} + +func (s *RemoteSuite) TestFetchFastForwardMem(c *C) { + s.testFetchFastForward(c, memory.NewStorage()) +} + +func (s *RemoteSuite) TestFetchFastForwardFS(c *C) { + dir, err := ioutil.TempDir("", "fetch") + c.Assert(err, IsNil) + + defer os.RemoveAll(dir) // clean up + + fss, err := filesystem.NewStorage(osfs.New(dir)) + c.Assert(err, IsNil) + + // This exercises `storage.filesystem.Storage.CheckAndSetReference()`. + s.testFetchFastForward(c, fss) +} + func (s *RemoteSuite) TestString(c *C) { r := newRemote(nil, &config.RemoteConfig{ Name: "foo", @@ -512,6 +572,7 @@ func (s *RemoteSuite) TestPushNewReference(c *C) { server, err := PlainClone(url, true, &CloneOptions{ URL: fs.Root(), }) + c.Assert(err, IsNil) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: url, @@ -544,6 +605,7 @@ func (s *RemoteSuite) TestPushNewReferenceAndDeleteInBatch(c *C) { server, err := PlainClone(url, true, &CloneOptions{ URL: fs.Root(), }) + c.Assert(err, IsNil) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: url, @@ -625,20 +687,57 @@ func (s *RemoteSuite) TestPushWrongRemoteName(c *C) { } func (s *RemoteSuite) TestGetHaves(c *C) { - st := memory.NewStorage() - st.SetReference(plumbing.NewReferenceFromStrings( - "foo", "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", - )) - - st.SetReference(plumbing.NewReferenceFromStrings( - "bar", "fe6cb94756faa81e5ed9240f9191b833db5f40ae", - )) + f := fixtures.Basic().One() + sto, err := filesystem.NewStorage(f.DotGit()) + c.Assert(err, IsNil) - st.SetReference(plumbing.NewReferenceFromStrings( - "qux", "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", - )) + var localRefs = []*plumbing.Reference{ + plumbing.NewReferenceFromStrings( + "foo", + "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", + ), + plumbing.NewReferenceFromStrings( + "bar", + "fe6cb94756faa81e5ed9240f9191b833db5f40ae", + ), + plumbing.NewReferenceFromStrings( + "qux", + "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", + ), + } - l, err := getHaves(st) + l, err := getHaves(localRefs, memory.NewStorage(), sto) c.Assert(err, IsNil) c.Assert(l, HasLen, 2) } + +func (s *RemoteSuite) TestList(c *C) { + repo := fixtures.Basic().One() + remote := newRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{repo.URL}, + }) + + refs, err := remote.List(&ListOptions{}) + c.Assert(err, IsNil) + + expected := []*plumbing.Reference{ + plumbing.NewSymbolicReference("HEAD", "refs/heads/master"), + plumbing.NewReferenceFromStrings("refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + plumbing.NewReferenceFromStrings("refs/heads/branch", "e8d3ffab552895c19b9fcf7aa264d277cde33881"), + plumbing.NewReferenceFromStrings("refs/pull/1/head", "b8e471f58bcbca63b07bda20e428190409c2db47"), + plumbing.NewReferenceFromStrings("refs/pull/2/head", "9632f02833b2f9613afb5e75682132b0b22e4a31"), + plumbing.NewReferenceFromStrings("refs/pull/2/merge", "c37f58a130ca555e42ff96a071cb9ccb3f437504"), + } + c.Assert(len(refs), Equals, len(expected)) + for _, e := range expected { + found := false + for _, r := range refs { + if r.Name() == e.Name() { + found = true + c.Assert(r, DeepEquals, e) + } + } + c.Assert(found, Equals, true) + } +} diff --git a/repository.go b/repository.go index b86054f..b159ff0 100644 --- a/repository.go +++ b/repository.go @@ -18,13 +18,13 @@ import ( "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy.v3" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/osfs" ) var ( ErrInvalidReference = errors.New("invalid reference, should be a tag or a branch") - ErrRepositoryNotExists = errors.New("repository not exists") + ErrRepositoryNotExists = errors.New("repository does not exist") ErrRepositoryAlreadyExists = errors.New("repository already exists") ErrRemoteNotFound = errors.New("remote not found") ErrRemoteExists = errors.New("remote already exists ") @@ -323,7 +323,7 @@ func newRepository(s storage.Storer, worktree billy.Filesystem) *Repository { return &Repository{ Storer: s, wt: worktree, - r: make(map[string]*Remote, 0), + r: make(map[string]*Remote), } } @@ -625,9 +625,9 @@ func (r *Repository) calculateRemoteHeadReference(spec []config.RefSpec, return refs } -func updateReferenceStorerIfNeeded( - s storer.ReferenceStorer, r *plumbing.Reference) (updated bool, err error) { - +func checkAndUpdateReferenceStorerIfNeeded( + s storer.ReferenceStorer, r, old *plumbing.Reference) ( + updated bool, err error) { p, err := s.Reference(r.Name()) if err != nil && err != plumbing.ErrReferenceNotFound { return false, err @@ -635,7 +635,7 @@ func updateReferenceStorerIfNeeded( // we use the string method to compare references, is the easiest way if err == plumbing.ErrReferenceNotFound || r.String() != p.String() { - if err := s.SetReference(r); err != nil { + if err := s.CheckAndSetReference(r, old); err != nil { return false, err } @@ -645,6 +645,11 @@ func updateReferenceStorerIfNeeded( return false, nil } +func updateReferenceStorerIfNeeded( + s storer.ReferenceStorer, r *plumbing.Reference) (updated bool, err error) { + return checkAndUpdateReferenceStorerIfNeeded(s, r, nil) +} + // Fetch fetches references along with the objects necessary to complete // their histories, from the remote named as FetchOptions.RemoteName. // @@ -720,7 +725,7 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, err } - return object.NewCommitPreorderIter(commit, nil), nil + return object.NewCommitPreorderIter(commit, nil, nil), nil } // Tags returns all the References from Tags. This method returns all the tag @@ -949,7 +954,7 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err commit = c } case revision.CaretReg: - history := object.NewCommitPreorderIter(commit, nil) + history := object.NewCommitPreorderIter(commit, nil, nil) re := item.(revision.CaretReg).Regexp negate := item.(revision.CaretReg).Negate @@ -979,7 +984,7 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err commit = c case revision.AtDate: - history := object.NewCommitPreorderIter(commit, nil) + history := object.NewCommitPreorderIter(commit, nil, nil) date := item.(revision.AtDate).Date diff --git a/repository_test.go b/repository_test.go index 4480484..9d82651 100644 --- a/repository_test.go +++ b/repository_test.go @@ -11,7 +11,6 @@ import ( "path/filepath" "strings" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -19,9 +18,10 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/osfs" - "gopkg.in/src-d/go-billy.v3/util" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-billy.v4/util" + "gopkg.in/src-d/go-git-fixtures.v3" ) type RepositorySuite struct { @@ -82,6 +82,7 @@ func (s *RepositorySuite) TestInitStandardDotGit(c *C) { c.Assert(r, NotNil) l, err := fs.ReadDir(".git") + c.Assert(err, IsNil) c.Assert(len(l) > 0, Equals, true) cfg, err := r.Config() @@ -439,6 +440,7 @@ func (s *RepositorySuite) TestPlainCloneWithRecurseSubmodules(c *C) { c.Assert(err, IsNil) cfg, err := r.Config() + c.Assert(err, IsNil) c.Assert(cfg.Remotes, HasLen, 1) c.Assert(cfg.Submodules, HasLen, 2) } diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go index 4226b33..cc03119 100644 --- a/storage/filesystem/config_test.go +++ b/storage/filesystem/config_test.go @@ -4,12 +4,12 @@ import ( "io/ioutil" "os" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ConfigSuite struct { diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go index 2840bc7..1cb97bd 100644 --- a/storage/filesystem/internal/dotgit/dotgit.go +++ b/storage/filesystem/internal/dotgit/dotgit.go @@ -5,15 +5,15 @@ import ( "bufio" "errors" "fmt" + "io" stdioutil "io/ioutil" "os" "strings" - "time" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) const ( @@ -57,16 +57,14 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem - cachedPackedRefs refCache - packedRefsLastMod time.Time + fs billy.Filesystem } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs, cachedPackedRefs: make(refCache)} + return &DotGit{fs: fs} } // Initialize creates all the folder scaffolding. @@ -242,7 +240,35 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { return d.fs.Open(file) } -func (d *DotGit) SetRef(r *plumbing.Reference) error { +func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { + b, err := stdioutil.ReadAll(rd) + if err != nil { + return nil, err + } + + line := strings.TrimSpace(string(b)) + return plumbing.NewReferenceFromStrings(name, line), nil +} + +func (d *DotGit) checkReferenceAndTruncate(f billy.File, old *plumbing.Reference) error { + if old == nil { + return nil + } + ref, err := d.readReferenceFrom(f, old.Name().String()) + if err != nil { + return err + } + if ref.Hash() != old.Hash() { + return fmt.Errorf("reference has changed concurrently") + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return err + } + return f.Truncate(0) +} + +func (d *DotGit) SetRef(r, old *plumbing.Reference) error { var content string switch r.Type() { case plumbing.SymbolicReference: @@ -251,13 +277,34 @@ func (d *DotGit) SetRef(r *plumbing.Reference) error { content = fmt.Sprintln(r.Hash().String()) } - f, err := d.fs.Create(r.Name().String()) + // If we are not checking an old ref, just truncate the file. + mode := os.O_RDWR | os.O_CREATE + if old == nil { + mode |= os.O_TRUNC + } + + f, err := d.fs.OpenFile(r.Name().String(), mode, 0666) if err != nil { return err } defer ioutil.CheckClose(f, &err) + // Lock is unlocked by the deferred Close above. This is because Unlock + // does not imply a fsync and thus there would be a race between + // Unlock+Close and other concurrent writers. Adding Sync to go-billy + // could work, but this is better (and avoids superfluous syncs). + err = f.Lock() + if err != nil { + return err + } + + // this is a no-op to call even when old is nil. + err = d.checkReferenceAndTruncate(f, old) + if err != nil { + return err + } + _, err = f.Write([]byte(content)) return err } @@ -292,54 +339,43 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) { return d.packedRef(name) } -func (d *DotGit) syncPackedRefs() error { - fi, err := d.fs.Stat(packedRefsPath) - if os.IsNotExist(err) { - return nil - } - +func (d *DotGit) findPackedRefs() ([]*plumbing.Reference, error) { + f, err := d.fs.Open(packedRefsPath) if err != nil { - return err + if os.IsNotExist(err) { + return nil, nil + } + return nil, err } - if d.packedRefsLastMod.Before(fi.ModTime()) { - d.cachedPackedRefs = make(refCache) - f, err := d.fs.Open(packedRefsPath) + defer ioutil.CheckClose(f, &err) + + s := bufio.NewScanner(f) + var refs []*plumbing.Reference + for s.Scan() { + ref, err := d.processLine(s.Text()) if err != nil { - if os.IsNotExist(err) { - return nil - } - return err + return nil, err } - defer ioutil.CheckClose(f, &err) - - s := bufio.NewScanner(f) - for s.Scan() { - ref, err := d.processLine(s.Text()) - if err != nil { - return err - } - if ref != nil { - d.cachedPackedRefs[ref.Name()] = ref - } + if ref != nil { + refs = append(refs, ref) } - - d.packedRefsLastMod = fi.ModTime() - - return s.Err() } - return nil + return refs, s.Err() } func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) { - if err := d.syncPackedRefs(); err != nil { + refs, err := d.findPackedRefs() + if err != nil { return nil, err } - if ref, ok := d.cachedPackedRefs[name]; ok { - return ref, nil + for _, ref := range refs { + if ref.Name() == name { + return ref, nil + } } return nil, plumbing.ErrReferenceNotFound @@ -350,7 +386,8 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { path := d.fs.Join(".", name.String()) _, err := d.fs.Stat(path) if err == nil { - return d.fs.Remove(path) + err = d.fs.Remove(path) + // Drop down to remove it from the packed refs file, too. } if err != nil && !os.IsNotExist(err) { @@ -361,17 +398,17 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { } func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) { - if err := d.syncPackedRefs(); err != nil { + packedRefs, err := d.findPackedRefs() + if err != nil { return err } - for name, ref := range d.cachedPackedRefs { - if !seen[name] { + for _, ref := range packedRefs { + if !seen[ref.Name()] { *refs = append(*refs, ref) - seen[name] = true + seen[ref.Name()] = true } } - return nil } @@ -384,6 +421,17 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e return err } + doCloseF := true + defer func() { + if doCloseF { + ioutil.CheckClose(f, &err) + } + }() + + err = f.Lock() + if err != nil { + return err + } // Creating the temp file in the same directory as the target file // improves our chances for rename operation to be atomic. @@ -391,6 +439,12 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e if err != nil { return err } + doCloseTmp := true + defer func() { + if doCloseTmp { + ioutil.CheckClose(tmp, &err) + } + }() s := bufio.NewScanner(f) found := false @@ -416,14 +470,21 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e } if !found { - return nil + doCloseTmp = false + ioutil.CheckClose(tmp, &err) + if err != nil { + return err + } + // Delete the temp file if nothing needed to be removed. + return d.fs.Remove(tmp.Name()) } + doCloseF = false if err := f.Close(); err != nil { - ioutil.CheckClose(tmp, &err) return err } + doCloseTmp = false if err := tmp.Close(); err != nil { return err } @@ -512,13 +573,7 @@ func (d *DotGit) readReferenceFile(path, name string) (ref *plumbing.Reference, } defer ioutil.CheckClose(f, &err) - b, err := stdioutil.ReadAll(f) - if err != nil { - return nil, err - } - - line := strings.TrimSpace(string(b)) - return plumbing.NewReferenceFromStrings(name, line), nil + return d.readReferenceFrom(f, name) } // Module return a billy.Filesystem poiting to the module folder diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go index a7f16f4..446a204 100644 --- a/storage/filesystem/internal/dotgit/dotgit_test.go +++ b/storage/filesystem/internal/dotgit/dotgit_test.go @@ -8,11 +8,11 @@ import ( "strings" "testing" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -55,24 +55,25 @@ func (s *SuiteDotGit) TestSetRefs(c *C) { fs := osfs.New(tmp) dir := New(fs) - err = dir.SetRef(plumbing.NewReferenceFromStrings( + firstFoo := plumbing.NewReferenceFromStrings( "refs/heads/foo", "e8d3ffab552895c19b9fcf7aa264d277cde33881", - )) + ) + err = dir.SetRef(firstFoo, nil) c.Assert(err, IsNil) err = dir.SetRef(plumbing.NewReferenceFromStrings( "refs/heads/symbolic", "ref: refs/heads/foo", - )) + ), nil) c.Assert(err, IsNil) err = dir.SetRef(plumbing.NewReferenceFromStrings( "bar", "e8d3ffab552895c19b9fcf7aa264d277cde33881", - )) + ), nil) c.Assert(err, IsNil) refs, err := dir.Refs() @@ -105,6 +106,20 @@ func (s *SuiteDotGit) TestSetRefs(c *C) { c.Assert(ref, NotNil) c.Assert(ref.Hash().String(), Equals, "e8d3ffab552895c19b9fcf7aa264d277cde33881") + // Check that SetRef with a non-nil `old` works. + err = dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/heads/foo", + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + ), firstFoo) + c.Assert(err, IsNil) + + // `firstFoo` is no longer the right `old` reference, so this + // should fail. + err = dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/heads/foo", + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + ), firstFoo) + c.Assert(err, NotNil) } func (s *SuiteDotGit) TestRefsFromPackedRefs(c *C) { @@ -184,6 +199,46 @@ func (s *SuiteDotGit) TestRemoveRefFromPackedRefs(c *C) { "e8d3ffab552895c19b9fcf7aa264d277cde33881 refs/remotes/origin/branch\n") } +func (s *SuiteDotGit) TestRemoveRefFromReferenceFileAndPackedRefs(c *C) { + fs := fixtures.Basic().ByTag(".git").One().DotGit() + dir := New(fs) + + // Make a ref file for a ref that's already in `packed-refs`. + err := dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/remotes/origin/branch", + "e8d3ffab552895c19b9fcf7aa264d277cde33881", + ), nil) + + // Make sure it only appears once in the refs list. + refs, err := dir.Refs() + c.Assert(err, IsNil) + found := false + for _, ref := range refs { + if ref.Name() == "refs/remotes/origin/branch" { + c.Assert(found, Equals, false) + found = true + } + } + + name := plumbing.ReferenceName("refs/remotes/origin/branch") + err = dir.RemoveRef(name) + c.Assert(err, IsNil) + + b, err := ioutil.ReadFile(filepath.Join(fs.Root(), packedRefsPath)) + c.Assert(err, IsNil) + + c.Assert(string(b), Equals, ""+ + "# pack-refs with: peeled fully-peeled \n"+ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5 refs/heads/master\n"+ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5 refs/remotes/origin/master\n") + + refs, err = dir.Refs() + c.Assert(err, IsNil) + + ref := findReference(refs, string(name)) + c.Assert(ref, IsNil) +} + func (s *SuiteDotGit) TestRemoveRefNonExistent(c *C) { fs := fixtures.Basic().ByTag(".git").One().DotGit() dir := New(fs) @@ -418,6 +473,7 @@ func (s *SuiteDotGit) TestNewObject(c *C) { c.Assert(err, IsNil) err = w.WriteHeader(plumbing.BlobObject, 14) + c.Assert(err, IsNil) n, err := w.Write([]byte("this is a test")) c.Assert(err, IsNil) c.Assert(n, Equals, 14) diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/internal/dotgit/writers.go index 46d3619..c2b420f 100644 --- a/storage/filesystem/internal/dotgit/writers.go +++ b/storage/filesystem/internal/dotgit/writers.go @@ -10,7 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/objfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) // PackWriter is a io.Writer that generates the packfile index simultaneously, diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/internal/dotgit/writers_test.go index 1544de8..bf00762 100644 --- a/storage/filesystem/internal/dotgit/writers_test.go +++ b/storage/filesystem/internal/dotgit/writers_test.go @@ -8,12 +8,12 @@ import ( "os" "strconv" - "github.com/src-d/go-git-fixtures" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func (s *SuiteDotGit) TestNewObjectPack(c *C) { @@ -85,6 +85,7 @@ func (s *SuiteDotGit) TestNewObjectPackUnused(c *C) { // check clean up of temporary files info, err = fs.ReadDir("") + c.Assert(err, IsNil) for _, fi := range info { c.Assert(fi.IsDir(), Equals, true) } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 5073a38..9690c0e 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -14,7 +14,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) const DefaultMaxDeltaBaseCacheSize = 92 * cache.MiByte @@ -41,7 +41,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index, 0) + s.index = make(map[plumbing.Hash]*packfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -319,7 +319,7 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return nil, err } - seen := make(map[plumbing.Hash]bool, 0) + seen := make(map[plumbing.Hash]bool) var iters []storer.EncodedObjectIter if len(objects) != 0 { iters = append(iters, &objectsIter{s: s, t: t, h: objects}) diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 504bd45..de8f2b2 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,11 +1,11 @@ package filesystem import ( - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type FsSuite struct { diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go index 49627d3..54cdf56 100644 --- a/storage/filesystem/reference.go +++ b/storage/filesystem/reference.go @@ -11,7 +11,11 @@ type ReferenceStorage struct { } func (r *ReferenceStorage) SetReference(ref *plumbing.Reference) error { - return r.dir.SetRef(ref) + return r.dir.SetRef(ref, nil) +} + +func (r *ReferenceStorage) CheckAndSetReference(ref, old *plumbing.Reference) error { + return r.dir.SetRef(ref, old) } func (r *ReferenceStorage) Reference(n plumbing.ReferenceName) (*plumbing.Reference, error) { diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 2c7c107..82b137c 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -4,7 +4,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) // Storage is an implementation of git.Storer that stores data on disk in the diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index b165c5e..4d9ba6f 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -8,8 +8,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/test" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" ) func Test(t *testing.T) { TestingT(t) } diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 2380fed..927ec41 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -12,6 +12,7 @@ import ( ) var ErrUnsupportedObjectType = fmt.Errorf("unsupported object type") +var ErrRefHasChanged = fmt.Errorf("reference has changed concurrently") // Storage is an implementation of git.Storer that stores data on memory, being // ephemeral. The use of this storage should be done in controlled envoriments, @@ -29,17 +30,17 @@ type Storage struct { // NewStorage returns a new Storage base on memory func NewStorage() *Storage { return &Storage{ - ReferenceStorage: make(ReferenceStorage, 0), + ReferenceStorage: make(ReferenceStorage), ConfigStorage: ConfigStorage{}, ShallowStorage: ShallowStorage{}, ObjectStorage: ObjectStorage{ - Objects: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Commits: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Trees: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Blobs: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Tags: make(map[plumbing.Hash]plumbing.EncodedObject, 0), + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), + Commits: make(map[plumbing.Hash]plumbing.EncodedObject), + Trees: make(map[plumbing.Hash]plumbing.EncodedObject), + Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), + Tags: make(map[plumbing.Hash]plumbing.EncodedObject), }, - ModuleStorage: make(ModuleStorage, 0), + ModuleStorage: make(ModuleStorage), } } @@ -151,7 +152,7 @@ func flattenObjectMap(m map[plumbing.Hash]plumbing.EncodedObject) []plumbing.Enc func (o *ObjectStorage) Begin() storer.Transaction { return &TxObjectStorage{ Storage: o, - Objects: make(map[plumbing.Hash]plumbing.EncodedObject, 0), + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), } } @@ -188,7 +189,7 @@ func (tx *TxObjectStorage) Commit() error { } func (tx *TxObjectStorage) Rollback() error { - tx.Objects = make(map[plumbing.Hash]plumbing.EncodedObject, 0) + tx.Objects = make(map[plumbing.Hash]plumbing.EncodedObject) return nil } @@ -202,6 +203,21 @@ func (r ReferenceStorage) SetReference(ref *plumbing.Reference) error { return nil } +func (r ReferenceStorage) CheckAndSetReference(ref, old *plumbing.Reference) error { + if ref == nil { + return nil + } + + if old != nil { + tmp := r[ref.Name()] + if tmp != nil && tmp.Hash() != old.Hash() { + return ErrRefHasChanged + } + } + r[ref.Name()] = ref + return nil +} + func (r ReferenceStorage) Reference(n plumbing.ReferenceName) (*plumbing.Reference, error) { ref, ok := r[n] if !ok { diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go index bf93515..18e0086 100644 --- a/storage/test/storage_suite.go +++ b/storage/test/storage_suite.go @@ -13,7 +13,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) diff --git a/submodule.go b/submodule.go index de8ac73..a4eb7de 100644 --- a/submodule.go +++ b/submodule.go @@ -6,7 +6,7 @@ import ( "errors" "fmt" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/index" diff --git a/submodule_test.go b/submodule_test.go index e4f3013..bea5a0f 100644 --- a/submodule_test.go +++ b/submodule_test.go @@ -6,10 +6,10 @@ import ( "os" "path/filepath" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type SubmoduleSuite struct { diff --git a/utils/ioutil/common.go b/utils/ioutil/common.go index 66044e2..e9dcbfe 100644 --- a/utils/ioutil/common.go +++ b/utils/ioutil/common.go @@ -123,13 +123,13 @@ type readerOnError struct { } // NewReaderOnError returns a io.Reader that call the notify function when an -// unexpected (!io.EOF) error happends, after call Read function. +// unexpected (!io.EOF) error happens, after call Read function. func NewReaderOnError(r io.Reader, notify func(error)) io.Reader { return &readerOnError{r, notify} } // NewReadCloserOnError returns a io.ReadCloser that call the notify function -// when an unexpected (!io.EOF) error happends, after call Read function. +// when an unexpected (!io.EOF) error happens, after call Read function. func NewReadCloserOnError(r io.ReadCloser, notify func(error)) io.ReadCloser { return NewReadCloser(NewReaderOnError(r, notify), r) } @@ -149,13 +149,13 @@ type writerOnError struct { } // NewWriterOnError returns a io.Writer that call the notify function when an -// unexpected (!io.EOF) error happends, after call Write function. +// unexpected (!io.EOF) error happens, after call Write function. func NewWriterOnError(w io.Writer, notify func(error)) io.Writer { return &writerOnError{w, notify} } // NewWriteCloserOnError returns a io.WriteCloser that call the notify function -//when an unexpected (!io.EOF) error happends, after call Write function. +//when an unexpected (!io.EOF) error happens, after call Write function. func NewWriteCloserOnError(w io.WriteCloser, notify func(error)) io.WriteCloser { return NewWriteCloser(NewWriterOnError(w, notify), w) } diff --git a/utils/merkletrie/difftree.go b/utils/merkletrie/difftree.go index 0748865..2294096 100644 --- a/utils/merkletrie/difftree.go +++ b/utils/merkletrie/difftree.go @@ -54,7 +54,7 @@ package merkletrie // // Here is a full list of all the cases that are similar and how to // merge them together into more general cases. Each general case -// is labeled wiht an uppercase letter for further reference, and it +// is labeled with an uppercase letter for further reference, and it // is followed by the pseudocode of the checks you have to perfrom // on both noders to see if you are in such a case, the actions to // perform (i.e. what changes to output) and how to advance the diff --git a/utils/merkletrie/filesystem/node.go b/utils/merkletrie/filesystem/node.go index f763e08..12d0018 100644 --- a/utils/merkletrie/filesystem/node.go +++ b/utils/merkletrie/filesystem/node.go @@ -5,10 +5,11 @@ import ( "os" "path" - "gopkg.in/src-d/go-billy.v3" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" + + "gopkg.in/src-d/go-billy.v4" ) var ignore = map[string]bool{ @@ -77,6 +78,10 @@ func (n *node) NumChildren() (int, error) { } func (n *node) calculateChildren() error { + if !n.IsDir() { + return nil + } + if len(n.children) != 0 { return nil } diff --git a/utils/merkletrie/filesystem/node_test.go b/utils/merkletrie/filesystem/node_test.go index 42dd82e..12f3412 100644 --- a/utils/merkletrie/filesystem/node_test.go +++ b/utils/merkletrie/filesystem/node_test.go @@ -8,8 +8,8 @@ import ( "testing" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3" - "gopkg.in/src-d/go-billy.v3/memfs" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/merkletrie" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" @@ -82,6 +82,42 @@ func (s *NoderSuite) TestDiffChangeContent(c *C) { c.Assert(ch, HasLen, 1) } +func (s *NoderSuite) TestDiffSymlinkDirOnA(c *C) { + fsA := memfs.New() + WriteFile(fsA, "qux/qux", []byte("foo"), 0644) + + fsB := memfs.New() + fsB.Symlink("qux", "foo") + WriteFile(fsB, "qux/qux", []byte("foo"), 0644) + + ch, err := merkletrie.DiffTree( + NewRootNode(fsA, nil), + NewRootNode(fsB, nil), + IsEquals, + ) + + c.Assert(err, IsNil) + c.Assert(ch, HasLen, 1) +} + +func (s *NoderSuite) TestDiffSymlinkDirOnB(c *C) { + fsA := memfs.New() + fsA.Symlink("qux", "foo") + WriteFile(fsA, "qux/qux", []byte("foo"), 0644) + + fsB := memfs.New() + WriteFile(fsB, "qux/qux", []byte("foo"), 0644) + + ch, err := merkletrie.DiffTree( + NewRootNode(fsA, nil), + NewRootNode(fsB, nil), + IsEquals, + ) + + c.Assert(err, IsNil) + c.Assert(ch, HasLen, 1) +} + func (s *NoderSuite) TestDiffChangeMissing(c *C) { fsA := memfs.New() WriteFile(fsA, "foo", []byte("foo"), 0644) diff --git a/utils/merkletrie/iter.go b/utils/merkletrie/iter.go index e3f3055..b4d4c99 100644 --- a/utils/merkletrie/iter.go +++ b/utils/merkletrie/iter.go @@ -198,7 +198,7 @@ func (iter *Iter) current() (noder.Path, error) { } // removes the current node if any, and all the frames that become empty as a -// consecuence of this action. +// consequence of this action. func (iter *Iter) drop() { frame, ok := iter.top() if !ok { diff --git a/utils/merkletrie/noder/path.go b/utils/merkletrie/noder/path.go index d2e2932..e9c905c 100644 --- a/utils/merkletrie/noder/path.go +++ b/utils/merkletrie/noder/path.go @@ -8,7 +8,7 @@ import ( ) // Path values represent a noder and its ancestors. The root goes first -// and the actual final noder the path is refering to will be the last. +// and the actual final noder the path is referring to will be the last. // // A path implements the Noder interface, redirecting all the interface // calls to its final noder. diff --git a/worktree.go b/worktree.go index e2f8562..67d7f08 100644 --- a/worktree.go +++ b/worktree.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" - "gopkg.in/src-d/go-billy.v3/util" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" @@ -19,13 +18,14 @@ import ( "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-git.v4/utils/merkletrie" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-billy.v4/util" ) var ( ErrWorktreeNotClean = errors.New("worktree is not clean") ErrSubmoduleNotFound = errors.New("submodule not found") - ErrUnstaggedChanges = errors.New("worktree contains unstagged changes") + ErrUnstagedChanges = errors.New("worktree contains unstaged changes") ) // Worktree represents a git worktree. @@ -69,6 +69,7 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { Depth: o.Depth, Auth: o.Auth, Progress: o.Progress, + Force: o.Force, }) updated := true @@ -152,7 +153,7 @@ func (w *Worktree) Checkout(opts *CheckoutOptions) error { } if unstaged { - return ErrUnstaggedChanges + return ErrUnstagedChanges } } @@ -269,7 +270,7 @@ func (w *Worktree) Reset(opts *ResetOptions) error { } if unstaged { - return ErrUnstaggedChanges + return ErrUnstagedChanges } } diff --git a/worktree_commit.go b/worktree_commit.go index e5d0a11..3145c8a 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -10,7 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/storage" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) // Commit stores the current contents of the index in a new commit along with diff --git a/worktree_commit_test.go b/worktree_commit_test.go index f6744bc..5575bca 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -9,8 +9,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/util" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/util" ) func (s *WorktreeSuite) TestCommitInvalidOptions(c *C) { @@ -99,6 +99,42 @@ func (s *WorktreeSuite) TestCommitAll(c *C) { assertStorageStatus(c, s.Repository, 13, 11, 10, expected) } +func (s *WorktreeSuite) TestRemoveAndCommitAll(c *C) { + expected := plumbing.NewHash("907cd576c6ced2ecd3dab34a72bf9cf65944b9a9") + + fs := memfs.New() + w := &Worktree{ + r: s.Repository, + Filesystem: fs, + } + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + util.WriteFile(fs, "foo", []byte("foo"), 0644) + _, err = w.Add("foo") + c.Assert(err, IsNil) + + _, errFirst := w.Commit("Add in Repo\n", &CommitOptions{ + Author: defaultSignature(), + }) + c.Assert(errFirst, IsNil) + + errRemove := fs.Remove("foo") + c.Assert(errRemove, IsNil) + + hash, errSecond := w.Commit("Remove foo\n", &CommitOptions{ + All: true, + Author: defaultSignature(), + }) + c.Assert(errSecond, IsNil) + + c.Assert(hash, Equals, expected) + c.Assert(err, IsNil) + + assertStorageStatus(c, s.Repository, 13, 11, 11, expected) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, diff --git a/worktree_status.go b/worktree_status.go index 24d0534..36f48eb 100644 --- a/worktree_status.go +++ b/worktree_status.go @@ -39,7 +39,7 @@ func (w *Worktree) Status() (Status, error) { } func (w *Worktree) status(commit plumbing.Hash) (Status, error) { - s := make(Status, 0) + s := make(Status) left, err := w.diffCommitWithStaging(commit, false) if err != nil { @@ -243,7 +243,7 @@ func diffTreeIsEquals(a, b noder.Hasher) bool { } // Add adds the file contents of a file in the worktree to the index. if the -// file is already stagged in the index no error is returned. +// file is already staged in the index no error is returned. func (w *Worktree) Add(path string) (plumbing.Hash, error) { s, err := w.Status() if err != nil { @@ -252,6 +252,9 @@ func (w *Worktree) Add(path string) (plumbing.Hash, error) { h, err := w.copyFileToStorage(path) if err != nil { + if os.IsNotExist(err) { + h, err = w.deleteFromIndex(path) + } return h, err } diff --git a/worktree_test.go b/worktree_test.go index 1eb305d..1bdf946 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -8,8 +8,6 @@ import ( "path/filepath" "runtime" - "golang.org/x/text/unicode/norm" - "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" @@ -17,11 +15,12 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/storage/memory" - "github.com/src-d/go-git-fixtures" + "golang.org/x/text/unicode/norm" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/osfs" - "gopkg.in/src-d/go-billy.v3/util" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-billy.v4/util" + "gopkg.in/src-d/go-git-fixtures.v3" ) type WorktreeSuite struct { @@ -61,10 +60,12 @@ func (s *WorktreeSuite) TestPullFastForward(c *C) { server, err := PlainClone(url, false, &CloneOptions{ URL: path, }) + c.Assert(err, IsNil) r, err := PlainClone(c.MkDir(), false, &CloneOptions{ URL: url, }) + c.Assert(err, IsNil) w, err := server.Worktree() c.Assert(err, IsNil) @@ -91,10 +92,12 @@ func (s *WorktreeSuite) TestPullNonFastForward(c *C) { server, err := PlainClone(url, false, &CloneOptions{ URL: path, }) + c.Assert(err, IsNil) r, err := PlainClone(c.MkDir(), false, &CloneOptions{ URL: url, }) + c.Assert(err, IsNil) w, err := server.Worktree() c.Assert(err, IsNil) @@ -213,6 +216,7 @@ func (s *WorktreeSuite) TestPullProgressWithRecursion(c *C) { c.Assert(err, IsNil) cfg, err := r.Config() + c.Assert(err, IsNil) c.Assert(cfg.Submodules, HasLen, 2) } @@ -307,6 +311,7 @@ func (s *WorktreeSuite) TestCheckoutSymlink(c *C) { } dir, err := ioutil.TempDir("", "checkout") + c.Assert(err, IsNil) defer os.RemoveAll(dir) r, err := PlainInit(dir, false) @@ -345,6 +350,7 @@ func (s *WorktreeSuite) TestFilenameNormalization(c *C) { server, err := PlainClone(url, false, &CloneOptions{ URL: path, }) + c.Assert(err, IsNil) filename := "페" @@ -359,6 +365,7 @@ func (s *WorktreeSuite) TestFilenameNormalization(c *C) { r, err := Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ URL: url, }) + c.Assert(err, IsNil) w, err = r.Worktree() c.Assert(err, IsNil) @@ -372,7 +379,11 @@ func (s *WorktreeSuite) TestFilenameNormalization(c *C) { modFilename := norm.Form(norm.NFKD).String(filename) util.WriteFile(w.Filesystem, modFilename, []byte("foo"), 0755) + _, err = w.Add(filename) + c.Assert(err, IsNil) + _, err = w.Add(modFilename) + c.Assert(err, IsNil) status, err = w.Status() c.Assert(err, IsNil) @@ -441,6 +452,7 @@ func (s *WorktreeSuite) TestCheckoutIndexMem(c *C) { func (s *WorktreeSuite) TestCheckoutIndexOS(c *C) { dir, err := ioutil.TempDir("", "checkout") + c.Assert(err, IsNil) defer os.RemoveAll(dir) fs := osfs.New(filepath.Join(dir, "worktree")) @@ -809,7 +821,7 @@ func (s *WorktreeSuite) TestResetMerge(c *C) { c.Assert(err, IsNil) err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commitB}) - c.Assert(err, Equals, ErrUnstaggedChanges) + c.Assert(err, Equals, ErrUnstagedChanges) branch, err = w.r.Reference(plumbing.Master, false) c.Assert(err, IsNil) @@ -861,6 +873,7 @@ func (s *WorktreeSuite) TestStatusAfterCheckout(c *C) { func (s *WorktreeSuite) TestStatusModified(c *C) { dir, err := ioutil.TempDir("", "status") + c.Assert(err, IsNil) defer os.RemoveAll(dir) fs := osfs.New(filepath.Join(dir, "worktree")) @@ -954,6 +967,7 @@ func (s *WorktreeSuite) TestStatusUntracked(c *C) { func (s *WorktreeSuite) TestStatusDeleted(c *C) { dir, err := ioutil.TempDir("", "status") + c.Assert(err, IsNil) defer os.RemoveAll(dir) fs := osfs.New(filepath.Join(dir, "worktree")) @@ -1102,6 +1116,7 @@ func (s *WorktreeSuite) TestAddUnmodified(c *C) { func (s *WorktreeSuite) TestAddSymlink(c *C) { dir, err := ioutil.TempDir("", "checkout") + c.Assert(err, IsNil) defer os.RemoveAll(dir) r, err := PlainInit(dir, false) |