diff options
70 files changed, 2425 insertions, 507 deletions
diff --git a/.travis.yml b/.travis.yml index c81b17e..ee4a591 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.8 - - tip + - 1.8.x + - 1.9.x go_import_path: gopkg.in/src-d/go-git.v4 @@ -11,11 +11,6 @@ env: - GIT_VERSION=v1.9.3 - GIT_VERSION=v2.11.0 -matrix: - fast_finish: true - allow_failures: - - go: tip - cache: directories: - $HOME/.git-dist @@ -48,4 +43,4 @@ script: - go vet ./... after_success: - - bash <(curl -s https://codecov.io/bash)
\ No newline at end of file + - bash <(curl -s https://codecov.io/bash) @@ -1,4 +1,4 @@ -# go-git [](https://godoc.org/github.com/src-d/go-git) [](https://travis-ci.org/src-d/go-git) [](https://codecov.io/github/src-d/go-git) [](https://codebeat.co/projects/github-com-src-d-go-git) +# go-git [](https://godoc.org/github.com/src-d/go-git) [](https://travis-ci.org/src-d/go-git) [](https://ci.appveyor.com/project/mcuadros/go-git) [](https://codecov.io/github/src-d/go-git) [](https://codebeat.co/projects/github-com-src-d-go-git) A highly extensible git implementation in **pure Go**. @@ -120,7 +120,7 @@ table of git with go-git. Contribute ---------- -If you are interested on contributing to go-git, open an [issue](https://github.com/src-d/go-git/issues) explaining which missing functionality you want to work in, and we will guide you through the implementation. +If you are interested in contributing to go-git, open an [issue](https://github.com/src-d/go-git/issues) explaining which missing functionality you want to work on, and we will guide you through the implementation. License ------- diff --git a/_examples/README.md b/_examples/README.md index 2e7d514..10594ab 100644 --- a/_examples/README.md +++ b/_examples/README.md @@ -6,12 +6,14 @@ Here you can find a list of annotated _go-git_ examples: - [showcase](showcase/main.go) - A small showcase of the capabilities of _go-git_ - [open](open/main.go) - Opening a existing repository cloned by _git_ - [clone](clone/main.go) - Cloning a repository +- [clone with context](context/main.go) - Cloning a repository with graceful cancellation. - [log](log/main.go) - Emulate `git log` command output iterating all the commit history from HEAD reference - [remotes](remotes/main.go) - Working with remotes: adding, removing, etc - [progress](progress/main.go) - Printing the progress information from the sideband - [push](push/main.go) - Push repository to default remote (origin) - [checkout](checkout/main.go) - check out a specific commit from a repository - [tag](tag/main.go) - list/print repository tags +- [pull](pull/main.go) - pull changes from a remote repository ### Advanced - [custom_http](custom_http/main.go) - Replacing the HTTP client using a custom one - [storage](storage/README.md) - Implementing a custom storage system diff --git a/_examples/common_test.go b/_examples/common_test.go index 86205fe..9eeb643 100644 --- a/_examples/common_test.go +++ b/_examples/common_test.go @@ -17,6 +17,7 @@ var defaultURL = "https://github.com/git-fixtures/basic.git" var args = map[string][]string{ "checkout": []string{defaultURL, tempFolder(), "35e85108805c84807bc66a02d91535e1e24b38b9"}, "clone": []string{defaultURL, tempFolder()}, + "context": []string{defaultURL, tempFolder()}, "commit": []string{cloneRepository(defaultURL, tempFolder())}, "custom_http": []string{defaultURL}, "open": []string{cloneRepository(defaultURL, tempFolder())}, @@ -24,6 +25,7 @@ var args = map[string][]string{ "push": []string{setEmptyRemote(cloneRepository(defaultURL, tempFolder()))}, "showcase": []string{defaultURL, tempFolder()}, "tag": []string{cloneRepository(defaultURL, tempFolder())}, + "pull": []string{createRepositoryWithRemote(tempFolder(), defaultURL)}, } var ignored = map[string]bool{} @@ -88,13 +90,28 @@ func cloneRepository(url, folder string) string { } func createBareRepository(dir string) string { - cmd := exec.Command("git", "init", "--bare", dir) + return createRepository(dir, true) +} + +func createRepository(dir string, isBare bool) string { + var cmd *exec.Cmd + if isBare { + cmd = exec.Command("git", "init", "--bare", dir) + } else { + cmd = exec.Command("git", "init", dir) + } err := cmd.Run() CheckIfError(err) return dir } +func createRepositoryWithRemote(local, remote string) string { + createRepository(local, false) + addRemote(local, remote) + return local +} + func setEmptyRemote(dir string) string { remote := createBareRepository(tempFolder()) setRemote(dir, remote) @@ -108,6 +125,13 @@ func setRemote(local, remote string) { CheckIfError(err) } +func addRemote(local, remote string) { + cmd := exec.Command("git", "remote", "add", "origin", remote) + cmd.Dir = local + err := cmd.Run() + CheckIfError(err) +} + func testExample(t *testing.T, name, example string) { cmd := exec.Command("go", append([]string{ "run", filepath.Join(example), diff --git a/_examples/context/main.go b/_examples/context/main.go new file mode 100644 index 0000000..72885ff --- /dev/null +++ b/_examples/context/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "os" + "os/signal" + + "gopkg.in/src-d/go-git.v4" + . "gopkg.in/src-d/go-git.v4/_examples" +) + +// Gracefull cancellation example of a basic git operation such as Clone. +func main() { + CheckArgs("<url>", "<directory>") + url := os.Args[1] + directory := os.Args[2] + + // Clone the given repository to the given directory + Info("git clone %s %s", url, directory) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt) + + // The context is the mechanism used by go-git, to support deadlines and + // cancellation signals. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // cancel when we are finished consuming integers + + go func() { + <-stop + Warning("\nSignal detected, canceling operation...") + cancel() + }() + + Warning("To gracefully stop the clone operation, push Crtl-C.") + + // Using PlainCloneContext we can provide to a context, if the context + // is cancelled, the clone operation stops gracefully. + _, err := git.PlainCloneContext(ctx, directory, false, &git.CloneOptions{ + URL: url, + Progress: os.Stdout, + }) + + // If the context was cancelled, an error is returned. + CheckIfError(err) +} diff --git a/_examples/pull/main.go b/_examples/pull/main.go new file mode 100644 index 0000000..ae751d2 --- /dev/null +++ b/_examples/pull/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "os" + + "gopkg.in/src-d/go-git.v4" + . "gopkg.in/src-d/go-git.v4/_examples" +) + +// Pull changes from a remote repository +func main() { + CheckArgs("<path>") + path := os.Args[1] + + // We instance a new repository targeting the given path (the .git folder) + r, err := git.PlainOpen(path) + CheckIfError(err) + + // Get the working directory for the repository + w, err := r.Worktree() + CheckIfError(err) + + // Pull the latest changes from the origin remote and merge into the current branch + Info("git pull origin") + err = w.Pull(&git.PullOptions{RemoteName: "origin"}) + CheckIfError(err) + + // Print the latest commit that was just pulled + ref, err := r.Head() + CheckIfError(err) + commit, err := r.CommitObject(ref.Hash()) + CheckIfError(err) + + fmt.Println(commit) +} diff --git a/_examples/storage/README.md b/_examples/storage/README.md index b7207ee..fc72e6f 100644 --- a/_examples/storage/README.md +++ b/_examples/storage/README.md @@ -8,7 +8,7 @@ ### and what this means ... *git* has as very well defined storage system, the `.git` directory, present on any repository. This is the place where `git` stores al the [`objects`](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), [`references`](https://git-scm.com/book/es/v2/Git-Internals-Git-References) and [`configuration`](https://git-scm.com/docs/git-config#_configuration_file). This information is stored in plain files. -Our original **go-git** version was designed to work in memory, some time after we added support to read the `.git`, and now we have added support for fully customized [storages](https://godoc.org/github.com/src-d/go-git#Storer). +Our original **go-git** version was designed to work in memory, some time after we added support to read the `.git`, and now we have added support for fully customized [storages](https://godoc.org/gopkg.in/src-d/go-git.v4/storage#Storer). This means that the internal database of any repository can be saved and accessed on any support, databases, distributed filesystems, etc. This functionality is pretty similar to the [libgit2 backends](http://blog.deveo.com/your-git-repository-in-a-database-pluggable-backends-in-libgit2/) diff --git a/config/config.go b/config/config.go index cb10738..477eb35 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,8 @@ import ( "bytes" "errors" "fmt" + "sort" + "strconv" format "gopkg.in/src-d/go-git.v4/plumbing/format/config" ) @@ -39,6 +41,14 @@ type Config struct { // Worktree is the path to the root of the working tree. Worktree string } + + Pack struct { + // Window controls the size of the sliding window for delta + // compression. The default is 10. A value of 0 turns off + // delta compression entirely. + Window uint + } + // Remotes list of repository remotes, the key of the map is the name // of the remote, should equal to RemoteConfig.Name. Remotes map[string]*RemoteConfig @@ -80,10 +90,14 @@ const ( remoteSection = "remote" submoduleSection = "submodule" coreSection = "core" + packSection = "pack" fetchKey = "fetch" urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + windowKey = "window" + + defaultPackWindow = uint(10) ) // Unmarshal parses a git-config file and stores it. @@ -97,6 +111,9 @@ func (c *Config) Unmarshal(b []byte) error { } c.unmarshalCore() + if err := c.unmarshalPack(); err != nil { + return err + } c.unmarshalSubmodules() return c.unmarshalRemotes() } @@ -110,6 +127,21 @@ func (c *Config) unmarshalCore() { c.Core.Worktree = s.Options.Get(worktreeKey) } +func (c *Config) unmarshalPack() error { + s := c.Raw.Section(packSection) + window := s.Options.Get(windowKey) + if window == "" { + c.Pack.Window = defaultPackWindow + } else { + winUint, err := strconv.ParseUint(window, 10, 32) + if err != nil { + return err + } + c.Pack.Window = uint(winUint) + } + return nil +} + func (c *Config) unmarshalRemotes() error { s := c.Raw.Section(remoteSection) for _, sub := range s.Subsections { @@ -137,6 +169,7 @@ func (c *Config) unmarshalSubmodules() { // Marshal returns Config encoded as a git-config file. func (c *Config) Marshal() ([]byte, error) { c.marshalCore() + c.marshalPack() c.marshalRemotes() c.marshalSubmodules() @@ -157,6 +190,13 @@ func (c *Config) marshalCore() { } } +func (c *Config) marshalPack() { + s := c.Raw.Section(packSection) + if c.Pack.Window != defaultPackWindow { + s.SetOption(windowKey, fmt.Sprintf("%d", c.Pack.Window)) + } +} + func (c *Config) marshalRemotes() { s := c.Raw.Section(remoteSection) newSubsections := make(format.Subsections, 0, len(c.Remotes)) @@ -168,9 +208,16 @@ func (c *Config) marshalRemotes() { } } - for name, remote := range c.Remotes { + remoteNames := make([]string, 0, len(c.Remotes)) + for name := range c.Remotes { + remoteNames = append(remoteNames, name) + } + + sort.Strings(remoteNames) + + for _, name := range remoteNames { if !added[name] { - newSubsections = append(newSubsections, remote.marshal()) + newSubsections = append(newSubsections, c.Remotes[name].marshal()) } } diff --git a/config/config_test.go b/config/config_test.go index 97f4bbf..019cee6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -10,6 +10,8 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { input := []byte(`[core] bare = true worktree = foo +[pack] + window = 20 [remote "origin"] url = git@github.com:mcuadros/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* @@ -33,6 +35,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) c.Assert(cfg.Core.Worktree, Equals, "foo") + c.Assert(cfg.Pack.Window, Equals, uint(20)) c.Assert(cfg.Remotes, HasLen, 2) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") c.Assert(cfg.Remotes["origin"].URLs, DeepEquals, []string{"git@github.com:mcuadros/go-git.git"}) @@ -51,13 +54,15 @@ func (s *ConfigSuite) TestMarshall(c *C) { output := []byte(`[core] bare = true worktree = bar -[remote "origin"] - url = git@github.com:mcuadros/go-git.git +[pack] + window = 20 [remote "alt"] url = git@github.com:mcuadros/go-git.git url = git@github.com:src-d/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* fetch = +refs/pull/*:refs/remotes/origin/pull/* +[remote "origin"] + url = git@github.com:mcuadros/go-git.git [submodule "qux"] url = https://github.com/foo/qux.git `) @@ -65,6 +70,7 @@ func (s *ConfigSuite) TestMarshall(c *C) { cfg := NewConfig() cfg.Core.IsBare = true cfg.Core.Worktree = "bar" + cfg.Pack.Window = 20 cfg.Remotes["origin"] = &RemoteConfig{ Name: "origin", URLs: []string{"git@github.com:mcuadros/go-git.git"}, @@ -92,6 +98,8 @@ func (s *ConfigSuite) TestUnmarshallMarshall(c *C) { bare = true worktree = foo custom = ignored +[pack] + window = 20 [remote "origin"] url = git@github.com:mcuadros/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* @@ -50,6 +50,9 @@ type CloneOptions struct { // stored, if nil nothing is stored and the capability (if supported) // no-progress, is sent to the server to avoid send this information. Progress sideband.Progress + // Tags describe how the tags will be fetched from the remote repository, + // by default is AllTags. + Tags TagMode } // Validate validates the fields and sets the default values. @@ -66,6 +69,10 @@ func (o *CloneOptions) Validate() error { o.ReferenceName = plumbing.HEAD } + if o.Tags == InvalidTagMode { + o.Tags = AllTags + } + return nil } @@ -103,18 +110,19 @@ func (o *PullOptions) Validate() error { return nil } -type TagFetchMode int +type TagMode int -var ( +const ( + InvalidTagMode TagMode = iota // TagFollowing any tag that points into the histories being fetched is also // fetched. TagFollowing requires a server with `include-tag` capability // in order to fetch the annotated tags objects. - TagFollowing TagFetchMode = 0 + TagFollowing // AllTags fetch all tags from the remote (i.e., fetch remote tags // refs/tags/* into local tags with the same name) - AllTags TagFetchMode = 1 + AllTags //NoTags fetch no tags from the remote at all - NoTags TagFetchMode = 2 + NoTags ) // FetchOptions describes how a fetch should be performed @@ -133,7 +141,7 @@ type FetchOptions struct { Progress sideband.Progress // Tags describe how the tags will be fetched from the remote repository, // by default is TagFollowing. - Tags TagFetchMode + Tags TagMode } // Validate validates the fields and sets the default values. @@ -142,6 +150,10 @@ func (o *FetchOptions) Validate() error { o.RemoteName = DefaultRemoteName } + if o.Tags == InvalidTagMode { + o.Tags = TagFollowing + } + for _, r := range o.RefSpecs { if err := r.Validate(); err != nil { return err @@ -160,6 +172,9 @@ type PushOptions struct { RefSpecs []config.RefSpec // Auth credentials, if required, to use with the remote repository. Auth transport.AuthMethod + // Progress is where the human readable information sent by the server is + // stored, if nil nothing is stored. + Progress sideband.Progress } // Validate validates the fields and sets the default values. @@ -238,13 +253,13 @@ func (o *CheckoutOptions) Validate() error { type ResetMode int8 const ( - // HardReset resets the index and working tree. Any changes to tracked files - // in the working tree are discarded. - HardReset ResetMode = iota // MixedReset resets the index but not the working tree (i.e., the changed // files are preserved but not marked for commit) and reports what has not // been updated. This is the default action. - MixedReset + MixedReset ResetMode = iota + // HardReset resets the index and working tree. Any changes to tracked files + // in the working tree are discarded. + HardReset // MergeReset resets the index and updates the files in the working tree // that are different between Commit and HEAD, but keeps those which are // different between the index and working tree (i.e. which have changes @@ -253,6 +268,10 @@ const ( // If a file that is different between Commit and the index has unstaged // changes, reset is aborted. MergeReset + // SoftReset does not touch the index file or the working tree at all (but + // resets the head to <commit>, just like all modes do). This leaves all + // your changed files "Changes to be committed", as git status would put it. + SoftReset ) // ResetOptions describes how a reset operation should be performed. @@ -329,3 +348,9 @@ func (o *CommitOptions) Validate(r *Repository) error { return nil } + +// ListOptions describes how a remote list should be performed. +type ListOptions struct { + // Auth credentials, if required, to use with the remote repository. + Auth transport.AuthMethod +} diff --git a/plumbing/cache/object_lru.go b/plumbing/cache/object_lru.go index e4c3160..e8414ab 100644 --- a/plumbing/cache/object_lru.go +++ b/plumbing/cache/object_lru.go @@ -2,6 +2,7 @@ package cache import ( "container/list" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -14,6 +15,7 @@ type ObjectLRU struct { actualSize FileSize ll *list.List cache map[interface{}]*list.Element + mut sync.Mutex } // NewObjectLRU creates a new ObjectLRU with the given maximum size. The maximum @@ -26,6 +28,9 @@ func NewObjectLRU(maxSize FileSize) *ObjectLRU { // will be marked as used. Otherwise, it will be inserted. A single object might // be evicted to make room for the new object. func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { + c.mut.Lock() + defer c.mut.Unlock() + if c.cache == nil { c.actualSize = 0 c.cache = make(map[interface{}]*list.Element, 1000) @@ -67,6 +72,9 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { // Get returns an object by its hash. It marks the object as used. If the object // is not in the cache, (nil, false) will be returned. func (c *ObjectLRU) Get(k plumbing.Hash) (plumbing.EncodedObject, bool) { + c.mut.Lock() + defer c.mut.Unlock() + ee, ok := c.cache[k] if !ok { return nil, false @@ -78,6 +86,9 @@ func (c *ObjectLRU) Get(k plumbing.Hash) (plumbing.EncodedObject, bool) { // Clear the content of this object cache. func (c *ObjectLRU) Clear() { + c.mut.Lock() + defer c.mut.Unlock() + c.ll = nil c.cache = nil c.actualSize = 0 diff --git a/plumbing/cache/object_test.go b/plumbing/cache/object_test.go index 9359455..b38272f 100644 --- a/plumbing/cache/object_test.go +++ b/plumbing/cache/object_test.go @@ -1,7 +1,9 @@ package cache import ( + "fmt" "io" + "sync" "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -67,6 +69,32 @@ func (s *ObjectSuite) TestClear(c *C) { c.Assert(obj, IsNil) } +func (s *ObjectSuite) TestConcurrentAccess(c *C) { + var wg sync.WaitGroup + + for i := 0; i < 1000; i++ { + wg.Add(3) + go func(i int) { + s.c.Put(newObject(fmt.Sprint(i), FileSize(i))) + wg.Done() + }(i) + + go func(i int) { + if i%30 == 0 { + s.c.Clear() + } + wg.Done() + }(i) + + go func(i int) { + s.c.Get(plumbing.NewHash(fmt.Sprint(i))) + wg.Done() + }(i) + } + + wg.Wait() +} + type dummyObject struct { hash plumbing.Hash size FileSize diff --git a/plumbing/format/index/decoder_test.go b/plumbing/format/index/decoder_test.go index fd83ffb..c3fa590 100644 --- a/plumbing/format/index/decoder_test.go +++ b/plumbing/format/index/decoder_test.go @@ -21,6 +21,7 @@ var _ = Suite(&IndexSuite{}) func (s *IndexSuite) TestDecode(c *C) { f, err := fixtures.Basic().One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -34,6 +35,7 @@ func (s *IndexSuite) TestDecode(c *C) { func (s *IndexSuite) TestDecodeEntries(c *C) { f, err := fixtures.Basic().One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -64,6 +66,7 @@ func (s *IndexSuite) TestDecodeEntries(c *C) { func (s *IndexSuite) TestDecodeCacheTree(c *C) { f, err := fixtures.Basic().One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -93,6 +96,7 @@ var expectedEntries = []TreeEntry{ func (s *IndexSuite) TestDecodeMergeConflict(c *C) { f, err := fixtures.Basic().ByTag("merge-conflict").One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -130,6 +134,7 @@ func (s *IndexSuite) TestDecodeMergeConflict(c *C) { func (s *IndexSuite) TestDecodeExtendedV3(c *C) { f, err := fixtures.Basic().ByTag("intent-to-add").One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -147,6 +152,7 @@ func (s *IndexSuite) TestDecodeExtendedV3(c *C) { func (s *IndexSuite) TestDecodeResolveUndo(c *C) { f, err := fixtures.Basic().ByTag("resolve-undo").One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) @@ -172,6 +178,7 @@ func (s *IndexSuite) TestDecodeResolveUndo(c *C) { func (s *IndexSuite) TestDecodeV4(c *C) { f, err := fixtures.Basic().ByTag("index-v4").One().DotGit().Open("index") c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() idx := &Index{} d := NewDecoder(f) diff --git a/plumbing/format/index/encoder_test.go b/plumbing/format/index/encoder_test.go index bc5df0f..78cbbba 100644 --- a/plumbing/format/index/encoder_test.go +++ b/plumbing/format/index/encoder_test.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/google/go-cmp/cmp" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -46,7 +47,7 @@ func (s *IndexSuite) TestEncode(c *C) { err = d.Decode(output) c.Assert(err, IsNil) - c.Assert(idx, DeepEquals, output) + c.Assert(cmp.Equal(idx, output), Equals, true) c.Assert(output.Entries[0].Name, Equals, strings.Repeat(" ", 20)) c.Assert(output.Entries[1].Name, Equals, "bar") diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index 728cb16..7dad1f6 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -1,7 +1,9 @@ package packfile import ( + "bytes" "io" + "sync" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" @@ -49,3 +51,9 @@ func writePackfileToObjectStorage(sw storer.PackfileWriter, packfile io.Reader) _, err = io.Copy(w, packfile) return err } + +var bufPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(nil) + }, +} diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index e49de51..3d475b2 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -347,7 +347,8 @@ func (d *Decoder) fillRegularObjectContent(obj plumbing.EncodedObject) (uint32, } func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) (uint32, error) { - buf := bytes.NewBuffer(nil) + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() _, crc, err := d.s.NextObject(buf) if err != nil { return 0, err @@ -364,6 +365,7 @@ func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plum obj.SetType(base.Type()) err = ApplyDelta(obj, base, buf.Bytes()) d.cachePut(obj) + bufPool.Put(buf) return crc, err } diff --git a/plumbing/format/packfile/delta_index.go b/plumbing/format/packfile/delta_index.go new file mode 100644 index 0000000..349bedf --- /dev/null +++ b/plumbing/format/packfile/delta_index.go @@ -0,0 +1,299 @@ +package packfile + +const blksz = 16 +const maxChainLength = 64 + +// deltaIndex is a modified version of JGit's DeltaIndex adapted to our current +// design. +type deltaIndex struct { + table []int + entries []int + mask int +} + +func (idx *deltaIndex) init(buf []byte) { + scanner := newDeltaIndexScanner(buf, len(buf)) + idx.mask = scanner.mask + idx.table = scanner.table + idx.entries = make([]int, countEntries(scanner)+1) + idx.copyEntries(scanner) +} + +// findMatch returns the offset of src where the block starting at tgtOffset +// is and the length of the match. A length of 0 means there was no match. A +// length of -1 means the src length is lower than the blksz and whatever +// other positive length is the length of the match in bytes. +func (idx *deltaIndex) findMatch(src, tgt []byte, tgtOffset int) (srcOffset, l int) { + if len(tgt) < tgtOffset+s { + return 0, len(tgt) - tgtOffset + } + + if len(src) < blksz { + return 0, -1 + } + + if len(tgt) >= tgtOffset+s && len(src) >= blksz { + h := hashBlock(tgt, tgtOffset) + tIdx := h & idx.mask + eIdx := idx.table[tIdx] + if eIdx != 0 { + srcOffset = idx.entries[eIdx] + } else { + return + } + + l = matchLength(src, tgt, tgtOffset, srcOffset) + } + + return +} + +func matchLength(src, tgt []byte, otgt, osrc int) (l int) { + lensrc := len(src) + lentgt := len(tgt) + for (osrc < lensrc && otgt < lentgt) && src[osrc] == tgt[otgt] { + l++ + osrc++ + otgt++ + } + return +} + +func countEntries(scan *deltaIndexScanner) (cnt int) { + // Figure out exactly how many entries we need. As we do the + // enumeration truncate any delta chains longer than what we + // are willing to scan during encode. This keeps the encode + // logic linear in the size of the input rather than quadratic. + for i := 0; i < len(scan.table); i++ { + h := scan.table[i] + if h == 0 { + continue + } + + size := 0 + for { + size++ + if size == maxChainLength { + scan.next[h] = 0 + break + } + h = scan.next[h] + + if h == 0 { + break + } + } + cnt += size + } + + return +} + +func (idx *deltaIndex) copyEntries(scanner *deltaIndexScanner) { + // Rebuild the entries list from the scanner, positioning all + // blocks in the same hash chain next to each other. We can + // then later discard the next list, along with the scanner. + // + next := 1 + for i := 0; i < len(idx.table); i++ { + h := idx.table[i] + if h == 0 { + continue + } + + idx.table[i] = next + for { + idx.entries[next] = scanner.entries[h] + next++ + h = scanner.next[h] + + if h == 0 { + break + } + } + } +} + +type deltaIndexScanner struct { + table []int + entries []int + next []int + mask int + count int +} + +func newDeltaIndexScanner(buf []byte, size int) *deltaIndexScanner { + size -= size % blksz + worstCaseBlockCnt := size / blksz + if worstCaseBlockCnt < 1 { + return new(deltaIndexScanner) + } + + tableSize := tableSize(worstCaseBlockCnt) + scanner := &deltaIndexScanner{ + table: make([]int, tableSize), + mask: tableSize - 1, + entries: make([]int, worstCaseBlockCnt+1), + next: make([]int, worstCaseBlockCnt+1), + } + + scanner.scan(buf, size) + return scanner +} + +// slightly modified version of JGit's DeltaIndexScanner. We store the offset on the entries +// instead of the entries and the key, so we avoid operations to retrieve the offset later, as +// we don't use the key. +// See: https://github.com/eclipse/jgit/blob/005e5feb4ecd08c4e4d141a38b9e7942accb3212/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaIndexScanner.java +func (s *deltaIndexScanner) scan(buf []byte, end int) { + lastHash := 0 + ptr := end - blksz + + for { + key := hashBlock(buf, ptr) + tIdx := key & s.mask + head := s.table[tIdx] + if head != 0 && lastHash == key { + s.entries[head] = ptr + } else { + s.count++ + eIdx := s.count + s.entries[eIdx] = ptr + s.next[eIdx] = head + s.table[tIdx] = eIdx + } + + lastHash = key + ptr -= blksz + + if 0 > ptr { + break + } + } +} + +func tableSize(worstCaseBlockCnt int) int { + shift := 32 - leadingZeros(uint32(worstCaseBlockCnt)) + sz := 1 << uint(shift-1) + if sz < worstCaseBlockCnt { + sz <<= 1 + } + return sz +} + +// use https://golang.org/pkg/math/bits/#LeadingZeros32 in the future +func leadingZeros(x uint32) (n int) { + if x >= 1<<16 { + x >>= 16 + n = 16 + } + if x >= 1<<8 { + x >>= 8 + n += 8 + } + n += int(len8tab[x]) + return 32 - n +} + +var len8tab = [256]uint8{ + 0x00, 0x01, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, +} + +func hashBlock(raw []byte, ptr int) int { + var hash uint32 + + // The first 4 steps collapse out into a 4 byte big-endian decode, + // with a larger right shift as we combined shift lefts together. + // + hash = ((uint32(raw[ptr]) & 0xff) << 24) | + ((uint32(raw[ptr+1]) & 0xff) << 16) | + ((uint32(raw[ptr+2]) & 0xff) << 8) | + (uint32(raw[ptr+3]) & 0xff) + hash ^= T[hash>>31] + + hash = ((hash << 8) | (uint32(raw[ptr+4]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+5]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+6]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+7]) & 0xff)) ^ T[hash>>23] + + hash = ((hash << 8) | (uint32(raw[ptr+8]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+9]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+10]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+11]) & 0xff)) ^ T[hash>>23] + + hash = ((hash << 8) | (uint32(raw[ptr+12]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+13]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+14]) & 0xff)) ^ T[hash>>23] + hash = ((hash << 8) | (uint32(raw[ptr+15]) & 0xff)) ^ T[hash>>23] + + return int(hash) +} + +var T = []uint32{0x00000000, 0xd4c6b32d, 0x7d4bd577, + 0xa98d665a, 0x2e5119c3, 0xfa97aaee, 0x531accb4, 0x87dc7f99, + 0x5ca23386, 0x886480ab, 0x21e9e6f1, 0xf52f55dc, 0x72f32a45, + 0xa6359968, 0x0fb8ff32, 0xdb7e4c1f, 0x6d82d421, 0xb944670c, + 0x10c90156, 0xc40fb27b, 0x43d3cde2, 0x97157ecf, 0x3e981895, + 0xea5eabb8, 0x3120e7a7, 0xe5e6548a, 0x4c6b32d0, 0x98ad81fd, + 0x1f71fe64, 0xcbb74d49, 0x623a2b13, 0xb6fc983e, 0x0fc31b6f, + 0xdb05a842, 0x7288ce18, 0xa64e7d35, 0x219202ac, 0xf554b181, + 0x5cd9d7db, 0x881f64f6, 0x536128e9, 0x87a79bc4, 0x2e2afd9e, + 0xfaec4eb3, 0x7d30312a, 0xa9f68207, 0x007be45d, 0xd4bd5770, + 0x6241cf4e, 0xb6877c63, 0x1f0a1a39, 0xcbcca914, 0x4c10d68d, + 0x98d665a0, 0x315b03fa, 0xe59db0d7, 0x3ee3fcc8, 0xea254fe5, + 0x43a829bf, 0x976e9a92, 0x10b2e50b, 0xc4745626, 0x6df9307c, + 0xb93f8351, 0x1f8636de, 0xcb4085f3, 0x62cde3a9, 0xb60b5084, + 0x31d72f1d, 0xe5119c30, 0x4c9cfa6a, 0x985a4947, 0x43240558, + 0x97e2b675, 0x3e6fd02f, 0xeaa96302, 0x6d751c9b, 0xb9b3afb6, + 0x103ec9ec, 0xc4f87ac1, 0x7204e2ff, 0xa6c251d2, 0x0f4f3788, + 0xdb8984a5, 0x5c55fb3c, 0x88934811, 0x211e2e4b, 0xf5d89d66, + 0x2ea6d179, 0xfa606254, 0x53ed040e, 0x872bb723, 0x00f7c8ba, + 0xd4317b97, 0x7dbc1dcd, 0xa97aaee0, 0x10452db1, 0xc4839e9c, + 0x6d0ef8c6, 0xb9c84beb, 0x3e143472, 0xead2875f, 0x435fe105, + 0x97995228, 0x4ce71e37, 0x9821ad1a, 0x31accb40, 0xe56a786d, + 0x62b607f4, 0xb670b4d9, 0x1ffdd283, 0xcb3b61ae, 0x7dc7f990, + 0xa9014abd, 0x008c2ce7, 0xd44a9fca, 0x5396e053, 0x8750537e, + 0x2edd3524, 0xfa1b8609, 0x2165ca16, 0xf5a3793b, 0x5c2e1f61, + 0x88e8ac4c, 0x0f34d3d5, 0xdbf260f8, 0x727f06a2, 0xa6b9b58f, + 0x3f0c6dbc, 0xebcade91, 0x4247b8cb, 0x96810be6, 0x115d747f, + 0xc59bc752, 0x6c16a108, 0xb8d01225, 0x63ae5e3a, 0xb768ed17, + 0x1ee58b4d, 0xca233860, 0x4dff47f9, 0x9939f4d4, 0x30b4928e, + 0xe47221a3, 0x528eb99d, 0x86480ab0, 0x2fc56cea, 0xfb03dfc7, + 0x7cdfa05e, 0xa8191373, 0x01947529, 0xd552c604, 0x0e2c8a1b, + 0xdaea3936, 0x73675f6c, 0xa7a1ec41, 0x207d93d8, 0xf4bb20f5, + 0x5d3646af, 0x89f0f582, 0x30cf76d3, 0xe409c5fe, 0x4d84a3a4, + 0x99421089, 0x1e9e6f10, 0xca58dc3d, 0x63d5ba67, 0xb713094a, + 0x6c6d4555, 0xb8abf678, 0x11269022, 0xc5e0230f, 0x423c5c96, + 0x96faefbb, 0x3f7789e1, 0xebb13acc, 0x5d4da2f2, 0x898b11df, + 0x20067785, 0xf4c0c4a8, 0x731cbb31, 0xa7da081c, 0x0e576e46, + 0xda91dd6b, 0x01ef9174, 0xd5292259, 0x7ca44403, 0xa862f72e, + 0x2fbe88b7, 0xfb783b9a, 0x52f55dc0, 0x8633eeed, 0x208a5b62, + 0xf44ce84f, 0x5dc18e15, 0x89073d38, 0x0edb42a1, 0xda1df18c, + 0x739097d6, 0xa75624fb, 0x7c2868e4, 0xa8eedbc9, 0x0163bd93, + 0xd5a50ebe, 0x52797127, 0x86bfc20a, 0x2f32a450, 0xfbf4177d, + 0x4d088f43, 0x99ce3c6e, 0x30435a34, 0xe485e919, 0x63599680, + 0xb79f25ad, 0x1e1243f7, 0xcad4f0da, 0x11aabcc5, 0xc56c0fe8, + 0x6ce169b2, 0xb827da9f, 0x3ffba506, 0xeb3d162b, 0x42b07071, + 0x9676c35c, 0x2f49400d, 0xfb8ff320, 0x5202957a, 0x86c42657, + 0x011859ce, 0xd5deeae3, 0x7c538cb9, 0xa8953f94, 0x73eb738b, + 0xa72dc0a6, 0x0ea0a6fc, 0xda6615d1, 0x5dba6a48, 0x897cd965, + 0x20f1bf3f, 0xf4370c12, 0x42cb942c, 0x960d2701, 0x3f80415b, + 0xeb46f276, 0x6c9a8def, 0xb85c3ec2, 0x11d15898, 0xc517ebb5, + 0x1e69a7aa, 0xcaaf1487, 0x632272dd, 0xb7e4c1f0, 0x3038be69, + 0xe4fe0d44, 0x4d736b1e, 0x99b5d833, +} diff --git a/plumbing/format/packfile/delta_selector.go b/plumbing/format/packfile/delta_selector.go index efcbd53..51adcdf 100644 --- a/plumbing/format/packfile/delta_selector.go +++ b/plumbing/format/packfile/delta_selector.go @@ -2,6 +2,7 @@ package packfile import ( "sort" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -27,27 +28,75 @@ func newDeltaSelector(s storer.EncodedObjectStorer) *deltaSelector { return &deltaSelector{s} } -// ObjectsToPack creates a list of ObjectToPack from the hashes provided, -// creating deltas if it's suitable, using an specific internal logic -func (dw *deltaSelector) ObjectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, error) { - otp, err := dw.objectsToPack(hashes) +// ObjectsToPack creates a list of ObjectToPack from the hashes +// provided, creating deltas if it's suitable, using an specific +// internal logic. `packWindow` specifies the size of the sliding +// window used to compare objects for delta compression; 0 turns off +// delta compression entirely. +func (dw *deltaSelector) ObjectsToPack( + hashes []plumbing.Hash, + packWindow uint, +) ([]*ObjectToPack, error) { + otp, err := dw.objectsToPack(hashes, packWindow) if err != nil { return nil, err } + if packWindow == 0 { + return otp, nil + } + dw.sort(otp) - if err := dw.walk(otp); err != nil { + var objectGroups [][]*ObjectToPack + var prev *ObjectToPack + i := -1 + for _, obj := range otp { + if prev == nil || prev.Type() != obj.Type() { + objectGroups = append(objectGroups, []*ObjectToPack{obj}) + i++ + prev = obj + } else { + objectGroups[i] = append(objectGroups[i], obj) + } + } + + var wg sync.WaitGroup + var once sync.Once + for _, objs := range objectGroups { + objs := objs + wg.Add(1) + go func() { + if walkErr := dw.walk(objs, packWindow); walkErr != nil { + once.Do(func() { + err = walkErr + }) + } + wg.Done() + }() + } + wg.Wait() + + if err != nil { return nil, err } return otp, nil } -func (dw *deltaSelector) objectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, error) { +func (dw *deltaSelector) objectsToPack( + hashes []plumbing.Hash, + packWindow uint, +) ([]*ObjectToPack, error) { var objectsToPack []*ObjectToPack for _, h := range hashes { - o, err := dw.encodedDeltaObject(h) + var o plumbing.EncodedObject + var err error + if packWindow == 0 { + o, err = dw.encodedObject(h) + } else { + o, err = dw.encodedDeltaObject(h) + } if err != nil { return nil, err } @@ -60,6 +109,10 @@ func (dw *deltaSelector) objectsToPack(hashes []plumbing.Hash) ([]*ObjectToPack, objectsToPack = append(objectsToPack, otp) } + if packWindow == 0 { + return objectsToPack, nil + } + if err := dw.fixAndBreakChains(objectsToPack); err != nil { return nil, err } @@ -168,8 +221,18 @@ func (dw *deltaSelector) sort(objectsToPack []*ObjectToPack) { sort.Sort(byTypeAndSize(objectsToPack)) } -func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { +func (dw *deltaSelector) walk( + objectsToPack []*ObjectToPack, + packWindow uint, +) error { + indexMap := make(map[plumbing.Hash]*deltaIndex) for i := 0; i < len(objectsToPack); i++ { + // Clean up the index map for anything outside our pack + // window, to save memory. + if i > int(packWindow) { + delete(indexMap, objectsToPack[i-int(packWindow)].Hash()) + } + target := objectsToPack[i] // If we already have a delta, we don't try to find a new one for this @@ -184,7 +247,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { continue } - for j := i - 1; j >= 0; j-- { + for j := i - 1; j >= 0 && i-j < int(packWindow); j-- { base := objectsToPack[j] // Objects must use only the same type as their delta base. // Since objectsToPack is sorted by type and size, once we find @@ -193,7 +256,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { break } - if err := dw.tryToDeltify(base, target); err != nil { + if err := dw.tryToDeltify(indexMap, base, target); err != nil { return err } } @@ -202,7 +265,7 @@ func (dw *deltaSelector) walk(objectsToPack []*ObjectToPack) error { return nil } -func (dw *deltaSelector) tryToDeltify(base, target *ObjectToPack) error { +func (dw *deltaSelector) tryToDeltify(indexMap map[plumbing.Hash]*deltaIndex, base, target *ObjectToPack) error { // If the sizes are radically different, this is a bad pairing. if target.Size() < base.Size()>>4 { return nil @@ -235,8 +298,12 @@ func (dw *deltaSelector) tryToDeltify(base, target *ObjectToPack) error { return err } + if _, ok := indexMap[base.Hash()]; !ok { + indexMap[base.Hash()] = new(deltaIndex) + } + // Now we can generate the delta using originals - delta, err := GetDelta(base.Original, target.Original) + delta, err := getDelta(indexMap[base.Hash()], base.Original, target.Original) if err != nil { return err } diff --git a/plumbing/format/packfile/delta_selector_test.go b/plumbing/format/packfile/delta_selector_test.go index cbbbc89..7d7fd0c 100644 --- a/plumbing/format/packfile/delta_selector_test.go +++ b/plumbing/format/packfile/delta_selector_test.go @@ -146,7 +146,8 @@ func (s *DeltaSelectorSuite) createTestObjects() { func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Different type hashes := []plumbing.Hash{s.hashes["base"], s.hashes["treeType"]} - otp, err := s.ds.ObjectsToPack(hashes) + deltaWindowSize := uint(10) + otp, err := s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["base"]]) @@ -154,7 +155,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Size radically different hashes = []plumbing.Hash{s.hashes["bigBase"], s.hashes["target"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["bigBase"]]) @@ -162,7 +163,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // Delta Size Limit with no best delta yet hashes = []plumbing.Hash{s.hashes["smallBase"], s.hashes["smallTarget"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["smallBase"]]) @@ -170,7 +171,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { // It will create the delta hashes = []plumbing.Hash{s.hashes["base"], s.hashes["target"]} - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 2) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["target"]]) @@ -185,7 +186,7 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { s.hashes["o2"], s.hashes["o3"], } - otp, err = s.ds.ObjectsToPack(hashes) + otp, err = s.ds.ObjectsToPack(hashes, deltaWindowSize) c.Assert(err, IsNil) c.Assert(len(otp), Equals, 3) c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["o1"]]) @@ -196,6 +197,37 @@ func (s *DeltaSelectorSuite) TestObjectsToPack(c *C) { c.Assert(otp[2].Original, Equals, s.store.Objects[s.hashes["o3"]]) c.Assert(otp[2].IsDelta(), Equals, true) c.Assert(otp[2].Depth, Equals, 2) + + // Check that objects outside of the sliding window don't produce + // a delta. + hashes = make([]plumbing.Hash, 0, deltaWindowSize+2) + hashes = append(hashes, s.hashes["base"]) + for i := uint(0); i < deltaWindowSize; i++ { + hashes = append(hashes, s.hashes["smallTarget"]) + } + hashes = append(hashes, s.hashes["target"]) + + // Don't sort so we can easily check the sliding window without + // creating a bunch of new objects. + otp, err = s.ds.objectsToPack(hashes, deltaWindowSize) + c.Assert(err, IsNil) + err = s.ds.walk(otp, deltaWindowSize) + c.Assert(err, IsNil) + c.Assert(len(otp), Equals, int(deltaWindowSize)+2) + targetIdx := len(otp) - 1 + c.Assert(otp[targetIdx].IsDelta(), Equals, false) + + // Check that no deltas are created, and the objects are unsorted, + // if compression is off. + hashes = []plumbing.Hash{s.hashes["base"], s.hashes["target"]} + otp, err = s.ds.ObjectsToPack(hashes, 0) + c.Assert(err, IsNil) + c.Assert(len(otp), Equals, 2) + c.Assert(otp[0].Object, Equals, s.store.Objects[s.hashes["base"]]) + c.Assert(otp[0].IsDelta(), Equals, false) + c.Assert(otp[1].Original, Equals, s.store.Objects[s.hashes["target"]]) + c.Assert(otp[1].IsDelta(), Equals, false) + c.Assert(otp[1].Depth, Equals, 0) } func (s *DeltaSelectorSuite) TestMaxDepth(c *C) { diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 60a04d9..4d56dc1 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -2,8 +2,6 @@ package packfile import ( "bytes" - "hash/adler32" - "io/ioutil" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -26,26 +24,40 @@ const ( // To generate target again, you will need the obtained object and "base" one. // Error will be returned if base or target object cannot be read. func GetDelta(base, target plumbing.EncodedObject) (plumbing.EncodedObject, error) { + return getDelta(new(deltaIndex), base, target) +} + +func getDelta(index *deltaIndex, base, target plumbing.EncodedObject) (plumbing.EncodedObject, error) { br, err := base.Reader() if err != nil { return nil, err } + defer br.Close() tr, err := target.Reader() if err != nil { return nil, err } + defer tr.Close() + + bb := bufPool.Get().(*bytes.Buffer) + bb.Reset() + defer bufPool.Put(bb) - bb, err := ioutil.ReadAll(br) + _, err = bb.ReadFrom(br) if err != nil { return nil, err } - tb, err := ioutil.ReadAll(tr) + tb := bufPool.Get().(*bytes.Buffer) + tb.Reset() + defer bufPool.Put(tb) + + _, err = tb.ReadFrom(tr) if err != nil { return nil, err } - db := DiffDelta(bb, tb) + db := diffDelta(index, bb.Bytes(), tb.Bytes()) delta := &plumbing.MemoryObject{} _, err = delta.Write(db) if err != nil { @@ -59,19 +71,41 @@ func GetDelta(base, target plumbing.EncodedObject) (plumbing.EncodedObject, erro } // DiffDelta returns the delta that transforms src into tgt. -func DiffDelta(src []byte, tgt []byte) []byte { - buf := bytes.NewBuffer(nil) +func DiffDelta(src, tgt []byte) []byte { + return diffDelta(new(deltaIndex), src, tgt) +} + +func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() buf.Write(deltaEncodeSize(len(src))) buf.Write(deltaEncodeSize(len(tgt))) - sindex := initMatch(src) + if len(index.entries) == 0 { + index.init(src) + } - ibuf := bytes.NewBuffer(nil) + ibuf := bufPool.Get().(*bytes.Buffer) + ibuf.Reset() for i := 0; i < len(tgt); i++ { - offset, l := findMatch(src, tgt, sindex, i) + offset, l := index.findMatch(src, tgt, i) - if l < s { + if l == 0 { + // couldn't find a match, just write the current byte and continue ibuf.WriteByte(tgt[i]) + } else if l < 0 { + // src is less than blksz, copy the rest of the target to avoid + // calls to findMatch + for ; i < len(tgt); i++ { + ibuf.WriteByte(tgt[i]) + } + } else if l < s { + // remaining target is less than blksz, copy what's left of it + // and avoid calls to findMatch + for j := i; j < i+l; j++ { + ibuf.WriteByte(tgt[j]) + } + i += l - 1 } else { encodeInsertOperation(ibuf, buf) @@ -93,8 +127,12 @@ func DiffDelta(src []byte, tgt []byte) []byte { } encodeInsertOperation(ibuf, buf) + bytes := buf.Bytes() - return buf.Bytes() + bufPool.Put(buf) + bufPool.Put(ibuf) + + return bytes } func encodeInsertOperation(ibuf, buf *bytes.Buffer) { @@ -120,52 +158,6 @@ func encodeInsertOperation(ibuf, buf *bytes.Buffer) { ibuf.Reset() } -func initMatch(src []byte) map[uint32]int { - i := 0 - index := make(map[uint32]int) - for { - if i+s > len(src) { - break - } - - ch := adler32.Checksum(src[i : i+s]) - index[ch] = i - i += s - } - - return index -} - -func findMatch(src, tgt []byte, sindex map[uint32]int, tgtOffset int) (srcOffset, l int) { - if len(tgt) >= tgtOffset+s { - ch := adler32.Checksum(tgt[tgtOffset : tgtOffset+s]) - var ok bool - srcOffset, ok = sindex[ch] - if !ok { - return - } - - l = matchLength(src, tgt, tgtOffset, srcOffset) - } - - return -} - -func matchLength(src, tgt []byte, otgt, osrc int) int { - l := 0 - for { - if (osrc >= len(src) || otgt >= len(tgt)) || src[osrc] != tgt[otgt] { - break - } - - l++ - osrc++ - otgt++ - } - - return l -} - func deltaEncodeSize(size int) []byte { var ret []byte c := size & 0x7f diff --git a/plumbing/format/packfile/encoder.go b/plumbing/format/packfile/encoder.go index 1426559..7ee6546 100644 --- a/plumbing/format/packfile/encoder.go +++ b/plumbing/format/packfile/encoder.go @@ -14,10 +14,10 @@ import ( // Encoder gets the data from the storage and write it into the writer in PACK // format type Encoder struct { - selector *deltaSelector - w *offsetWriter - zw *zlib.Writer - hasher plumbing.Hasher + selector *deltaSelector + w *offsetWriter + zw *zlib.Writer + hasher plumbing.Hasher // offsets is a map of object hashes to corresponding offsets in the packfile. // It is used to determine offset of the base of a delta when a OFS_DELTA is // used. @@ -45,10 +45,15 @@ func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *E } } -// Encode creates a packfile containing all the objects referenced in hashes -// and writes it to the writer in the Encoder. -func (e *Encoder) Encode(hashes []plumbing.Hash) (plumbing.Hash, error) { - objects, err := e.selector.ObjectsToPack(hashes) +// Encode creates a packfile containing all the objects referenced in +// hashes and writes it to the writer in the Encoder. `packWindow` +// specifies the size of the sliding window used to compare objects +// for delta compression; 0 turns off delta compression entirely. +func (e *Encoder) Encode( + hashes []plumbing.Hash, + packWindow uint, +) (plumbing.Hash, error) { + objects, err := e.selector.ObjectsToPack(hashes, packWindow) if err != nil { return plumbing.ZeroHash, err } @@ -137,7 +142,7 @@ func (e *Encoder) writeOfsDeltaHeader(deltaOffset int64, base plumbing.Hash) err // for OFS_DELTA, offset of the base is interpreted as negative offset // relative to the type-byte of the header of the ofs-delta entry. - relativeOffset := deltaOffset-baseOffset + relativeOffset := deltaOffset - baseOffset if relativeOffset <= 0 { return fmt.Errorf("bad offset for OFS_DELTA entry: %d", relativeOffset) } diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index d92e2c4..39c0700 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -27,12 +27,23 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { fixs.Test(c, func(f *fixtures.Fixture) { storage, err := filesystem.NewStorage(f.DotGit()) c.Assert(err, IsNil) - s.testEncodeDecode(c, storage) + s.testEncodeDecode(c, storage, 10) }) } -func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer) { +func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { + fixs := fixtures.Basic().ByTag("packfile").ByTag(".git") + fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). + ByTag("packfile").ByTag(".git").One()) + fixs.Test(c, func(f *fixtures.Fixture) { + storage, err := filesystem.NewStorage(f.DotGit()) + c.Assert(err, IsNil) + s.testEncodeDecode(c, storage, 0) + }) +} + +func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer, packWindow uint) { objIter, err := storage.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -57,7 +68,7 @@ func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer) { buf := bytes.NewBuffer(nil) enc := NewEncoder(buf, storage, false) - _, err = enc.Encode(hashes) + _, err = enc.Encode(hashes, packWindow) c.Assert(err, IsNil) scanner := NewScanner(buf) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index b5b0c42..2cb9094 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -26,7 +26,7 @@ func (s *EncoderSuite) SetUpTest(c *C) { } func (s *EncoderSuite) TestCorrectPackHeader(c *C) { - hash, err := s.enc.Encode([]plumbing.Hash{}) + hash, err := s.enc.Encode([]plumbing.Hash{}, 10) c.Assert(err, IsNil) hb := [20]byte(hash) @@ -47,7 +47,7 @@ func (s *EncoderSuite) TestCorrectPackWithOneEmptyObject(c *C) { _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) // PACK + VERSION(2) + OBJECT NUMBER(1) @@ -74,13 +74,13 @@ func (s *EncoderSuite) TestMaxObjectSize(c *C) { o.SetType(plumbing.CommitObject) _, err := s.store.SetEncodedObject(o) c.Assert(err, IsNil) - hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}) + hash, err := s.enc.Encode([]plumbing.Hash{o.Hash()}, 10) c.Assert(err, IsNil) c.Assert(hash.IsZero(), Not(Equals), true) } func (s *EncoderSuite) TestHashNotFound(c *C) { - h, err := s.enc.Encode([]plumbing.Hash{plumbing.NewHash("BAD")}) + h, err := s.enc.Encode([]plumbing.Hash{plumbing.NewHash("BAD")}, 10) c.Assert(h, Equals, plumbing.ZeroHash) c.Assert(err, NotNil) c.Assert(err, Equals, plumbing.ErrObjectNotFound) diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 1dab2f2..d2d776f 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -9,6 +9,7 @@ import ( "hash/crc32" "io" stdioutil "io/ioutil" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" @@ -291,10 +292,18 @@ func (s *Scanner) copyObject(w io.Writer) (n int64, err error) { } defer ioutil.CheckClose(s.zr, &err) - n, err = io.Copy(w, s.zr) + buf := byteSlicePool.Get().([]byte) + n, err = io.CopyBuffer(w, s.zr, buf) + byteSlicePool.Put(buf) return } +var byteSlicePool = sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024) + }, +} + // SeekFromStart sets a new offset from start, returns the old position before // the change. func (s *Scanner) SeekFromStart(offset int64) (previous int64, err error) { @@ -324,7 +333,9 @@ func (s *Scanner) Checksum() (plumbing.Hash, error) { // Close reads the reader until io.EOF func (s *Scanner) Close() error { - _, err := io.Copy(stdioutil.Discard, s.r) + buf := byteSlicePool.Get().([]byte) + _, err := io.CopyBuffer(stdioutil.Discard, s.r, buf) + byteSlicePool.Put(buf) return err } diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index a994efb..b2f1f15 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -13,6 +13,11 @@ import ( "gopkg.in/src-d/go-git.v4/utils/ioutil" ) +const ( + beginpgp string = "-----BEGIN PGP SIGNATURE-----" + endpgp string = "-----END PGP SIGNATURE-----" +) + // Hash represents the hash of an object type Hash plumbing.Hash @@ -20,7 +25,7 @@ type Hash plumbing.Hash // at a certain point in time. It contains meta-information about that point // in time, such as a timestamp, the author of the changes since the last // commit, a pointer to the previous commit(s), etc. -// http://schacon.github.io/gitbook/1_the_git_object_model.html +// http://shafiulazam.com/gitbook/1_the_git_object_model.html type Commit struct { // Hash of the commit object. Hash plumbing.Hash @@ -29,6 +34,8 @@ type Commit struct { // Committer is the one performing the commit, might be different from // Author. Committer Signature + // PGPSignature is the PGP signature of the commit. + PGPSignature string // Message is the commit message, contains arbitrary text. Message string // TreeHash is the hash of the root tree of the commit. @@ -157,12 +164,33 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { r := bufio.NewReader(reader) var message bool + var pgpsig bool for { line, err := r.ReadBytes('\n') if err != nil && err != io.EOF { return err } + if pgpsig { + // Check if it's the end of a PGP signature. + if bytes.Contains(line, []byte(endpgp)) { + c.PGPSignature += endpgp + "\n" + pgpsig = false + } else { + // Trim the left padding. + line = bytes.TrimLeft(line, " ") + c.PGPSignature += string(line) + } + continue + } + + // Check if it's the beginning of a PGP signature. + if bytes.Contains(line, []byte(beginpgp)) { + c.PGPSignature += beginpgp + "\n" + pgpsig = true + continue + } + if !message { line = bytes.TrimSpace(line) if len(line) == 0 { @@ -227,6 +255,21 @@ func (b *Commit) Encode(o plumbing.EncodedObject) error { return err } + if b.PGPSignature != "" { + if _, err = fmt.Fprint(w, "pgpsig"); err != nil { + return err + } + + // Split all the signature lines and write with a left padding and + // newline at the end. + lines := strings.Split(b.PGPSignature, "\n") + for _, line := range lines { + if _, err = fmt.Fprintf(w, " %s\n", line); err != nil { + return err + } + } + } + if _, err = fmt.Fprintf(w, "\n\n%s", b.Message); err != nil { return err } @@ -234,6 +277,32 @@ func (b *Commit) Encode(o plumbing.EncodedObject) error { return err } +// Stats shows the status of commit. +func (c *Commit) Stats() (FileStats, error) { + // Get the previous commit. + ci := c.Parents() + parentCommit, err := ci.Next() + if err != nil { + if err == io.EOF { + emptyNoder := treeNoder{} + parentCommit = &Commit{ + Hash: emptyNoder.hash, + // TreeHash: emptyNoder.parent.Hash, + s: c.s, + } + } else { + return nil, err + } + } + + patch, err := parentCommit.Patch(c) + if err != nil { + return nil, err + } + + return getFileStatsFromFilePatches(patch.FilePatches()), nil +} + func (c *Commit) String() string { return fmt.Sprintf( "%s %s\nAuthor: %s\nDate: %s\n\n%s\n", @@ -290,7 +359,7 @@ func (iter *storerCommitIter) Next() (*Commit, error) { // ForEach call the cb function for each commit contained on this iter until // an error appends or the end of the iter is reached. If ErrStop is sent -// the iteration is stop but no error is returned. The iterator is closed. +// the iteration is stopped but no error is returned. The iterator is closed. func (iter *storerCommitIter) ForEach(cb func(*Commit) error) error { return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error { c, err := DecodeCommit(iter.s, obj) diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index 213dfba..f0792e6 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -244,3 +244,55 @@ func (s *SuiteCommit) TestLongCommitMessageSerialization(c *C) { c.Assert(err, IsNil) c.Assert(decoded.Message, Equals, longMessage) } + +func (s *SuiteCommit) TestPGPSignatureSerialization(c *C) { + encoded := &plumbing.MemoryObject{} + decoded := &Commit{} + commit := *s.Commit + + pgpsignature := `-----BEGIN PGP SIGNATURE----- + +iQEcBAABAgAGBQJTZbQlAAoJEF0+sviABDDrZbQH/09PfE51KPVPlanr6q1v4/Ut +LQxfojUWiLQdg2ESJItkcuweYg+kc3HCyFejeDIBw9dpXt00rY26p05qrpnG+85b +hM1/PswpPLuBSr+oCIDj5GMC2r2iEKsfv2fJbNW8iWAXVLoWZRF8B0MfqX/YTMbm +ecorc4iXzQu7tupRihslbNkfvfciMnSDeSvzCpWAHl7h8Wj6hhqePmLm9lAYqnKp +8S5B/1SSQuEAjRZgI4IexpZoeKGVDptPHxLLS38fozsyi0QyDyzEgJxcJQVMXxVi +RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= +=EFTF +-----END PGP SIGNATURE----- +` + commit.PGPSignature = pgpsignature + + err := commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, pgpsignature) +} + +func (s *SuiteCommit) TestStat(c *C) { + aCommit := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) + fileStats, err := aCommit.Stats() + c.Assert(err, IsNil) + + c.Assert(fileStats[0].Name, Equals, "vendor/foo.go") + c.Assert(fileStats[0].Addition, Equals, 7) + c.Assert(fileStats[0].Deletion, Equals, 0) + c.Assert(fileStats[0].String(), Equals, " vendor/foo.go | 7 +++++++\n") + + // Stats for another commit. + aCommit = s.commit(c, plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294")) + fileStats, err = aCommit.Stats() + c.Assert(err, IsNil) + + c.Assert(fileStats[0].Name, Equals, "go/example.go") + c.Assert(fileStats[0].Addition, Equals, 142) + c.Assert(fileStats[0].Deletion, Equals, 0) + c.Assert(fileStats[0].String(), Equals, " go/example.go | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + + c.Assert(fileStats[1].Name, Equals, "php/crappy.php") + c.Assert(fileStats[1].Addition, Equals, 259) + c.Assert(fileStats[1].Deletion, Equals, 0) + c.Assert(fileStats[1].String(), Equals, " php/crappy.php | 259 ++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} diff --git a/plumbing/object/commit_walker.go b/plumbing/object/commit_walker.go index 797c17a..40ad258 100644 --- a/plumbing/object/commit_walker.go +++ b/plumbing/object/commit_walker.go @@ -8,9 +8,10 @@ import ( ) type commitPreIterator struct { - seen map[plumbing.Hash]bool - stack []CommitIter - start *Commit + seenExternal map[plumbing.Hash]bool + seen map[plumbing.Hash]bool + stack []CommitIter + start *Commit } // NewCommitPreorderIter returns a CommitIter that walks the commit history, @@ -20,16 +21,21 @@ type commitPreIterator struct { // and will return the error. Other errors might be returned if the history // cannot be traversed (e.g. missing objects). Ignore allows to skip some // commits from being iterated. -func NewCommitPreorderIter(c *Commit, ignore []plumbing.Hash) CommitIter { +func NewCommitPreorderIter( + c *Commit, + seenExternal map[plumbing.Hash]bool, + ignore []plumbing.Hash, +) CommitIter { seen := make(map[plumbing.Hash]bool) for _, h := range ignore { seen[h] = true } return &commitPreIterator{ - seen: seen, - stack: make([]CommitIter, 0), - start: c, + seenExternal: seenExternal, + seen: seen, + stack: make([]CommitIter, 0), + start: c, } } @@ -57,7 +63,7 @@ func (w *commitPreIterator) Next() (*Commit, error) { } } - if w.seen[c.Hash] { + if w.seen[c.Hash] || w.seenExternal[c.Hash] { continue } diff --git a/plumbing/object/commit_walker_test.go b/plumbing/object/commit_walker_test.go index 48b504d..a27104e 100644 --- a/plumbing/object/commit_walker_test.go +++ b/plumbing/object/commit_walker_test.go @@ -16,7 +16,7 @@ func (s *CommitWalkerSuite) TestCommitPreIterator(c *C) { commit := s.commit(c, s.Fixture.Head) var commits []*Commit - NewCommitPreorderIter(commit, nil).ForEach(func(c *Commit) error { + NewCommitPreorderIter(commit, nil, nil).ForEach(func(c *Commit) error { commits = append(commits, c) return nil }) @@ -42,7 +42,7 @@ func (s *CommitWalkerSuite) TestCommitPreIteratorWithIgnore(c *C) { commit := s.commit(c, s.Fixture.Head) var commits []*Commit - NewCommitPreorderIter(commit, []plumbing.Hash{ + NewCommitPreorderIter(commit, nil, []plumbing.Hash{ plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), }).ForEach(func(c *Commit) error { commits = append(commits, c) @@ -60,6 +60,30 @@ func (s *CommitWalkerSuite) TestCommitPreIteratorWithIgnore(c *C) { } } +func (s *CommitWalkerSuite) TestCommitPreIteratorWithSeenExternal(c *C) { + commit := s.commit(c, s.Fixture.Head) + + var commits []*Commit + seenExternal := map[plumbing.Hash]bool{ + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"): true, + } + NewCommitPreorderIter(commit, seenExternal, nil). + ForEach(func(c *Commit) error { + commits = append(commits, c) + return nil + }) + + c.Assert(commits, HasLen, 2) + + expected := []string{ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + "918c48b83bd081e863dbe1b80f8998f058cd8294", + } + for i, commit := range commits { + c.Assert(commit.Hash.String(), Equals, expected[i]) + } +} + func (s *CommitWalkerSuite) TestCommitPostIterator(c *C) { commit := s.commit(c, s.Fixture.Head) diff --git a/plumbing/object/file.go b/plumbing/object/file.go index 79f57fe..40b5206 100644 --- a/plumbing/object/file.go +++ b/plumbing/object/file.go @@ -81,7 +81,7 @@ type FileIter struct { // NewFileIter takes a storer.EncodedObjectStorer and a Tree and returns a // *FileIter that iterates over all files contained in the tree, recursively. func NewFileIter(s storer.EncodedObjectStorer, t *Tree) *FileIter { - return &FileIter{s: s, w: *NewTreeWalker(t, true)} + return &FileIter{s: s, w: *NewTreeWalker(t, true, nil)} } // Next moves the iterator to the next file and returns a pointer to it. If diff --git a/plumbing/object/patch.go b/plumbing/object/patch.go index d413114..a920631 100644 --- a/plumbing/object/patch.go +++ b/plumbing/object/patch.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" "io" + "math" + "strings" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" @@ -105,6 +107,10 @@ func (p *Patch) Encode(w io.Writer) error { return ue.Encode(p) } +func (p *Patch) Stats() FileStats { + return getFileStatsFromFilePatches(p.FilePatches()) +} + func (p *Patch) String() string { buf := bytes.NewBuffer(nil) err := p.Encode(buf) @@ -185,3 +191,112 @@ func (t *textChunk) Content() string { func (t *textChunk) Type() fdiff.Operation { return t.op } + +// FileStat stores the status of changes in content of a file. +type FileStat struct { + Name string + Addition int + Deletion int +} + +func (fs FileStat) String() string { + return printStat([]FileStat{fs}) +} + +// FileStats is a collection of FileStat. +type FileStats []FileStat + +func (fileStats FileStats) String() string { + return printStat(fileStats) +} + +func printStat(fileStats []FileStat) string { + padLength := float64(len(" ")) + newlineLength := float64(len("\n")) + separatorLength := float64(len("|")) + // Soft line length limit. The text length calculation below excludes + // length of the change number. Adding that would take it closer to 80, + // but probably not more than 80, until it's a huge number. + lineLength := 72.0 + + // Get the longest filename and longest total change. + var longestLength float64 + var longestTotalChange float64 + for _, fs := range fileStats { + if int(longestLength) < len(fs.Name) { + longestLength = float64(len(fs.Name)) + } + totalChange := fs.Addition + fs.Deletion + if int(longestTotalChange) < totalChange { + longestTotalChange = float64(totalChange) + } + } + + // Parts of the output: + // <pad><filename><pad>|<pad><changeNumber><pad><+++/---><newline> + // example: " main.go | 10 +++++++--- " + + // <pad><filename><pad> + leftTextLength := padLength + longestLength + padLength + + // <pad><number><pad><+++++/-----><newline> + // Excluding number length here. + rightTextLength := padLength + padLength + newlineLength + + totalTextArea := leftTextLength + separatorLength + rightTextLength + heightOfHistogram := lineLength - totalTextArea + + // Scale the histogram. + var scaleFactor float64 + if longestTotalChange > heightOfHistogram { + // Scale down to heightOfHistogram. + scaleFactor = float64(longestTotalChange / heightOfHistogram) + } else { + scaleFactor = 1.0 + } + + finalOutput := "" + for _, fs := range fileStats { + addn := float64(fs.Addition) + deln := float64(fs.Deletion) + adds := strings.Repeat("+", int(math.Floor(addn/scaleFactor))) + dels := strings.Repeat("-", int(math.Floor(deln/scaleFactor))) + finalOutput += fmt.Sprintf(" %s | %d %s%s\n", fs.Name, (fs.Addition + fs.Deletion), adds, dels) + } + + return finalOutput +} + +func getFileStatsFromFilePatches(filePatches []fdiff.FilePatch) FileStats { + var fileStats FileStats + + for _, fp := range filePatches { + cs := FileStat{} + from, to := fp.Files() + if from == nil { + // New File is created. + cs.Name = to.Path() + } else if to == nil { + // File is deleted. + cs.Name = from.Path() + } else if from.Path() != to.Path() { + // File is renamed. Not supported. + // cs.Name = fmt.Sprintf("%s => %s", from.Path(), to.Path()) + } else { + cs.Name = from.Path() + } + + for _, chunk := range fp.Chunks() { + switch chunk.Type() { + case fdiff.Add: + cs.Addition += strings.Count(chunk.Content(), "\n") + case fdiff.Delete: + cs.Deletion += strings.Count(chunk.Content(), "\n") + } + } + + fileStats = append(fileStats, cs) + } + + return fileStats +} diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index 512db9f..44ac720 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -297,9 +297,10 @@ func (iter *treeEntryIter) Next() (TreeEntry, error) { // TreeWalker provides a means of walking through all of the entries in a Tree. type TreeWalker struct { - stack []treeEntryIter + stack []*treeEntryIter base string recursive bool + seen map[plumbing.Hash]bool s storer.EncodedObjectStorer t *Tree @@ -309,13 +310,14 @@ type TreeWalker struct { // // It is the caller's responsibility to call Close() when finished with the // tree walker. -func NewTreeWalker(t *Tree, recursive bool) *TreeWalker { - stack := make([]treeEntryIter, 0, startingStackSize) - stack = append(stack, treeEntryIter{t, 0}) +func NewTreeWalker(t *Tree, recursive bool, seen map[plumbing.Hash]bool) *TreeWalker { + stack := make([]*treeEntryIter, 0, startingStackSize) + stack = append(stack, &treeEntryIter{t, 0}) return &TreeWalker{ stack: stack, recursive: recursive, + seen: seen, s: t.s, t: t, @@ -358,6 +360,10 @@ func (w *TreeWalker) Next() (name string, entry TreeEntry, err error) { return } + if w.seen[entry.Hash] { + continue + } + if entry.Mode == filemode.Dir { obj, err = GetTree(w.s, entry.Hash) } @@ -377,7 +383,7 @@ func (w *TreeWalker) Next() (name string, entry TreeEntry, err error) { } if t, ok := obj.(*Tree); ok { - w.stack = append(w.stack, treeEntryIter{t, 0}) + w.stack = append(w.stack, &treeEntryIter{t, 0}) w.base = path.Join(w.base, entry.Name) } diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index aa86517..796d979 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -228,7 +228,7 @@ func (s *TreeSuite) TestTreeWalkerNext(c *C) { tree, err := commit.Tree() c.Assert(err, IsNil) - walker := NewTreeWalker(tree, true) + walker := NewTreeWalker(tree, true, nil) for _, e := range treeWalkerExpects { name, entry, err := walker.Next() if err == io.EOF { @@ -245,13 +245,39 @@ func (s *TreeSuite) TestTreeWalkerNext(c *C) { } } +func (s *TreeSuite) TestTreeWalkerNextSkipSeen(c *C) { + commit, err := GetCommit(s.Storer, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) + c.Assert(err, IsNil) + tree, err := commit.Tree() + c.Assert(err, IsNil) + + seen := map[plumbing.Hash]bool{ + plumbing.NewHash(treeWalkerExpects[0].Hash): true, + } + walker := NewTreeWalker(tree, true, seen) + for _, e := range treeWalkerExpects[1:] { + name, entry, err := walker.Next() + if err == io.EOF { + break + } + + c.Assert(err, IsNil) + c.Assert(name, Equals, e.Path) + c.Assert(entry.Name, Equals, e.Name) + c.Assert(entry.Mode, Equals, e.Mode) + c.Assert(entry.Hash.String(), Equals, e.Hash) + + c.Assert(walker.Tree().ID().String(), Equals, e.Tree) + } +} + func (s *TreeSuite) TestTreeWalkerNextNonRecursive(c *C) { commit := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) tree, err := commit.Tree() c.Assert(err, IsNil) var count int - walker := NewTreeWalker(tree, false) + walker := NewTreeWalker(tree, false, nil) for { name, entry, err := walker.Next() if err == io.EOF { @@ -290,7 +316,7 @@ func (s *TreeSuite) TestTreeWalkerNextSubmodule(c *C) { } var count int - walker := NewTreeWalker(tree, true) + walker := NewTreeWalker(tree, true, nil) defer walker.Close() for { diff --git a/plumbing/object/treenoder.go b/plumbing/object/treenoder.go index bd65abc..52f0e61 100644 --- a/plumbing/object/treenoder.go +++ b/plumbing/object/treenoder.go @@ -99,7 +99,7 @@ func transformChildren(t *Tree) ([]noder.Noder, error) { // is bigger than needed. ret := make([]noder.Noder, 0, len(t.Entries)) - walker := NewTreeWalker(t, false) // don't recurse + walker := NewTreeWalker(t, false, nil) // don't recurse // don't defer walker.Close() for efficiency reasons. for { _, e, err = walker.Next() diff --git a/plumbing/protocol/packp/advrefs_encode.go b/plumbing/protocol/packp/advrefs_encode.go index e981120..cb93d46 100644 --- a/plumbing/protocol/packp/advrefs_encode.go +++ b/plumbing/protocol/packp/advrefs_encode.go @@ -2,6 +2,7 @@ package packp import ( "bytes" + "fmt" "io" "sort" @@ -21,9 +22,13 @@ func (a *AdvRefs) Encode(w io.Writer) error { } type advRefsEncoder struct { - data *AdvRefs // data to encode - pe *pktline.Encoder // where to write the encoded data - err error // sticky error + data *AdvRefs // data to encode + pe *pktline.Encoder // where to write the encoded data + firstRefName string // reference name to encode in the first pkt-line (HEAD if present) + firstRefHash plumbing.Hash // hash referenced to encode in the first pkt-line (HEAD if present) + sortedRefs []string // hash references to encode ordered by increasing order + err error // sticky error + } func newAdvRefsEncoder(w io.Writer) *advRefsEncoder { @@ -34,6 +39,8 @@ func newAdvRefsEncoder(w io.Writer) *advRefsEncoder { func (e *advRefsEncoder) Encode(v *AdvRefs) error { e.data = v + e.sortRefs() + e.setFirstRef() for state := encodePrefix; state != nil; { state = state(e) @@ -42,6 +49,32 @@ func (e *advRefsEncoder) Encode(v *AdvRefs) error { return e.err } +func (e *advRefsEncoder) sortRefs() { + if len(e.data.References) > 0 { + refs := make([]string, 0, len(e.data.References)) + for refName := range e.data.References { + refs = append(refs, refName) + } + + sort.Strings(refs) + e.sortedRefs = refs + } +} + +func (e *advRefsEncoder) setFirstRef() { + if e.data.Head != nil { + e.firstRefName = head + e.firstRefHash = *e.data.Head + return + } + + if len(e.sortedRefs) > 0 { + refName := e.sortedRefs[0] + e.firstRefName = refName + e.firstRefHash = e.data.References[refName] + } +} + type encoderStateFn func(*advRefsEncoder) encoderStateFn func encodePrefix(e *advRefsEncoder) encoderStateFn { @@ -61,33 +94,27 @@ func encodePrefix(e *advRefsEncoder) encoderStateFn { } // Adds the first pkt-line payload: head hash, head ref and capabilities. -// Also handle the special case when no HEAD ref is found. +// If HEAD ref is not found, the first reference ordered in increasing order will be used. +// If there aren't HEAD neither refs, the first line will be "PKT-LINE(zero-id SP "capabilities^{}" NUL capability-list)". +// See: https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt +// See: https://github.com/git/git/blob/master/Documentation/technical/protocol-common.txt func encodeFirstLine(e *advRefsEncoder) encoderStateFn { - head := formatHead(e.data.Head) - separator := formatSeparator(e.data.Head) + const formatFirstLine = "%s %s\x00%s\n" + var firstLine string capabilities := formatCaps(e.data.Capabilities) - if e.err = e.pe.Encodef("%s %s\x00%s\n", head, separator, capabilities); e.err != nil { - return nil - } + if e.firstRefName == "" { + firstLine = fmt.Sprintf(formatFirstLine, plumbing.ZeroHash.String(), "capabilities^{}", capabilities) + } else { + firstLine = fmt.Sprintf(formatFirstLine, e.firstRefHash.String(), e.firstRefName, capabilities) - return encodeRefs -} - -func formatHead(h *plumbing.Hash) string { - if h == nil { - return plumbing.ZeroHash.String() } - return h.String() -} - -func formatSeparator(h *plumbing.Hash) string { - if h == nil { - return noHead + if e.err = e.pe.EncodeString(firstLine); e.err != nil { + return nil } - return head + return encodeRefs } func formatCaps(c *capability.List) string { @@ -101,8 +128,11 @@ func formatCaps(c *capability.List) string { // Adds the (sorted) refs: hash SP refname EOL // and their peeled refs if any. func encodeRefs(e *advRefsEncoder) encoderStateFn { - refs := sortRefs(e.data.References) - for _, r := range refs { + for _, r := range e.sortedRefs { + if r == e.firstRefName { + continue + } + hash, _ := e.data.References[r] if e.err = e.pe.Encodef("%s %s\n", hash.String(), r); e.err != nil { return nil @@ -118,16 +148,6 @@ func encodeRefs(e *advRefsEncoder) encoderStateFn { return encodeShallow } -func sortRefs(m map[string]plumbing.Hash) []string { - ret := make([]string, 0, len(m)) - for k := range m { - ret = append(ret, k) - } - sort.Strings(ret) - - return ret -} - // Adds the (sorted) shallows: "shallow" SP hash EOL func encodeShallow(e *advRefsEncoder) encoderStateFn { sorted := sortShallows(e.data.Shallows) diff --git a/plumbing/protocol/packp/advrefs_encode_test.go b/plumbing/protocol/packp/advrefs_encode_test.go index f901440..3ae84a7 100644 --- a/plumbing/protocol/packp/advrefs_encode_test.go +++ b/plumbing/protocol/packp/advrefs_encode_test.go @@ -99,8 +99,7 @@ func (s *AdvRefsEncodeSuite) TestRefs(c *C) { } expected := pktlines(c, - "0000000000000000000000000000000000000000 capabilities^{}\x00\n", - "a6930aaee06755d1bdcfd943fbf614e4d92bb0c7 refs/heads/master\n", + "a6930aaee06755d1bdcfd943fbf614e4d92bb0c7 refs/heads/master\x00\n", "5dc01c595e6c6ec9ccda4f6f69c131c0dd945f8c refs/tags/v2.6.11-tree\n", "1111111111111111111111111111111111111111 refs/tags/v2.6.12-tree\n", "2222222222222222222222222222222222222222 refs/tags/v2.6.13-tree\n", @@ -129,8 +128,7 @@ func (s *AdvRefsEncodeSuite) TestPeeled(c *C) { } expected := pktlines(c, - "0000000000000000000000000000000000000000 capabilities^{}\x00\n", - "a6930aaee06755d1bdcfd943fbf614e4d92bb0c7 refs/heads/master\n", + "a6930aaee06755d1bdcfd943fbf614e4d92bb0c7 refs/heads/master\x00\n", "5dc01c595e6c6ec9ccda4f6f69c131c0dd945f8c refs/tags/v2.6.11-tree\n", "1111111111111111111111111111111111111111 refs/tags/v2.6.12-tree\n", "5555555555555555555555555555555555555555 refs/tags/v2.6.12-tree^{}\n", diff --git a/plumbing/protocol/packp/capability/capability.go b/plumbing/protocol/packp/capability/capability.go index 96d93f6..a129781 100644 --- a/plumbing/protocol/packp/capability/capability.go +++ b/plumbing/protocol/packp/capability/capability.go @@ -234,7 +234,7 @@ const ( const DefaultAgent = "go-git/4.x" -var valid = map[Capability]bool{ +var known = map[Capability]bool{ MultiACK: true, MultiACKDetailed: true, NoDone: true, ThinPack: true, Sideband: true, Sideband64k: true, OFSDelta: true, Agent: true, Shallow: true, DeepenSince: true, DeepenNot: true, DeepenRelative: true, diff --git a/plumbing/protocol/packp/capability/list.go b/plumbing/protocol/packp/capability/list.go index 3904a4e..26a79b6 100644 --- a/plumbing/protocol/packp/capability/list.go +++ b/plumbing/protocol/packp/capability/list.go @@ -108,7 +108,7 @@ func (l *List) Add(c Capability, values ...string) error { return nil } - if !multipleArgument[c] && len(l.m[c].Values) > 0 { + if known[c] && !multipleArgument[c] && len(l.m[c].Values) > 0 { return ErrMultipleArguments } @@ -116,7 +116,19 @@ func (l *List) Add(c Capability, values ...string) error { return nil } +func (l *List) validateNoEmptyArgs(values []string) error { + for _, v := range values { + if v == "" { + return ErrEmtpyArgument + } + } + return nil +} + func (l *List) validate(c Capability, values []string) error { + if !known[c] { + return l.validateNoEmptyArgs(values) + } if requiresArgument[c] && len(values) == 0 { return ErrArgumentsRequired } @@ -128,14 +140,7 @@ func (l *List) validate(c Capability, values []string) error { if !multipleArgument[c] && len(values) > 1 { return ErrMultipleArguments } - - for _, v := range values { - if v == "" { - return ErrEmtpyArgument - } - } - - return nil + return l.validateNoEmptyArgs(values) } // Supports returns true if capability is present diff --git a/plumbing/protocol/packp/capability/list_test.go b/plumbing/protocol/packp/capability/list_test.go index 9665e89..82dd63f 100644 --- a/plumbing/protocol/packp/capability/list_test.go +++ b/plumbing/protocol/packp/capability/list_test.go @@ -65,6 +65,26 @@ func (s *SuiteCapabilities) TestDecodeWithUnknownCapability(c *check.C) { c.Assert(cap.Supports(Capability("foo")), check.Equals, true) } +func (s *SuiteCapabilities) TestDecodeWithUnknownCapabilityWithArgument(c *check.C) { + cap := NewList() + err := cap.Decode([]byte("oldref=HEAD:refs/heads/v2 thin-pack")) + c.Assert(err, check.IsNil) + + c.Assert(cap.m, check.HasLen, 2) + c.Assert(cap.Get("oldref"), check.DeepEquals, []string{"HEAD:refs/heads/v2"}) + c.Assert(cap.Get(ThinPack), check.IsNil) +} + +func (s *SuiteCapabilities) TestDecodeWithUnknownCapabilityWithMultipleArgument(c *check.C) { + cap := NewList() + err := cap.Decode([]byte("foo=HEAD:refs/heads/v2 foo=HEAD:refs/heads/v1 thin-pack")) + c.Assert(err, check.IsNil) + + c.Assert(cap.m, check.HasLen, 2) + c.Assert(cap.Get("foo"), check.DeepEquals, []string{"HEAD:refs/heads/v2", "HEAD:refs/heads/v1"}) + c.Assert(cap.Get(ThinPack), check.IsNil) +} + func (s *SuiteCapabilities) TestString(c *check.C) { cap := NewList() cap.Set(Agent, "bar") @@ -153,7 +173,7 @@ func (s *SuiteCapabilities) TestAddErrArgumentsNotAllowed(c *check.C) { c.Assert(err, check.Equals, ErrArguments) } -func (s *SuiteCapabilities) TestAddErrArgumendts(c *check.C) { +func (s *SuiteCapabilities) TestAddErrArguments(c *check.C) { cap := NewList() err := cap.Add(SymRef, "") c.Assert(err, check.Equals, ErrEmtpyArgument) diff --git a/plumbing/protocol/packp/updreq.go b/plumbing/protocol/packp/updreq.go index b246613..73be117 100644 --- a/plumbing/protocol/packp/updreq.go +++ b/plumbing/protocol/packp/updreq.go @@ -6,6 +6,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" ) var ( @@ -21,6 +22,9 @@ type ReferenceUpdateRequest struct { Shallow *plumbing.Hash // Packfile contains an optional packfile reader. Packfile io.ReadCloser + + // Progress receives sideband progress messages from the server + Progress sideband.Progress } // New returns a pointer to a new ReferenceUpdateRequest value. diff --git a/plumbing/revlist/revlist.go b/plumbing/revlist/revlist.go index f56cf28..0a9d1e8 100644 --- a/plumbing/revlist/revlist.go +++ b/plumbing/revlist/revlist.go @@ -35,9 +35,9 @@ func objects( ignore []plumbing.Hash, allowMissingObjects bool, ) ([]plumbing.Hash, error) { - seen := hashListToSet(ignore) result := make(map[plumbing.Hash]bool) + visited := make(map[plumbing.Hash]bool) walkerFunc := func(h plumbing.Hash) { if !seen[h] { @@ -47,7 +47,7 @@ func objects( } for _, h := range objects { - if err := processObject(s, h, seen, ignore, walkerFunc); err != nil { + if err := processObject(s, h, seen, visited, ignore, walkerFunc); err != nil { if allowMissingObjects && err == plumbing.ErrObjectNotFound { continue } @@ -64,6 +64,7 @@ func processObject( s storer.EncodedObjectStorer, h plumbing.Hash, seen map[plumbing.Hash]bool, + visited map[plumbing.Hash]bool, ignore []plumbing.Hash, walkerFunc func(h plumbing.Hash), ) error { @@ -83,12 +84,12 @@ func processObject( switch do := do.(type) { case *object.Commit: - return reachableObjects(do, seen, ignore, walkerFunc) + return reachableObjects(do, seen, visited, ignore, walkerFunc) case *object.Tree: return iterateCommitTrees(seen, do, walkerFunc) case *object.Tag: walkerFunc(do.Hash) - return processObject(s, do.Target, seen, ignore, walkerFunc) + return processObject(s, do.Target, seen, visited, ignore, walkerFunc) case *object.Blob: walkerFunc(do.Hash) default: @@ -106,13 +107,36 @@ func processObject( func reachableObjects( commit *object.Commit, seen map[plumbing.Hash]bool, + visited map[plumbing.Hash]bool, ignore []plumbing.Hash, - cb func(h plumbing.Hash)) error { + cb func(h plumbing.Hash), +) error { + i := object.NewCommitPreorderIter(commit, seen, ignore) + pending := make(map[plumbing.Hash]bool) + addPendingParents(pending, visited, commit) + + for { + commit, err := i.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + if pending[commit.Hash] { + delete(pending, commit.Hash) + } + + addPendingParents(pending, visited, commit) + + if visited[commit.Hash] && len(pending) == 0 { + break + } - i := object.NewCommitPreorderIter(commit, ignore) - return i.ForEach(func(commit *object.Commit) error { if seen[commit.Hash] { - return nil + continue } cb(commit.Hash) @@ -122,22 +146,35 @@ func reachableObjects( return err } - return iterateCommitTrees(seen, tree, cb) - }) + if err := iterateCommitTrees(seen, tree, cb); err != nil { + return err + } + } + + return nil +} + +func addPendingParents(pending, visited map[plumbing.Hash]bool, commit *object.Commit) { + for _, p := range commit.ParentHashes { + if !visited[p] { + pending[p] = true + } + } } // iterateCommitTrees iterate all reachable trees from the given commit func iterateCommitTrees( seen map[plumbing.Hash]bool, tree *object.Tree, - cb func(h plumbing.Hash)) error { + cb func(h plumbing.Hash), +) error { if seen[tree.Hash] { return nil } cb(tree.Hash) - treeWalker := object.NewTreeWalker(tree, true) + treeWalker := object.NewTreeWalker(tree, true, seen) for { _, e, err := treeWalker.Next() diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go index dd1e8c1..643e3eb 100644 --- a/plumbing/revlist/revlist_test.go +++ b/plumbing/revlist/revlist_test.go @@ -217,3 +217,60 @@ func (s *RevListSuite) TestRevListObjectsNewBranch(c *C) { } c.Assert(len(remoteHist), Equals, len(revList)) } + +// This tests will ensure that a5b8b09 and b8e471f will be visited even if +// 35e8510 has already been visited and will not stop iterating until they +// have been as well. +// +// * af2d6a6 some json +// * 1669dce Merge branch 'master' +// |\ +// | * a5b8b09 Merge pull request #1 +// | |\ +// | | * b8e471f Creating changelog +// | |/ +// * | 35e8510 binary file +// |/ +// * b029517 Initial commit +func (s *RevListSuite) TestReachableObjectsNoRevisit(c *C) { + obj, err := s.Storer.EncodedObject(plumbing.CommitObject, plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a")) + c.Assert(err, IsNil) + + do, err := object.DecodeObject(s.Storer, obj) + c.Assert(err, IsNil) + + commit, ok := do.(*object.Commit) + c.Assert(ok, Equals, true) + + var visited []plumbing.Hash + err = reachableObjects( + commit, + map[plumbing.Hash]bool{ + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"): true, + }, + map[plumbing.Hash]bool{ + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"): true, + }, + nil, + func(h plumbing.Hash) { + obj, err := s.Storer.EncodedObject(plumbing.AnyObject, h) + c.Assert(err, IsNil) + + do, err := object.DecodeObject(s.Storer, obj) + c.Assert(err, IsNil) + + if _, ok := do.(*object.Commit); ok { + visited = append(visited, h) + } + }, + ) + c.Assert(err, IsNil) + + c.Assert(visited, DeepEquals, []plumbing.Hash{ + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), + plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea"), + plumbing.NewHash("a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"), + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"), + }) +} diff --git a/plumbing/transport/common.go b/plumbing/transport/common.go index 2088500..ac71bb3 100644 --- a/plumbing/transport/common.go +++ b/plumbing/transport/common.go @@ -187,6 +187,7 @@ func (e urlEndpoint) Path() string { type scpEndpoint struct { user string host string + port string path string } @@ -194,8 +195,14 @@ func (e *scpEndpoint) Protocol() string { return "ssh" } func (e *scpEndpoint) User() string { return e.user } func (e *scpEndpoint) Password() string { return "" } func (e *scpEndpoint) Host() string { return e.host } -func (e *scpEndpoint) Port() int { return 22 } func (e *scpEndpoint) Path() string { return e.path } +func (e *scpEndpoint) Port() int { + i, err := strconv.Atoi(e.port) + if err != nil { + return 22 + } + return i +} func (e *scpEndpoint) String() string { var user string @@ -220,7 +227,7 @@ func (e *fileEndpoint) String() string { return e.path } var ( isSchemeRegExp = regexp.MustCompile(`^[^:]+://`) - scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?P<path>[^\\].*)$`) + scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?:(?P<port>[0-9]{1,5})/)?(?P<path>[^\\].*)$`) ) func parseSCPLike(endpoint string) (Endpoint, bool) { @@ -232,7 +239,8 @@ func parseSCPLike(endpoint string) (Endpoint, bool) { return &scpEndpoint{ user: m[1], host: m[2], - path: m[3], + port: m[3], + path: m[4], }, true } diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index ec617bd..52759e6 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -74,6 +74,18 @@ func (s *SuiteCommon) TestNewEndpointSCPLike(c *C) { c.Assert(e.String(), Equals, "git@github.com:user/repository.git") } +func (s *SuiteCommon) TestNewEndpointSCPLikeWithPort(c *C) { + e, err := NewEndpoint("git@github.com:9999/user/repository.git") + c.Assert(err, IsNil) + c.Assert(e.Protocol(), Equals, "ssh") + c.Assert(e.User(), Equals, "git") + c.Assert(e.Password(), Equals, "") + c.Assert(e.Host(), Equals, "github.com") + c.Assert(e.Port(), Equals, 9999) + c.Assert(e.Path(), Equals, "user/repository.git") + c.Assert(e.String(), Equals, "git@github.com:user/repository.git") +} + func (s *SuiteCommon) TestNewEndpointFileAbs(c *C) { e, err := NewEndpoint("/foo.git") c.Assert(err, IsNil) diff --git a/plumbing/transport/file/client.go b/plumbing/transport/file/client.go index 0b42abf..d229fdd 100644 --- a/plumbing/transport/file/client.go +++ b/plumbing/transport/file/client.go @@ -2,9 +2,13 @@ package file import ( + "bufio" + "errors" "io" "os" "os/exec" + "path/filepath" + "strings" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common" @@ -30,6 +34,45 @@ func NewClient(uploadPackBin, receivePackBin string) transport.Transport { }) } +func prefixExecPath(cmd string) (string, error) { + // Use `git --exec-path` to find the exec path. + execCmd := exec.Command("git", "--exec-path") + + stdout, err := execCmd.StdoutPipe() + if err != nil { + return "", err + } + stdoutBuf := bufio.NewReader(stdout) + + err = execCmd.Start() + if err != nil { + return "", err + } + + execPathBytes, isPrefix, err := stdoutBuf.ReadLine() + if err != nil { + return "", err + } + if isPrefix { + return "", errors.New("Couldn't read exec-path line all at once") + } + + err = execCmd.Wait() + if err != nil { + return "", err + } + execPath := string(execPathBytes) + execPath = strings.TrimSpace(execPath) + cmd = filepath.Join(execPath, cmd) + + // Make sure it actually exists. + _, err = exec.LookPath(cmd) + if err != nil { + return "", err + } + return cmd, nil +} + func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthMethod, ) (common.Command, error) { @@ -40,8 +83,16 @@ func (r *runner) Command(cmd string, ep transport.Endpoint, auth transport.AuthM cmd = r.ReceivePackBin } - if _, err := exec.LookPath(cmd); err != nil { - return nil, err + _, err := exec.LookPath(cmd) + if err != nil { + if e, ok := err.(*exec.Error); ok && e.Err == exec.ErrNotFound { + cmd, err = prefixExecPath(cmd) + if err != nil { + return nil, err + } + } else { + return nil, err + } } return &command{cmd: exec.Command(cmd, ep.Path())}, nil diff --git a/plumbing/transport/file/client_test.go b/plumbing/transport/file/client_test.go index 030175e..864cddc 100644 --- a/plumbing/transport/file/client_test.go +++ b/plumbing/transport/file/client_test.go @@ -14,6 +14,28 @@ import ( func Test(t *testing.T) { TestingT(t) } +type ClientSuite struct { + CommonSuite +} + +var _ = Suite(&ClientSuite{}) + +func (s *ClientSuite) TestCommand(c *C) { + runner := &runner{ + UploadPackBin: transport.UploadPackServiceName, + ReceivePackBin: transport.ReceivePackServiceName, + } + ep, err := transport.NewEndpoint(filepath.Join("fake", "repo")) + c.Assert(err, IsNil) + var emptyAuth transport.AuthMethod + _, err = runner.Command("git-receive-pack", ep, emptyAuth) + c.Assert(err, IsNil) + + // Make sure we get an error for one that doesn't exist. + _, err = runner.Command("git-fake-command", ep, emptyAuth) + c.Assert(err, NotNil) +} + const bareConfig = `[core] repositoryformatversion = 0 filemode = true diff --git a/plumbing/transport/file/server_test.go b/plumbing/transport/file/server_test.go index ee72282..080beef 100644 --- a/plumbing/transport/file/server_test.go +++ b/plumbing/transport/file/server_test.go @@ -35,6 +35,10 @@ func (s *ServerSuite) SetUpSuite(c *C) { } func (s *ServerSuite) TestPush(c *C) { + if !s.checkExecPerm(c) { + c.Skip("go-git binary has not execution permissions") + } + // git <2.0 cannot push to an empty repository without a refspec. cmd := exec.Command("git", "push", "--receive-pack", s.ReceivePackBin, @@ -48,6 +52,10 @@ func (s *ServerSuite) TestPush(c *C) { } func (s *ServerSuite) TestClone(c *C) { + if !s.checkExecPerm(c) { + c.Skip("go-git binary has not execution permissions") + } + pathToClone := c.MkDir() cmd := exec.Command("git", "clone", @@ -59,3 +67,10 @@ func (s *ServerSuite) TestClone(c *C) { out, err := cmd.CombinedOutput() c.Assert(err, IsNil, Commentf("combined stdout and stderr:\n%s\n", out)) } + +func (s *ServerSuite) checkExecPerm(c *C) bool { + const userExecPermMask = 0100 + info, err := os.Stat(s.ReceivePackBin) + c.Assert(err, IsNil) + return (info.Mode().Perm() & userExecPermMask) == userExecPermMask +} diff --git a/plumbing/transport/git/receive_pack_test.go b/plumbing/transport/git/receive_pack_test.go index f9afede..7b0fa46 100644 --- a/plumbing/transport/git/receive_pack_test.go +++ b/plumbing/transport/git/receive_pack_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "time" @@ -29,6 +30,11 @@ type ReceivePackSuite struct { var _ = Suite(&ReceivePackSuite{}) func (s *ReceivePackSuite) SetUpTest(c *C) { + if runtime.GOOS == "windows" { + c.Skip(`git for windows has issues with write operations through git:// protocol. + See https://github.com/git-for-windows/git/issues/907`) + } + s.ReceivePackSuite.Client = DefaultClient port, err := freePort() diff --git a/plumbing/transport/http/receive_pack.go b/plumbing/transport/http/receive_pack.go index b54b70f..d2dfeb7 100644 --- a/plumbing/transport/http/receive_pack.go +++ b/plumbing/transport/http/receive_pack.go @@ -9,6 +9,8 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -52,6 +54,17 @@ func (s *rpSession) ReceivePack(ctx context.Context, req *packp.ReferenceUpdateR return nil, err } + var d *sideband.Demuxer + if req.Capabilities.Supports(capability.Sideband64k) { + d = sideband.NewDemuxer(sideband.Sideband64k, r) + } else if req.Capabilities.Supports(capability.Sideband) { + d = sideband.NewDemuxer(sideband.Sideband, r) + } + if d != nil { + d.Progress = req.Progress + r = d + } + rc := ioutil.NewReadCloser(r, res.Body) report := packp.NewReportStatus() diff --git a/plumbing/transport/http/receive_pack_test.go b/plumbing/transport/http/receive_pack_test.go index d870e5d..970121d 100644 --- a/plumbing/transport/http/receive_pack_test.go +++ b/plumbing/transport/http/receive_pack_test.go @@ -25,6 +25,8 @@ type ReceivePackSuite struct { fixtures.Suite base string + host string + port int } var _ = Suite(&ReceivePackSuite{}) @@ -32,52 +34,35 @@ var _ = Suite(&ReceivePackSuite{}) func (s *ReceivePackSuite) SetUpTest(c *C) { s.ReceivePackSuite.Client = DefaultClient - port, err := freePort() + l, err := net.Listen("tcp", "localhost:0") c.Assert(err, IsNil) base, err := ioutil.TempDir(os.TempDir(), "go-git-http-backend-test") c.Assert(err, IsNil) - s.base = base - host := fmt.Sprintf("localhost_%d", port) - interpolatedBase := filepath.Join(base, host) - err = os.MkdirAll(interpolatedBase, 0755) - c.Assert(err, IsNil) - - dotgit := fixtures.Basic().One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "basic.git")) - c.Assert(err, IsNil) - - ep, err := transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/basic.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.Endpoint = ep - - dotgit = fixtures.ByTag("empty").One().DotGit().Root() - prepareRepo(c, dotgit) - err = os.Rename(dotgit, filepath.Join(interpolatedBase, "empty.git")) - c.Assert(err, IsNil) + s.port = l.Addr().(*net.TCPAddr).Port + s.host = fmt.Sprintf("localhost_%d", s.port) + s.base = filepath.Join(base, s.host) - ep, err = transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/empty.git", port)) + err = os.MkdirAll(s.base, 0755) c.Assert(err, IsNil) - s.ReceivePackSuite.EmptyEndpoint = ep - ep, err = transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/non-existent.git", port)) - c.Assert(err, IsNil) - s.ReceivePackSuite.NonExistentEndpoint = ep + s.ReceivePackSuite.Endpoint = s.prepareRepository(c, fixtures.Basic().One(), "basic.git") + s.ReceivePackSuite.EmptyEndpoint = s.prepareRepository(c, fixtures.ByTag("empty").One(), "empty.git") + s.ReceivePackSuite.NonExistentEndpoint = s.newEndpoint(c, "non-existent.git") cmd := exec.Command("git", "--exec-path") out, err := cmd.CombinedOutput() c.Assert(err, IsNil) - p := filepath.Join(strings.Trim(string(out), "\n"), "git-http-backend") - h := &cgi.Handler{ - Path: p, - Env: []string{"GIT_HTTP_EXPORT_ALL=true", fmt.Sprintf("GIT_PROJECT_ROOT=%s", interpolatedBase)}, + server := &http.Server{ + Handler: &cgi.Handler{ + Path: filepath.Join(strings.Trim(string(out), "\n"), "git-http-backend"), + Env: []string{"GIT_HTTP_EXPORT_ALL=true", fmt.Sprintf("GIT_PROJECT_ROOT=%s", s.base)}, + }, } - go func() { - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), h)) + log.Fatal(server.Serve(l)) }() } @@ -86,37 +71,44 @@ func (s *ReceivePackSuite) TearDownTest(c *C) { c.Assert(err, IsNil) } -func freePort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - return 0, err - } +func (s *ReceivePackSuite) prepareRepository(c *C, f *fixtures.Fixture, name string) transport.Endpoint { + path := filepath.Join(s.base, name) - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, err - } + err := os.Rename(f.DotGit().Root(), path) + c.Assert(err, IsNil) - return l.Addr().(*net.TCPAddr).Port, l.Close() + s.setConfigToRepository(c, path) + return s.newEndpoint(c, name) } -const bareConfig = `[core] -repositoryformatversion = 0 -filemode = true -bare = true -[http] -receivepack = true` - -func prepareRepo(c *C, path string) { - // git-receive-pack refuses to update refs/heads/master on non-bare repo - // so we ensure bare repo config. - config := filepath.Join(path, "config") - if _, err := os.Stat(config); err == nil { - f, err := os.OpenFile(config, os.O_TRUNC|os.O_WRONLY, 0) - c.Assert(err, IsNil) - content := strings.NewReader(bareConfig) - _, err = io.Copy(f, content) - c.Assert(err, IsNil) - c.Assert(f.Close(), IsNil) - } +// git-receive-pack refuses to update refs/heads/master on non-bare repo +// so we ensure bare repo config. +func (s *ReceivePackSuite) setConfigToRepository(c *C, path string) { + cfgPath := filepath.Join(path, "config") + _, err := os.Stat(cfgPath) + c.Assert(err, IsNil) + + cfg, err := os.OpenFile(cfgPath, os.O_TRUNC|os.O_WRONLY, 0) + c.Assert(err, IsNil) + + content := strings.NewReader("" + + "[core]\n" + + "repositoryformatversion = 0\n" + + "filemode = true\n" + + "bare = true\n" + + "[http]\n" + + "receivepack = true\n", + ) + + _, err = io.Copy(cfg, content) + c.Assert(err, IsNil) + + c.Assert(cfg.Close(), IsNil) +} + +func (s *ReceivePackSuite) newEndpoint(c *C, name string) transport.Endpoint { + ep, err := transport.NewEndpoint(fmt.Sprintf("http://localhost:%d/%s", s.port, name)) + c.Assert(err, IsNil) + + return ep } diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index 2db8d54..598c6b1 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -18,6 +18,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/pktline" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability" + "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -298,13 +299,26 @@ func (s *session) ReceivePack(ctx context.Context, req *packp.ReferenceUpdateReq } if !req.Capabilities.Supports(capability.ReportStatus) { - // If we have neither report-status or sideband, we can only + // If we don't have report-status, we can only // check return value error. return nil, s.Command.Close() } + r := s.StdoutContext(ctx) + + var d *sideband.Demuxer + if req.Capabilities.Supports(capability.Sideband64k) { + d = sideband.NewDemuxer(sideband.Sideband64k, r) + } else if req.Capabilities.Supports(capability.Sideband) { + d = sideband.NewDemuxer(sideband.Sideband, r) + } + if d != nil { + d.Progress = req.Progress + r = d + } + report := packp.NewReportStatus() - if err := report.Decode(s.StdoutContext(ctx)); err != nil { + if err := report.Decode(r); err != nil { return nil, err } diff --git a/plumbing/transport/server/server.go b/plumbing/transport/server/server.go index be36de5..f896f7a 100644 --- a/plumbing/transport/server/server.go +++ b/plumbing/transport/server/server.go @@ -165,7 +165,8 @@ func (s *upSession) UploadPack(ctx context.Context, req *packp.UploadPackRequest pr, pw := io.Pipe() e := packfile.NewEncoder(pw, s.storer, false) go func() { - _, err := e.Encode(objs) + // TODO: plumb through a pack window. + _, err := e.Encode(objs, 10) pw.CloseWithError(err) }() diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index f95235b..baae181 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -3,6 +3,7 @@ package ssh import ( "crypto/x509" "encoding/pem" + "errors" "fmt" "io/ioutil" "os" @@ -11,6 +12,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/transport" + "github.com/mitchellh/go-homedir" "github.com/xanzy/ssh-agent" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" @@ -164,6 +166,19 @@ func (a *PublicKeys) clientConfig() *ssh.ClientConfig { } } +func username() (string, error) { + var username string + if user, err := user.Current(); err == nil { + username = user.Username + } else { + username = os.Getenv("USER") + } + if username == "" { + return "", errors.New("failed to get username") + } + return username, nil +} + // PublicKeysCallback implements AuthMethod by asking a // ssh.agent.Agent to act as a signer. type PublicKeysCallback struct { @@ -176,13 +191,12 @@ type PublicKeysCallback struct { // a pipe with the SSH agent and uses the pipe as the implementer of the public // key callback function. func NewSSHAgentAuth(u string) (AuthMethod, error) { + var err error if u == "" { - usr, err := user.Current() + u, err = username() if err != nil { - return nil, fmt.Errorf("error getting current user: %q", err) + return nil, err } - - u = usr.Username } a, _, err := sshagent.New() @@ -241,13 +255,13 @@ func getDefaultKnownHostsFiles() ([]string, error) { return files, nil } - user, err := user.Current() + homeDirPath, err := homedir.Dir() if err != nil { return nil, err } return []string{ - filepath.Join(user.HomeDir, "/.ssh/known_hosts"), + filepath.Join(homeDirPath, "/.ssh/known_hosts"), "/etc/ssh/ssh_known_hosts", }, nil } diff --git a/plumbing/transport/ssh/auth_method_test.go b/plumbing/transport/ssh/auth_method_test.go index aa05f7f..2ee5100 100644 --- a/plumbing/transport/ssh/auth_method_test.go +++ b/plumbing/transport/ssh/auth_method_test.go @@ -115,7 +115,7 @@ func (s *SuiteCommon) TestNewSSHAgentAuthNoAgent(c *C) { k, err := NewSSHAgentAuth("foo") c.Assert(k, IsNil) - c.Assert(err, ErrorMatches, ".*SSH_AUTH_SOCK.*") + c.Assert(err, ErrorMatches, ".*SSH_AUTH_SOCK.*|.*SSH agent .* not running.*") } func (*SuiteCommon) TestNewPublicKeys(c *C) { diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index d29d9ca..ed0f517 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -348,7 +348,7 @@ func (s *ReceivePackSuite) testSendPackDeleteReference(c *C) { func (s *ReceivePackSuite) emptyPackfile() io.ReadCloser { var buf bytes.Buffer e := packfile.NewEncoder(&buf, memory.NewStorage(), false) - _, err := e.Encode(nil) + _, err := e.Encode(nil, 10) if err != nil { panic(err) } diff --git a/plumbing/transport/test/upload_pack.go b/plumbing/transport/test/upload_pack.go index ade6cdc..b3acc4f 100644 --- a/plumbing/transport/test/upload_pack.go +++ b/plumbing/transport/test/upload_pack.go @@ -31,6 +31,8 @@ type UploadPackSuite struct { func (s *UploadPackSuite) TestAdvertisedReferencesEmpty(c *C) { r, err := s.Client.NewUploadPackSession(s.EmptyEndpoint, s.EmptyAuth) c.Assert(err, IsNil) + defer func() { c.Assert(r.Close(), IsNil) }() + ar, err := r.AdvertisedReferences() c.Assert(err, Equals, transport.ErrEmptyRemoteRepository) c.Assert(ar, IsNil) @@ -39,6 +41,8 @@ func (s *UploadPackSuite) TestAdvertisedReferencesEmpty(c *C) { func (s *UploadPackSuite) TestAdvertisedReferencesNotExists(c *C) { r, err := s.Client.NewUploadPackSession(s.NonExistentEndpoint, s.EmptyAuth) c.Assert(err, IsNil) + defer func() { c.Assert(r.Close(), IsNil) }() + ar, err := r.AdvertisedReferences() c.Assert(err, Equals, transport.ErrRepositoryNotFound) c.Assert(ar, IsNil) @@ -55,6 +59,8 @@ func (s *UploadPackSuite) TestAdvertisedReferencesNotExists(c *C) { func (s *UploadPackSuite) TestCallAdvertisedReferenceTwice(c *C) { r, err := s.Client.NewUploadPackSession(s.Endpoint, s.EmptyAuth) c.Assert(err, IsNil) + defer func() { c.Assert(r.Close(), IsNil) }() + ar1, err := r.AdvertisedReferences() c.Assert(err, IsNil) c.Assert(ar1, NotNil) @@ -27,6 +27,14 @@ var ( ErrDeleteRefNotSupported = errors.New("server does not support delete-refs") ) +const ( + // This describes the maximum number of commits to walk when + // computing the haves to send to a server, for each ref in the + // repo containing this remote, when not using the multi-ack + // protocol. Setting this to 0 means there is no limit. + maxHavesToVisitPerRef = 100 +) + // Remote represents a connection to a remote repository. type Remote struct { c *config.RemoteConfig @@ -65,7 +73,6 @@ func (r *Remote) Push(o *PushOptions) error { // operation is complete, an error is returned. The context only affects to the // transport operations. func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { - // TODO: Sideband support if err := o.Validate(); err != nil { return err } @@ -92,9 +99,14 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { } isDelete := false + allDelete := true for _, rs := range o.RefSpecs { if rs.IsDelete() { isDelete = true + } else { + allDelete = false + } + if isDelete && !allDelete { break } } @@ -103,9 +115,13 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { return ErrDeleteRefNotSupported } - req := packp.NewReferenceUpdateRequestFromCapabilities(ar.Capabilities) - if err := r.addReferencesToUpdate(o.RefSpecs, remoteRefs, req); err != nil { + localRefs, err := r.references() + if err != nil { + return err + } + req, err := r.newReferenceUpdateRequest(o, localRefs, remoteRefs, ar) + if err != nil { return err } @@ -132,9 +148,13 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { // we are aware. haves = append(haves, stop...) - hashesToPush, err := revlist.Objects(r.s, objects, haves) - if err != nil { - return err + var hashesToPush []plumbing.Hash + // Avoid the expensive revlist operation if we're only doing deletes. + if !allDelete { + hashesToPush, err = revlist.Objects(r.s, objects, haves) + if err != nil { + return err + } } rs, err := pushHashes(ctx, s, r.s, req, hashesToPush) @@ -149,6 +169,30 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error { return r.updateRemoteReferenceStorage(req, rs) } +func (r *Remote) newReferenceUpdateRequest( + o *PushOptions, + localRefs []*plumbing.Reference, + remoteRefs storer.ReferenceStorer, + ar *packp.AdvRefs, +) (*packp.ReferenceUpdateRequest, error) { + req := packp.NewReferenceUpdateRequestFromCapabilities(ar.Capabilities) + + if o.Progress != nil { + req.Progress = o.Progress + if ar.Capabilities.Supports(capability.Sideband64k) { + req.Capabilities.Set(capability.Sideband64k) + } else if ar.Capabilities.Supports(capability.Sideband) { + req.Capabilities.Set(capability.Sideband) + } + } + + if err := r.addReferencesToUpdate(o.RefSpecs, localRefs, remoteRefs, req); err != nil { + return nil, err + } + + return req, nil +} + func (r *Remote) updateRemoteReferenceStorage( req *packp.ReferenceUpdateRequest, result *packp.ReportStatus, @@ -236,6 +280,11 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt return nil, err } + localRefs, err := r.references() + if err != nil { + return nil, err + } + refs, err := calculateRefs(o.RefSpecs, remoteRefs, o.Tags) if err != nil { return nil, err @@ -243,7 +292,7 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt req.Wants, err = getWants(r.s, refs) if len(req.Wants) > 0 { - req.Haves, err = getHaves(r.s) + req.Haves, err = getHaves(localRefs, remoteRefs, r.s) if err != nil { return nil, err } @@ -253,7 +302,7 @@ func (r *Remote) fetch(ctx context.Context, o *FetchOptions) (storer.ReferenceSt } } - updated, err := r.updateLocalReferenceStorage(o.RefSpecs, refs, remoteRefs) + updated, err := r.updateLocalReferenceStorage(o.RefSpecs, refs, remoteRefs, o.Tags) if err != nil { return nil, err } @@ -320,17 +369,18 @@ func (r *Remote) fetchPack(ctx context.Context, o *FetchOptions, s transport.Upl return err } -func (r *Remote) addReferencesToUpdate(refspecs []config.RefSpec, +func (r *Remote) addReferencesToUpdate( + refspecs []config.RefSpec, + localRefs []*plumbing.Reference, remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest) error { - for _, rs := range refspecs { if rs.IsDelete() { if err := r.deleteReferences(rs, remoteRefs, req); err != nil { return err } } else { - if err := r.addOrUpdateReferences(rs, remoteRefs, req); err != nil { + if err := r.addOrUpdateReferences(rs, localRefs, remoteRefs, req); err != nil { return err } } @@ -339,18 +389,20 @@ func (r *Remote) addReferencesToUpdate(refspecs []config.RefSpec, return nil } -func (r *Remote) addOrUpdateReferences(rs config.RefSpec, - remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest) error { - iter, err := r.s.IterReferences() - if err != nil { - return err +func (r *Remote) addOrUpdateReferences( + rs config.RefSpec, + localRefs []*plumbing.Reference, + remoteRefs storer.ReferenceStorer, + req *packp.ReferenceUpdateRequest, +) error { + for _, ref := range localRefs { + err := r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) + if err != nil { + return err + } } - return iter.ForEach(func(ref *plumbing.Reference) error { - return r.addReferenceIfRefSpecMatches( - rs, remoteRefs, ref, req, - ) - }) + return nil } func (r *Remote) deleteReferences(rs config.RefSpec, @@ -423,30 +475,124 @@ func (r *Remote) addReferenceIfRefSpecMatches(rs config.RefSpec, return nil } -func getHaves(localRefs storer.ReferenceStorer) ([]plumbing.Hash, error) { - iter, err := localRefs.IterReferences() +func (r *Remote) references() ([]*plumbing.Reference, error) { + var localRefs []*plumbing.Reference + iter, err := r.s.IterReferences() if err != nil { return nil, err } - haves := map[plumbing.Hash]bool{} - err = iter.ForEach(func(ref *plumbing.Reference) error { - if haves[ref.Hash()] == true { - return nil + for { + ref, err := iter.Next() + if err == io.EOF { + break } + if err != nil { + return nil, err + } + + localRefs = append(localRefs, ref) + } + + return localRefs, nil +} + +func getRemoteRefsFromStorer(remoteRefStorer storer.ReferenceStorer) ( + map[plumbing.Hash]bool, error) { + remoteRefs := map[plumbing.Hash]bool{} + iter, err := remoteRefStorer.IterReferences() + if err != nil { + return nil, err + } + err = iter.ForEach(func(ref *plumbing.Reference) error { if ref.Type() != plumbing.HashReference { return nil } + remoteRefs[ref.Hash()] = true + return nil + }) + if err != nil { + return nil, err + } + return remoteRefs, nil +} + +// getHavesFromRef populates the given `haves` map with the given +// reference, and up to `maxHavesToVisitPerRef` ancestor commits. +func getHavesFromRef( + ref *plumbing.Reference, + remoteRefs map[plumbing.Hash]bool, + s storage.Storer, + haves map[plumbing.Hash]bool, +) error { + h := ref.Hash() + if haves[h] { + return nil + } + + // No need to load the commit if we know the remote already + // has this hash. + if remoteRefs[h] { + haves[h] = true + return nil + } + commit, err := object.GetCommit(s, h) + if err != nil { + // Ignore the error if this isn't a commit. haves[ref.Hash()] = true return nil + } + + // Until go-git supports proper commit negotiation during an + // upload pack request, include up to `maxHavesToVisitPerRef` + // commits from the history of each ref. + walker := object.NewCommitPreorderIter(commit, haves, nil) + toVisit := maxHavesToVisitPerRef + return walker.ForEach(func(c *object.Commit) error { + haves[c.Hash] = true + toVisit-- + // If toVisit starts out at 0 (indicating there is no + // max), then it will be negative here and we won't stop + // early. + if toVisit == 0 || remoteRefs[c.Hash] { + return storer.ErrStop + } + return nil }) +} + +func getHaves( + localRefs []*plumbing.Reference, + remoteRefStorer storer.ReferenceStorer, + s storage.Storer, +) ([]plumbing.Hash, error) { + haves := map[plumbing.Hash]bool{} + // Build a map of all the remote references, to avoid loading too + // many parent commits for references we know don't need to be + // transferred. + remoteRefs, err := getRemoteRefsFromStorer(remoteRefStorer) if err != nil { return nil, err } + for _, ref := range localRefs { + if haves[ref.Hash()] == true { + continue + } + + if ref.Type() != plumbing.HashReference { + continue + } + + err = getHavesFromRef(ref, remoteRefs, s, haves) + if err != nil { + return nil, err + } + } + var result []plumbing.Hash for h := range haves { result = append(result, h) @@ -455,10 +601,17 @@ func getHaves(localRefs storer.ReferenceStorer) ([]plumbing.Hash, error) { return result, nil } -func calculateRefs(spec []config.RefSpec, +const refspecTag = "+refs/tags/*:refs/tags/*" + +func calculateRefs( + spec []config.RefSpec, remoteRefs storer.ReferenceStorer, - tags TagFetchMode, + tagMode TagMode, ) (memory.ReferenceStorage, error) { + if tagMode == AllTags { + spec = append(spec, refspecTag) + } + iter, err := remoteRefs.IterReferences() if err != nil { return nil, err @@ -467,9 +620,7 @@ func calculateRefs(spec []config.RefSpec, refs := make(memory.ReferenceStorage, 0) return refs, iter.ForEach(func(ref *plumbing.Reference) error { if !config.MatchAny(spec, ref.Name()) { - if !ref.Name().IsTag() || tags != AllTags { - return nil - } + return nil } if ref.Type() == plumbing.SymbolicReference { @@ -553,7 +704,7 @@ func isFastForward(s storer.EncodedObjectStorer, old, new plumbing.Hash) (bool, } found := false - iter := object.NewCommitPreorderIter(c, nil) + iter := object.NewCommitPreorderIter(c, nil, nil) return found, iter.ForEach(func(c *object.Commit) error { if c.Hash != old { return nil @@ -586,6 +737,7 @@ func (r *Remote) newUploadPackRequest(o *FetchOptions, for _, s := range o.RefSpecs { if !s.IsWildcard() { isWildcard = false + break } } @@ -619,6 +771,7 @@ func buildSidebandIfSupported(l *capability.List, reader io.Reader, p sideband.P func (r *Remote) updateLocalReferenceStorage( specs []config.RefSpec, fetchedRefs, remoteRefs memory.ReferenceStorage, + tagMode TagMode, ) (updated bool, err error) { isWildcard := true for _, spec := range specs { @@ -648,6 +801,10 @@ func (r *Remote) updateLocalReferenceStorage( } } + if tagMode == NoTags { + return updated, nil + } + tags := fetchedRefs if isWildcard { tags = remoteRefs @@ -692,6 +849,39 @@ func (r *Remote) buildFetchedTags(refs memory.ReferenceStorage) (updated bool, e return } +// List the references on the remote repository. +func (r *Remote) List(o *ListOptions) ([]*plumbing.Reference, error) { + s, err := newUploadPackSession(r.c.URLs[0], o.Auth) + if err != nil { + return nil, err + } + + defer ioutil.CheckClose(s, &err) + + ar, err := s.AdvertisedReferences() + if err != nil { + return nil, err + } + + allRefs, err := ar.AllReferences() + if err != nil { + return nil, err + } + + refs, err := allRefs.IterReferences() + if err != nil { + return nil, err + } + + var resultRefs []*plumbing.Reference + refs.ForEach(func(ref *plumbing.Reference) error { + resultRefs = append(resultRefs, ref) + return nil + }) + + return resultRefs, nil +} + func objectsToPush(commands []*packp.Command) ([]plumbing.Hash, error) { var objects []plumbing.Hash for _, cmd := range commands { @@ -730,17 +920,21 @@ func referencesToHashes(refs storer.ReferenceStorer) ([]plumbing.Hash, error) { func pushHashes( ctx context.Context, sess transport.ReceivePackSession, - sto storer.EncodedObjectStorer, + s storage.Storer, req *packp.ReferenceUpdateRequest, hs []plumbing.Hash, ) (*packp.ReportStatus, error) { rd, wr := io.Pipe() req.Packfile = rd + config, err := s.Config() + if err != nil { + return nil, err + } done := make(chan error) go func() { - e := packfile.NewEncoder(wr, sto, false) - if _, err := e.Encode(hs); err != nil { + e := packfile.NewEncoder(wr, s, false) + if _, err := e.Encode(hs, config.Pack.Window); err != nil { done <- wr.CloseWithError(err) return } diff --git a/remote_test.go b/remote_test.go index e2fd8ae..a38675a 100644 --- a/remote_test.go +++ b/remote_test.go @@ -143,7 +143,7 @@ func (s *RemoteSuite) TestFetchWithNoTags(c *C) { s.testFetch(c, r, &FetchOptions{ Tags: NoTags, RefSpecs: []config.RefSpec{ - config.RefSpec("+refs/heads/master:refs/remotes/origin/master"), + config.RefSpec("+refs/heads/*:refs/remotes/origin/*"), }, }, []*plumbing.Reference{ plumbing.NewReferenceFromStrings("refs/remotes/origin/master", "f7b877701fbf855b44c0a9e86f3fdce2c298b07f"), @@ -538,6 +538,42 @@ func (s *RemoteSuite) TestPushNewReference(c *C) { }) } +func (s *RemoteSuite) TestPushNewReferenceAndDeleteInBatch(c *C) { + fs := fixtures.Basic().One().DotGit() + url := c.MkDir() + server, err := PlainClone(url, true, &CloneOptions{ + URL: fs.Root(), + }) + + r, err := PlainClone(c.MkDir(), true, &CloneOptions{ + URL: url, + }) + c.Assert(err, IsNil) + + remote, err := r.Remote(DefaultRemoteName) + c.Assert(err, IsNil) + + ref, err := r.Reference(plumbing.ReferenceName("refs/heads/master"), true) + c.Assert(err, IsNil) + + err = remote.Push(&PushOptions{RefSpecs: []config.RefSpec{ + "refs/heads/master:refs/heads/branch2", + ":refs/heads/branch", + }}) + c.Assert(err, IsNil) + + AssertReferences(c, server, map[string]string{ + "refs/heads/branch2": ref.Hash().String(), + }) + + AssertReferences(c, r, map[string]string{ + "refs/remotes/origin/branch2": ref.Hash().String(), + }) + + _, err = server.Storer.Reference(plumbing.ReferenceName("refs/heads/branch")) + c.Assert(err, Equals, plumbing.ErrReferenceNotFound) +} + func (s *RemoteSuite) TestPushInvalidEndpoint(c *C) { r := newRemote(nil, &config.RemoteConfig{Name: "foo", URLs: []string{"http://\\"}}) err := r.Push(&PushOptions{RemoteName: "foo"}) @@ -589,20 +625,57 @@ func (s *RemoteSuite) TestPushWrongRemoteName(c *C) { } func (s *RemoteSuite) TestGetHaves(c *C) { - st := memory.NewStorage() - st.SetReference(plumbing.NewReferenceFromStrings( - "foo", "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", - )) - - st.SetReference(plumbing.NewReferenceFromStrings( - "bar", "fe6cb94756faa81e5ed9240f9191b833db5f40ae", - )) + f := fixtures.Basic().One() + sto, err := filesystem.NewStorage(f.DotGit()) + c.Assert(err, IsNil) - st.SetReference(plumbing.NewReferenceFromStrings( - "qux", "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", - )) + var localRefs = []*plumbing.Reference{ + plumbing.NewReferenceFromStrings( + "foo", + "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", + ), + plumbing.NewReferenceFromStrings( + "bar", + "fe6cb94756faa81e5ed9240f9191b833db5f40ae", + ), + plumbing.NewReferenceFromStrings( + "qux", + "f7b877701fbf855b44c0a9e86f3fdce2c298b07f", + ), + } - l, err := getHaves(st) + l, err := getHaves(localRefs, memory.NewStorage(), sto) c.Assert(err, IsNil) c.Assert(l, HasLen, 2) } + +func (s *RemoteSuite) TestList(c *C) { + repo := fixtures.Basic().One() + remote := newRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: DefaultRemoteName, + URLs: []string{repo.URL}, + }) + + refs, err := remote.List(&ListOptions{}) + c.Assert(err, IsNil) + + expected := []*plumbing.Reference{ + plumbing.NewSymbolicReference("HEAD", "refs/heads/master"), + plumbing.NewReferenceFromStrings("refs/heads/master", "6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + plumbing.NewReferenceFromStrings("refs/heads/branch", "e8d3ffab552895c19b9fcf7aa264d277cde33881"), + plumbing.NewReferenceFromStrings("refs/pull/1/head", "b8e471f58bcbca63b07bda20e428190409c2db47"), + plumbing.NewReferenceFromStrings("refs/pull/2/head", "9632f02833b2f9613afb5e75682132b0b22e4a31"), + plumbing.NewReferenceFromStrings("refs/pull/2/merge", "c37f58a130ca555e42ff96a071cb9ccb3f437504"), + } + c.Assert(len(refs), Equals, len(expected)) + for _, e := range expected { + found := false + for _, r := range refs { + if r.Name() == e.Name() { + found = true + c.Assert(r, DeepEquals, e) + } + } + c.Assert(found, Equals, true) + } +} diff --git a/repository.go b/repository.go index 8110cf1..6694517 100644 --- a/repository.go +++ b/repository.go @@ -24,12 +24,13 @@ import ( var ( ErrInvalidReference = errors.New("invalid reference, should be a tag or a branch") - ErrRepositoryNotExists = errors.New("repository not exists") + ErrRepositoryNotExists = errors.New("repository does not exist") ErrRepositoryAlreadyExists = errors.New("repository already exists") ErrRemoteNotFound = errors.New("remote not found") ErrRemoteExists = errors.New("remote already exists ") ErrWorktreeNotProvided = errors.New("worktree should be provided") ErrIsBareRepository = errors.New("worktree not available in a bare repository") + ErrUnableToResolveCommit = errors.New("unable to resolve commit") ) // Repository represents a git repository @@ -400,6 +401,25 @@ func (r *Repository) DeleteRemote(name string) error { return r.Storer.SetConfig(cfg) } +func (r *Repository) resolveToCommitHash(h plumbing.Hash) (plumbing.Hash, error) { + obj, err := r.Storer.EncodedObject(plumbing.AnyObject, h) + if err != nil { + return plumbing.ZeroHash, err + } + switch obj.Type() { + case plumbing.TagObject: + t, err := object.DecodeTag(r.Storer, obj) + if err != nil { + return plumbing.ZeroHash, err + } + return r.resolveToCommitHash(t.Target) + case plumbing.CommitObject: + return h, nil + default: + return plumbing.ZeroHash, ErrUnableToResolveCommit + } +} + // Clone clones a remote repository func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { if err := o.Validate(); err != nil { @@ -415,11 +435,12 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { return err } - head, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ + ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ RefSpecs: r.cloneRefSpec(o, c), Depth: o.Depth, Auth: o.Auth, Progress: o.Progress, + Tags: o.Tags, }, o.ReferenceName) if err != nil { return err @@ -431,7 +452,15 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { return err } - if err := w.Reset(&ResetOptions{Commit: head.Hash()}); err != nil { + head, err := r.Head() + if err != nil { + return err + } + + if err := w.Reset(&ResetOptions{ + Mode: MergeReset, + Commit: head.Hash(), + }); err != nil { return err } @@ -445,7 +474,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } } - return r.updateRemoteConfigIfNeeded(o, c, head) + return r.updateRemoteConfigIfNeeded(o, c, ref) } const ( @@ -520,12 +549,12 @@ func (r *Repository) fetchAndUpdateReferences( return nil, err } - head, err := storer.ResolveReference(remoteRefs, ref) + resolvedRef, err := storer.ResolveReference(remoteRefs, ref) if err != nil { return nil, err } - refsUpdated, err := r.updateReferences(remote.c.Fetch, head) + refsUpdated, err := r.updateReferences(remote.c.Fetch, resolvedRef) if err != nil { return nil, err } @@ -534,26 +563,30 @@ func (r *Repository) fetchAndUpdateReferences( return nil, NoErrAlreadyUpToDate } - return head, nil + return resolvedRef, nil } func (r *Repository) updateReferences(spec []config.RefSpec, - resolvedHead *plumbing.Reference) (updated bool, err error) { + resolvedRef *plumbing.Reference) (updated bool, err error) { - if !resolvedHead.Name().IsBranch() { + if !resolvedRef.Name().IsBranch() { // Detached HEAD mode - head := plumbing.NewHashReference(plumbing.HEAD, resolvedHead.Hash()) + h, err := r.resolveToCommitHash(resolvedRef.Hash()) + if err != nil { + return false, err + } + head := plumbing.NewHashReference(plumbing.HEAD, h) return updateReferenceStorerIfNeeded(r.Storer, head) } refs := []*plumbing.Reference{ - // Create local reference for the resolved head - resolvedHead, + // Create local reference for the resolved ref + resolvedRef, // Create local symbolic HEAD - plumbing.NewSymbolicReference(plumbing.HEAD, resolvedHead.Name()), + plumbing.NewSymbolicReference(plumbing.HEAD, resolvedRef.Name()), } - refs = append(refs, r.calculateRemoteHeadReference(spec, resolvedHead)...) + refs = append(refs, r.calculateRemoteHeadReference(spec, resolvedRef)...) for _, ref := range refs { u, err := updateReferenceStorerIfNeeded(r.Storer, ref) @@ -687,7 +720,7 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, err } - return object.NewCommitPreorderIter(commit, nil), nil + return object.NewCommitPreorderIter(commit, nil, nil), nil } // Tags returns all the References from Tags. This method returns all the tag @@ -916,7 +949,7 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err commit = c } case revision.CaretReg: - history := object.NewCommitPreorderIter(commit, nil) + history := object.NewCommitPreorderIter(commit, nil, nil) re := item.(revision.CaretReg).Regexp negate := item.(revision.CaretReg).Negate @@ -946,7 +979,7 @@ func (r *Repository) ResolveRevision(rev plumbing.Revision) (*plumbing.Hash, err commit = c case revision.AtDate: - history := object.NewCommitPreorderIter(commit, nil) + history := object.NewCommitPreorderIter(commit, nil, nil) date := item.(revision.AtDate).Date diff --git a/repository_test.go b/repository_test.go index 3da11f6..4480484 100644 --- a/repository_test.go +++ b/repository_test.go @@ -177,6 +177,27 @@ func (s *RepositorySuite) TestCloneContext(c *C) { c.Assert(err, NotNil) } +func (s *RepositorySuite) TestCloneWithTags(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, err := Clone(memory.NewStorage(), nil, &CloneOptions{URL: url, Tags: NoTags}) + c.Assert(err, IsNil) + + remotes, err := r.Remotes() + c.Assert(err, IsNil) + c.Assert(remotes, HasLen, 1) + + i, err := r.References() + c.Assert(err, IsNil) + + var count int + i.ForEach(func(r *plumbing.Reference) error { count++; return nil }) + + c.Assert(count, Equals, 3) +} + func (s *RepositorySuite) TestCreateRemoteAndRemote(c *C) { r, _ := Init(memory.NewStorage(), nil) remote, err := r.CreateRemote(&config.RemoteConfig{ @@ -651,6 +672,27 @@ func (s *RepositorySuite) TestCloneDetachedHEADAndShallow(c *C) { c.Assert(count, Equals, 15) } +func (s *RepositorySuite) TestCloneDetachedHEADAnnotatedTag(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetLocalRepositoryURL(fixtures.ByTag("tags").One()), + ReferenceName: plumbing.ReferenceName("refs/tags/annotated-tag"), + }) + c.Assert(err, IsNil) + + head, err := r.Reference(plumbing.HEAD, false) + c.Assert(err, IsNil) + c.Assert(head, NotNil) + c.Assert(head.Type(), Equals, plumbing.HashReference) + c.Assert(head.Hash().String(), Equals, "f7b877701fbf855b44c0a9e86f3fdce2c298b07f") + + count := 0 + objects, err := r.Objects() + c.Assert(err, IsNil) + objects.ForEach(func(object.Object) error { count++; return nil }) + c.Assert(count, Equals, 7) +} + func (s *RepositorySuite) TestPush(c *C) { url := c.MkDir() server, err := PlainInit(url, true) @@ -698,6 +740,47 @@ func (s *RepositorySuite) TestPushContext(c *C) { c.Assert(err, NotNil) } +// installPreReceiveHook installs a pre-receive hook in the .git +// directory at path which prints message m before exiting +// successfully. +func installPreReceiveHook(c *C, path, m string) { + hooks := filepath.Join(path, "hooks") + err := os.MkdirAll(hooks, 0777) + c.Assert(err, IsNil) + + err = ioutil.WriteFile(filepath.Join(hooks, "pre-receive"), preReceiveHook(m), 0777) + c.Assert(err, IsNil) +} + +func (s *RepositorySuite) TestPushWithProgress(c *C) { + url := c.MkDir() + server, err := PlainInit(url, true) + c.Assert(err, IsNil) + + m := "Receiving..." + installPreReceiveHook(c, url, m) + + _, err = s.Repository.CreateRemote(&config.RemoteConfig{ + Name: "bar", + URLs: []string{url}, + }) + c.Assert(err, IsNil) + + var p bytes.Buffer + err = s.Repository.Push(&PushOptions{ + RemoteName: "bar", + Progress: &p, + }) + c.Assert(err, IsNil) + + AssertReferences(c, server, map[string]string{ + "refs/heads/master": "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + "refs/heads/branch": "e8d3ffab552895c19b9fcf7aa264d277cde33881", + }) + + c.Assert((&p).Bytes(), DeepEquals, []byte(m)) +} + func (s *RepositorySuite) TestPushDepth(c *C) { url := c.MkDir() server, err := PlainClone(url, true, &CloneOptions{ diff --git a/repository_unix_test.go b/repository_unix_test.go new file mode 100644 index 0000000..75682ae --- /dev/null +++ b/repository_unix_test.go @@ -0,0 +1,11 @@ +// +build !plan9,!windows + +package git + +import "fmt" + +// preReceiveHook returns the bytes of a pre-receive hook script +// that prints m before exiting successfully +func preReceiveHook(m string) []byte { + return []byte(fmt.Sprintf("#!/bin/sh\nprintf '%s'\n", m)) +} diff --git a/repository_windows_test.go b/repository_windows_test.go new file mode 100644 index 0000000..bec0acd --- /dev/null +++ b/repository_windows_test.go @@ -0,0 +1,9 @@ +package git + +import "fmt" + +// preReceiveHook returns the bytes of a pre-receive hook script +// that prints m before exiting successfully +func preReceiveHook(m string) []byte { + return []byte(fmt.Sprintf("#!C:/Program\\ Files/Git/usr/bin/sh.exe\nprintf '%s'\n", m)) +} diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go index e2ff51b..2840bc7 100644 --- a/storage/filesystem/internal/dotgit/dotgit.go +++ b/storage/filesystem/internal/dotgit/dotgit.go @@ -8,6 +8,7 @@ import ( stdioutil "io/ioutil" "os" "strings" + "time" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/ioutil" @@ -56,14 +57,16 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem + fs billy.Filesystem + cachedPackedRefs refCache + packedRefsLastMod time.Time } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return &DotGit{fs: fs, cachedPackedRefs: make(refCache)} } // Initialize creates all the folder scaffolding. @@ -263,11 +266,12 @@ func (d *DotGit) SetRef(r *plumbing.Reference) error { // Symbolic references are resolved and included in the output. func (d *DotGit) Refs() ([]*plumbing.Reference, error) { var refs []*plumbing.Reference - if err := d.addRefsFromPackedRefs(&refs); err != nil { + var seen = make(map[plumbing.ReferenceName]bool) + if err := d.addRefsFromRefDir(&refs, seen); err != nil { return nil, err } - if err := d.addRefsFromRefDir(&refs); err != nil { + if err := d.addRefsFromPackedRefs(&refs, seen); err != nil { return nil, err } @@ -285,15 +289,57 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) { return ref, nil } - refs, err := d.Refs() + return d.packedRef(name) +} + +func (d *DotGit) syncPackedRefs() error { + fi, err := d.fs.Stat(packedRefsPath) + if os.IsNotExist(err) { + return nil + } + if err != nil { - return nil, err + return err } - for _, ref := range refs { - if ref.Name() == name { - return ref, nil + if d.packedRefsLastMod.Before(fi.ModTime()) { + d.cachedPackedRefs = make(refCache) + f, err := d.fs.Open(packedRefsPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err } + defer ioutil.CheckClose(f, &err) + + s := bufio.NewScanner(f) + for s.Scan() { + ref, err := d.processLine(s.Text()) + if err != nil { + return err + } + + if ref != nil { + d.cachedPackedRefs[ref.Name()] = ref + } + } + + d.packedRefsLastMod = fi.ModTime() + + return s.Err() + } + + return nil +} + +func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) { + if err := d.syncPackedRefs(); err != nil { + return nil, err + } + + if ref, ok := d.cachedPackedRefs[name]; ok { + return ref, nil } return nil, plumbing.ErrReferenceNotFound @@ -314,29 +360,19 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { return d.rewritePackedRefsWithoutRef(name) } -func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference) (err error) { - f, err := d.fs.Open(packedRefsPath) - if err != nil { - if os.IsNotExist(err) { - return nil - } +func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) { + if err := d.syncPackedRefs(); err != nil { return err } - defer ioutil.CheckClose(f, &err) - - s := bufio.NewScanner(f) - for s.Scan() { - ref, err := d.processLine(s.Text()) - if err != nil { - return err - } - if ref != nil { + for name, ref := range d.cachedPackedRefs { + if !seen[name] { *refs = append(*refs, ref) + seen[name] = true } } - return s.Err() + return nil } func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err error) { @@ -416,11 +452,11 @@ func (d *DotGit) processLine(line string) (*plumbing.Reference, error) { } } -func (d *DotGit) addRefsFromRefDir(refs *[]*plumbing.Reference) error { - return d.walkReferencesTree(refs, []string{refsPath}) +func (d *DotGit) addRefsFromRefDir(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) error { + return d.walkReferencesTree(refs, []string{refsPath}, seen) } -func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []string) error { +func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []string, seen map[plumbing.ReferenceName]bool) error { files, err := d.fs.ReadDir(d.fs.Join(relPath...)) if err != nil { if os.IsNotExist(err) { @@ -433,7 +469,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin for _, f := range files { newRelPath := append(append([]string(nil), relPath...), f.Name()) if f.IsDir() { - if err = d.walkReferencesTree(refs, newRelPath); err != nil { + if err = d.walkReferencesTree(refs, newRelPath, seen); err != nil { return err } @@ -445,8 +481,9 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin return err } - if ref != nil { + if ref != nil && !seen[ref.Name()] { *refs = append(*refs, ref) + seen[ref.Name()] = true } } @@ -511,3 +548,5 @@ func isNum(b byte) bool { func isHexAlpha(b byte) bool { return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F' } + +type refCache map[plumbing.ReferenceName]*plumbing.Reference diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go index d935ec5..a7f16f4 100644 --- a/storage/filesystem/internal/dotgit/dotgit_test.go +++ b/storage/filesystem/internal/dotgit/dotgit_test.go @@ -134,6 +134,24 @@ func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) { } +func BenchmarkRefMultipleTimes(b *testing.B) { + fs := fixtures.Basic().ByTag(".git").One().DotGit() + refname := plumbing.ReferenceName("refs/remotes/origin/branch") + + dir := New(fs) + _, err := dir.Ref(refname) + if err != nil { + b.Fatalf("unexpected error: %s", err) + } + + for i := 0; i < b.N; i++ { + _, err := dir.Ref(refname) + if err != nil { + b.Fatalf("unexpected error: %s", err) + } + } +} + func (s *SuiteDotGit) TestRemoveRefFromReferenceFile(c *C) { fs := fixtures.Basic().ByTag(".git").One().DotGit() dir := New(fs) diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/internal/dotgit/writers.go index a7525d4..46d3619 100644 --- a/storage/filesystem/internal/dotgit/writers.go +++ b/storage/filesystem/internal/dotgit/writers.go @@ -92,7 +92,7 @@ func (w *PackWriter) Write(p []byte) (int, error) { // was written, the tempfiles are deleted without writing a packfile. func (w *PackWriter) Close() error { defer func() { - if w.Notify != nil { + if w.Notify != nil && w.index != nil && w.index.Size() > 0 { w.Notify(w.checksum, w.index) } diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/internal/dotgit/writers_test.go index 1342396..1544de8 100644 --- a/storage/filesystem/internal/dotgit/writers_test.go +++ b/storage/filesystem/internal/dotgit/writers_test.go @@ -12,6 +12,7 @@ import ( . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" ) @@ -132,3 +133,23 @@ func (s *SuiteDotGit) TestSyncedReader(c *C) { c.Assert(n, Equals, 3) c.Assert(string(head), Equals, "280") } + +func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) { + dir, err := ioutil.TempDir("", "example") + if err != nil { + c.Assert(err, IsNil) + } + + defer os.RemoveAll(dir) + + fs := osfs.New(dir) + + w, err := newPackWrite(fs) + c.Assert(err, IsNil) + + w.Notify = func(h plumbing.Hash, idx *packfile.Index) { + c.Fatal("unexpected call to PackWriter.Notify") + } + + c.Assert(w.Close(), IsNil) +} diff --git a/submodule.go b/submodule.go index fd3d173..de8ac73 100644 --- a/submodule.go +++ b/submodule.go @@ -62,14 +62,17 @@ func (s *Submodule) Status() (*SubmoduleStatus, error) { } func (s *Submodule) status(idx *index.Index) (*SubmoduleStatus, error) { + status := &SubmoduleStatus{ + Path: s.c.Path, + } + e, err := idx.Entry(s.c.Path) - if err != nil { + if err != nil && err != index.ErrEntryNotFound { return nil, err } - status := &SubmoduleStatus{ - Path: s.c.Path, - Expected: e.Hash, + if e != nil { + status.Expected = e.Hash } if !s.initialized { diff --git a/worktree.go b/worktree.go index 4f8e740..d2630b7 100644 --- a/worktree.go +++ b/worktree.go @@ -25,7 +25,7 @@ import ( var ( ErrWorktreeNotClean = errors.New("worktree is not clean") ErrSubmoduleNotFound = errors.New("submodule not found") - ErrUnstaggedChanges = errors.New("worktree contains unstagged changes") + ErrUnstagedChanges = errors.New("worktree contains unstaged changes") ) // Worktree represents a git worktree. @@ -107,7 +107,10 @@ func (w *Worktree) PullContext(ctx context.Context, o *PullOptions) error { return err } - if err := w.Reset(&ResetOptions{Commit: ref.Hash()}); err != nil { + if err := w.Reset(&ResetOptions{ + Mode: MergeReset, + Commit: ref.Hash(), + }); err != nil { return err } @@ -149,7 +152,7 @@ func (w *Worktree) Checkout(opts *CheckoutOptions) error { } if unstaged { - return ErrUnstaggedChanges + return ErrUnstagedChanges } } @@ -266,18 +269,16 @@ func (w *Worktree) Reset(opts *ResetOptions) error { } if unstaged { - return ErrUnstaggedChanges + return ErrUnstagedChanges } } - changes, err := w.diffCommitWithStaging(opts.Commit, true) - if err != nil { + if err := w.setHEADCommit(opts.Commit); err != nil { return err } - idx, err := w.r.Storer.Index() - if err != nil { - return err + if opts.Mode == SoftReset { + return nil } t, err := w.getTreeFromCommitHash(opts.Commit) @@ -285,50 +286,86 @@ func (w *Worktree) Reset(opts *ResetOptions) error { return err } - for _, ch := range changes { - if err := w.checkoutChange(ch, t, idx); err != nil { + if opts.Mode == MixedReset || opts.Mode == MergeReset || opts.Mode == HardReset { + if err := w.resetIndex(t); err != nil { return err } } - if err := w.r.Storer.SetIndex(idx); err != nil { - return err + if opts.Mode == MergeReset || opts.Mode == HardReset { + if err := w.resetWorktree(t); err != nil { + return err + } } - return w.setHEADCommit(opts.Commit) + return nil } -func (w *Worktree) containsUnstagedChanges() (bool, error) { - ch, err := w.diffStagingWithWorktree() +func (w *Worktree) resetIndex(t *object.Tree) error { + idx, err := w.r.Storer.Index() if err != nil { - return false, err + return err } - return len(ch) != 0, nil -} - -func (w *Worktree) setHEADCommit(commit plumbing.Hash) error { - head, err := w.r.Reference(plumbing.HEAD, false) + changes, err := w.diffTreeWithStaging(t, true) if err != nil { return err } - if head.Type() == plumbing.HashReference { - head = plumbing.NewHashReference(plumbing.HEAD, commit) - return w.r.Storer.SetReference(head) + for _, ch := range changes { + a, err := ch.Action() + if err != nil { + return err + } + + var name string + var e *object.TreeEntry + + switch a { + case merkletrie.Modify, merkletrie.Insert: + name = ch.To.String() + e, err = t.FindEntry(name) + if err != nil { + return err + } + case merkletrie.Delete: + name = ch.From.String() + } + + _, _ = idx.Remove(name) + if e == nil { + continue + } + + idx.Entries = append(idx.Entries, &index.Entry{ + Name: name, + Hash: e.Hash, + Mode: e.Mode, + }) + } - branch, err := w.r.Reference(head.Target(), false) + return w.r.Storer.SetIndex(idx) +} + +func (w *Worktree) resetWorktree(t *object.Tree) error { + changes, err := w.diffStagingWithWorktree(true) if err != nil { return err } - if !branch.Name().IsBranch() { - return fmt.Errorf("invalid HEAD target should be a branch, found %s", branch.Type()) + idx, err := w.r.Storer.Index() + if err != nil { + return err } - branch = plumbing.NewHashReference(branch.Name(), commit) - return w.r.Storer.SetReference(branch) + for _, ch := range changes { + if err := w.checkoutChange(ch, t, idx); err != nil { + return err + } + } + + return w.r.Storer.SetIndex(idx) } func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *index.Index) error { @@ -351,13 +388,7 @@ func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *ind isSubmodule = e.Mode == filemode.Submodule case merkletrie.Delete: - name = ch.From.String() - ie, err := idx.Entry(name) - if err != nil { - return err - } - - isSubmodule = ie.Mode == filemode.Submodule + return rmFileAndDirIfEmpty(w.Filesystem, ch.From.String()) } if isSubmodule { @@ -367,6 +398,52 @@ func (w *Worktree) checkoutChange(ch merkletrie.Change, t *object.Tree, idx *ind return w.checkoutChangeRegularFile(name, a, t, e, idx) } +func (w *Worktree) containsUnstagedChanges() (bool, error) { + ch, err := w.diffStagingWithWorktree(false) + if err != nil { + return false, err + } + + for _, c := range ch { + a, err := c.Action() + if err != nil { + return false, err + } + + if a == merkletrie.Insert { + continue + } + + return true, nil + } + + return false, nil +} + +func (w *Worktree) setHEADCommit(commit plumbing.Hash) error { + head, err := w.r.Reference(plumbing.HEAD, false) + if err != nil { + return err + } + + if head.Type() == plumbing.HashReference { + head = plumbing.NewHashReference(plumbing.HEAD, commit) + return w.r.Storer.SetReference(head) + } + + branch, err := w.r.Reference(head.Target(), false) + if err != nil { + return err + } + + if !branch.Name().IsBranch() { + return fmt.Errorf("invalid HEAD target should be a branch, found %s", branch.Type()) + } + + branch = plumbing.NewHashReference(branch.Name(), commit) + return w.r.Storer.SetReference(branch) +} + func (w *Worktree) checkoutChangeSubmodule(name string, a merkletrie.Action, e *object.TreeEntry, @@ -383,17 +460,7 @@ func (w *Worktree) checkoutChangeSubmodule(name string, return nil } - if err := w.rmIndexFromFile(name, idx); err != nil { - return err - } - - if err := w.addIndexFromTreeEntry(name, e, idx); err != nil { - return err - } - - // TODO: the submodule update should be reviewed as reported at: - // https://github.com/src-d/go-git/issues/415 - return sub.update(context.TODO(), &SubmoduleUpdateOptions{}, e.Hash) + return w.addIndexFromTreeEntry(name, e, idx) case merkletrie.Insert: mode, err := e.Mode.ToOSFileMode() if err != nil { @@ -405,12 +472,6 @@ func (w *Worktree) checkoutChangeSubmodule(name string, } return w.addIndexFromTreeEntry(name, e, idx) - case merkletrie.Delete: - if err := rmFileAndDirIfEmpty(w.Filesystem, name); err != nil { - return err - } - - return w.rmIndexFromFile(name, idx) } return nil @@ -424,9 +485,7 @@ func (w *Worktree) checkoutChangeRegularFile(name string, ) error { switch a { case merkletrie.Modify: - if err := w.rmIndexFromFile(name, idx); err != nil { - return err - } + _, _ = idx.Remove(name) // to apply perm changes the file is deleted, billy doesn't implement // chmod @@ -446,12 +505,6 @@ func (w *Worktree) checkoutChangeRegularFile(name string, } return w.addIndexFromFile(name, e.Hash, idx) - case merkletrie.Delete: - if err := rmFileAndDirIfEmpty(w.Filesystem, name); err != nil { - return err - } - - return w.rmIndexFromFile(name, idx) } return nil @@ -503,6 +556,7 @@ func (w *Worktree) checkoutFileSymlink(f *object.File) (err error) { } func (w *Worktree) addIndexFromTreeEntry(name string, f *object.TreeEntry, idx *index.Index) error { + _, _ = idx.Remove(name) idx.Entries = append(idx.Entries, &index.Entry{ Hash: f.Hash, Name: name, @@ -513,6 +567,7 @@ func (w *Worktree) addIndexFromTreeEntry(name string, f *object.TreeEntry, idx * } func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *index.Index) error { + _, _ = idx.Remove(name) fi, err := w.Filesystem.Lstat(name) if err != nil { return err @@ -541,19 +596,6 @@ func (w *Worktree) addIndexFromFile(name string, h plumbing.Hash, idx *index.Ind return nil } -func (w *Worktree) rmIndexFromFile(name string, idx *index.Index) error { - for i, e := range idx.Entries { - if e.Name != name { - continue - } - - idx.Entries = append(idx.Entries[:i], idx.Entries[i+1:]...) - return nil - } - - return nil -} - func (w *Worktree) getTreeFromCommitHash(commit plumbing.Hash) (*object.Tree, error) { c, err := w.r.CommitObject(commit) if err != nil { diff --git a/worktree_commit_test.go b/worktree_commit_test.go index f6744bc..09360af 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -99,6 +99,42 @@ func (s *WorktreeSuite) TestCommitAll(c *C) { assertStorageStatus(c, s.Repository, 13, 11, 10, expected) } +func (s *WorktreeSuite) TestRemoveAndCommitAll(c *C) { + expected := plumbing.NewHash("907cd576c6ced2ecd3dab34a72bf9cf65944b9a9") + + fs := memfs.New() + w := &Worktree{ + r: s.Repository, + Filesystem: fs, + } + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + util.WriteFile(fs, "foo", []byte("foo"), 0644) + _, err = w.Add("foo") + c.Assert(err, IsNil) + + _, errFirst := w.Commit("Add in Repo\n", &CommitOptions{ + Author: defaultSignature(), + }) + c.Assert(errFirst, IsNil) + + errRemove := fs.Remove("foo") + c.Assert(errRemove, IsNil) + + hash, errSecond := w.Commit("Remove foo\n", &CommitOptions{ + All: true, + Author: defaultSignature(), + }) + c.Assert(errSecond, IsNil) + + c.Assert(hash, Equals, expected) + c.Assert(err, IsNil) + + assertStorageStatus(c, s.Repository, 13, 11, 11, expected) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, diff --git a/worktree_status.go b/worktree_status.go index 9b0773e..ac4be3a 100644 --- a/worktree_status.go +++ b/worktree_status.go @@ -65,7 +65,7 @@ func (w *Worktree) status(commit plumbing.Hash) (Status, error) { } } - right, err := w.diffStagingWithWorktree() + right, err := w.diffStagingWithWorktree(false) if err != nil { return nil, err } @@ -104,7 +104,7 @@ func nameFromAction(ch *merkletrie.Change) string { return name } -func (w *Worktree) diffStagingWithWorktree() (merkletrie.Changes, error) { +func (w *Worktree) diffStagingWithWorktree(reverse bool) (merkletrie.Changes, error) { idx, err := w.r.Storer.Index() if err != nil { return nil, err @@ -117,11 +117,19 @@ func (w *Worktree) diffStagingWithWorktree() (merkletrie.Changes, error) { } to := filesystem.NewRootNode(w.Filesystem, submodules) - res, err := merkletrie.DiffTree(from, to, diffTreeIsEquals) - if err == nil { - res = w.excludeIgnoredChanges(res) + + var c merkletrie.Changes + if reverse { + c, err = merkletrie.DiffTree(to, from, diffTreeIsEquals) + } else { + c, err = merkletrie.DiffTree(from, to, diffTreeIsEquals) } - return res, err + + if err != nil { + return nil, err + } + + return w.excludeIgnoredChanges(c), nil } func (w *Worktree) excludeIgnoredChanges(changes merkletrie.Changes) merkletrie.Changes { @@ -179,27 +187,35 @@ func (w *Worktree) getSubmodulesStatus() (map[string]plumbing.Hash, error) { } func (w *Worktree) diffCommitWithStaging(commit plumbing.Hash, reverse bool) (merkletrie.Changes, error) { - idx, err := w.r.Storer.Index() - if err != nil { - return nil, err - } - - var from noder.Noder + var t *object.Tree if !commit.IsZero() { c, err := w.r.CommitObject(commit) if err != nil { return nil, err } - t, err := c.Tree() + t, err = c.Tree() if err != nil { return nil, err } + } + return w.diffTreeWithStaging(t, reverse) +} + +func (w *Worktree) diffTreeWithStaging(t *object.Tree, reverse bool) (merkletrie.Changes, error) { + var from noder.Noder + if t != nil { from = object.NewTreeRootNode(t) } + idx, err := w.r.Storer.Index() + if err != nil { + return nil, err + } + to := mindex.NewRootNode(idx) + if reverse { return merkletrie.DiffTree(to, from, diffTreeIsEquals) } @@ -227,7 +243,7 @@ func diffTreeIsEquals(a, b noder.Hasher) bool { } // Add adds the file contents of a file in the worktree to the index. if the -// file is already stagged in the index no error is returned. +// file is already staged in the index no error is returned. func (w *Worktree) Add(path string) (plumbing.Hash, error) { s, err := w.Status() if err != nil { @@ -236,6 +252,9 @@ func (w *Worktree) Add(path string) (plumbing.Hash, error) { h, err := w.copyFileToStorage(path) if err != nil { + if os.IsNotExist(err) { + h, err = w.deleteFromIndex(path) + } return h, err } diff --git a/worktree_test.go b/worktree_test.go index 0a1c2d1..fb71873 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -259,7 +259,9 @@ func (s *WorktreeSuite) TestCheckout(c *C) { Filesystem: fs, } - err := w.Checkout(&CheckoutOptions{}) + err := w.Checkout(&CheckoutOptions{ + Force: true, + }) c.Assert(err, IsNil) entries, err := fs.ReadDir("/") @@ -278,6 +280,27 @@ func (s *WorktreeSuite) TestCheckout(c *C) { c.Assert(idx.Entries, HasLen, 9) } +func (s *WorktreeSuite) TestCheckoutForce(c *C) { + w := &Worktree{ + r: s.Repository, + Filesystem: memfs.New(), + } + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + w.Filesystem = memfs.New() + + err = w.Checkout(&CheckoutOptions{ + Force: true, + }) + c.Assert(err, IsNil) + + entries, err := w.Filesystem.ReadDir("/") + c.Assert(err, IsNil) + c.Assert(entries, HasLen, 8) +} + func (s *WorktreeSuite) TestCheckoutSymlink(c *C) { if runtime.GOOS == "windows" { c.Skip("git doesn't support symlinks by default in windows") @@ -312,6 +335,10 @@ func (s *WorktreeSuite) TestCheckoutSymlink(c *C) { } func (s *WorktreeSuite) TestFilenameNormalization(c *C) { + if runtime.GOOS == "windows" { + c.Skip("windows paths may contain non utf-8 sequences") + } + url := c.MkDir() path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() @@ -345,7 +372,11 @@ func (s *WorktreeSuite) TestFilenameNormalization(c *C) { modFilename := norm.Form(norm.NFKD).String(filename) util.WriteFile(w.Filesystem, modFilename, []byte("foo"), 0755) + _, err = w.Add(filename) + c.Assert(err, IsNil) + _, err = w.Add(modFilename) + c.Assert(err, IsNil) status, err = w.Status() c.Assert(err, IsNil) @@ -604,35 +635,6 @@ func (s *WorktreeSuite) testCheckoutBisect(c *C, url string) { }) } -func (s *WorktreeSuite) TestCheckoutWithGitignore(c *C) { - fs := memfs.New() - w := &Worktree{ - r: s.Repository, - Filesystem: fs, - } - - err := w.Checkout(&CheckoutOptions{}) - c.Assert(err, IsNil) - - f, _ := fs.Create("file") - f.Close() - - err = w.Checkout(&CheckoutOptions{}) - c.Assert(err.Error(), Equals, "worktree contains unstagged changes") - - f, _ = fs.Create(".gitignore") - f.Write([]byte("file")) - f.Close() - - err = w.Checkout(&CheckoutOptions{}) - c.Assert(err.Error(), Equals, "worktree contains unstagged changes") - - w.Add(".gitignore") - - err = w.Checkout(&CheckoutOptions{}) - c.Assert(err, IsNil) -} - func (s *WorktreeSuite) TestStatus(c *C) { fs := memfs.New() w := &Worktree{ @@ -698,15 +700,67 @@ func (s *WorktreeSuite) TestReset(c *C) { c.Assert(err, IsNil) c.Assert(branch.Hash(), Not(Equals), commit) - err = w.Reset(&ResetOptions{Commit: commit}) + err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commit}) c.Assert(err, IsNil) branch, err = w.r.Reference(plumbing.Master, false) c.Assert(err, IsNil) c.Assert(branch.Hash(), Equals, commit) + + status, err := w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, true) } -func (s *WorktreeSuite) TestResetMerge(c *C) { +func (s *WorktreeSuite) TestResetWithUntracked(c *C) { + fs := memfs.New() + w := &Worktree{ + r: s.Repository, + Filesystem: fs, + } + + commit := plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9") + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + err = util.WriteFile(fs, "foo", nil, 0755) + c.Assert(err, IsNil) + + err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commit}) + c.Assert(err, IsNil) + + status, err := w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, true) +} + +func (s *WorktreeSuite) TestResetSoft(c *C) { + fs := memfs.New() + w := &Worktree{ + r: s.Repository, + Filesystem: fs, + } + + commit := plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9") + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + err = w.Reset(&ResetOptions{Mode: SoftReset, Commit: commit}) + c.Assert(err, IsNil) + + branch, err := w.r.Reference(plumbing.Master, false) + c.Assert(err, IsNil) + c.Assert(branch.Hash(), Equals, commit) + + status, err := w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, false) + c.Assert(status.File("CHANGELOG").Staging, Equals, Added) +} + +func (s *WorktreeSuite) TestResetMixed(c *C) { fs := memfs.New() w := &Worktree{ r: s.Repository, @@ -718,6 +772,39 @@ func (s *WorktreeSuite) TestResetMerge(c *C) { err := w.Checkout(&CheckoutOptions{}) c.Assert(err, IsNil) + err = w.Reset(&ResetOptions{Mode: MixedReset, Commit: commit}) + c.Assert(err, IsNil) + + branch, err := w.r.Reference(plumbing.Master, false) + c.Assert(err, IsNil) + c.Assert(branch.Hash(), Equals, commit) + + status, err := w.Status() + c.Assert(err, IsNil) + c.Assert(status.IsClean(), Equals, false) + c.Assert(status.File("CHANGELOG").Staging, Equals, Untracked) +} + +func (s *WorktreeSuite) TestResetMerge(c *C) { + fs := memfs.New() + w := &Worktree{ + r: s.Repository, + Filesystem: fs, + } + + commitA := plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294") + commitB := plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9") + + err := w.Checkout(&CheckoutOptions{}) + c.Assert(err, IsNil) + + err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commitA}) + c.Assert(err, IsNil) + + branch, err := w.r.Reference(plumbing.Master, false) + c.Assert(err, IsNil) + c.Assert(branch.Hash(), Equals, commitA) + f, err := fs.Create(".gitignore") c.Assert(err, IsNil) _, err = f.Write([]byte("foo")) @@ -725,12 +812,12 @@ func (s *WorktreeSuite) TestResetMerge(c *C) { err = f.Close() c.Assert(err, IsNil) - err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commit}) - c.Assert(err, Equals, ErrUnstaggedChanges) + err = w.Reset(&ResetOptions{Mode: MergeReset, Commit: commitB}) + c.Assert(err, Equals, ErrUnstagedChanges) - branch, err := w.r.Reference(plumbing.Master, false) + branch, err = w.r.Reference(plumbing.Master, false) c.Assert(err, IsNil) - c.Assert(branch.Hash(), Not(Equals), commit) + c.Assert(branch.Hash(), Equals, commitA) } func (s *WorktreeSuite) TestResetHard(c *C) { |