diff options
44 files changed, 1340 insertions, 306 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 92b7b8c..bdb5f73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,8 @@ This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d The official support channels, for both users and contributors, are: -- GitHub [issues](https://github.com/src-d/go-git/issues)* +- [StackOverflow go-git tag](https://stackoverflow.com/questions/tagged/go-git) for user questions. +- GitHub [Issues](https://github.com/src-d/go-git/issues)* for bug reports and feature requests. - Slack: #go-git room in the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) *Before opening a new issue or submitting a new pull request, it's helpful to @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Sourced Technologies, S.L. + Copyright 2018 Sourced Technologies, S.L. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -198,4 +198,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License.
\ No newline at end of file + limitations under the License. @@ -5,14 +5,14 @@ It can be used to manipulate git repositories at low level *(plumbing)* or high level *(porcelain)*, through an idiomatic Go API. It also supports several types of storage, such as in-memory filesystems, or custom implementations thanks to the [`Storer`](https://godoc.org/gopkg.in/src-d/go-git.v4/plumbing/storer) interface. -It's being actively develop since 2015 and is being use extensively by [source{d}](https://sourced.tech/) and [Keybase](https://keybase.io/blog/encrypted-git-for-everyone), and by many other libraries and tools. +It's being actively developed since 2015 and is being used extensively by [source{d}](https://sourced.tech/) and [Keybase](https://keybase.io/blog/encrypted-git-for-everyone), and by many other libraries and tools. Comparison with git ------------------- *go-git* aims to be fully compatible with [git](https://github.com/git/git), all the *porcelain* operations are implemented to work exactly as *git* does. -*git* is a humongous project with years of development by thousands of contributors, making it challenging for *go-git* implement all the features. You can find a comparison of *go-git* vs *git* in the [compatibility documentation](COMPATIBILITY.md). +*git* is a humongous project with years of development by thousands of contributors, making it challenging for *go-git* to implement all the features. You can find a comparison of *go-git* vs *git* in the [compatibility documentation](COMPATIBILITY.md). Installation @@ -24,12 +24,12 @@ The recommended way to install *go-git* is: go get -u gopkg.in/src-d/go-git.v4/... ``` -> We use [gopkg.in](http://labix.org/gopkg.in) for having a versioned API, this means that when `go get` clones the package, is the latest tag matching `v4.*` cloned and not the master branch. +> We use [gopkg.in](http://labix.org/gopkg.in) to version the API, this means that when `go get` clones the package, it's the latest tag matching `v4.*` that is cloned and not the master branch. Examples -------- -> Please note that the functions `CheckIfError` and `Info` used in the examples are from the [examples package](https://github.com/src-d/go-git/blob/master/_examples/common.go#L17) just to be used in the examples. +> Please note that the `CheckIfError` and `Info` functions used in the examples are from the [examples package](https://github.com/src-d/go-git/blob/master/_examples/common.go#L17) just to be used in the examples. ### Basic example @@ -71,7 +71,7 @@ r, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ CheckIfError(err) -// Gets the HEAD history from HEAD, just like does: +// Gets the HEAD history from HEAD, just like this command: Info("git log") // ... retrieves the branch pointed by HEAD @@ -110,7 +110,7 @@ Date: Fri Nov 11 13:23:22 2016 +0100 ... ``` -You can find this [example](_examples/log/main.go) and many others at the [examples](_examples) folder +You can find this [example](_examples/log/main.go) and many others in the [examples](_examples) folder. Contribute ---------- diff --git a/_examples/README.md b/_examples/README.md index 26639b1..cf9c2d3 100644 --- a/_examples/README.md +++ b/_examples/README.md @@ -6,6 +6,10 @@ Here you can find a list of annotated _go-git_ examples: - [showcase](showcase/main.go) - A small showcase of the capabilities of _go-git_ - [open](open/main.go) - Opening a existing repository cloned by _git_ - [clone](clone/main.go) - Cloning a repository + - [username and password](clone/auth/basic/username_password/main.go) - Cloning a repository + using a username and password + - [personal access token](clone/auth/basic/access_token/main.go) - Cloning + a repository using a GitHub personal access token - [commit](commit/main.go) - Commit changes to the current branch to an existent repository - [push](push/main.go) - Push repository to default remote (origin) - [pull](pull/main.go) - Pull changes from a remote repository diff --git a/_examples/checkout/main.go b/_examples/checkout/main.go index 2c54550..5969eb4 100644 --- a/_examples/checkout/main.go +++ b/_examples/checkout/main.go @@ -38,8 +38,8 @@ func main() { }) CheckIfError(err) - // ... retrieving the commit being pointed by HEAD, it's shows that the - // repository is poiting to the giving commit in detached mode + // ... retrieving the commit being pointed by HEAD, it shows that the + // repository is pointing to the giving commit in detached mode Info("git show-ref --head HEAD") ref, err = r.Head() CheckIfError(err) diff --git a/_examples/clone/auth/basic/access_token/main.go b/_examples/clone/auth/basic/access_token/main.go new file mode 100644 index 0000000..7f6d121 --- /dev/null +++ b/_examples/clone/auth/basic/access_token/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "os" + + git "gopkg.in/src-d/go-git.v4" + . "gopkg.in/src-d/go-git.v4/_examples" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" +) + +func main() { + CheckArgs("<url>", "<directory>", "<github_access_token>") + url, directory, token := os.Args[1], os.Args[2], os.Args[3] + + // Clone the given repository to the given directory + Info("git clone %s %s", url, directory) + + r, err := git.PlainClone(directory, false, &git.CloneOptions{ + // The intended use of a GitHub personal access token is in replace of your password + // because access tokens can easily be revoked. + // https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ + Auth: &http.BasicAuth{ + Username: "abc123", // yes, this can be anything except an empty string + Password: token, + }, + URL: url, + Progress: os.Stdout, + }) + CheckIfError(err) + + // ... retrieving the branch being pointed by HEAD + ref, err := r.Head() + CheckIfError(err) + // ... retrieving the commit object + commit, err := r.CommitObject(ref.Hash()) + CheckIfError(err) + + fmt.Println(commit) +} diff --git a/_examples/clone/auth/basic/username_password/main.go b/_examples/clone/auth/basic/username_password/main.go new file mode 100644 index 0000000..754558c --- /dev/null +++ b/_examples/clone/auth/basic/username_password/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "os" + + git "gopkg.in/src-d/go-git.v4" + . "gopkg.in/src-d/go-git.v4/_examples" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" +) + +func main() { + CheckArgs("<url>", "<directory>", "<github_username>", "<github_password>") + url, directory, username, password := os.Args[1], os.Args[2], os.Args[3], os.Args[4] + + // Clone the given repository to the given directory + Info("git clone %s %s", url, directory) + + r, err := git.PlainClone(directory, false, &git.CloneOptions{ + Auth: &http.BasicAuth{ + Username: username, + Password: password, + }, + URL: url, + Progress: os.Stdout, + }) + CheckIfError(err) + + // ... retrieving the branch being pointed by HEAD + ref, err := r.Head() + CheckIfError(err) + // ... retrieving the commit object + commit, err := r.CommitObject(ref.Hash()) + CheckIfError(err) + + fmt.Println(commit) +} diff --git a/_examples/commit/main.go b/_examples/commit/main.go index 556cb9c..ec296b9 100644 --- a/_examples/commit/main.go +++ b/_examples/commit/main.go @@ -12,13 +12,13 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/object" ) -// Basic example of how to commit changes to the current branch to an existent +// Basic example of how to commit changes to the current branch to an existing // repository. func main() { CheckArgs("<directory>") directory := os.Args[1] - // Opens an already existent repository. + // Opens an already existing repository. r, err := git.PlainOpen(directory) CheckIfError(err) @@ -44,7 +44,7 @@ func main() { fmt.Println(status) - // Commits the current staging are to the repository, with the new file + // Commits the current staging area to the repository, with the new file // just created. We should provide the object.Signature of Author of the // commit. Info("git commit -m \"example go-git commit\"") diff --git a/_examples/log/main.go b/_examples/log/main.go index 714d58f..ba0597a 100644 --- a/_examples/log/main.go +++ b/_examples/log/main.go @@ -23,7 +23,7 @@ func main() { }) CheckIfError(err) - // Gets the HEAD history from HEAD, just like does: + // Gets the HEAD history from HEAD, just like this command: Info("git log") // ... retrieves the branch pointed by HEAD diff --git a/_examples/open/main.go b/_examples/open/main.go index b890423..dec183e 100644 --- a/_examples/open/main.go +++ b/_examples/open/main.go @@ -14,7 +14,7 @@ func main() { CheckArgs("<path>") path := os.Args[1] - // We instance a new repository targeting the given path (the .git folder) + // We instanciate a new repository targeting the given path (the .git folder) r, err := git.PlainOpen(path) CheckIfError(err) diff --git a/_examples/pull/main.go b/_examples/pull/main.go index ae751d2..06369fa 100644 --- a/_examples/pull/main.go +++ b/_examples/pull/main.go @@ -13,7 +13,7 @@ func main() { CheckArgs("<path>") path := os.Args[1] - // We instance a new repository targeting the given path (the .git folder) + // We instance\iate a new repository targeting the given path (the .git folder) r, err := git.PlainOpen(path) CheckIfError(err) diff --git a/_examples/showcase/main.go b/_examples/showcase/main.go index aeeddb8..85f2b58 100644 --- a/_examples/showcase/main.go +++ b/_examples/showcase/main.go @@ -16,7 +16,7 @@ import ( // - Get the HEAD reference // - Using the HEAD reference, obtain the commit this reference is pointing to // - Print the commit content -// - Using the commit, iterate all its files and print them +// - Using the commit, iterate over all its files and print them // - Print all the commit history with commit messages, short hash and the // first line of the commit message func main() { diff --git a/_examples/storage/README.md b/_examples/storage/README.md index fc72e6f..b002515 100644 --- a/_examples/storage/README.md +++ b/_examples/storage/README.md @@ -6,7 +6,7 @@ ### and what this means ... -*git* has as very well defined storage system, the `.git` directory, present on any repository. This is the place where `git` stores al the [`objects`](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), [`references`](https://git-scm.com/book/es/v2/Git-Internals-Git-References) and [`configuration`](https://git-scm.com/docs/git-config#_configuration_file). This information is stored in plain files. +*git* has a very well defined storage system, the `.git` directory, present on any repository. This is the place where `git` stores all the [`objects`](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects), [`references`](https://git-scm.com/book/es/v2/Git-Internals-Git-References) and [`configuration`](https://git-scm.com/docs/git-config#_configuration_file). This information is stored in plain files. Our original **go-git** version was designed to work in memory, some time after we added support to read the `.git`, and now we have added support for fully customized [storages](https://godoc.org/gopkg.in/src-d/go-git.v4/storage#Storer). diff --git a/_examples/tag/main.go b/_examples/tag/main.go index 190c3ad..1e6212b 100644 --- a/_examples/tag/main.go +++ b/_examples/tag/main.go @@ -15,7 +15,7 @@ func main() { CheckArgs("<path>") path := os.Args[1] - // We instance a new repository targeting the given path (the .git folder) + // We instanciate a new repository targeting the given path (the .git folder) r, err := git.PlainOpen(path) CheckIfError(err) diff --git a/example_test.go b/example_test.go index ef7e3d3..691b4ac 100644 --- a/example_test.go +++ b/example_test.go @@ -11,6 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-billy.v4/memfs" @@ -69,6 +70,52 @@ func ExamplePlainClone() { // Output: Initial changelog } +func ExamplePlainClone_usernamePassword() { + // Tempdir to clone the repository + dir, err := ioutil.TempDir("", "clone-example") + if err != nil { + log.Fatal(err) + } + + defer os.RemoveAll(dir) // clean up + + // Clones the repository into the given dir, just as a normal git clone does + _, err = git.PlainClone(dir, false, &git.CloneOptions{ + URL: "https://github.com/git-fixtures/basic.git", + Auth: &http.BasicAuth{ + Username: "username", + Password: "password", + }, + }) + + if err != nil { + log.Fatal(err) + } +} + +func ExamplePlainClone_accessToken() { + // Tempdir to clone the repository + dir, err := ioutil.TempDir("", "clone-example") + if err != nil { + log.Fatal(err) + } + + defer os.RemoveAll(dir) // clean up + + // Clones the repository into the given dir, just as a normal git clone does + _, err = git.PlainClone(dir, false, &git.CloneOptions{ + URL: "https://github.com/git-fixtures/basic.git", + Auth: &http.BasicAuth{ + Username: "abc123", // anything except an empty string + Password: "github_access_token", + }, + }) + + if err != nil { + log.Fatal(err) + } +} + func ExampleRepository_References() { r, _ := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ URL: "https://github.com/git-fixtures/basic.git", @@ -16,7 +16,7 @@ require ( github.com/pkg/errors v0.8.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.0.0 - github.com/src-d/gcfg v1.3.0 + github.com/src-d/gcfg v1.4.0 github.com/stretchr/testify v1.2.2 // indirect github.com/xanzy/ssh-agent v0.2.0 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 @@ -35,6 +35,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg= github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= @@ -335,6 +335,11 @@ type LogOptions struct { // Show only those commits in which the specified file was inserted/updated. // It is equivalent to running `git log -- <file-name>`. FileName *string + + // Pretend as if all the refs in refs/, along with HEAD, are listed on the command line as <commit>. + // It is equivalent to running `git log --all`. + // If set on true, the From option will be ignored. + All bool } var ( diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index df25530..ac57d08 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -261,6 +261,17 @@ func (d *Decoder) readExtension(idx *Index, header []byte) error { if err := d.Decode(idx.ResolveUndo); err != nil { return err } + case bytes.Equal(header, endOfIndexEntryExtSignature): + r, err := d.getExtensionReader() + if err != nil { + return err + } + + idx.EndOfIndexEntry = &EndOfIndexEntry{} + d := &endOfIndexEntryDecoder{r} + if err := d.Decode(idx.EndOfIndexEntry); err != nil { + return err + } default: return errUnknownExtension } @@ -449,3 +460,17 @@ func (d *resolveUndoDecoder) readStage(e *ResolveUndoEntry, s Stage) error { return nil } + +type endOfIndexEntryDecoder struct { + r io.Reader +} + +func (d *endOfIndexEntryDecoder) Decode(e *EndOfIndexEntry) error { + var err error + e.Offset, err = binary.ReadUint32(d.r) + if err != nil { + return err + } + + return binary.Read(d.r, &e.Hash) +} diff --git a/plumbing/format/index/decoder_test.go b/plumbing/format/index/decoder_test.go index b612ebb..7468ad0 100644 --- a/plumbing/format/index/decoder_test.go +++ b/plumbing/format/index/decoder_test.go @@ -202,3 +202,19 @@ func (s *IndexSuite) TestDecodeV4(c *C) { c.Assert(idx.Entries[6].IntentToAdd, Equals, true) c.Assert(idx.Entries[6].SkipWorktree, Equals, false) } + +func (s *IndexSuite) TestDecodeEndOfIndexEntry(c *C) { + f, err := fixtures.Basic().ByTag("end-of-index-entry").One().DotGit().Open("index") + c.Assert(err, IsNil) + defer func() { c.Assert(f.Close(), IsNil) }() + + idx := &Index{} + d := NewDecoder(f) + err = d.Decode(idx) + c.Assert(err, IsNil) + + c.Assert(idx.Version, Equals, uint32(2)) + c.Assert(idx.EndOfIndexEntry, NotNil) + c.Assert(idx.EndOfIndexEntry.Offset, Equals, uint32(716)) + c.Assert(idx.EndOfIndexEntry.Hash.String(), Equals, "922e89d9ffd7cefce93a211615b2053c0f42bd78") +} diff --git a/plumbing/format/index/doc.go b/plumbing/format/index/doc.go index d1e7b33..f2b3d76 100644 --- a/plumbing/format/index/doc.go +++ b/plumbing/format/index/doc.go @@ -297,5 +297,64 @@ // in the previous ewah bitmap. // // - One NUL. -// Source https://www.kernel.org/pub/software/scm/git/docs/technical/index-format.txt +// +// == File System Monitor cache +// +// The file system monitor cache tracks files for which the core.fsmonitor +// hook has told us about changes. The signature for this extension is +// { 'F', 'S', 'M', 'N' }. +// +// The extension starts with +// +// - 32-bit version number: the current supported version is 1. +// +// - 64-bit time: the extension data reflects all changes through the given +// time which is stored as the nanoseconds elapsed since midnight, +// January 1, 1970. +// +// - 32-bit bitmap size: the size of the CE_FSMONITOR_VALID bitmap. +// +// - An ewah bitmap, the n-th bit indicates whether the n-th index entry +// is not CE_FSMONITOR_VALID. +// +// == End of Index Entry +// +// The End of Index Entry (EOIE) is used to locate the end of the variable +// length index entries and the begining of the extensions. Code can take +// advantage of this to quickly locate the index extensions without having +// to parse through all of the index entries. +// +// Because it must be able to be loaded before the variable length cache +// entries and other index extensions, this extension must be written last. +// The signature for this extension is { 'E', 'O', 'I', 'E' }. +// +// The extension consists of: +// +// - 32-bit offset to the end of the index entries +// +// - 160-bit SHA-1 over the extension types and their sizes (but not +// their contents). E.g. if we have "TREE" extension that is N-bytes +// long, "REUC" extension that is M-bytes long, followed by "EOIE", +// then the hash would be: +// +// SHA-1("TREE" + <binary representation of N> + +// "REUC" + <binary representation of M>) +// +// == Index Entry Offset Table +// +// The Index Entry Offset Table (IEOT) is used to help address the CPU +// cost of loading the index by enabling multi-threading the process of +// converting cache entries from the on-disk format to the in-memory format. +// The signature for this extension is { 'I', 'E', 'O', 'T' }. +// +// The extension consists of: +// +// - 32-bit version (currently 1) +// +// - A number of index offset entries each consisting of: +// +// - 32-bit offset from the begining of the file to the first cache entry +// in this block of entries. +// +// - 32-bit count of cache entries in this blockpackage index package index diff --git a/plumbing/format/index/index.go b/plumbing/format/index/index.go index fc7b8cd..6c4b7ca 100644 --- a/plumbing/format/index/index.go +++ b/plumbing/format/index/index.go @@ -18,9 +18,10 @@ var ( // ErrEntryNotFound is returned by Index.Entry, if an entry is not found. ErrEntryNotFound = errors.New("entry not found") - indexSignature = []byte{'D', 'I', 'R', 'C'} - treeExtSignature = []byte{'T', 'R', 'E', 'E'} - resolveUndoExtSignature = []byte{'R', 'E', 'U', 'C'} + indexSignature = []byte{'D', 'I', 'R', 'C'} + treeExtSignature = []byte{'T', 'R', 'E', 'E'} + resolveUndoExtSignature = []byte{'R', 'E', 'U', 'C'} + endOfIndexEntryExtSignature = []byte{'E', 'O', 'I', 'E'} ) // Stage during merge @@ -50,6 +51,8 @@ type Index struct { Cache *Tree // ResolveUndo represents the 'Resolve undo' extension ResolveUndo *ResolveUndo + // EndOfIndexEntry represents the 'End of Index Entry' extension + EndOfIndexEntry *EndOfIndexEntry } // Add creates a new Entry and returns it. The caller should first check that @@ -193,3 +196,18 @@ type ResolveUndoEntry struct { Path string Stages map[Stage]plumbing.Hash } + +// EndOfIndexEntry is the End of Index Entry (EOIE) is used to locate the end of +// the variable length index entries and the begining of the extensions. Code +// can take advantage of this to quickly locate the index extensions without +// having to parse through all of the index entries. +// +// Because it must be able to be loaded before the variable length cache +// entries and other index extensions, this extension must be written last. +type EndOfIndexEntry struct { + // Offset to the end of the index entries + Offset uint32 + // Hash is a SHA-1 over the extension types and their sizes (but not + // their contents). + Hash plumbing.Hash +} diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index 2b4aceb..0d9ed54 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -51,7 +51,13 @@ func WritePackfileToObjectStorage( } defer ioutil.CheckClose(w, &err) - _, err = io.Copy(w, packfile) + + var n int64 + n, err = io.Copy(w, packfile) + if err == nil && n == 0 { + return ErrEmptyPackfile + } + return err } diff --git a/plumbing/format/packfile/common_test.go b/plumbing/format/packfile/common_test.go index 387c0d1..eafc617 100644 --- a/plumbing/format/packfile/common_test.go +++ b/plumbing/format/packfile/common_test.go @@ -1,15 +1,29 @@ package packfile import ( + "bytes" "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" ) func Test(t *testing.T) { TestingT(t) } +type CommonSuite struct{} + +var _ = Suite(&CommonSuite{}) + +func (s *CommonSuite) TestEmptyUpdateObjectStorage(c *C) { + var buf bytes.Buffer + sto := memory.NewStorage() + + err := UpdateObjectStorage(sto, &buf) + c.Assert(err, Equals, ErrEmptyPackfile) +} + func newObject(t plumbing.ObjectType, cont []byte) plumbing.EncodedObject { o := plumbing.MemoryObject{} o.SetType(t) diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index 330cb73..a268bce 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -48,7 +48,7 @@ func NewFSObject( // Reader implements the plumbing.EncodedObject interface. func (o *FSObject) Reader() (io.ReadCloser, error) { obj, ok := o.cache.Get(o.hash) - if ok { + if ok && obj != o { reader, err := obj.Reader() if err != nil { return nil, err diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 0d13066..1e7ef26 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -21,6 +21,16 @@ var ( ErrZLib = NewError("zlib reading error") ) +// When reading small objects from packfile it is beneficial to do so at +// once to exploit the buffered I/O. In many cases the objects are so small +// that they were already loaded to memory when the object header was +// loaded from the packfile. Wrapping in FSObject would cause this buffered +// data to be thrown away and then re-read later, with the additional +// seeking causing reloads from disk. Objects smaller than this threshold +// are now always read into memory and stored in cache instead of being +// wrapped in FSObject. +const smallObjectThreshold = 16 * 1024 + // Packfile allows retrieving information from inside a packfile. type Packfile struct { idxfile.Index @@ -79,15 +89,7 @@ func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { } } - if _, err := p.s.SeekFromStart(o); err != nil { - if err == io.EOF || isInvalid(err) { - return nil, plumbing.ErrObjectNotFound - } - - return nil, err - } - - return p.nextObject() + return p.objectAtOffset(o) } // GetSizeByOffset retrieves the size of the encoded object from the @@ -108,66 +110,16 @@ func (p *Packfile) GetSizeByOffset(o int64) (size int64, err error) { return h.Length, nil } -func (p *Packfile) nextObjectHeader() (*ObjectHeader, error) { - h, err := p.s.NextObjectHeader() +func (p *Packfile) objectHeaderAtOffset(offset int64) (*ObjectHeader, error) { + h, err := p.s.SeekObjectHeader(offset) p.s.pendingObject = nil return h, err } -func (p *Packfile) getObjectData( - h *ObjectHeader, -) (typ plumbing.ObjectType, size int64, err error) { - switch h.Type { - case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: - typ = h.Type - size = h.Length - case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: - buf := bufPool.Get().(*bytes.Buffer) - buf.Reset() - defer bufPool.Put(buf) - - _, _, err = p.s.NextObject(buf) - if err != nil { - return - } - - delta := buf.Bytes() - _, delta = decodeLEB128(delta) // skip src size - sz, _ := decodeLEB128(delta) - size = int64(sz) - - var offset int64 - if h.Type == plumbing.REFDeltaObject { - offset, err = p.FindOffset(h.Reference) - if err != nil { - return - } - } else { - offset = h.OffsetReference - } - - if baseType, ok := p.offsetToType[offset]; ok { - typ = baseType - } else { - if _, err = p.s.SeekFromStart(offset); err != nil { - return - } - - h, err = p.nextObjectHeader() - if err != nil { - return - } - - typ, _, err = p.getObjectData(h) - if err != nil { - return - } - } - default: - err = ErrInvalidObject.AddDetails("type %q", h.Type) - } - - return +func (p *Packfile) nextObjectHeader() (*ObjectHeader, error) { + h, err := p.s.NextObjectHeader() + p.s.pendingObject = nil + return h, err } func (p *Packfile) getObjectSize(h *ObjectHeader) (int64, error) { @@ -210,11 +162,7 @@ func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err if baseType, ok := p.offsetToType[offset]; ok { typ = baseType } else { - if _, err = p.s.SeekFromStart(offset); err != nil { - return - } - - h, err = p.nextObjectHeader() + h, err = p.objectHeaderAtOffset(offset) if err != nil { return } @@ -231,8 +179,8 @@ func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err return } -func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { - h, err := p.nextObjectHeader() +func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) { + h, err := p.objectHeaderAtOffset(offset) if err != nil { if err == io.EOF || isInvalid(err) { return nil, plumbing.ErrObjectNotFound @@ -246,6 +194,13 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { return p.getNextObject(h) } + // If the object is not a delta and it's small enough then read it + // completely into memory now since it is already read from disk + // into buffer anyway. + if h.Length <= smallObjectThreshold && h.Type != plumbing.OFSDeltaObject && h.Type != plumbing.REFDeltaObject { + return p.getNextObject(h) + } + hash, err := p.FindHash(h.Offset) if err != nil { return nil, err @@ -289,11 +244,7 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { } } - if _, err := p.s.SeekFromStart(offset); err != nil { - return nil, err - } - - h, err := p.nextObjectHeader() + h, err := p.objectHeaderAtOffset(offset) if err != nil { return nil, err } @@ -385,8 +336,6 @@ func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset if err != nil { return err } - - p.cachePut(base) } obj.SetType(base.Type()) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 28582b5..71cbba9 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -38,15 +38,14 @@ type Observer interface { // Parser decodes a packfile and calls any observer associated to it. Is used // to generate indexes. type Parser struct { - storage storer.EncodedObjectStorer - scanner *Scanner - count uint32 - oi []*objectInfo - oiByHash map[plumbing.Hash]*objectInfo - oiByOffset map[int64]*objectInfo - hashOffset map[plumbing.Hash]int64 - pendingRefDeltas map[plumbing.Hash][]*objectInfo - checksum plumbing.Hash + storage storer.EncodedObjectStorer + scanner *Scanner + count uint32 + oi []*objectInfo + oiByHash map[plumbing.Hash]*objectInfo + oiByOffset map[int64]*objectInfo + hashOffset map[plumbing.Hash]int64 + checksum plumbing.Hash cache *cache.BufferLRU // delta content by offset, only used if source is not seekable @@ -78,13 +77,12 @@ func NewParserWithStorage( } return &Parser{ - storage: storage, - scanner: scanner, - ob: ob, - count: 0, - cache: cache.NewBufferLRUDefault(), - pendingRefDeltas: make(map[plumbing.Hash][]*objectInfo), - deltas: deltas, + storage: storage, + scanner: scanner, + ob: ob, + count: 0, + cache: cache.NewBufferLRUDefault(), + deltas: deltas, }, nil } @@ -150,10 +148,6 @@ func (p *Parser) Parse() (plumbing.Hash, error) { return plumbing.ZeroHash, err } - if len(p.pendingRefDeltas) > 0 { - return plumbing.ZeroHash, ErrReferenceDeltaNotFound - } - if err := p.onFooter(p.checksum); err != nil { return plumbing.ZeroHash, err } @@ -205,18 +199,21 @@ func (p *Parser) indexObjects() error { parent.Children = append(parent.Children, ota) case plumbing.REFDeltaObject: delta = true - parent, ok := p.oiByHash[oh.Reference] - if ok { - ota = newDeltaObject(oh.Offset, oh.Length, t, parent) - parent.Children = append(parent.Children, ota) - } else { - ota = newBaseObject(oh.Offset, oh.Length, t) - p.pendingRefDeltas[oh.Reference] = append( - p.pendingRefDeltas[oh.Reference], - ota, - ) + if !ok { + // can't find referenced object in this pack file + // this must be a "thin" pack. + parent = &objectInfo{ //Placeholder parent + SHA1: oh.Reference, + ExternalRef: true, // mark as an external reference that must be resolved + Type: plumbing.AnyObject, + DiskType: plumbing.AnyObject, + } + p.oiByHash[oh.Reference] = parent } + ota = newDeltaObject(oh.Offset, oh.Length, t, parent) + parent.Children = append(parent.Children, ota) + default: ota = newBaseObject(oh.Offset, oh.Length, t) } @@ -297,16 +294,20 @@ func (p *Parser) resolveDeltas() error { return nil } -func (p *Parser) get(o *objectInfo) ([]byte, error) { - b, ok := p.cache.Get(o.Offset) +func (p *Parser) get(o *objectInfo) (b []byte, err error) { + var ok bool + if !o.ExternalRef { // skip cache check for placeholder parents + b, ok = p.cache.Get(o.Offset) + } + // If it's not on the cache and is not a delta we can try to find it in the - // storage, if there's one. + // storage, if there's one. External refs must enter here. if !ok && p.storage != nil && !o.Type.IsDelta() { - var err error e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) if err != nil { return nil, err } + o.Type = e.Type() r, err := e.Reader() if err != nil { @@ -323,6 +324,11 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { return b, nil } + if o.ExternalRef { + // we were not able to resolve a ref in a thin pack + return nil, ErrReferenceDeltaNotFound + } + var data []byte if o.DiskType.IsDelta() { base, err := p.get(o.Parent) @@ -335,7 +341,6 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { return nil, err } } else { - var err error data, err = p.readData(o) if err != nil { return nil, err @@ -367,14 +372,6 @@ func (p *Parser) resolveObject( return nil, err } - if pending, ok := p.pendingRefDeltas[o.SHA1]; ok { - for _, po := range pending { - po.Parent = o - o.Children = append(o.Children, po) - } - delete(p.pendingRefDeltas, o.SHA1) - } - if p.storage != nil { obj := new(plumbing.MemoryObject) obj.SetSize(o.Size()) @@ -401,11 +398,7 @@ func (p *Parser) readData(o *objectInfo) ([]byte, error) { return data, nil } - if _, err := p.scanner.SeekFromStart(o.Offset); err != nil { - return nil, err - } - - if _, err := p.scanner.NextObjectHeader(); err != nil { + if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil { return nil, err } @@ -447,10 +440,11 @@ func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) { } type objectInfo struct { - Offset int64 - Length int64 - Type plumbing.ObjectType - DiskType plumbing.ObjectType + Offset int64 + Length int64 + Type plumbing.ObjectType + DiskType plumbing.ObjectType + ExternalRef bool // indicates this is an external reference in a thin pack file Crc32 uint32 diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index 012a140..6e7c84b 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -1,10 +1,13 @@ package packfile_test import ( + "io" "testing" + git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + "gopkg.in/src-d/go-git.v4/plumbing/storer" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -74,6 +77,53 @@ func (s *ParserSuite) TestParserHashes(c *C) { c.Assert(obs.objects, DeepEquals, objs) } +func (s *ParserSuite) TestThinPack(c *C) { + + // Initialize an empty repository + fs, err := git.PlainInit(c.MkDir(), true) + c.Assert(err, IsNil) + + // Try to parse a thin pack without having the required objects in the repo to + // see if the correct errors are returned + thinpack := fixtures.ByTag("thinpack").One() + scanner := packfile.NewScanner(thinpack.Packfile()) + parser, err := packfile.NewParserWithStorage(scanner, fs.Storer) // ParserWithStorage writes to the storer all parsed objects! + c.Assert(err, IsNil) + + _, err = parser.Parse() + c.Assert(err, Equals, plumbing.ErrObjectNotFound) + + // start over with a clean repo + fs, err = git.PlainInit(c.MkDir(), true) + c.Assert(err, IsNil) + + // Now unpack a base packfile into our empty repo: + f := fixtures.ByURL("https://github.com/spinnaker/spinnaker.git").One() + w, err := fs.Storer.(storer.PackfileWriter).PackfileWriter() + c.Assert(err, IsNil) + _, err = io.Copy(w, f.Packfile()) + c.Assert(err, IsNil) + w.Close() + + // Check that the test object that will come with our thin pack is *not* in the repo + _, err = fs.Storer.EncodedObject(plumbing.CommitObject, thinpack.Head) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) + + // Now unpack the thin pack: + scanner = packfile.NewScanner(thinpack.Packfile()) + parser, err = packfile.NewParserWithStorage(scanner, fs.Storer) // ParserWithStorage writes to the storer all parsed objects! + c.Assert(err, IsNil) + + h, err := parser.Parse() + c.Assert(err, IsNil) + c.Assert(h, Equals, plumbing.NewHash("1288734cbe0b95892e663221d94b95de1f5d7be8")) + + // Check that our test object is now accessible + _, err = fs.Storer.EncodedObject(plumbing.CommitObject, thinpack.Head) + c.Assert(err, IsNil) + +} + type observerObject struct { hash string otype plumbing.ObjectType diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 6fc183b..614b0d1 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -138,14 +138,52 @@ func (s *Scanner) readCount() (uint32, error) { return binary.ReadUint32(s.r) } +// SeekObjectHeader seeks to specified offset and returns the ObjectHeader +// for the next object in the reader +func (s *Scanner) SeekObjectHeader(offset int64) (*ObjectHeader, error) { + // if seeking we assume that you are not interested in the header + if s.version == 0 { + s.version = VersionSupported + } + + if _, err := s.r.Seek(offset, io.SeekStart); err != nil { + return nil, err + } + + h, err := s.nextObjectHeader() + if err != nil { + return nil, err + } + + h.Offset = offset + return h, nil +} + // NextObjectHeader returns the ObjectHeader for the next object in the reader func (s *Scanner) NextObjectHeader() (*ObjectHeader, error) { - defer s.Flush() - if err := s.doPending(); err != nil { return nil, err } + offset, err := s.r.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + + h, err := s.nextObjectHeader() + if err != nil { + return nil, err + } + + h.Offset = offset + return h, nil +} + +// nextObjectHeader returns the ObjectHeader for the next object in the reader +// without the Offset field +func (s *Scanner) nextObjectHeader() (*ObjectHeader, error) { + defer s.Flush() + s.crc.Reset() h := &ObjectHeader{} @@ -308,7 +346,7 @@ var byteSlicePool = sync.Pool{ // SeekFromStart sets a new offset from start, returns the old position before // the change. func (s *Scanner) SeekFromStart(offset int64) (previous int64, err error) { - // if seeking we assume that you are not interested on the header + // if seeking we assume that you are not interested in the header if s.version == 0 { s.version = VersionSupported } @@ -385,7 +423,7 @@ type bufferedSeeker struct { } func (r *bufferedSeeker) Seek(offset int64, whence int) (int64, error) { - if whence == io.SeekCurrent { + if whence == io.SeekCurrent && offset == 0 { current, err := r.r.Seek(offset, whence) if err != nil { return current, err diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 644d0eb..091b457 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -118,6 +118,23 @@ func (s *ScannerSuite) TestNextObjectHeaderWithOutReadObjectNonSeekable(c *C) { c.Assert(n, Equals, f.PackfileHash) } +func (s *ScannerSuite) TestSeekObjectHeader(c *C) { + r := fixtures.Basic().One().Packfile() + p := NewScanner(r) + + h, err := p.SeekObjectHeader(expectedHeadersOFS[4].Offset) + c.Assert(err, IsNil) + c.Assert(h, DeepEquals, &expectedHeadersOFS[4]) +} + +func (s *ScannerSuite) TestSeekObjectHeaderNonSeekable(c *C) { + r := io.MultiReader(fixtures.Basic().One().Packfile()) + p := NewScanner(r) + + _, err := p.SeekObjectHeader(expectedHeadersOFS[4].Offset) + c.Assert(err, Equals, ErrSeekNotSupported) +} + var expectedHeadersOFS = []ObjectHeader{ {Type: plumbing.CommitObject, Offset: 12, Length: 254}, {Type: plumbing.OFSDeltaObject, Offset: 186, Length: 93, OffsetReference: 12}, diff --git a/plumbing/object/commit_walker.go b/plumbing/object/commit_walker.go index 40ad258..8c76557 100644 --- a/plumbing/object/commit_walker.go +++ b/plumbing/object/commit_walker.go @@ -1,10 +1,12 @@ package object import ( + "container/list" "io" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" + "gopkg.in/src-d/go-git.v4/storage" ) type commitPreIterator struct { @@ -181,3 +183,133 @@ func (w *commitPostIterator) ForEach(cb func(*Commit) error) error { } func (w *commitPostIterator) Close() {} + +// commitAllIterator stands for commit iterator for all refs. +type commitAllIterator struct { + // currCommit points to the current commit. + currCommit *list.Element +} + +// NewCommitAllIter returns a new commit iterator for all refs. +// repoStorer is a repo Storer used to get commits and references. +// commitIterFunc is a commit iterator function, used to iterate through ref commits in chosen order +func NewCommitAllIter(repoStorer storage.Storer, commitIterFunc func(*Commit) CommitIter) (CommitIter, error) { + commitsPath := list.New() + commitsLookup := make(map[plumbing.Hash]*list.Element) + head, err := storer.ResolveReference(repoStorer, plumbing.HEAD) + if err != nil { + return nil, err + } + + // add all references along with the HEAD + if err = addReference(repoStorer, commitIterFunc, head, commitsPath, commitsLookup); err != nil { + return nil, err + } + refIter, err := repoStorer.IterReferences() + if err != nil { + return nil, err + } + defer refIter.Close() + err = refIter.ForEach( + func(ref *plumbing.Reference) error { + return addReference(repoStorer, commitIterFunc, ref, commitsPath, commitsLookup) + }, + ) + if err != nil { + return nil, err + } + + return &commitAllIterator{commitsPath.Front()}, nil +} + +func addReference( + repoStorer storage.Storer, + commitIterFunc func(*Commit) CommitIter, + ref *plumbing.Reference, + commitsPath *list.List, + commitsLookup map[plumbing.Hash]*list.Element) error { + + _, exists := commitsLookup[ref.Hash()] + if exists { + // we already have it - skip the reference. + return nil + } + + refCommit, _ := GetCommit(repoStorer, ref.Hash()) + if refCommit == nil { + // if it's not a commit - skip it. + return nil + } + + var ( + refCommits []*Commit + parent *list.Element + ) + // collect all ref commits to add + commitIter := commitIterFunc(refCommit) + for c, e := commitIter.Next(); e == nil; { + parent, exists = commitsLookup[c.Hash] + if exists { + break + } + refCommits = append(refCommits, c) + c, e = commitIter.Next() + } + commitIter.Close() + + if parent == nil { + // common parent - not found + // add all commits to the path from this ref (maybe it's a HEAD and we don't have anything, yet) + for _, c := range refCommits { + parent = commitsPath.PushBack(c) + commitsLookup[c.Hash] = parent + } + } else { + // add ref's commits to the path in reverse order (from the latest) + for i := len(refCommits) - 1; i >= 0; i-- { + c := refCommits[i] + // insert before found common parent + parent = commitsPath.InsertBefore(c, parent) + commitsLookup[c.Hash] = parent + } + } + + return nil +} + +func (it *commitAllIterator) Next() (*Commit, error) { + if it.currCommit == nil { + return nil, io.EOF + } + + c := it.currCommit.Value.(*Commit) + it.currCommit = it.currCommit.Next() + + return c, nil +} + +func (it *commitAllIterator) ForEach(cb func(*Commit) error) error { + for { + c, err := it.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + err = cb(c) + if err == storer.ErrStop { + break + } + if err != nil { + return err + } + } + + return nil +} + +func (it *commitAllIterator) Close() { + it.currCommit = nil +} diff --git a/plumbing/object/commit_walker_file.go b/plumbing/object/commit_walker_file.go index 84e738a..6f16e61 100644 --- a/plumbing/object/commit_walker_file.go +++ b/plumbing/object/commit_walker_file.go @@ -1,23 +1,30 @@ package object import ( - "gopkg.in/src-d/go-git.v4/plumbing/storer" "io" + + "gopkg.in/src-d/go-git.v4/plumbing" + + "gopkg.in/src-d/go-git.v4/plumbing/storer" ) type commitFileIter struct { fileName string sourceIter CommitIter currentCommit *Commit + checkParent bool } // NewCommitFileIterFromIter returns a commit iterator which performs diffTree between // successive trees returned from the commit iterator from the argument. The purpose of this is // to find the commits that explain how the files that match the path came to be. -func NewCommitFileIterFromIter(fileName string, commitIter CommitIter) CommitIter { +// If checkParent is true then the function double checks if potential parent (next commit in a path) +// is one of the parents in the tree (it's used by `git log --all`). +func NewCommitFileIterFromIter(fileName string, commitIter CommitIter, checkParent bool) CommitIter { iterator := new(commitFileIter) iterator.sourceIter = commitIter iterator.fileName = fileName + iterator.checkParent = checkParent return iterator } @@ -71,20 +78,14 @@ func (c *commitFileIter) getNextFileCommit() (*Commit, error) { return nil, diffErr } - foundChangeForFile := false - for _, change := range changes { - if change.name() == c.fileName { - foundChangeForFile = true - break - } - } + found := c.hasFileChange(changes, parentCommit) // Storing the current-commit in-case a change is found, and // Updating the current-commit for the next-iteration prevCommit := c.currentCommit c.currentCommit = parentCommit - if foundChangeForFile == true { + if found { return prevCommit, nil } @@ -95,6 +96,35 @@ func (c *commitFileIter) getNextFileCommit() (*Commit, error) { } } +func (c *commitFileIter) hasFileChange(changes Changes, parent *Commit) bool { + for _, change := range changes { + if change.name() != c.fileName { + continue + } + + // filename matches, now check if source iterator contains all commits (from all refs) + if c.checkParent { + if parent != nil && isParentHash(parent.Hash, c.currentCommit) { + return true + } + continue + } + + return true + } + + return false +} + +func isParentHash(hash plumbing.Hash, commit *Commit) bool { + for _, h := range commit.ParentHashes { + if h == hash { + return true + } + } + return false +} + func (c *commitFileIter) ForEach(cb func(*Commit) error) error { for { commit, nextErr := c.Next() diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index c034846..5d3535e 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -4,6 +4,7 @@ package http import ( "bytes" "fmt" + "net" "net/http" "strconv" "strings" @@ -151,6 +152,18 @@ func (s *session) ModifyEndpointIfRedirect(res *http.Response) { return } + h, p, err := net.SplitHostPort(r.URL.Host) + if err != nil { + h = r.URL.Host + } + if p != "" { + port, err := strconv.Atoi(p) + if err == nil { + s.endpoint.Port = port + } + } + s.endpoint.Host = h + s.endpoint.Protocol = r.URL.Scheme s.endpoint.Path = r.URL.Path[:len(r.URL.Path)-len(infoRefsPath)] } @@ -201,7 +214,14 @@ func (a *BasicAuth) String() string { return fmt.Sprintf("%s - %s:%s", a.Name(), a.Username, masked) } -// TokenAuth implements the go-git http.AuthMethod and transport.AuthMethod interfaces +// TokenAuth implements an http.AuthMethod that can be used with http transport +// to authenticate with HTTP token authentication (also known as bearer +// authentication). +// +// IMPORTANT: If you are looking to use OAuth tokens with popular servers (e.g. +// GitHub, Bitbucket, GitLab) you should use BasicAuth instead. These servers +// use basic HTTP authentication, with the OAuth token as user or password. +// Check the documentation of your git server for details. type TokenAuth struct { Token string } diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index 71eede4..8b300e8 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/http/cgi" + "net/url" "os" "os/exec" "path/filepath" @@ -119,6 +120,42 @@ func (s *ClientSuite) TestSetAuthWrongType(c *C) { c.Assert(err, Equals, transport.ErrInvalidAuthMethod) } +func (s *ClientSuite) TestModifyEndpointIfRedirect(c *C) { + sess := &session{endpoint: nil} + u, _ := url.Parse("https://example.com/info/refs") + res := &http.Response{Request: &http.Request{URL: u}} + c.Assert(func() { + sess.ModifyEndpointIfRedirect(res) + }, PanicMatches, ".*nil pointer dereference.*") + + sess = &session{endpoint: nil} + // no-op - should return and not panic + sess.ModifyEndpointIfRedirect(&http.Response{}) + + data := []struct { + url string + endpoint *transport.Endpoint + expected *transport.Endpoint + }{ + {"https://example.com/foo/bar", nil, nil}, + {"https://example.com/foo.git/info/refs", + &transport.Endpoint{}, + &transport.Endpoint{Protocol: "https", Host: "example.com", Path: "/foo.git"}}, + {"https://example.com:8080/foo.git/info/refs", + &transport.Endpoint{}, + &transport.Endpoint{Protocol: "https", Host: "example.com", Port: 8080, Path: "/foo.git"}}, + } + + for _, d := range data { + u, _ := url.Parse(d.url) + sess := &session{endpoint: d.endpoint} + sess.ModifyEndpointIfRedirect(&http.Response{ + Request: &http.Request{URL: u}, + }) + c.Assert(d.endpoint, DeepEquals, d.expected) + } +} + type BaseSuite struct { fixtures.Suite diff --git a/plumbing/transport/ssh/upload_pack_test.go b/plumbing/transport/ssh/upload_pack_test.go index 87fd4f5..2685ff0 100644 --- a/plumbing/transport/ssh/upload_pack_test.go +++ b/plumbing/transport/ssh/upload_pack_test.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/test" @@ -97,13 +98,20 @@ func handlerSSH(s ssh.Session) { io.Copy(stdin, s) }() + var wg sync.WaitGroup + wg.Add(2) + go func() { - defer stderr.Close() + defer wg.Done() io.Copy(s.Stderr(), stderr) }() - defer stdout.Close() - io.Copy(s, stdout) + go func() { + defer wg.Done() + io.Copy(s, stdout) + }() + + wg.Wait() if err := cmd.Wait(); err != nil { return diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index 5aea1c0..8dcde8b 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -262,11 +262,7 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, req.Packfile = s.emptyPackfile() } - if s, err := r.ReceivePack(context.Background(), req); err != nil { - return s, err - } else { - return s, err - } + return r.ReceivePack(context.Background(), req) } func (s *ReceivePackSuite) receivePack(c *C, ep *transport.Endpoint, diff --git a/repository.go b/repository.go index 507ff44..de92d64 100644 --- a/repository.go +++ b/repository.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" stdioutil "io/ioutil" "os" "path" @@ -40,6 +41,8 @@ var ( ErrTagExists = errors.New("tag already exists") // ErrTagNotFound an error stating the specified tag does not exist ErrTagNotFound = errors.New("tag not found") + // ErrFetching is returned when the packfile could not be downloaded + ErrFetching = errors.New("unable to fetch packfile") ErrInvalidReference = errors.New("invalid reference, should be a tag or a branch") ErrRepositoryNotExists = errors.New("repository does not exist") @@ -341,13 +344,26 @@ func PlainClone(path string, isBare bool, o *CloneOptions) (*Repository, error) // transport operations. // // TODO(mcuadros): move isBare to CloneOptions in v5 +// TODO(smola): refuse upfront to clone on a non-empty directory in v5, see #1027 func PlainCloneContext(ctx context.Context, path string, isBare bool, o *CloneOptions) (*Repository, error) { + cleanup, cleanupParent, err := checkIfCleanupIsNeeded(path) + if err != nil { + return nil, err + } + r, err := PlainInit(path, isBare) if err != nil { return nil, err } - return r, r.clone(ctx, o) + err = r.clone(ctx, o) + if err != nil && err != ErrRepositoryAlreadyExists { + if cleanup { + cleanUpDir(path, cleanupParent) + } + } + + return r, err } func newRepository(s storage.Storer, worktree billy.Filesystem) *Repository { @@ -358,6 +374,65 @@ func newRepository(s storage.Storer, worktree billy.Filesystem) *Repository { } } +func checkIfCleanupIsNeeded(path string) (cleanup bool, cleanParent bool, err error) { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return true, true, nil + } + + return false, false, err + } + + if !fi.IsDir() { + return false, false, fmt.Errorf("path is not a directory: %s", path) + } + + f, err := os.Open(path) + if err != nil { + return false, false, err + } + + defer ioutil.CheckClose(f, &err) + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, false, nil + } + + if err != nil { + return false, false, err + } + + return false, false, nil +} + +func cleanUpDir(path string, all bool) error { + if all { + return os.RemoveAll(path) + } + + f, err := os.Open(path) + if err != nil { + return err + } + + defer ioutil.CheckClose(f, &err) + + names, err := f.Readdirnames(-1) + if err != nil { + return err + } + + for _, name := range names { + if err := os.RemoveAll(filepath.Join(path, name)); err != nil { + return err + } + } + + return err +} + // Config return the repository config func (r *Repository) Config() (*config.Config, error) { return r.Storer.Config() @@ -640,8 +715,9 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } c := &config.RemoteConfig{ - Name: o.RemoteName, - URLs: []string{o.URL}, + Name: o.RemoteName, + URLs: []string{o.URL}, + Fetch: r.cloneRefSpec(o), } if _, err := r.CreateRemote(c); err != nil { @@ -649,7 +725,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ - RefSpecs: r.cloneRefSpec(o, c), + RefSpecs: c.Fetch, Depth: o.Depth, Auth: o.Auth, Progress: o.Progress, @@ -719,21 +795,26 @@ const ( refspecSingleBranchHEAD = "+HEAD:refs/remotes/%s/HEAD" ) -func (r *Repository) cloneRefSpec(o *CloneOptions, c *config.RemoteConfig) []config.RefSpec { - var rs string - +func (r *Repository) cloneRefSpec(o *CloneOptions) []config.RefSpec { switch { case o.ReferenceName.IsTag(): - rs = fmt.Sprintf(refspecTag, o.ReferenceName.Short()) + return []config.RefSpec{ + config.RefSpec(fmt.Sprintf(refspecTag, o.ReferenceName.Short())), + } case o.SingleBranch && o.ReferenceName == plumbing.HEAD: - rs = fmt.Sprintf(refspecSingleBranchHEAD, c.Name) + return []config.RefSpec{ + config.RefSpec(fmt.Sprintf(refspecSingleBranchHEAD, o.RemoteName)), + config.RefSpec(fmt.Sprintf(refspecSingleBranch, plumbing.Master.Short(), o.RemoteName)), + } case o.SingleBranch: - rs = fmt.Sprintf(refspecSingleBranch, o.ReferenceName.Short(), c.Name) + return []config.RefSpec{ + config.RefSpec(fmt.Sprintf(refspecSingleBranch, o.ReferenceName.Short(), o.RemoteName)), + } default: - return c.Fetch + return []config.RefSpec{ + config.RefSpec(fmt.Sprintf(config.DefaultFetchRefSpec, o.RemoteName)), + } } - - return []config.RefSpec{config.RefSpec(rs)} } func (r *Repository) setIsBare(isBare bool) error { @@ -751,9 +832,7 @@ func (r *Repository) updateRemoteConfigIfNeeded(o *CloneOptions, c *config.Remot return nil } - c.Fetch = []config.RefSpec{config.RefSpec(fmt.Sprintf( - refspecSingleBranch, head.Name().Short(), c.Name, - ))} + c.Fetch = r.cloneRefSpec(o) cfg, err := r.Storer.Config() if err != nil { @@ -781,6 +860,8 @@ func (r *Repository) fetchAndUpdateReferences( remoteRefs, err := remote.fetch(ctx, o) if err == NoErrAlreadyUpToDate { objsUpdated = false + } else if err == packfile.ErrEmptyPackfile { + return nil, ErrFetching } else if err != nil { return nil, err } @@ -946,8 +1027,36 @@ func (r *Repository) PushContext(ctx context.Context, o *PushOptions) error { // Log returns the commit history from the given LogOptions. func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { - h := o.From - if o.From == plumbing.ZeroHash { + fn := commitIterFunc(o.Order) + if fn == nil { + return nil, fmt.Errorf("invalid Order=%v", o.Order) + } + + var ( + it object.CommitIter + err error + ) + if o.All { + it, err = r.logAll(fn) + } else { + it, err = r.log(o.From, fn) + } + + if err != nil { + return nil, err + } + + if o.FileName != nil { + // for `git log --all` also check parent (if the next commit comes from the real parent) + it = r.logWithFile(*o.FileName, it, o.All) + } + + return it, nil +} + +func (r *Repository) log(from plumbing.Hash, commitIterFunc func(*object.Commit) object.CommitIter) (object.CommitIter, error) { + h := from + if from == plumbing.ZeroHash { head, err := r.Head() if err != nil { return nil, err @@ -960,27 +1069,41 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { if err != nil { return nil, err } + return commitIterFunc(commit), nil +} + +func (r *Repository) logAll(commitIterFunc func(*object.Commit) object.CommitIter) (object.CommitIter, error) { + return object.NewCommitAllIter(r.Storer, commitIterFunc) +} + +func (*Repository) logWithFile(fileName string, commitIter object.CommitIter, checkParent bool) object.CommitIter { + return object.NewCommitFileIterFromIter(fileName, commitIter, checkParent) +} - var commitIter object.CommitIter - switch o.Order { +func commitIterFunc(order LogOrder) func(c *object.Commit) object.CommitIter { + switch order { case LogOrderDefault: - commitIter = object.NewCommitPreorderIter(commit, nil, nil) + return func(c *object.Commit) object.CommitIter { + return object.NewCommitPreorderIter(c, nil, nil) + } case LogOrderDFS: - commitIter = object.NewCommitPreorderIter(commit, nil, nil) + return func(c *object.Commit) object.CommitIter { + return object.NewCommitPreorderIter(c, nil, nil) + } case LogOrderDFSPost: - commitIter = object.NewCommitPostorderIter(commit, nil) + return func(c *object.Commit) object.CommitIter { + return object.NewCommitPostorderIter(c, nil) + } case LogOrderBSF: - commitIter = object.NewCommitIterBSF(commit, nil, nil) + return func(c *object.Commit) object.CommitIter { + return object.NewCommitIterBSF(c, nil, nil) + } case LogOrderCommitterTime: - commitIter = object.NewCommitIterCTime(commit, nil, nil) - default: - return nil, fmt.Errorf("invalid Order=%v", o.Order) - } - - if o.FileName == nil { - return commitIter, nil + return func(c *object.Commit) object.CommitIter { + return object.NewCommitIterCTime(c, nil, nil) + } } - return object.NewCommitFileIterFromIter(*o.FileName, commitIter), nil + return nil } // Tags returns all the tag References in a repository. diff --git a/repository_test.go b/repository_test.go index 07c3570..2a56dd2 100644 --- a/repository_test.go +++ b/repository_test.go @@ -21,6 +21,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" + "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -177,11 +178,12 @@ func (s *RepositorySuite) TestCloneContext(c *C) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := CloneContext(ctx, memory.NewStorage(), nil, &CloneOptions{ + r, err := CloneContext(ctx, memory.NewStorage(), nil, &CloneOptions{ URL: s.GetBasicLocalRepositoryURL(), }) - c.Assert(err, NotNil) + c.Assert(r, NotNil) + c.Assert(err, ErrorMatches, ".* context canceled") } func (s *RepositorySuite) TestCloneWithTags(c *C) { @@ -581,15 +583,128 @@ func (s *RepositorySuite) TestPlainCloneWithRemoteName(c *C) { c.Assert(remote, NotNil) } -func (s *RepositorySuite) TestPlainCloneContext(c *C) { +func (s *RepositorySuite) TestPlainCloneOverExistingGitDirectory(c *C) { + tmpDir := c.MkDir() + r, err := PlainInit(tmpDir, false) + c.Assert(r, NotNil) + c.Assert(err, IsNil) + + r, err = PlainClone(tmpDir, false, &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(r, IsNil) + c.Assert(err, Equals, ErrRepositoryAlreadyExists) +} + +func (s *RepositorySuite) TestPlainCloneContextCancel(c *C) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := PlainCloneContext(ctx, c.MkDir(), false, &CloneOptions{ + r, err := PlainCloneContext(ctx, c.MkDir(), false, &CloneOptions{ URL: s.GetBasicLocalRepositoryURL(), }) - c.Assert(err, NotNil) + c.Assert(r, NotNil) + c.Assert(err, ErrorMatches, ".* context canceled") +} + +func (s *RepositorySuite) TestPlainCloneContextNonExistentWithExistentDir(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + tmpDir := c.MkDir() + repoDir := tmpDir + + r, err := PlainCloneContext(ctx, repoDir, false, &CloneOptions{ + URL: "incorrectOnPurpose", + }) + c.Assert(r, NotNil) + c.Assert(err, Equals, transport.ErrRepositoryNotFound) + + _, err = os.Stat(repoDir) + c.Assert(os.IsNotExist(err), Equals, false) + + names, err := ioutil.ReadDir(repoDir) + c.Assert(err, IsNil) + c.Assert(names, HasLen, 0) +} + +func (s *RepositorySuite) TestPlainCloneContextNonExistentWithNonExistentDir(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + tmpDir := c.MkDir() + repoDir := filepath.Join(tmpDir, "repoDir") + + r, err := PlainCloneContext(ctx, repoDir, false, &CloneOptions{ + URL: "incorrectOnPurpose", + }) + c.Assert(r, NotNil) + c.Assert(err, Equals, transport.ErrRepositoryNotFound) + + _, err = os.Stat(repoDir) + c.Assert(os.IsNotExist(err), Equals, true) +} + +func (s *RepositorySuite) TestPlainCloneContextNonExistentWithNotDir(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + tmpDir := c.MkDir() + repoDir := filepath.Join(tmpDir, "repoDir") + f, err := os.Create(repoDir) + c.Assert(err, IsNil) + c.Assert(f.Close(), IsNil) + + r, err := PlainCloneContext(ctx, repoDir, false, &CloneOptions{ + URL: "incorrectOnPurpose", + }) + c.Assert(r, IsNil) + c.Assert(err, ErrorMatches, ".*not a directory.*") + + fi, err := os.Stat(repoDir) + c.Assert(err, IsNil) + c.Assert(fi.IsDir(), Equals, false) +} + +func (s *RepositorySuite) TestPlainCloneContextNonExistentWithNotEmptyDir(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + tmpDir := c.MkDir() + repoDirPath := filepath.Join(tmpDir, "repoDir") + err := os.Mkdir(repoDirPath, 0777) + c.Assert(err, IsNil) + + dummyFile := filepath.Join(repoDirPath, "dummyFile") + err = ioutil.WriteFile(dummyFile, []byte(fmt.Sprint("dummyContent")), 0644) + c.Assert(err, IsNil) + + r, err := PlainCloneContext(ctx, repoDirPath, false, &CloneOptions{ + URL: "incorrectOnPurpose", + }) + c.Assert(r, NotNil) + c.Assert(err, Equals, transport.ErrRepositoryNotFound) + + _, err = os.Stat(dummyFile) + c.Assert(err, IsNil) + +} + +func (s *RepositorySuite) TestPlainCloneContextNonExistingOverExistingGitDirectory(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + tmpDir := c.MkDir() + r, err := PlainInit(tmpDir, false) + c.Assert(r, NotNil) + c.Assert(err, IsNil) + + r, err = PlainCloneContext(ctx, tmpDir, false, &CloneOptions{ + URL: "incorrectOnPurpose", + }) + c.Assert(r, IsNil) + c.Assert(err, Equals, ErrRepositoryAlreadyExists) } func (s *RepositorySuite) TestPlainCloneWithRecurseSubmodules(c *C) { @@ -839,6 +954,32 @@ func (s *RepositorySuite) TestCloneSingleBranch(c *C) { c.Assert(branch.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") } +func (s *RepositorySuite) TestCloneSingleTag(c *C) { + r, _ := Init(memory.NewStorage(), nil) + + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + err := r.clone(context.Background(), &CloneOptions{ + URL: url, + SingleBranch: true, + ReferenceName: plumbing.ReferenceName("refs/tags/commit-tag"), + }) + c.Assert(err, IsNil) + + branch, err := r.Reference("refs/tags/commit-tag", false) + c.Assert(err, IsNil) + c.Assert(branch, NotNil) + + conf, err := r.Config() + c.Assert(err, IsNil) + originRemote := conf.Remotes["origin"] + c.Assert(originRemote, NotNil) + c.Assert(originRemote.Fetch, HasLen, 1) + c.Assert(originRemote.Fetch[0].String(), Equals, "+refs/tags/commit-tag:refs/tags/commit-tag") +} + func (s *RepositorySuite) TestCloneDetachedHEAD(c *C) { r, _ := Init(memory.NewStorage(), nil) err := r.clone(context.Background(), &CloneOptions{ @@ -1110,6 +1251,77 @@ func (s *RepositorySuite) TestLog(c *C) { c.Assert(err, Equals, io.EOF) } +func (s *RepositorySuite) TestLogAll(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + cIter, err := r.Log(&LogOptions{ + All: true, + }) + c.Assert(err, IsNil) + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"), + plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294"), + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), + plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea"), + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + plumbing.NewHash("a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"), + plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"), + } + + for _, o := range commitOrder { + commit, err := cIter.Next() + c.Assert(err, IsNil) + c.Assert(commit.Hash, Equals, o) + } + _, err = cIter.Next() + c.Assert(err, Equals, io.EOF) + cIter.Close() +} + +func (s *RepositorySuite) TestLogAllOrderByTime(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + cIter, err := r.Log(&LogOptions{ + Order: LogOrderCommitterTime, + All: true, + }) + c.Assert(err, IsNil) + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"), + plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294"), + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"), + plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea"), + plumbing.NewHash("a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"), + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"), + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + } + + for _, o := range commitOrder { + commit, err := cIter.Next() + c.Assert(err, IsNil) + c.Assert(commit.Hash, Equals, o) + } + _, err = cIter.Next() + c.Assert(err, Equals, io.EOF) + cIter.Close() +} + func (s *RepositorySuite) TestLogHead(c *C) { r, _ := Init(memory.NewStorage(), nil) err := r.clone(context.Background(), &CloneOptions{ @@ -1192,8 +1404,8 @@ func (s *RepositorySuite) TestLogFileForEach(c *C) { fileName := "php/crappy.php" cIter, err := r.Log(&LogOptions{FileName: &fileName}) - c.Assert(err, IsNil) + defer cIter.Close() commitOrder := []plumbing.Hash{ plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294"), @@ -1203,7 +1415,51 @@ func (s *RepositorySuite) TestLogFileForEach(c *C) { cIter.ForEach(func(commit *object.Commit) error { expectedCommitHash := commitOrder[expectedIndex] c.Assert(commit.Hash.String(), Equals, expectedCommitHash.String()) - expectedIndex += 1 + expectedIndex++ + return nil + }) + c.Assert(expectedIndex, Equals, 1) +} + +func (s *RepositorySuite) TestLogNonHeadFile(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + fileName := "README" + cIter, err := r.Log(&LogOptions{FileName: &fileName}) + c.Assert(err, IsNil) + defer cIter.Close() + + _, err = cIter.Next() + c.Assert(err, Equals, io.EOF) +} + +func (s *RepositorySuite) TestLogAllFileForEach(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + fileName := "README" + cIter, err := r.Log(&LogOptions{FileName: &fileName, All: true}) + c.Assert(err, IsNil) + defer cIter.Close() + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"), + } + + expectedIndex := 0 + cIter.ForEach(func(commit *object.Commit) error { + expectedCommitHash := commitOrder[expectedIndex] + c.Assert(commit.Hash.String(), Equals, expectedCommitHash.String()) + expectedIndex++ return nil }) c.Assert(expectedIndex, Equals, 1) @@ -1221,6 +1477,7 @@ func (s *RepositorySuite) TestLogInvalidFile(c *C) { cIter, err := r.Log(&LogOptions{FileName: &fileName}) // Not raising an error since `git log -- vendor/foo12.go` responds silently c.Assert(err, IsNil) + defer cIter.Close() _, err = cIter.Next() c.Assert(err, Equals, io.EOF) @@ -1238,8 +1495,8 @@ func (s *RepositorySuite) TestLogFileInitialCommit(c *C) { Order: LogOrderCommitterTime, FileName: &fileName, }) - c.Assert(err, IsNil) + defer cIter.Close() commitOrder := []plumbing.Hash{ plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), @@ -1249,7 +1506,7 @@ func (s *RepositorySuite) TestLogFileInitialCommit(c *C) { cIter.ForEach(func(commit *object.Commit) error { expectedCommitHash := commitOrder[expectedIndex] c.Assert(commit.Hash.String(), Equals, expectedCommitHash.String()) - expectedIndex += 1 + expectedIndex++ return nil }) c.Assert(expectedIndex, Equals, 1) @@ -1269,6 +1526,8 @@ func (s *RepositorySuite) TestLogFileWithOtherParamsFail(c *C) { From: plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), }) c.Assert(err, IsNil) + defer cIter.Close() + _, iterErr := cIter.Next() c.Assert(iterErr, Equals, io.EOF) } @@ -2104,9 +2363,9 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { c.Assert(err, IsNil) datas := map[string]string{ - "efs/heads/master~": "reference not found", - "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "efs/heads/master~": "reference not found", + "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, + "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", "918c48b83bd081e863dbe1b80f8998f058cd8294": `refname "918c48b83bd081e863dbe1b80f8998f058cd8294" is ambiguous`, } @@ -2202,8 +2461,6 @@ func executeOnPath(path, cmd string) error { c.Stderr = buf c.Stdout = buf - //defer func() { fmt.Println(buf.String()) }() - return c.Run() } diff --git a/storage/filesystem/dotgit/dotgit_setref.go b/storage/filesystem/dotgit/dotgit_setref.go index d27c1a3..9da2f31 100644 --- a/storage/filesystem/dotgit/dotgit_setref.go +++ b/storage/filesystem/dotgit/dotgit_setref.go @@ -1,15 +1,24 @@ -// +build !norwfs - package dotgit import ( + "fmt" "os" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/ioutil" + + "gopkg.in/src-d/go-billy.v4" ) func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) (err error) { + if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) { + return d.setRefRwfs(fileName, content, old) + } + + return d.setRefNorwfs(fileName, content, old) +} + +func (d *DotGit) setRefRwfs(fileName, content string, old *plumbing.Reference) (err error) { // If we are not checking an old ref, just truncate the file. mode := os.O_RDWR | os.O_CREATE if old == nil { @@ -41,3 +50,41 @@ func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) (err _, err = f.Write([]byte(content)) return err } + +// There are some filesystems that don't support opening files in RDWD mode. +// In these filesystems the standard SetRef function can not be used as it +// reads the reference file to check that it's not modified before updating it. +// +// This version of the function writes the reference without extra checks +// making it compatible with these simple filesystems. This is usually not +// a problem as they should be accessed by only one process at a time. +func (d *DotGit) setRefNorwfs(fileName, content string, old *plumbing.Reference) error { + _, err := d.fs.Stat(fileName) + if err == nil && old != nil { + fRead, err := d.fs.Open(fileName) + if err != nil { + return err + } + + ref, err := d.readReferenceFrom(fRead, old.Name().String()) + fRead.Close() + + if err != nil { + return err + } + + if ref.Hash() != old.Hash() { + return fmt.Errorf("reference has changed concurrently") + } + } + + f, err := d.fs.Create(fileName) + if err != nil { + return err + } + + defer f.Close() + + _, err = f.Write([]byte(content)) + return err +} diff --git a/storage/filesystem/dotgit/dotgit_setref_norwfs.go b/storage/filesystem/dotgit/dotgit_setref_norwfs.go deleted file mode 100644 index 5695bd3..0000000 --- a/storage/filesystem/dotgit/dotgit_setref_norwfs.go +++ /dev/null @@ -1,47 +0,0 @@ -// +build norwfs - -package dotgit - -import ( - "fmt" - - "gopkg.in/src-d/go-git.v4/plumbing" -) - -// There are some filesystems that don't support opening files in RDWD mode. -// In these filesystems the standard SetRef function can not be used as i -// reads the reference file to check that it's not modified before updating it. -// -// This version of the function writes the reference without extra checks -// making it compatible with these simple filesystems. This is usually not -// a problem as they should be accessed by only one process at a time. -func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) error { - _, err := d.fs.Stat(fileName) - if err == nil && old != nil { - fRead, err := d.fs.Open(fileName) - if err != nil { - return err - } - - ref, err := d.readReferenceFrom(fRead, old.Name().String()) - fRead.Close() - - if err != nil { - return err - } - - if ref.Hash() != old.Hash() { - return fmt.Errorf("reference has changed concurrently") - } - } - - f, err := d.fs.Create(fileName) - if err != nil { - return err - } - - defer f.Close() - - _, err = f.Write([]byte(content)) - return err -} diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 308c6b7..73b0291 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -57,11 +57,26 @@ func (s *SuiteDotGit) TestSetRefs(c *C) { fs := osfs.New(tmp) dir := New(fs) + testSetRefs(c, dir) +} + +func (s *SuiteDotGit) TestSetRefsNorwfs(c *C) { + tmp, err := ioutil.TempDir("", "dot-git") + c.Assert(err, IsNil) + defer os.RemoveAll(tmp) + + fs := osfs.New(tmp) + dir := New(&norwfs{fs}) + + testSetRefs(c, dir) +} + +func testSetRefs(c *C, dir *DotGit) { firstFoo := plumbing.NewReferenceFromStrings( "refs/heads/foo", "e8d3ffab552895c19b9fcf7aa264d277cde33881", ) - err = dir.SetRef(firstFoo, nil) + err := dir.SetRef(firstFoo, nil) c.Assert(err, IsNil) @@ -795,3 +810,11 @@ func (s *SuiteDotGit) TestAlternates(c *C) { } c.Assert(dotgits[1].fs.Root(), Equals, expectedPath) } + +type norwfs struct { + billy.Filesystem +} + +func (f *norwfs) Capabilities() billy.Capability { + return billy.Capabilities(f.Filesystem) &^ billy.ReadAndWriteCapability +} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 6cd2d4c..3eb62a2 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -20,24 +20,25 @@ import ( type ObjectStorage struct { options Options - // deltaBaseCache is an object cache uses to cache delta's bases when - deltaBaseCache cache.Object + // objectCache is an object cache uses to cache delta's bases and also recently + // loaded loose objects + objectCache cache.Object dir *dotgit.DotGit index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. -func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { - return NewObjectStorageWithOptions(dir, cache, Options{}) +func NewObjectStorage(dir *dotgit.DotGit, objectCache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, objectCache, Options{}) } // NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options -func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { +func NewObjectStorageWithOptions(dir *dotgit.DotGit, objectCache cache.Object, ops Options) *ObjectStorage { return &ObjectStorage{ - options: ops, - deltaBaseCache: cache, - dir: dir, + options: ops, + objectCache: objectCache, + dir: dir, } } @@ -61,6 +62,11 @@ func (s *ObjectStorage) requireIndex() error { return nil } +// Reindex indexes again all packfiles. Useful if git changed packfiles externally +func (s *ObjectStorage) Reindex() { + s.index = nil +} + func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { f, err := s.dir.ObjectPackIdx(h) if err != nil { @@ -201,7 +207,7 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( idx := s.index[pack] hash, err := idx.FindHash(offset) if err == nil { - obj, ok := s.deltaBaseCache.Get(hash) + obj, ok := s.objectCache.Get(hash) if ok { return obj.Size(), nil } @@ -210,8 +216,8 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( } var p *packfile.Packfile - if s.deltaBaseCache != nil { - p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + if s.objectCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } @@ -236,9 +242,19 @@ func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( // EncodedObject returns the object with the given hash, by searching for it in // the packfile and the git object directories. func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { - obj, err := s.getFromUnpacked(h) - if err == plumbing.ErrObjectNotFound { + var obj plumbing.EncodedObject + var err error + + if s.index != nil { obj, err = s.getFromPackfile(h, false) + if err == plumbing.ErrObjectNotFound { + obj, err = s.getFromUnpacked(h) + } + } else { + obj, err = s.getFromUnpacked(h) + if err == plumbing.ErrObjectNotFound { + obj, err = s.getFromPackfile(h, false) + } } // If the error is still object not found, check if it's a shared object @@ -249,7 +265,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o := NewObjectStorage(dg, s.deltaBaseCache) + o := NewObjectStorage(dg, s.objectCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -299,9 +315,12 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } - defer ioutil.CheckClose(f, &err) + if cacheObj, found := s.objectCache.Get(h); found { + return cacheObj, nil + } + obj = s.NewEncodedObject() r, err := objfile.NewReader(f) if err != nil { @@ -322,6 +341,8 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } + s.objectCache.Put(obj) + _, err = io.Copy(w, r) return obj, err } @@ -364,7 +385,7 @@ func (s *ObjectStorage) decodeObjectAt( ) (plumbing.EncodedObject, error) { hash, err := idx.FindHash(offset) if err == nil { - obj, ok := s.deltaBaseCache.Get(hash) + obj, ok := s.objectCache.Get(hash) if ok { return obj, nil } @@ -375,8 +396,8 @@ func (s *ObjectStorage) decodeObjectAt( } var p *packfile.Packfile - if s.deltaBaseCache != nil { - p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + if s.objectCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } @@ -395,11 +416,7 @@ func (s *ObjectStorage) decodeDeltaObjectAt( } p := packfile.NewScanner(f) - if _, err := p.SeekFromStart(offset); err != nil { - return nil, err - } - - header, err := p.NextObjectHeader() + header, err := p.SeekObjectHeader(offset) if err != nil { return nil, err } @@ -490,7 +507,7 @@ func (s *ObjectStorage) buildPackfileIters( } return newPackfileIter( s.dir.Fs(), pack, t, seen, s.index[h], - s.deltaBaseCache, s.options.KeepDescriptors, + s.objectCache, s.options.KeepDescriptors, ) }, }, nil diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 4e6bbfb..5cfb227 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,8 +1,11 @@ package filesystem import ( + "fmt" + "io" "io/ioutil" "os" + "path/filepath" "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -204,6 +207,59 @@ func (s *FsSuite) TestPackfileIter(c *C) { }) } +func copyFile(c *C, dstDir, dstFilename string, srcFile *os.File) { + _, err := srcFile.Seek(0, 0) + c.Assert(err, IsNil) + + err = os.MkdirAll(dstDir, 0750|os.ModeDir) + c.Assert(err, IsNil) + + dst, err := os.OpenFile(filepath.Join(dstDir, dstFilename), os.O_CREATE|os.O_WRONLY, 0666) + c.Assert(err, IsNil) + defer dst.Close() + + _, err = io.Copy(dst, srcFile) + c.Assert(err, IsNil) +} + +// TestPackfileReindex tests that externally-added packfiles are considered by go-git +// after calling the Reindex method +func (s *FsSuite) TestPackfileReindex(c *C) { + // obtain a standalone packfile that is not part of any other repository + // in the fixtures: + packFixture := fixtures.ByTag("packfile").ByTag("standalone").One() + packFile := packFixture.Packfile() + idxFile := packFixture.Idx() + packFilename := packFixture.PackfileHash.String() + testObjectHash := plumbing.NewHash("a771b1e94141480861332fd0e4684d33071306c6") // this is an object we know exists in the standalone packfile + fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + storer := NewStorage(fs, cache.NewObjectLRUDefault()) + + // check that our test object is NOT found + _, err := storer.EncodedObject(plumbing.CommitObject, testObjectHash) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) + + // add the external packfile+idx to the packs folder + // this simulates a git bundle unbundle command, or a repack, for example. + copyFile(c, filepath.Join(storer.Filesystem().Root(), "objects", "pack"), + fmt.Sprintf("pack-%s.pack", packFilename), packFile) + copyFile(c, filepath.Join(storer.Filesystem().Root(), "objects", "pack"), + fmt.Sprintf("pack-%s.idx", packFilename), idxFile) + + // check that we cannot still retrieve the test object + _, err = storer.EncodedObject(plumbing.CommitObject, testObjectHash) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) + + storer.Reindex() // actually reindex + + // Now check that the test object can be retrieved + _, err = storer.EncodedObject(plumbing.CommitObject, testObjectHash) + c.Assert(err, IsNil) + + }) +} + func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() @@ -241,6 +297,23 @@ func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) { }) } +func (s *FsSuite) TestGetFromObjectFileSharedCache(c *C) { + f1 := fixtures.ByTag("worktree").One().DotGit() + f2 := fixtures.ByTag("worktree").ByTag("submodule").One().DotGit() + + ch := cache.NewObjectLRUDefault() + o1 := NewObjectStorage(dotgit.New(f1), ch) + o2 := NewObjectStorage(dotgit.New(f2), ch) + + expected := plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a") + obj, err := o1.EncodedObject(plumbing.CommitObject, expected) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + + obj, err = o2.EncodedObject(plumbing.CommitObject, expected) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) +} + func BenchmarkPackfileIter(b *testing.B) { if err := fixtures.Init(); err != nil { b.Fatal(err) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 14a772a..370f7bd 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -51,11 +51,7 @@ func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) fs: fs, dir: dir, - ObjectStorage: ObjectStorage{ - options: ops, - deltaBaseCache: cache, - dir: dir, - }, + ObjectStorage: *NewObjectStorageWithOptions(dir, cache, ops), ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, |