diff options
author | Máximo Cuadros <mcuadros@gmail.com> | 2018-09-10 11:33:35 +0200 |
---|---|---|
committer | Máximo Cuadros <mcuadros@gmail.com> | 2018-09-10 11:33:35 +0200 |
commit | bf0593d86ef73f28d3eb097c02d53a6a036f52f7 (patch) | |
tree | 2cfe20ab4b2ee85280aeabba40fd004d88e8b47f | |
parent | f8adfff71d844df7efa1367b7958e8f26411aaf9 (diff) | |
parent | a2d62f58ea626bb9f41de6431f6f18ca92cf78a0 (diff) | |
download | go-git-bf0593d86ef73f28d3eb097c02d53a6a036f52f7.tar.gz |
Merge branch 'master' of github.com:src-d/go-git into f-add-tagging-support
44 files changed, 1028 insertions, 187 deletions
diff --git a/common_test.go b/common_test.go index efe1ecc..dad0a37 100644 --- a/common_test.go +++ b/common_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -59,10 +60,7 @@ func (s *BaseSuite) NewRepository(f *fixtures.Fixture) *Repository { dotgit = f.DotGit() worktree = memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { @@ -89,10 +87,7 @@ func (s *BaseSuite) NewRepositoryWithEmptyWorktree(f *fixtures.Fixture) *Reposit worktree := memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { diff --git a/config/config.go b/config/config.go index ce6506d..a637f6d 100644 --- a/config/config.go +++ b/config/config.go @@ -40,6 +40,9 @@ type Config struct { IsBare bool // Worktree is the path to the root of the working tree. Worktree string + // CommentChar is the character indicating the start of a + // comment for commands like commit and tag + CommentChar string } Pack struct { @@ -113,6 +116,7 @@ const ( urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + commentCharKey = "commentChar" windowKey = "window" mergeKey = "merge" @@ -151,6 +155,7 @@ func (c *Config) unmarshalCore() { } c.Core.Worktree = s.Options.Get(worktreeKey) + c.Core.CommentChar = s.Options.Get(commentCharKey) } func (c *Config) unmarshalPack() error { diff --git a/config/config_test.go b/config/config_test.go index 5cd713e..fe73de8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -13,6 +13,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { input := []byte(`[core] bare = true worktree = foo + commentchar = bar [pack] window = 20 [remote "origin"] @@ -38,6 +39,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) c.Assert(cfg.Core.Worktree, Equals, "foo") + c.Assert(cfg.Core.CommentChar, Equals, "bar") c.Assert(cfg.Pack.Window, Equals, uint(20)) c.Assert(cfg.Remotes, HasLen, 2) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") diff --git a/plumbing/format/diff/unified_encoder.go b/plumbing/format/diff/unified_encoder.go index 58edd95..8bd6d8a 100644 --- a/plumbing/format/diff/unified_encoder.go +++ b/plumbing/format/diff/unified_encoder.go @@ -237,9 +237,13 @@ func (c *hunksGenerator) addLineNumbers(la, lb int, linesBefore int, i int, op O // we need to search for a reference for the next diff switch { case linesBefore != 0 && c.ctxLines != 0: - clb = lb - c.ctxLines + 1 + if lb > c.ctxLines { + clb = lb - c.ctxLines + 1 + } else { + clb = 1 + } case c.ctxLines == 0: - clb = lb - c.ctxLines + clb = lb case i != len(c.chunks)-1: next := c.chunks[i+1] if next.Type() == op || next.Type() == Equal { diff --git a/plumbing/format/diff/unified_encoder_test.go b/plumbing/format/diff/unified_encoder_test.go index 0e419ca..7736af1 100644 --- a/plumbing/format/diff/unified_encoder_test.go +++ b/plumbing/format/diff/unified_encoder_test.go @@ -155,6 +155,43 @@ var fixtures []*fixture = []*fixture{{ filePatches: []testFilePatch{{ from: &testFile{ mode: filemode.Regular, + path: "README.md", + seed: "hello\nworld\n", + }, + to: &testFile{ + mode: filemode.Regular, + path: "README.md", + seed: "hello\nbug\n", + }, + chunks: []testChunk{{ + content: "hello", + op: Equal, + }, { + content: "world", + op: Delete, + }, { + content: "bug", + op: Add, + }}, + }}, + }, + desc: "positive negative number", + context: 2, + diff: `diff --git a/README.md b/README.md +index 94954abda49de8615a048f8d2e64b5de848e27a1..f3dad9514629b9ff9136283ae331ad1fc95748a8 100644 +--- a/README.md ++++ b/README.md +@@ -1,2 +1,2 @@ + hello +-world ++bug +`, +}, { + patch: testPatch{ + message: "", + filePatches: []testFilePatch{{ + from: &testFile{ + mode: filemode.Regular, path: "test.txt", seed: "test", }, diff --git a/plumbing/format/gitignore/pattern.go b/plumbing/format/gitignore/pattern.go index 2603352..098cb50 100644 --- a/plumbing/format/gitignore/pattern.go +++ b/plumbing/format/gitignore/pattern.go @@ -133,6 +133,9 @@ func (p *pattern) globMatch(path []string, isDir bool) bool { } else if match { matched = true break + } else if len(path) == 0 { + // if nothing left then fail + matched = false } } } else { diff --git a/plumbing/format/gitignore/pattern_test.go b/plumbing/format/gitignore/pattern_test.go index f94cef3..c410442 100644 --- a/plumbing/format/gitignore/pattern_test.go +++ b/plumbing/format/gitignore/pattern_test.go @@ -281,3 +281,9 @@ func (s *PatternSuite) TestGlobMatch_wrongPattern_onTraversal_mismatch(c *C) { r := p.Match([]string{"value", "head", "vol["}, false) c.Assert(r, Equals, NoMatch) } + +func (s *PatternSuite) TestGlobMatch_issue_923(c *C) { + p := ParsePattern("**/android/**/GeneratedPluginRegistrant.java", nil) + r := p.Match([]string{"packages", "flutter_tools", "lib", "src", "android", "gradle.dart"}, false) + c.Assert(r, Equals, NoMatch) +} diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index c977bee..5fed278 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -3,6 +3,7 @@ package idxfile import ( "bytes" "io" + "sort" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" @@ -34,6 +35,9 @@ type Index interface { Count() (int64, error) // Entries returns an iterator to retrieve all index entries. Entries() (EntryIter, error) + // EntriesByOffset returns an iterator to retrieve all index entries ordered + // by offset. + EntriesByOffset() (EntryIter, error) } // MemoryIndex is the in memory representation of an idx file. @@ -215,6 +219,36 @@ func (idx *MemoryIndex) Entries() (EntryIter, error) { return &idxfileEntryIter{idx, 0, 0, 0}, nil } +// EntriesByOffset implements the Index interface. +func (idx *MemoryIndex) EntriesByOffset() (EntryIter, error) { + count, err := idx.Count() + if err != nil { + return nil, err + } + + iter := &idxfileEntryOffsetIter{ + entries: make(entriesByOffset, count), + } + + entries, err := idx.Entries() + if err != nil { + return nil, err + } + + for pos := 0; int64(pos) < count; pos++ { + entry, err := entries.Next() + if err != nil { + return nil, err + } + + iter.entries[pos] = entry + } + + sort.Sort(iter.entries) + + return iter, nil +} + // EntryIter is an iterator that will return the entries in a packfile index. type EntryIter interface { // Next returns the next entry in the packfile index. @@ -276,3 +310,38 @@ type Entry struct { CRC32 uint32 Offset uint64 } + +type idxfileEntryOffsetIter struct { + entries entriesByOffset + pos int +} + +func (i *idxfileEntryOffsetIter) Next() (*Entry, error) { + if i.pos >= len(i.entries) { + return nil, io.EOF + } + + entry := i.entries[i.pos] + i.pos++ + + return entry, nil +} + +func (i *idxfileEntryOffsetIter) Close() error { + i.pos = len(i.entries) + 1 + return nil +} + +type entriesByOffset []*Entry + +func (o entriesByOffset) Len() int { + return len(o) +} + +func (o entriesByOffset) Less(i int, j int) bool { + return o[i].Offset < o[j].Offset +} + +func (o entriesByOffset) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] +} diff --git a/plumbing/format/idxfile/idxfile_test.go b/plumbing/format/idxfile/idxfile_test.go index d15accf..0e0ca2a 100644 --- a/plumbing/format/idxfile/idxfile_test.go +++ b/plumbing/format/idxfile/idxfile_test.go @@ -115,6 +115,21 @@ func (s *IndexSuite) TestFindHash(c *C) { } } +func (s *IndexSuite) TestEntriesByOffset(c *C) { + idx, err := fixtureIndex() + c.Assert(err, IsNil) + + entries, err := idx.EntriesByOffset() + c.Assert(err, IsNil) + + for _, pos := range fixtureOffsets { + e, err := entries.Next() + c.Assert(err, IsNil) + + c.Assert(e.Offset, Equals, uint64(pos)) + } +} + var fixtureHashes = []plumbing.Hash{ plumbing.NewHash("303953e5aa461c203a324821bc1717f9b4fff895"), plumbing.NewHash("5296768e3d9f661387ccbff18c4dea6c997fd78c"), diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index fc1419e..e15126e 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -8,6 +8,7 @@ import ( "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -32,8 +33,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 10) }) } @@ -47,8 +47,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 0) }) } diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index 6fd3ca5..330cb73 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -47,6 +47,16 @@ func NewFSObject( // Reader implements the plumbing.EncodedObject interface. func (o *FSObject) Reader() (io.ReadCloser, error) { + obj, ok := o.cache.Get(o.hash) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + f, err := o.fs.Open(o.path) if err != nil { return nil, err diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 5feb781..852a834 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -258,6 +258,19 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { } func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { + ref, err := p.FindHash(offset) + if err == nil { + obj, ok := p.cacheGet(ref) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + } + if _, err := p.s.SeekFromStart(offset); err != nil { return nil, err } @@ -306,6 +319,8 @@ func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { } _, _, err = p.s.NextObject(w) + p.cachePut(obj) + return err } @@ -394,7 +409,7 @@ func (p *Packfile) GetByType(typ plumbing.ObjectType) (storer.EncodedObjectIter, plumbing.TreeObject, plumbing.CommitObject, plumbing.TagObject: - entries, err := p.Entries() + entries, err := p.EntriesByOffset() if err != nil { return nil, err } diff --git a/plumbing/object/change_adaptor_test.go b/plumbing/object/change_adaptor_test.go index 803c3b8..c7c003b 100644 --- a/plumbing/object/change_adaptor_test.go +++ b/plumbing/object/change_adaptor_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -23,8 +24,7 @@ type ChangeAdaptorSuite struct { func (s *ChangeAdaptorSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } diff --git a/plumbing/object/change_test.go b/plumbing/object/change_test.go index b0e89c7..e2f0a23 100644 --- a/plumbing/object/change_test.go +++ b/plumbing/object/change_test.go @@ -5,6 +5,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/diff" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ func (s *ChangeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag(".git").One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -253,8 +253,7 @@ func (s *ChangeSuite) TestNoFileFilemodes(c *C) { s.Suite.SetUpSuite(c) f := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) iter, err := sto.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 00ae3f1..e254342 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -199,17 +199,23 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } split := bytes.SplitN(line, []byte{' '}, 2) + + var data []byte + if len(split) == 2 { + data = split[1] + } + switch string(split[0]) { case "tree": - c.TreeHash = plumbing.NewHash(string(split[1])) + c.TreeHash = plumbing.NewHash(string(data)) case "parent": - c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(split[1]))) + c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(data))) case "author": - c.Author.Decode(split[1]) + c.Author.Decode(data) case "committer": - c.Committer.Decode(split[1]) + c.Committer.Decode(data) case headerpgp: - c.PGPSignature += string(split[1]) + "\n" + c.PGPSignature += string(data) + "\n" pgpsig = true } } else { diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index b5dfbe3..c9acf42 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -247,8 +248,7 @@ func (s *SuiteCommit) TestStringMultiLine(c *C) { hash := plumbing.NewHash("e7d896db87294e33ca3202e536d4d9bb16023db3") f := fixtures.ByURL("https://github.com/src-d/go-git.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) o, err := sto.EncodedObject(plumbing.CommitObject, hash) c.Assert(err, IsNil) @@ -325,6 +325,22 @@ RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= c.Assert(err, IsNil) c.Assert(decoded.PGPSignature, Equals, pgpsignature) + // signature with extra empty line, it caused "index out of range" when + // parsing it + + pgpsignature2 := "\n" + pgpsignature + + commit.PGPSignature = pgpsignature2 + encoded = &plumbing.MemoryObject{} + decoded = &Commit{} + + err = commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, pgpsignature2) + // signature in author name commit.PGPSignature = "" @@ -461,3 +477,21 @@ func (s *SuiteCommit) TestPatchCancel(c *C) { c.Assert(err, ErrorMatches, "operation canceled") } + +func (s *SuiteCommit) TestMalformedHeader(c *C) { + encoded := &plumbing.MemoryObject{} + decoded := &Commit{} + commit := *s.Commit + + commit.PGPSignature = "\n" + commit.Author.Name = "\n" + commit.Author.Email = "\n" + commit.Committer.Name = "\n" + commit.Committer.Email = "\n" + + err := commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) +} diff --git a/plumbing/object/difftree_test.go b/plumbing/object/difftree_test.go index ff9ecbc..4af8684 100644 --- a/plumbing/object/difftree_test.go +++ b/plumbing/object/difftree_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ type DiffTreeSuite struct { func (s *DiffTreeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto s.cache = make(map[string]storer.EncodedObjectStorer) } diff --git a/plumbing/object/file_test.go b/plumbing/object/file_test.go index edb82d0..4b92749 100644 --- a/plumbing/object/file_test.go +++ b/plumbing/object/file_test.go @@ -4,6 +4,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -44,8 +45,7 @@ var fileIterTests = []struct { func (s *FileSuite) TestIter(c *C) { for i, t := range fileIterTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -106,8 +106,7 @@ hs_err_pid* func (s *FileSuite) TestContents(c *C) { for i, t := range contentsTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -160,8 +159,7 @@ var linesTests = []struct { func (s *FileSuite) TestLines(c *C) { for i, t := range linesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -195,8 +193,7 @@ var ignoreEmptyDirEntriesTests = []struct { func (s *FileSuite) TestIgnoreEmptyDirEntries(c *C) { for i, t := range ignoreEmptyDirEntriesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -251,9 +248,7 @@ func (s *FileSuite) TestFileIter(c *C) { func (s *FileSuite) TestFileIterSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/plumbing/object/object.go b/plumbing/object/object.go index 4b59aba..e960e50 100644 --- a/plumbing/object/object.go +++ b/plumbing/object/object.go @@ -152,7 +152,11 @@ func (s *Signature) decodeTimeAndTimeZone(b []byte) { } func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error { - _, err := fmt.Fprintf(w, "%d %s", s.When.Unix(), s.When.Format("-0700")) + u := s.When.Unix() + if u < 0 { + u = 0 + } + _, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700")) return err } diff --git a/plumbing/object/object_test.go b/plumbing/object/object_test.go index 68aa1a1..8f0eede 100644 --- a/plumbing/object/object_test.go +++ b/plumbing/object/object_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -26,8 +27,7 @@ type BaseObjectsSuite struct { func (s *BaseObjectsSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - storer, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } diff --git a/plumbing/object/patch_test.go b/plumbing/object/patch_test.go index 8eb65ec..47057fb 100644 --- a/plumbing/object/patch_test.go +++ b/plumbing/object/patch_test.go @@ -4,6 +4,7 @@ import ( . "gopkg.in/check.v1" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -14,8 +15,8 @@ type PatchSuite struct { var _ = Suite(&PatchSuite{}) func (s *PatchSuite) TestStatsWithSubmodules(c *C) { - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit()) + storer := filesystem.NewStorage( + fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit(), cache.NewObjectLRUDefault()) commit, err := GetCommit(storer, plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4")) diff --git a/plumbing/object/tag_test.go b/plumbing/object/tag_test.go index 9900093..59c28b0 100644 --- a/plumbing/object/tag_test.go +++ b/plumbing/object/tag_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -22,9 +23,7 @@ var _ = Suite(&TagSuite{}) func (s *TagSuite) SetUpSuite(c *C) { s.BaseObjectsSuite.SetUpSuite(c) - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } @@ -265,7 +264,7 @@ func (s *TagSuite) TestStringNonCommit(c *C) { c.Assert(tag.String(), Equals, "tag TAG TWO\n"+ "Tagger: <>\n"+ - "Date: Mon Jan 01 00:00:00 0001 +0000\n"+ + "Date: Thu Jan 01 00:00:00 1970 +0000\n"+ "\n"+ "tag two\n") } diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index 59d5d21..7366421 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -5,6 +5,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -341,8 +342,7 @@ func (s *TreeSuite) TestTreeWalkerNextNonRecursive(c *C) { func (s *TreeSuite) TestTreeWalkerNextSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go index 55d9bca..dea1c73 100644 --- a/plumbing/revlist/revlist_test.go +++ b/plumbing/revlist/revlist_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -51,8 +52,7 @@ const ( func (s *RevListSuite) SetUpTest(c *C) { s.Suite.SetUpSuite(c) - sto, err := filesystem.NewStorage(fixtures.Basic().One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.Basic().One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -67,8 +67,7 @@ func (s *RevListSuite) TestRevListObjects_Submodules(c *C) { "6ecf0ef2c2dffb796033e5a02219af86ec6584e5": true, } - sto, err := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit(), cache.NewObjectLRUDefault()) ref, err := storer.ResolveReference(sto, plumbing.HEAD) c.Assert(err, IsNil) @@ -109,10 +108,9 @@ func (s *RevListSuite) TestRevListObjects(c *C) { } func (s *RevListSuite) TestRevListObjectsTagObject(c *C) { - sto, err := filesystem.NewStorage( + sto := filesystem.NewStorage( fixtures.ByTag("tags"). - ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) expected := map[string]bool{ "70846e9a10ef7b41064b40f07713d5b8b9a8fc73": true, diff --git a/plumbing/transport/server/loader.go b/plumbing/transport/server/loader.go index c83752c..13b3526 100644 --- a/plumbing/transport/server/loader.go +++ b/plumbing/transport/server/loader.go @@ -1,6 +1,7 @@ package server import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -43,7 +44,7 @@ func (l *fsLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { return nil, transport.ErrRepositoryNotFound } - return filesystem.NewStorage(fs) + return filesystem.NewStorage(fs, cache.NewObjectLRUDefault()), nil } // MapLoader is a Loader that uses a lookup map of storer.Storer by diff --git a/plumbing/transport/server/server_test.go b/plumbing/transport/server/server_test.go index 33d74d1..302ff48 100644 --- a/plumbing/transport/server/server_test.go +++ b/plumbing/transport/server/server_test.go @@ -3,6 +3,7 @@ package server_test import ( "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/client" "gopkg.in/src-d/go-git.v4/plumbing/transport/server" @@ -53,8 +54,7 @@ func (s *BaseSuite) prepareRepositories(c *C) { fs := fixtures.Basic().One().DotGit() s.Endpoint, err = transport.NewEndpoint(fs.Root()) c.Assert(err, IsNil) - s.loader[s.Endpoint.String()], err = filesystem.NewStorage(fs) - c.Assert(err, IsNil) + s.loader[s.Endpoint.String()] = filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) s.EmptyEndpoint, err = transport.NewEndpoint("/empty.git") c.Assert(err, IsNil) diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index 84cfab2..dbb47c5 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -236,7 +236,7 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a // known_hosts file. http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT // -// If files is empty, the list of files will be read from the SSH_KNOWN_HOSTS +// If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: // /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // @@ -244,13 +244,15 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // ~/.ssh/known_hosts // /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { - files, err := getDefaultKnownHostsFiles() - if err != nil { - return nil, err + var err error + + if len(files) == 0 { + if files, err = getDefaultKnownHostsFiles(); err != nil { + return nil, err + } } - files, err = filterKnownHostsFiles(files...) - if err != nil { + if files, err = filterKnownHostsFiles(files...); err != nil { return nil, err } diff --git a/plumbing/transport/ssh/auth_method_test.go b/plumbing/transport/ssh/auth_method_test.go index 0025669..0cde61e 100644 --- a/plumbing/transport/ssh/auth_method_test.go +++ b/plumbing/transport/ssh/auth_method_test.go @@ -1,16 +1,30 @@ package ssh import ( + "bufio" "fmt" "io/ioutil" "os" + "strings" + "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/testdata" . "gopkg.in/check.v1" ) -type SuiteCommon struct{} +type ( + SuiteCommon struct{} + + mockKnownHosts struct{} +) + +func (mockKnownHosts) host() string { return "github.com" } +func (mockKnownHosts) knownHosts() []byte { + return []byte(`github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`) +} +func (mockKnownHosts) Network() string { return "tcp" } +func (mockKnownHosts) String() string { return "github.com:22" } var _ = Suite(&SuiteCommon{}) @@ -149,3 +163,49 @@ func (*SuiteCommon) TestNewPublicKeysWithInvalidPEM(c *C) { c.Assert(err, NotNil) c.Assert(auth, IsNil) } + +func (*SuiteCommon) TestNewKnownHostsCallback(c *C) { + var mock = mockKnownHosts{} + + f, err := ioutil.TempFile("", "known-hosts") + c.Assert(err, IsNil) + + _, err = f.Write(mock.knownHosts()) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + defer os.RemoveAll(f.Name()) + + f, err = os.Open(f.Name()) + c.Assert(err, IsNil) + + defer f.Close() + + var hostKey ssh.PublicKey + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Split(scanner.Text(), " ") + if len(fields) != 3 { + continue + } + if strings.Contains(fields[0], mock.host()) { + var err error + hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes()) + if err != nil { + c.Fatalf("error parsing %q: %v", fields[2], err) + } + break + } + } + if hostKey == nil { + c.Fatalf("no hostkey for %s", mock.host()) + } + + clb, err := NewKnownHostsCallback(f.Name()) + c.Assert(err, IsNil) + + err = clb(mock.String(), mock, hostKey) + c.Assert(err, IsNil) +} diff --git a/prune_test.go b/prune_test.go index 60652ec..670cd07 100644 --- a/prune_test.go +++ b/prune_test.go @@ -4,6 +4,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -22,8 +23,7 @@ func (s *PruneSuite) testPrune(c *C, deleteTime time.Time) { srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) diff --git a/remote_test.go b/remote_test.go index dd386b0..175faed 100644 --- a/remote_test.go +++ b/remote_test.go @@ -9,6 +9,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -238,7 +239,7 @@ func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) c.Assert(err, IsNil) mock := &mockPackfileWriter{Storer: fss} @@ -375,8 +376,7 @@ func (s *RemoteSuite) TestFetchFastForwardFS(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) - c.Assert(err, IsNil) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) // This exercises `storage.filesystem.Storage.CheckAndSetReference()`. s.testFetchFastForward(c, fss) @@ -400,8 +400,7 @@ func (s *RemoteSuite) TestPushToEmptyRepository(c *C) { c.Assert(err, IsNil) srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -438,8 +437,7 @@ func (s *RemoteSuite) TestPushContext(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -461,8 +459,7 @@ func (s *RemoteSuite) TestPushTags(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -485,15 +482,14 @@ func (s *RemoteSuite) TestPushTags(c *C) { func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, URLs: []string{fs.Root()}, }) - err = r.Push(&PushOptions{ + err := r.Push(&PushOptions{ RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"}, }) c.Assert(err, Equals, NoErrAlreadyUpToDate) @@ -501,8 +497,7 @@ func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { func (s *RemoteSuite) TestPushDeleteReference(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -526,8 +521,7 @@ func (s *RemoteSuite) TestPushDeleteReference(c *C) { func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { fs := fixtures.Basic().One().DotGit() - server, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + server := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -554,12 +548,10 @@ func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { func (s *RemoteSuite) TestPushForce(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) dstFs := f.DotGit() - dstSto, err := filesystem.NewStorage(dstFs) - c.Assert(err, IsNil) + dstSto := filesystem.NewStorage(dstFs, cache.NewObjectLRUDefault()) url := dstFs.Root() r := newRemote(sto, &config.RemoteConfig{ @@ -703,8 +695,7 @@ func (s *RemoteSuite) TestPushWrongRemoteName(c *C) { func (s *RemoteSuite) TestGetHaves(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) var localRefs = []*plumbing.Reference{ plumbing.NewReferenceFromStrings( diff --git a/repository.go b/repository.go index a132c39..be1f057 100644 --- a/repository.go +++ b/repository.go @@ -16,6 +16,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/internal/revision" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -228,10 +229,7 @@ func PlainInit(path string, isBare bool) (*Repository, error) { dot, _ = wt.Chroot(GitDirName) } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Init(s, wt) } @@ -243,9 +241,8 @@ func PlainOpen(path string) (*Repository, error) { return PlainOpenWithOptions(path, &PlainOpenOptions{}) } -// PlainOpen opens a git repository from the given path. It detects if the -// repository is bare or a normal one. If the path doesn't contain a valid -// repository ErrRepositoryNotExists is returned +// PlainOpenWithOptions opens a git repository from the given path with specific +// options. See PlainOpen for more info. func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) { dot, wt, err := dotGitToOSFilesystems(path, o.DetectDotGit) if err != nil { @@ -260,10 +257,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Open(s, wt) } diff --git a/repository_test.go b/repository_test.go index 89911c0..e48f42d 100644 --- a/repository_test.go +++ b/repository_test.go @@ -18,6 +18,7 @@ import ( openpgperr "golang.org/x/crypto/openpgp/errors" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -54,8 +55,7 @@ func (s *RepositorySuite) TestInitNonStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot("storage") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) wt, _ := fs.Chroot("worktree") r, err := Init(storage, wt) @@ -81,8 +81,7 @@ func (s *RepositorySuite) TestInitStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot(".git") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) r, err := Init(storage, fs) c.Assert(err, IsNil) @@ -1061,8 +1060,7 @@ func (s *RepositorySuite) TestPushDepth(c *C) { func (s *RepositorySuite) TestPushNonExistentRemote(c *C) { srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r, err := Open(sto, srcFs) c.Assert(err, IsNil) @@ -1669,8 +1667,7 @@ func (s *RepositorySuite) TestDeleteTagAnnotatedUnpacked(c *C) { func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1887,8 +1884,7 @@ func (s *RepositorySuite) TestWorktreeBare(c *C) { func (s *RepositorySuite) TestResolveRevision(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/basic.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1959,8 +1955,7 @@ func (s *RepositorySuite) testRepackObjects( srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) @@ -2125,10 +2120,7 @@ func BenchmarkObjects(b *testing.B) { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - storer, err := filesystem.NewStorage(fs) - if err != nil { - b.Fatal(err) - } + storer := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) worktree, err := fs.Chroot(filepath.Dir(fs.Root())) if err != nil { @@ -1,7 +1,10 @@ package git -import "fmt" -import "bytes" +import ( + "bytes" + "fmt" + "path/filepath" +) // Status represents the current status of a Worktree. // The key of the map is the path of the file. @@ -17,6 +20,12 @@ func (s Status) File(path string) *FileStatus { return s[path] } +// IsUntracked checks if file for given path is 'Untracked' +func (s Status) IsUntracked(path string) bool { + stat, ok := (s)[filepath.ToSlash(path)] + return ok && stat.Worktree == Untracked +} + // IsClean returns true if all the files aren't in Unmodified status. func (s Status) IsClean() bool { for _, status := range s { diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index af07eb5..a58c248 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -57,17 +57,48 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem + options Options + fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, Options{}) +} + +// NewWithOptions sets non default configuration options. +// See New for complete help. +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { + return &DotGit{ + options: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -97,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -139,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.options.ExclusiveAccess { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -177,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + if d.files == nil { + d.files = make(map[string]billy.File) + } + + err := d.hasPack(hash) + if err != nil { + return nil, err + } + + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -186,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -220,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.options.ExclusiveAccess { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -237,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.options.ExclusiveAccess { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -274,24 +403,178 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack +func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { + hString := h.String() + + if d.incomingDirName == "" { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { + d.incomingDirName = file.Name() + } + } + } + + d.incomingChecked = true + } + + return d.incomingDirName != "" +} + // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { - return d.fs.Open(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Open(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { - return d.fs.Stat(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Stat(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { - return d.fs.Remove(d.objectPath(h)) + d.cleanObjectList() + + err1 := d.fs.Remove(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + err2 := d.fs.Remove(d.incomingObjectPath(h)) + if err2 != nil { + return err1 + } + return err2 + } + return err1 } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 7733eef..308c6b7 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" @@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) { fs := f.DotGit() dir := New(fs) + testObjectPacks(c, fs, dir, f) +} + +func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjectPacks(c, fs, dir, f) +} + +func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) { hashes, err := dir.ObjectPacks() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 1) @@ -452,6 +465,45 @@ func (s *SuiteDotGit) TestObjectPack(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") } +func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{KeepDescriptors: true}) + + pack, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + + // Move to an specific offset + pack.Seek(42, os.SEEK_SET) + + pack2, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is the same the offset should be the same + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) + + err = dir.Close() + c.Assert(err, IsNil) + + pack2, err = dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is opened again its offset should be 0 + offset, err = pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = pack2.Close() + c.Assert(err, IsNil) + + err = dir.Close() + c.Assert(err, NotNil) + +} + func (s *SuiteDotGit) TestObjectPackIdx(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() @@ -506,6 +558,17 @@ func (s *SuiteDotGit) TestObjects(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() dir := New(fs) + testObjects(c, fs, dir) +} + +func (s *SuiteDotGit) TestObjectsExclusive(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjects(c, fs, dir) +} + +func testObjects(c *C, fs billy.Filesystem, dir *DotGit) { hashes, err := dir.Objects() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 187) @@ -537,6 +600,57 @@ func (s *SuiteDotGit) TestObject(c *C) { file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")), Equals, true, ) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + file, err = dir.Object(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectStat(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + _, err := dir.ObjectStat(hash) + c.Assert(err, IsNil) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + _, err = dir.ObjectStat(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectDelete(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + err := dir.ObjectDelete(hash) + c.Assert(err, IsNil) + + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40]) + + err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + c.Assert(err, IsNil) + + f, err := fs.Create(incomingFilePath) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { diff --git a/storage/filesystem/module.go b/storage/filesystem/module.go index 7c8c8d8..9272206 100644 --- a/storage/filesystem/module.go +++ b/storage/filesystem/module.go @@ -1,6 +1,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) @@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) { return nil, err } - return NewStorage(fs) + return NewStorage(fs, cache.NewObjectLRUDefault()), nil } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 6958e32..9eb085f 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -18,6 +18,8 @@ import ( ) type ObjectStorage struct { + options Options + // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object @@ -25,14 +27,18 @@ type ObjectStorage struct { index map[plumbing.Hash]idxfile.Index } -// NewObjectStorage creates a new ObjectStorage with the given .git directory. -func NewObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { - s := ObjectStorage{ - deltaBaseCache: cache.NewObjectLRUDefault(), +// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. +func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, cache, Options{}) +} + +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options +func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { + return &ObjectStorage{ + options: ops, + deltaBaseCache: cache, dir: dir, } - - return s, nil } func (s *ObjectStorage) requireIndex() error { @@ -62,6 +68,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { @@ -169,10 +176,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := NewObjectStorage(dg) - if oe != nil { - continue - } + o := NewObjectStorage(dg, s.deltaBaseCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -268,7 +272,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( return nil, err } - defer ioutil.CheckClose(f, &err) + if !s.options.KeepDescriptors { + defer ioutil.CheckClose(f, &err) + } idx := s.index[pack] if canBeDelta { @@ -295,7 +301,14 @@ func (s *ObjectStorage) decodeObjectAt( return nil, err } - return packfile.NewPackfile(idx, s.dir.Fs(), f).GetByOffset(offset) + var p *packfile.Packfile + if s.deltaBaseCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + } else { + p = packfile.NewPackfile(idx, s.dir.Fs(), f) + } + + return p.GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( @@ -404,6 +417,11 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb }, nil } +// Close closes all opened files. +func (s *ObjectStorage) Close() error { + return s.dir.Close() +} + type lazyPackfilesIter struct { hashes []plumbing.Hash open func(h plumbing.Hash) (storer.EncodedObjectIter, error) @@ -486,7 +504,14 @@ func newPackfileIter( index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - iter, err := packfile.NewPackfile(index, fs, f).GetByType(t) + var p *packfile.Packfile + if cache != nil { + p = packfile.NewPackfileWithCache(index, fs, f, cache) + } else { + p = packfile.NewPackfile(index, fs, f) + } + + iter, err := p.GetByType(t) if err != nil { return nil, err } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index b1408b7..bd4a94b 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -2,9 +2,11 @@ package filesystem import ( "io/ioutil" + "os" "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" @@ -26,8 +28,7 @@ var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -38,20 +39,53 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) { func (s *FsSuite) TestGetFromPackfile(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) + + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + }) +} + +func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { + fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true}) + o := NewObjectStorageWithOptions(dg, cache.NewObjectLRUDefault(), Options{KeepDescriptors: true}) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) c.Assert(obj.Hash(), Equals, expected) + + packfiles, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + pack1, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + pack1.Seek(42, os.SEEK_SET) + + err = o.Close() + c.Assert(err, IsNil) + + pack2, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = o.Close() + c.Assert(err, IsNil) + }) } func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") obj, err := o.getFromPackfile(expected, false) @@ -67,8 +101,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -88,8 +121,7 @@ func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { for _, t := range objectTypes { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(t) c.Assert(err, IsNil) @@ -271,11 +303,7 @@ func BenchmarkGetObjectFromPackfile(b *testing.B) { for _, f := range fixtures.Basic() { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - if err != nil { - b.Fatal(err) - } - + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) for i := 0; i < b.N; i++ { expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 622bb4a..14a772a 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -22,25 +23,45 @@ type Storage struct { ModuleStorage } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) - o, err := NewObjectStorage(dir) - if err != nil { - return nil, err +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) +} + +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } + dir := dotgit.NewWithOptions(fs, dirOps) return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -48,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 4d9ba6f..6fa0d90 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/test" @@ -23,9 +24,12 @@ var _ = Suite(&StorageSuite{}) func (s *StorageSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorage(osfs.New(s.dir)) - c.Assert(err, IsNil) + storage := NewStorage(osfs.New(s.dir), cache.NewObjectLRUDefault()) + + setUpTest(s, c, storage) +} +func setUpTest(s *StorageSuite, c *C, storage *Storage) { // ensure that right interfaces are implemented var _ storer.EncodedObjectStorer = storage var _ storer.IndexStorer = storage @@ -40,8 +44,7 @@ func (s *StorageSuite) SetUpTest(c *C) { func (s *StorageSuite) TestFilesystem(c *C) { fs := memfs.New() - storage, err := NewStorage(fs) - c.Assert(err, IsNil) + storage := NewStorage(fs, cache.NewObjectLRUDefault()) c.Assert(storage.Filesystem(), Equals, fs) } @@ -51,3 +54,19 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) { c.Assert(err, IsNil) c.Assert(fis, HasLen, 0) } + +type StorageExclusiveSuite struct { + StorageSuite +} + +var _ = Suite(&StorageExclusiveSuite{}) + +func (s *StorageExclusiveSuite) SetUpTest(c *C) { + s.dir = c.MkDir() + storage := NewStorageWithOptions( + osfs.New(s.dir), + cache.NewObjectLRUDefault(), + Options{ExclusiveAccess: true}) + + setUpTest(&s.StorageSuite, c, storage) +} diff --git a/worktree.go b/worktree.go index 99b2cd1..e45d815 100644 --- a/worktree.go +++ b/worktree.go @@ -713,29 +713,54 @@ func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { } // Clean the worktree by removing untracked files. +// An empty dir could be removed - this is what `git clean -f -d .` does. func (w *Worktree) Clean(opts *CleanOptions) error { s, err := w.Status() if err != nil { return err } - // Check Worktree status to be Untracked, obtain absolute path and delete. - for relativePath, status := range s { - // Check if the path contains a directory and if Dir options is false, - // skip the path. - if relativePath != filepath.Base(relativePath) && !opts.Dir { + root := "" + files, err := w.Filesystem.ReadDir(root) + if err != nil { + return err + } + return w.doClean(s, opts, root, files) +} + +func (w *Worktree) doClean(status Status, opts *CleanOptions, dir string, files []os.FileInfo) error { + for _, fi := range files { + if fi.Name() == ".git" { continue } - // Remove the file only if it's an untracked file. - if status.Worktree == Untracked { - absPath := filepath.Join(w.Filesystem.Root(), relativePath) - if err := os.Remove(absPath); err != nil { + // relative path under the root + path := filepath.Join(dir, fi.Name()) + if fi.IsDir() { + if !opts.Dir { + continue + } + + subfiles, err := w.Filesystem.ReadDir(path) + if err != nil { + return err + } + err = w.doClean(status, opts, path, subfiles) + if err != nil { return err } + } else { + if status.IsUntracked(path) { + if err := w.Filesystem.Remove(path); err != nil { + return err + } + } } } + if opts.Dir { + return doCleanDirectories(w.Filesystem, dir) + } return nil } @@ -881,15 +906,18 @@ func rmFileAndDirIfEmpty(fs billy.Filesystem, name string) error { return err } - path := filepath.Dir(name) - files, err := fs.ReadDir(path) + dir := filepath.Dir(name) + return doCleanDirectories(fs, dir) +} + +// doCleanDirectories removes empty subdirs (without files) +func doCleanDirectories(fs billy.Filesystem, dir string) error { + files, err := fs.ReadDir(dir) if err != nil { return err } - if len(files) == 0 { - fs.Remove(path) + return fs.Remove(dir) } - return nil } diff --git a/worktree_commit.go b/worktree_commit.go index b83bf0c..673eb16 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -3,6 +3,7 @@ package git import ( "bytes" "path" + "sort" "strings" "golang.org/x/crypto/openpgp" @@ -188,7 +189,20 @@ func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) { h.trees[parent].Entries = append(h.trees[parent].Entries, te) } +type sortableEntries []object.TreeEntry + +func (sortableEntries) sortName(te object.TreeEntry) string { + if te.Mode == filemode.Dir { + return te.Name + "/" + } + return te.Name +} +func (se sortableEntries) Len() int { return len(se) } +func (se sortableEntries) Less(i int, j int) bool { return se.sortName(se[i]) < se.sortName(se[j]) } +func (se sortableEntries) Swap(i int, j int) { se[i], se[j] = se[j], se[i] } + func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) { + sort.Sort(sortableEntries(t.Entries)) for i, e := range t.Entries { if e.Mode != filemode.Dir && !e.Hash.IsZero() { continue diff --git a/worktree_commit_test.go b/worktree_commit_test.go index fdeaec0..da377c6 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -2,19 +2,25 @@ package git import ( "bytes" + "io/ioutil" + "os" + "os/exec" "strings" "time" - "golang.org/x/crypto/openpgp" - "golang.org/x/crypto/openpgp/armor" - "golang.org/x/crypto/openpgp/errors" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" + "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-billy.v4/util" ) @@ -196,6 +202,53 @@ func (s *WorktreeSuite) TestCommitSignBadKey(c *C) { c.Assert(err, Equals, errors.InvalidArgumentError("signing key is encrypted")) } +func (s *WorktreeSuite) TestCommitTreeSort(c *C) { + path, err := ioutil.TempDir(os.TempDir(), "test-commit-tree-sort") + c.Assert(err, IsNil) + fs := osfs.New(path) + st := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + r, err := Init(st, nil) + c.Assert(err, IsNil) + + r, err = Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + URL: path, + }) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + mfs := w.Filesystem + + err = mfs.MkdirAll("delta", 0755) + c.Assert(err, IsNil) + + for _, p := range []string{"delta_last", "Gamma", "delta/middle", "Beta", "delta-first", "alpha"} { + util.WriteFile(mfs, p, []byte("foo"), 0644) + _, err = w.Add(p) + c.Assert(err, IsNil) + } + + _, err = w.Commit("foo\n", &CommitOptions{ + All: true, + Author: defaultSignature(), + }) + c.Assert(err, IsNil) + + err = r.Push(&PushOptions{}) + c.Assert(err, IsNil) + + cmd := exec.Command("git", "fsck") + cmd.Dir = path + cmd.Env = os.Environ() + buf := &bytes.Buffer{} + cmd.Stderr = buf + cmd.Stdout = buf + + err = cmd.Run() + + c.Assert(err, IsNil, Commentf("%s", buf.Bytes())) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, diff --git a/worktree_test.go b/worktree_test.go index df191b0..c714011 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -1591,6 +1591,10 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(len(status), Equals, 1) + fi, err := fs.Lstat("pkgA") + c.Assert(err, IsNil) + c.Assert(fi.IsDir(), Equals, true) + // Clean with Dir: true. err = wt.Clean(&CleanOptions{Dir: true}) c.Assert(err, IsNil) @@ -1599,6 +1603,11 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(err, IsNil) c.Assert(len(status), Equals, 0) + + // An empty dir should be deleted, as well. + _, err = fs.Lstat("pkgA") + c.Assert(err, ErrorMatches, ".*(no such file or directory.*|.*file does not exist)*.") + } func (s *WorktreeSuite) TestAlternatesRepo(c *C) { |