diff options
Diffstat (limited to 'storage')
-rw-r--r-- | storage/filesystem/config_test.go | 4 | ||||
-rw-r--r-- | storage/filesystem/internal/dotgit/dotgit.go | 167 | ||||
-rw-r--r-- | storage/filesystem/internal/dotgit/dotgit_test.go | 68 | ||||
-rw-r--r-- | storage/filesystem/internal/dotgit/writers.go | 2 | ||||
-rw-r--r-- | storage/filesystem/internal/dotgit/writers_test.go | 9 | ||||
-rw-r--r-- | storage/filesystem/object.go | 6 | ||||
-rw-r--r-- | storage/filesystem/object_test.go | 2 | ||||
-rw-r--r-- | storage/filesystem/reference.go | 6 | ||||
-rw-r--r-- | storage/filesystem/storage.go | 2 | ||||
-rw-r--r-- | storage/filesystem/storage_test.go | 4 | ||||
-rw-r--r-- | storage/memory/storage.go | 34 | ||||
-rw-r--r-- | storage/test/storage_suite.go | 2 |
12 files changed, 219 insertions, 87 deletions
diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go index 4226b33..cc03119 100644 --- a/storage/filesystem/config_test.go +++ b/storage/filesystem/config_test.go @@ -4,12 +4,12 @@ import ( "io/ioutil" "os" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) type ConfigSuite struct { diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go index 2840bc7..1cb97bd 100644 --- a/storage/filesystem/internal/dotgit/dotgit.go +++ b/storage/filesystem/internal/dotgit/dotgit.go @@ -5,15 +5,15 @@ import ( "bufio" "errors" "fmt" + "io" stdioutil "io/ioutil" "os" "strings" - "time" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) const ( @@ -57,16 +57,14 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem - cachedPackedRefs refCache - packedRefsLastMod time.Time + fs billy.Filesystem } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs, cachedPackedRefs: make(refCache)} + return &DotGit{fs: fs} } // Initialize creates all the folder scaffolding. @@ -242,7 +240,35 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { return d.fs.Open(file) } -func (d *DotGit) SetRef(r *plumbing.Reference) error { +func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { + b, err := stdioutil.ReadAll(rd) + if err != nil { + return nil, err + } + + line := strings.TrimSpace(string(b)) + return plumbing.NewReferenceFromStrings(name, line), nil +} + +func (d *DotGit) checkReferenceAndTruncate(f billy.File, old *plumbing.Reference) error { + if old == nil { + return nil + } + ref, err := d.readReferenceFrom(f, old.Name().String()) + if err != nil { + return err + } + if ref.Hash() != old.Hash() { + return fmt.Errorf("reference has changed concurrently") + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return err + } + return f.Truncate(0) +} + +func (d *DotGit) SetRef(r, old *plumbing.Reference) error { var content string switch r.Type() { case plumbing.SymbolicReference: @@ -251,13 +277,34 @@ func (d *DotGit) SetRef(r *plumbing.Reference) error { content = fmt.Sprintln(r.Hash().String()) } - f, err := d.fs.Create(r.Name().String()) + // If we are not checking an old ref, just truncate the file. + mode := os.O_RDWR | os.O_CREATE + if old == nil { + mode |= os.O_TRUNC + } + + f, err := d.fs.OpenFile(r.Name().String(), mode, 0666) if err != nil { return err } defer ioutil.CheckClose(f, &err) + // Lock is unlocked by the deferred Close above. This is because Unlock + // does not imply a fsync and thus there would be a race between + // Unlock+Close and other concurrent writers. Adding Sync to go-billy + // could work, but this is better (and avoids superfluous syncs). + err = f.Lock() + if err != nil { + return err + } + + // this is a no-op to call even when old is nil. + err = d.checkReferenceAndTruncate(f, old) + if err != nil { + return err + } + _, err = f.Write([]byte(content)) return err } @@ -292,54 +339,43 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) { return d.packedRef(name) } -func (d *DotGit) syncPackedRefs() error { - fi, err := d.fs.Stat(packedRefsPath) - if os.IsNotExist(err) { - return nil - } - +func (d *DotGit) findPackedRefs() ([]*plumbing.Reference, error) { + f, err := d.fs.Open(packedRefsPath) if err != nil { - return err + if os.IsNotExist(err) { + return nil, nil + } + return nil, err } - if d.packedRefsLastMod.Before(fi.ModTime()) { - d.cachedPackedRefs = make(refCache) - f, err := d.fs.Open(packedRefsPath) + defer ioutil.CheckClose(f, &err) + + s := bufio.NewScanner(f) + var refs []*plumbing.Reference + for s.Scan() { + ref, err := d.processLine(s.Text()) if err != nil { - if os.IsNotExist(err) { - return nil - } - return err + return nil, err } - defer ioutil.CheckClose(f, &err) - - s := bufio.NewScanner(f) - for s.Scan() { - ref, err := d.processLine(s.Text()) - if err != nil { - return err - } - if ref != nil { - d.cachedPackedRefs[ref.Name()] = ref - } + if ref != nil { + refs = append(refs, ref) } - - d.packedRefsLastMod = fi.ModTime() - - return s.Err() } - return nil + return refs, s.Err() } func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) { - if err := d.syncPackedRefs(); err != nil { + refs, err := d.findPackedRefs() + if err != nil { return nil, err } - if ref, ok := d.cachedPackedRefs[name]; ok { - return ref, nil + for _, ref := range refs { + if ref.Name() == name { + return ref, nil + } } return nil, plumbing.ErrReferenceNotFound @@ -350,7 +386,8 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { path := d.fs.Join(".", name.String()) _, err := d.fs.Stat(path) if err == nil { - return d.fs.Remove(path) + err = d.fs.Remove(path) + // Drop down to remove it from the packed refs file, too. } if err != nil && !os.IsNotExist(err) { @@ -361,17 +398,17 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error { } func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) { - if err := d.syncPackedRefs(); err != nil { + packedRefs, err := d.findPackedRefs() + if err != nil { return err } - for name, ref := range d.cachedPackedRefs { - if !seen[name] { + for _, ref := range packedRefs { + if !seen[ref.Name()] { *refs = append(*refs, ref) - seen[name] = true + seen[ref.Name()] = true } } - return nil } @@ -384,6 +421,17 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e return err } + doCloseF := true + defer func() { + if doCloseF { + ioutil.CheckClose(f, &err) + } + }() + + err = f.Lock() + if err != nil { + return err + } // Creating the temp file in the same directory as the target file // improves our chances for rename operation to be atomic. @@ -391,6 +439,12 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e if err != nil { return err } + doCloseTmp := true + defer func() { + if doCloseTmp { + ioutil.CheckClose(tmp, &err) + } + }() s := bufio.NewScanner(f) found := false @@ -416,14 +470,21 @@ func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err e } if !found { - return nil + doCloseTmp = false + ioutil.CheckClose(tmp, &err) + if err != nil { + return err + } + // Delete the temp file if nothing needed to be removed. + return d.fs.Remove(tmp.Name()) } + doCloseF = false if err := f.Close(); err != nil { - ioutil.CheckClose(tmp, &err) return err } + doCloseTmp = false if err := tmp.Close(); err != nil { return err } @@ -512,13 +573,7 @@ func (d *DotGit) readReferenceFile(path, name string) (ref *plumbing.Reference, } defer ioutil.CheckClose(f, &err) - b, err := stdioutil.ReadAll(f) - if err != nil { - return nil, err - } - - line := strings.TrimSpace(string(b)) - return plumbing.NewReferenceFromStrings(name, line), nil + return d.readReferenceFrom(f, name) } // Module return a billy.Filesystem poiting to the module folder diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go index a7f16f4..446a204 100644 --- a/storage/filesystem/internal/dotgit/dotgit_test.go +++ b/storage/filesystem/internal/dotgit/dotgit_test.go @@ -8,11 +8,11 @@ import ( "strings" "testing" - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -55,24 +55,25 @@ func (s *SuiteDotGit) TestSetRefs(c *C) { fs := osfs.New(tmp) dir := New(fs) - err = dir.SetRef(plumbing.NewReferenceFromStrings( + firstFoo := plumbing.NewReferenceFromStrings( "refs/heads/foo", "e8d3ffab552895c19b9fcf7aa264d277cde33881", - )) + ) + err = dir.SetRef(firstFoo, nil) c.Assert(err, IsNil) err = dir.SetRef(plumbing.NewReferenceFromStrings( "refs/heads/symbolic", "ref: refs/heads/foo", - )) + ), nil) c.Assert(err, IsNil) err = dir.SetRef(plumbing.NewReferenceFromStrings( "bar", "e8d3ffab552895c19b9fcf7aa264d277cde33881", - )) + ), nil) c.Assert(err, IsNil) refs, err := dir.Refs() @@ -105,6 +106,20 @@ func (s *SuiteDotGit) TestSetRefs(c *C) { c.Assert(ref, NotNil) c.Assert(ref.Hash().String(), Equals, "e8d3ffab552895c19b9fcf7aa264d277cde33881") + // Check that SetRef with a non-nil `old` works. + err = dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/heads/foo", + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + ), firstFoo) + c.Assert(err, IsNil) + + // `firstFoo` is no longer the right `old` reference, so this + // should fail. + err = dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/heads/foo", + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + ), firstFoo) + c.Assert(err, NotNil) } func (s *SuiteDotGit) TestRefsFromPackedRefs(c *C) { @@ -184,6 +199,46 @@ func (s *SuiteDotGit) TestRemoveRefFromPackedRefs(c *C) { "e8d3ffab552895c19b9fcf7aa264d277cde33881 refs/remotes/origin/branch\n") } +func (s *SuiteDotGit) TestRemoveRefFromReferenceFileAndPackedRefs(c *C) { + fs := fixtures.Basic().ByTag(".git").One().DotGit() + dir := New(fs) + + // Make a ref file for a ref that's already in `packed-refs`. + err := dir.SetRef(plumbing.NewReferenceFromStrings( + "refs/remotes/origin/branch", + "e8d3ffab552895c19b9fcf7aa264d277cde33881", + ), nil) + + // Make sure it only appears once in the refs list. + refs, err := dir.Refs() + c.Assert(err, IsNil) + found := false + for _, ref := range refs { + if ref.Name() == "refs/remotes/origin/branch" { + c.Assert(found, Equals, false) + found = true + } + } + + name := plumbing.ReferenceName("refs/remotes/origin/branch") + err = dir.RemoveRef(name) + c.Assert(err, IsNil) + + b, err := ioutil.ReadFile(filepath.Join(fs.Root(), packedRefsPath)) + c.Assert(err, IsNil) + + c.Assert(string(b), Equals, ""+ + "# pack-refs with: peeled fully-peeled \n"+ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5 refs/heads/master\n"+ + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5 refs/remotes/origin/master\n") + + refs, err = dir.Refs() + c.Assert(err, IsNil) + + ref := findReference(refs, string(name)) + c.Assert(ref, IsNil) +} + func (s *SuiteDotGit) TestRemoveRefNonExistent(c *C) { fs := fixtures.Basic().ByTag(".git").One().DotGit() dir := New(fs) @@ -418,6 +473,7 @@ func (s *SuiteDotGit) TestNewObject(c *C) { c.Assert(err, IsNil) err = w.WriteHeader(plumbing.BlobObject, 14) + c.Assert(err, IsNil) n, err := w.Write([]byte("this is a test")) c.Assert(err, IsNil) c.Assert(n, Equals, 14) diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/internal/dotgit/writers.go index 46d3619..c2b420f 100644 --- a/storage/filesystem/internal/dotgit/writers.go +++ b/storage/filesystem/internal/dotgit/writers.go @@ -10,7 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/objfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) // PackWriter is a io.Writer that generates the packfile index simultaneously, diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/internal/dotgit/writers_test.go index 1544de8..bf00762 100644 --- a/storage/filesystem/internal/dotgit/writers_test.go +++ b/storage/filesystem/internal/dotgit/writers_test.go @@ -8,12 +8,12 @@ import ( "os" "strconv" - "github.com/src-d/go-git-fixtures" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/osfs" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func (s *SuiteDotGit) TestNewObjectPack(c *C) { @@ -85,6 +85,7 @@ func (s *SuiteDotGit) TestNewObjectPackUnused(c *C) { // check clean up of temporary files info, err = fs.ReadDir("") + c.Assert(err, IsNil) for _, fi := range info { c.Assert(fi.IsDir(), Equals, true) } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 5073a38..9690c0e 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -14,7 +14,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) const DefaultMaxDeltaBaseCacheSize = 92 * cache.MiByte @@ -41,7 +41,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index, 0) + s.index = make(map[plumbing.Hash]*packfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -319,7 +319,7 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return nil, err } - seen := make(map[plumbing.Hash]bool, 0) + seen := make(map[plumbing.Hash]bool) var iters []storer.EncodedObjectIter if len(objects) != 0 { iters = append(iters, &objectsIter{s: s, t: t, h: objects}) diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 504bd45..de8f2b2 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,11 +1,11 @@ package filesystem import ( - "github.com/src-d/go-git-fixtures" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) type FsSuite struct { diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go index 49627d3..54cdf56 100644 --- a/storage/filesystem/reference.go +++ b/storage/filesystem/reference.go @@ -11,7 +11,11 @@ type ReferenceStorage struct { } func (r *ReferenceStorage) SetReference(ref *plumbing.Reference) error { - return r.dir.SetRef(ref) + return r.dir.SetRef(ref, nil) +} + +func (r *ReferenceStorage) CheckAndSetReference(ref, old *plumbing.Reference) error { + return r.dir.SetRef(ref, old) } func (r *ReferenceStorage) Reference(n plumbing.ReferenceName) (*plumbing.Reference, error) { diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 2c7c107..82b137c 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -4,7 +4,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" - "gopkg.in/src-d/go-billy.v3" + "gopkg.in/src-d/go-billy.v4" ) // Storage is an implementation of git.Storer that stores data on disk in the diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index b165c5e..4d9ba6f 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -8,8 +8,8 @@ import ( "gopkg.in/src-d/go-git.v4/storage/test" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v3/memfs" - "gopkg.in/src-d/go-billy.v3/osfs" + "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" ) func Test(t *testing.T) { TestingT(t) } diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 2380fed..927ec41 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -12,6 +12,7 @@ import ( ) var ErrUnsupportedObjectType = fmt.Errorf("unsupported object type") +var ErrRefHasChanged = fmt.Errorf("reference has changed concurrently") // Storage is an implementation of git.Storer that stores data on memory, being // ephemeral. The use of this storage should be done in controlled envoriments, @@ -29,17 +30,17 @@ type Storage struct { // NewStorage returns a new Storage base on memory func NewStorage() *Storage { return &Storage{ - ReferenceStorage: make(ReferenceStorage, 0), + ReferenceStorage: make(ReferenceStorage), ConfigStorage: ConfigStorage{}, ShallowStorage: ShallowStorage{}, ObjectStorage: ObjectStorage{ - Objects: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Commits: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Trees: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Blobs: make(map[plumbing.Hash]plumbing.EncodedObject, 0), - Tags: make(map[plumbing.Hash]plumbing.EncodedObject, 0), + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), + Commits: make(map[plumbing.Hash]plumbing.EncodedObject), + Trees: make(map[plumbing.Hash]plumbing.EncodedObject), + Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), + Tags: make(map[plumbing.Hash]plumbing.EncodedObject), }, - ModuleStorage: make(ModuleStorage, 0), + ModuleStorage: make(ModuleStorage), } } @@ -151,7 +152,7 @@ func flattenObjectMap(m map[plumbing.Hash]plumbing.EncodedObject) []plumbing.Enc func (o *ObjectStorage) Begin() storer.Transaction { return &TxObjectStorage{ Storage: o, - Objects: make(map[plumbing.Hash]plumbing.EncodedObject, 0), + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), } } @@ -188,7 +189,7 @@ func (tx *TxObjectStorage) Commit() error { } func (tx *TxObjectStorage) Rollback() error { - tx.Objects = make(map[plumbing.Hash]plumbing.EncodedObject, 0) + tx.Objects = make(map[plumbing.Hash]plumbing.EncodedObject) return nil } @@ -202,6 +203,21 @@ func (r ReferenceStorage) SetReference(ref *plumbing.Reference) error { return nil } +func (r ReferenceStorage) CheckAndSetReference(ref, old *plumbing.Reference) error { + if ref == nil { + return nil + } + + if old != nil { + tmp := r[ref.Name()] + if tmp != nil && tmp.Hash() != old.Hash() { + return ErrRefHasChanged + } + } + r[ref.Name()] = ref + return nil +} + func (r ReferenceStorage) Reference(n plumbing.ReferenceName) (*plumbing.Reference, error) { ref, ok := r[n] if !ok { diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go index bf93515..18e0086 100644 --- a/storage/test/storage_suite.go +++ b/storage/test/storage_suite.go @@ -13,7 +13,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" - "github.com/src-d/go-git-fixtures" + "gopkg.in/src-d/go-git-fixtures.v3" . "gopkg.in/check.v1" ) |