aboutsummaryrefslogtreecommitdiffstats
path: root/storage/filesystem
diff options
context:
space:
mode:
Diffstat (limited to 'storage/filesystem')
-rw-r--r--storage/filesystem/dotgit/dotgit.go3
-rw-r--r--storage/filesystem/dotgit/dotgit_setref.go51
-rw-r--r--storage/filesystem/dotgit/dotgit_setref_norwfs.go47
-rw-r--r--storage/filesystem/dotgit/dotgit_test.go25
-rw-r--r--storage/filesystem/object.go65
-rw-r--r--storage/filesystem/object_test.go73
-rw-r--r--storage/filesystem/storage.go6
7 files changed, 190 insertions, 80 deletions
diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index a58c248..ba9667e 100644
--- a/storage/filesystem/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -14,6 +14,7 @@ import (
"gopkg.in/src-d/go-billy.v4/osfs"
"gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/storage"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
"gopkg.in/src-d/go-billy.v4"
@@ -596,7 +597,7 @@ func (d *DotGit) checkReferenceAndTruncate(f billy.File, old *plumbing.Reference
return err
}
if ref.Hash() != old.Hash() {
- return fmt.Errorf("reference has changed concurrently")
+ return storage.ErrReferenceHasChanged
}
_, err = f.Seek(0, io.SeekStart)
if err != nil {
diff --git a/storage/filesystem/dotgit/dotgit_setref.go b/storage/filesystem/dotgit/dotgit_setref.go
index d27c1a3..9da2f31 100644
--- a/storage/filesystem/dotgit/dotgit_setref.go
+++ b/storage/filesystem/dotgit/dotgit_setref.go
@@ -1,15 +1,24 @@
-// +build !norwfs
-
package dotgit
import (
+ "fmt"
"os"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
+
+ "gopkg.in/src-d/go-billy.v4"
)
func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) (err error) {
+ if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) {
+ return d.setRefRwfs(fileName, content, old)
+ }
+
+ return d.setRefNorwfs(fileName, content, old)
+}
+
+func (d *DotGit) setRefRwfs(fileName, content string, old *plumbing.Reference) (err error) {
// If we are not checking an old ref, just truncate the file.
mode := os.O_RDWR | os.O_CREATE
if old == nil {
@@ -41,3 +50,41 @@ func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) (err
_, err = f.Write([]byte(content))
return err
}
+
+// There are some filesystems that don't support opening files in RDWD mode.
+// In these filesystems the standard SetRef function can not be used as it
+// reads the reference file to check that it's not modified before updating it.
+//
+// This version of the function writes the reference without extra checks
+// making it compatible with these simple filesystems. This is usually not
+// a problem as they should be accessed by only one process at a time.
+func (d *DotGit) setRefNorwfs(fileName, content string, old *plumbing.Reference) error {
+ _, err := d.fs.Stat(fileName)
+ if err == nil && old != nil {
+ fRead, err := d.fs.Open(fileName)
+ if err != nil {
+ return err
+ }
+
+ ref, err := d.readReferenceFrom(fRead, old.Name().String())
+ fRead.Close()
+
+ if err != nil {
+ return err
+ }
+
+ if ref.Hash() != old.Hash() {
+ return fmt.Errorf("reference has changed concurrently")
+ }
+ }
+
+ f, err := d.fs.Create(fileName)
+ if err != nil {
+ return err
+ }
+
+ defer f.Close()
+
+ _, err = f.Write([]byte(content))
+ return err
+}
diff --git a/storage/filesystem/dotgit/dotgit_setref_norwfs.go b/storage/filesystem/dotgit/dotgit_setref_norwfs.go
deleted file mode 100644
index 5695bd3..0000000
--- a/storage/filesystem/dotgit/dotgit_setref_norwfs.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// +build norwfs
-
-package dotgit
-
-import (
- "fmt"
-
- "gopkg.in/src-d/go-git.v4/plumbing"
-)
-
-// There are some filesystems that don't support opening files in RDWD mode.
-// In these filesystems the standard SetRef function can not be used as i
-// reads the reference file to check that it's not modified before updating it.
-//
-// This version of the function writes the reference without extra checks
-// making it compatible with these simple filesystems. This is usually not
-// a problem as they should be accessed by only one process at a time.
-func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) error {
- _, err := d.fs.Stat(fileName)
- if err == nil && old != nil {
- fRead, err := d.fs.Open(fileName)
- if err != nil {
- return err
- }
-
- ref, err := d.readReferenceFrom(fRead, old.Name().String())
- fRead.Close()
-
- if err != nil {
- return err
- }
-
- if ref.Hash() != old.Hash() {
- return fmt.Errorf("reference has changed concurrently")
- }
- }
-
- f, err := d.fs.Create(fileName)
- if err != nil {
- return err
- }
-
- defer f.Close()
-
- _, err = f.Write([]byte(content))
- return err
-}
diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index 308c6b7..73b0291 100644
--- a/storage/filesystem/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -57,11 +57,26 @@ func (s *SuiteDotGit) TestSetRefs(c *C) {
fs := osfs.New(tmp)
dir := New(fs)
+ testSetRefs(c, dir)
+}
+
+func (s *SuiteDotGit) TestSetRefsNorwfs(c *C) {
+ tmp, err := ioutil.TempDir("", "dot-git")
+ c.Assert(err, IsNil)
+ defer os.RemoveAll(tmp)
+
+ fs := osfs.New(tmp)
+ dir := New(&norwfs{fs})
+
+ testSetRefs(c, dir)
+}
+
+func testSetRefs(c *C, dir *DotGit) {
firstFoo := plumbing.NewReferenceFromStrings(
"refs/heads/foo",
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
)
- err = dir.SetRef(firstFoo, nil)
+ err := dir.SetRef(firstFoo, nil)
c.Assert(err, IsNil)
@@ -795,3 +810,11 @@ func (s *SuiteDotGit) TestAlternates(c *C) {
}
c.Assert(dotgits[1].fs.Root(), Equals, expectedPath)
}
+
+type norwfs struct {
+ billy.Filesystem
+}
+
+func (f *norwfs) Capabilities() billy.Capability {
+ return billy.Capabilities(f.Filesystem) &^ billy.ReadAndWriteCapability
+}
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 6cd2d4c..3eb62a2 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -20,24 +20,25 @@ import (
type ObjectStorage struct {
options Options
- // deltaBaseCache is an object cache uses to cache delta's bases when
- deltaBaseCache cache.Object
+ // objectCache is an object cache uses to cache delta's bases and also recently
+ // loaded loose objects
+ objectCache cache.Object
dir *dotgit.DotGit
index map[plumbing.Hash]idxfile.Index
}
// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache.
-func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage {
- return NewObjectStorageWithOptions(dir, cache, Options{})
+func NewObjectStorage(dir *dotgit.DotGit, objectCache cache.Object) *ObjectStorage {
+ return NewObjectStorageWithOptions(dir, objectCache, Options{})
}
// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options
-func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage {
+func NewObjectStorageWithOptions(dir *dotgit.DotGit, objectCache cache.Object, ops Options) *ObjectStorage {
return &ObjectStorage{
- options: ops,
- deltaBaseCache: cache,
- dir: dir,
+ options: ops,
+ objectCache: objectCache,
+ dir: dir,
}
}
@@ -61,6 +62,11 @@ func (s *ObjectStorage) requireIndex() error {
return nil
}
+// Reindex indexes again all packfiles. Useful if git changed packfiles externally
+func (s *ObjectStorage) Reindex() {
+ s.index = nil
+}
+
func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) {
f, err := s.dir.ObjectPackIdx(h)
if err != nil {
@@ -201,7 +207,7 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) (
idx := s.index[pack]
hash, err := idx.FindHash(offset)
if err == nil {
- obj, ok := s.deltaBaseCache.Get(hash)
+ obj, ok := s.objectCache.Get(hash)
if ok {
return obj.Size(), nil
}
@@ -210,8 +216,8 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) (
}
var p *packfile.Packfile
- if s.deltaBaseCache != nil {
- p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache)
+ if s.objectCache != nil {
+ p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache)
} else {
p = packfile.NewPackfile(idx, s.dir.Fs(), f)
}
@@ -236,9 +242,19 @@ func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) (
// EncodedObject returns the object with the given hash, by searching for it in
// the packfile and the git object directories.
func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) {
- obj, err := s.getFromUnpacked(h)
- if err == plumbing.ErrObjectNotFound {
+ var obj plumbing.EncodedObject
+ var err error
+
+ if s.index != nil {
obj, err = s.getFromPackfile(h, false)
+ if err == plumbing.ErrObjectNotFound {
+ obj, err = s.getFromUnpacked(h)
+ }
+ } else {
+ obj, err = s.getFromUnpacked(h)
+ if err == plumbing.ErrObjectNotFound {
+ obj, err = s.getFromPackfile(h, false)
+ }
}
// If the error is still object not found, check if it's a shared object
@@ -249,7 +265,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p
// Create a new object storage with the DotGit(s) and check for the
// required hash object. Skip when not found.
for _, dg := range dotgits {
- o := NewObjectStorage(dg, s.deltaBaseCache)
+ o := NewObjectStorage(dg, s.objectCache)
enobj, enerr := o.EncodedObject(t, h)
if enerr != nil {
continue
@@ -299,9 +315,12 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
return nil, err
}
-
defer ioutil.CheckClose(f, &err)
+ if cacheObj, found := s.objectCache.Get(h); found {
+ return cacheObj, nil
+ }
+
obj = s.NewEncodedObject()
r, err := objfile.NewReader(f)
if err != nil {
@@ -322,6 +341,8 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
return nil, err
}
+ s.objectCache.Put(obj)
+
_, err = io.Copy(w, r)
return obj, err
}
@@ -364,7 +385,7 @@ func (s *ObjectStorage) decodeObjectAt(
) (plumbing.EncodedObject, error) {
hash, err := idx.FindHash(offset)
if err == nil {
- obj, ok := s.deltaBaseCache.Get(hash)
+ obj, ok := s.objectCache.Get(hash)
if ok {
return obj, nil
}
@@ -375,8 +396,8 @@ func (s *ObjectStorage) decodeObjectAt(
}
var p *packfile.Packfile
- if s.deltaBaseCache != nil {
- p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache)
+ if s.objectCache != nil {
+ p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache)
} else {
p = packfile.NewPackfile(idx, s.dir.Fs(), f)
}
@@ -395,11 +416,7 @@ func (s *ObjectStorage) decodeDeltaObjectAt(
}
p := packfile.NewScanner(f)
- if _, err := p.SeekFromStart(offset); err != nil {
- return nil, err
- }
-
- header, err := p.NextObjectHeader()
+ header, err := p.SeekObjectHeader(offset)
if err != nil {
return nil, err
}
@@ -490,7 +507,7 @@ func (s *ObjectStorage) buildPackfileIters(
}
return newPackfileIter(
s.dir.Fs(), pack, t, seen, s.index[h],
- s.deltaBaseCache, s.options.KeepDescriptors,
+ s.objectCache, s.options.KeepDescriptors,
)
},
}, nil
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index 4e6bbfb..5cfb227 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -1,8 +1,11 @@
package filesystem
import (
+ "fmt"
+ "io"
"io/ioutil"
"os"
+ "path/filepath"
"testing"
"gopkg.in/src-d/go-git.v4/plumbing"
@@ -204,6 +207,59 @@ func (s *FsSuite) TestPackfileIter(c *C) {
})
}
+func copyFile(c *C, dstDir, dstFilename string, srcFile *os.File) {
+ _, err := srcFile.Seek(0, 0)
+ c.Assert(err, IsNil)
+
+ err = os.MkdirAll(dstDir, 0750|os.ModeDir)
+ c.Assert(err, IsNil)
+
+ dst, err := os.OpenFile(filepath.Join(dstDir, dstFilename), os.O_CREATE|os.O_WRONLY, 0666)
+ c.Assert(err, IsNil)
+ defer dst.Close()
+
+ _, err = io.Copy(dst, srcFile)
+ c.Assert(err, IsNil)
+}
+
+// TestPackfileReindex tests that externally-added packfiles are considered by go-git
+// after calling the Reindex method
+func (s *FsSuite) TestPackfileReindex(c *C) {
+ // obtain a standalone packfile that is not part of any other repository
+ // in the fixtures:
+ packFixture := fixtures.ByTag("packfile").ByTag("standalone").One()
+ packFile := packFixture.Packfile()
+ idxFile := packFixture.Idx()
+ packFilename := packFixture.PackfileHash.String()
+ testObjectHash := plumbing.NewHash("a771b1e94141480861332fd0e4684d33071306c6") // this is an object we know exists in the standalone packfile
+ fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
+ fs := f.DotGit()
+ storer := NewStorage(fs, cache.NewObjectLRUDefault())
+
+ // check that our test object is NOT found
+ _, err := storer.EncodedObject(plumbing.CommitObject, testObjectHash)
+ c.Assert(err, Equals, plumbing.ErrObjectNotFound)
+
+ // add the external packfile+idx to the packs folder
+ // this simulates a git bundle unbundle command, or a repack, for example.
+ copyFile(c, filepath.Join(storer.Filesystem().Root(), "objects", "pack"),
+ fmt.Sprintf("pack-%s.pack", packFilename), packFile)
+ copyFile(c, filepath.Join(storer.Filesystem().Root(), "objects", "pack"),
+ fmt.Sprintf("pack-%s.idx", packFilename), idxFile)
+
+ // check that we cannot still retrieve the test object
+ _, err = storer.EncodedObject(plumbing.CommitObject, testObjectHash)
+ c.Assert(err, Equals, plumbing.ErrObjectNotFound)
+
+ storer.Reindex() // actually reindex
+
+ // Now check that the test object can be retrieved
+ _, err = storer.EncodedObject(plumbing.CommitObject, testObjectHash)
+ c.Assert(err, IsNil)
+
+ })
+}
+
func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) {
fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
fs := f.DotGit()
@@ -241,6 +297,23 @@ func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) {
})
}
+func (s *FsSuite) TestGetFromObjectFileSharedCache(c *C) {
+ f1 := fixtures.ByTag("worktree").One().DotGit()
+ f2 := fixtures.ByTag("worktree").ByTag("submodule").One().DotGit()
+
+ ch := cache.NewObjectLRUDefault()
+ o1 := NewObjectStorage(dotgit.New(f1), ch)
+ o2 := NewObjectStorage(dotgit.New(f2), ch)
+
+ expected := plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a")
+ obj, err := o1.EncodedObject(plumbing.CommitObject, expected)
+ c.Assert(err, IsNil)
+ c.Assert(obj.Hash(), Equals, expected)
+
+ obj, err = o2.EncodedObject(plumbing.CommitObject, expected)
+ c.Assert(err, Equals, plumbing.ErrObjectNotFound)
+}
+
func BenchmarkPackfileIter(b *testing.B) {
if err := fixtures.Init(); err != nil {
b.Fatal(err)
diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go
index 14a772a..370f7bd 100644
--- a/storage/filesystem/storage.go
+++ b/storage/filesystem/storage.go
@@ -51,11 +51,7 @@ func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options)
fs: fs,
dir: dir,
- ObjectStorage: ObjectStorage{
- options: ops,
- deltaBaseCache: cache,
- dir: dir,
- },
+ ObjectStorage: *NewObjectStorageWithOptions(dir, cache, ops),
ReferenceStorage: ReferenceStorage{dir: dir},
IndexStorage: IndexStorage{dir: dir},
ShallowStorage: ShallowStorage{dir: dir},
generated by cgit (git 2.34.1) at 2024-11-07 11:26:46 +0000