aboutsummaryrefslogtreecommitdiffstats
path: root/storage
diff options
context:
space:
mode:
Diffstat (limited to 'storage')
-rw-r--r--storage/filesystem/dotgit/dotgit.go104
-rw-r--r--storage/filesystem/dotgit/dotgit_test.go100
-rw-r--r--storage/filesystem/dotgit/writers.go5
-rw-r--r--storage/filesystem/object.go24
-rw-r--r--storage/filesystem/object_test.go18
-rw-r--r--storage/filesystem/shallow.go2
-rw-r--r--storage/test/storage_suite.go5
7 files changed, 193 insertions, 65 deletions
diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index 6c386f7..e02e6dd 100644
--- a/storage/filesystem/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"io"
- stdioutil "io/ioutil"
"os"
"path/filepath"
"sort"
@@ -16,6 +15,7 @@ import (
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/hash"
"github.com/go-git/go-git/v5/storage"
"github.com/go-git/go-git/v5/utils/ioutil"
@@ -552,8 +552,8 @@ func (d *DotGit) hasPack(h plumbing.Hash) error {
}
func (d *DotGit) objectPath(h plumbing.Hash) string {
- hash := h.String()
- return d.fs.Join(objectsPath, hash[0:2], hash[2:40])
+ hex := h.String()
+ return d.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize])
}
// incomingObjectPath is intended to add support for a git pre-receive hook
@@ -563,15 +563,16 @@ func (d *DotGit) objectPath(h plumbing.Hash) string {
//
// More on git hooks found here : https://git-scm.com/docs/githooks
// More on 'quarantine'/incoming directory here:
-// https://git-scm.com/docs/git-receive-pack
+//
+// https://git-scm.com/docs/git-receive-pack
func (d *DotGit) incomingObjectPath(h plumbing.Hash) string {
hString := h.String()
if d.incomingDirName == "" {
- return d.fs.Join(objectsPath, hString[0:2], hString[2:40])
+ return d.fs.Join(objectsPath, hString[0:2], hString[2:hash.HexSize])
}
- return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40])
+ return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:hash.HexSize])
}
// hasIncomingObjects searches for an incoming directory and keeps its name
@@ -581,7 +582,9 @@ func (d *DotGit) hasIncomingObjects() bool {
directoryContents, err := d.fs.ReadDir(objectsPath)
if err == nil {
for _, file := range directoryContents {
- if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() {
+ if file.IsDir() && (strings.HasPrefix(file.Name(), "tmp_objdir-incoming-") ||
+ // Before Git 2.35 incoming commits directory had another prefix
+ strings.HasPrefix(file.Name(), "incoming-")) {
d.incomingDirName = file.Name()
}
}
@@ -645,7 +648,7 @@ func (d *DotGit) ObjectDelete(h plumbing.Hash) error {
}
func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) {
- b, err := stdioutil.ReadAll(rd)
+ b, err := io.ReadAll(rd)
if err != nil {
return nil, err
}
@@ -716,48 +719,56 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) {
return d.packedRef(name)
}
-func (d *DotGit) findPackedRefsInFile(f billy.File) ([]*plumbing.Reference, error) {
+func (d *DotGit) findPackedRefsInFile(f billy.File, recv refsRecv) error {
s := bufio.NewScanner(f)
- var refs []*plumbing.Reference
for s.Scan() {
ref, err := d.processLine(s.Text())
if err != nil {
- return nil, err
+ return err
}
- if ref != nil {
- refs = append(refs, ref)
+ if !recv(ref) {
+ // skip parse
+ return nil
}
}
-
- return refs, s.Err()
+ if err := s.Err(); err != nil {
+ return err
+ }
+ return nil
}
-func (d *DotGit) findPackedRefs() (r []*plumbing.Reference, err error) {
+// refsRecv: returning true means that the reference continues to be resolved, otherwise it is stopped, which will speed up the lookup of a single reference.
+type refsRecv func(*plumbing.Reference) bool
+
+func (d *DotGit) findPackedRefs(recv refsRecv) error {
f, err := d.fs.Open(packedRefsPath)
if err != nil {
if os.IsNotExist(err) {
- return nil, nil
+ return nil
}
- return nil, err
+ return err
}
defer ioutil.CheckClose(f, &err)
- return d.findPackedRefsInFile(f)
+ return d.findPackedRefsInFile(f, recv)
}
func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) {
- refs, err := d.findPackedRefs()
- if err != nil {
+ var ref *plumbing.Reference
+ if err := d.findPackedRefs(func(r *plumbing.Reference) bool {
+ if r != nil && r.Name() == name {
+ ref = r
+ // ref found
+ return false
+ }
+ return true
+ }); err != nil {
return nil, err
}
-
- for _, ref := range refs {
- if ref.Name() == name {
- return ref, nil
- }
+ if ref != nil {
+ return ref, nil
}
-
return nil, plumbing.ErrReferenceNotFound
}
@@ -777,34 +788,22 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error {
return d.rewritePackedRefsWithoutRef(name)
}
-func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) {
- packedRefs, err := d.findPackedRefs()
- if err != nil {
- return err
- }
-
- for _, ref := range packedRefs {
- if !seen[ref.Name()] {
- *refs = append(*refs, ref)
- seen[ref.Name()] = true
+func refsRecvFunc(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) refsRecv {
+ return func(r *plumbing.Reference) bool {
+ if r != nil && !seen[r.Name()] {
+ *refs = append(*refs, r)
+ seen[r.Name()] = true
}
+ return true
}
- return nil
}
-func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) {
- packedRefs, err := d.findPackedRefsInFile(f)
- if err != nil {
- return err
- }
+func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) {
+ return d.findPackedRefs(refsRecvFunc(refs, seen))
+}
- for _, ref := range packedRefs {
- if !seen[ref.Name()] {
- *refs = append(*refs, ref)
- seen[ref.Name()] = true
- }
- }
- return nil
+func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) {
+ return d.findPackedRefsInFile(f, refsRecvFunc(refs, seen))
}
func (d *DotGit) openAndLockPackedRefs(doCreate bool) (
@@ -943,6 +942,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
files, err := d.fs.ReadDir(d.fs.Join(relPath...))
if err != nil {
if os.IsNotExist(err) {
+ // a race happened, and our directory is gone now
return nil
}
@@ -960,6 +960,10 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
}
ref, err := d.readReferenceFile(".", strings.Join(newRelPath, "/"))
+ if os.IsNotExist(err) {
+ // a race happened, and our file is gone now
+ continue
+ }
if err != nil {
return err
}
diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index a8f0eb7..1b6c113 100644
--- a/storage/filesystem/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -4,7 +4,6 @@ import (
"bufio"
"encoding/hex"
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -374,7 +373,7 @@ func (s *SuiteDotGit) TestConfigWriteAndConfig(c *C) {
f, err = dir.Config()
c.Assert(err, IsNil)
- cnt, err := ioutil.ReadAll(f)
+ cnt, err := io.ReadAll(f)
c.Assert(err, IsNil)
c.Assert(string(cnt), Equals, "foo")
@@ -404,7 +403,7 @@ func (s *SuiteDotGit) TestIndexWriteAndIndex(c *C) {
f, err = dir.Index()
c.Assert(err, IsNil)
- cnt, err := ioutil.ReadAll(f)
+ cnt, err := io.ReadAll(f)
c.Assert(err, IsNil)
c.Assert(string(cnt), Equals, "foo")
@@ -434,7 +433,7 @@ func (s *SuiteDotGit) TestShallowWriteAndShallow(c *C) {
f, err = dir.Shallow()
c.Assert(err, IsNil)
- cnt, err := ioutil.ReadAll(f)
+ cnt, err := io.ReadAll(f)
c.Assert(err, IsNil)
c.Assert(string(cnt), Equals, "foo")
@@ -649,6 +648,27 @@ func (s *SuiteDotGit) TestObject(c *C) {
Equals, true,
)
incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+ incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
+ incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+ fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+ fs.Create(incomingFilePath)
+
+ _, err = dir.Object(plumbing.NewHash(incomingHash))
+ c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestPreGit235Object(c *C) {
+ fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+ dir := New(fs)
+
+ hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+ file, err := dir.Object(hash)
+ c.Assert(err, IsNil)
+ c.Assert(strings.HasSuffix(
+ file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")),
+ Equals, true,
+ )
+ incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
incomingDirPath := fs.Join("objects", "incoming-123456")
incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
fs.MkdirAll(incomingDirPath, os.FileMode(0755))
@@ -666,7 +686,7 @@ func (s *SuiteDotGit) TestObjectStat(c *C) {
_, err := dir.ObjectStat(hash)
c.Assert(err, IsNil)
incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
- incomingDirPath := fs.Join("objects", "incoming-123456")
+ incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
fs.MkdirAll(incomingDirPath, os.FileMode(0755))
fs.Create(incomingFilePath)
@@ -684,7 +704,7 @@ func (s *SuiteDotGit) TestObjectDelete(c *C) {
c.Assert(err, IsNil)
incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
- incomingDirPath := fs.Join("objects", "incoming-123456")
+ incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2])
incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40])
@@ -864,3 +884,71 @@ func (s *SuiteDotGit) TestIncBytes(c *C) {
c.Assert(overflow, Equals, test.overflow)
}
}
+
+// this filesystem wrapper returns os.ErrNotExist if the file matches
+// the provided paths list
+type notExistsFS struct {
+ billy.Filesystem
+
+ paths []string
+}
+
+func (f *notExistsFS) matches(path string) bool {
+ p := filepath.ToSlash(path)
+ for _, n := range f.paths {
+ if p == n {
+ return true
+ }
+ }
+ return false
+}
+
+func (f *notExistsFS) Open(filename string) (billy.File, error) {
+ if f.matches(filename) {
+ return nil, os.ErrNotExist
+ }
+
+ return f.Filesystem.Open(filename)
+}
+
+func (f *notExistsFS) ReadDir(path string) ([]os.FileInfo, error) {
+ if f.matches(path) {
+ return nil, os.ErrNotExist
+ }
+
+ return f.Filesystem.ReadDir(path)
+}
+
+func (s *SuiteDotGit) TestDeletedRefs(c *C) {
+ fs, clean := s.TemporalFilesystem()
+ defer clean()
+
+ dir := New(&notExistsFS{
+ Filesystem: fs,
+ paths: []string{
+ "refs/heads/bar",
+ "refs/heads/baz",
+ },
+ })
+
+ err := dir.SetRef(plumbing.NewReferenceFromStrings(
+ "refs/heads/foo",
+ "e8d3ffab552895c19b9fcf7aa264d277cde33881",
+ ), nil)
+ c.Assert(err, IsNil)
+ err = dir.SetRef(plumbing.NewReferenceFromStrings(
+ "refs/heads/bar",
+ "a8d3ffab552895c19b9fcf7aa264d277cde33881",
+ ), nil)
+ c.Assert(err, IsNil)
+ err = dir.SetRef(plumbing.NewReferenceFromStrings(
+ "refs/heads/baz/baz",
+ "a8d3ffab552895c19b9fcf7aa264d277cde33881",
+ ), nil)
+ c.Assert(err, IsNil)
+
+ refs, err := dir.Refs()
+ c.Assert(err, IsNil)
+ c.Assert(refs, HasLen, 1)
+ c.Assert(refs[0].Name(), Equals, plumbing.ReferenceName("refs/heads/foo"))
+}
diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go
index e2ede93..849b7a1 100644
--- a/storage/filesystem/dotgit/writers.go
+++ b/storage/filesystem/dotgit/writers.go
@@ -9,6 +9,7 @@ import (
"github.com/go-git/go-git/v5/plumbing/format/idxfile"
"github.com/go-git/go-git/v5/plumbing/format/objfile"
"github.com/go-git/go-git/v5/plumbing/format/packfile"
+ "github.com/go-git/go-git/v5/plumbing/hash"
"github.com/go-git/go-billy/v5"
)
@@ -277,8 +278,8 @@ func (w *ObjectWriter) Close() error {
}
func (w *ObjectWriter) save() error {
- hash := w.Hash().String()
- file := w.fs.Join(objectsPath, hash[0:2], hash[2:40])
+ hex := w.Hash().String()
+ file := w.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize])
return w.fs.Rename(w.f.Name(), file)
}
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 5c91bcd..846a7b8 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -4,6 +4,7 @@ import (
"bytes"
"io"
"os"
+ "sync"
"time"
"github.com/go-git/go-git/v5/plumbing"
@@ -419,10 +420,21 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
s.objectCache.Put(obj)
- _, err = io.Copy(w, r)
+ bufp := copyBufferPool.Get().(*[]byte)
+ buf := *bufp
+ _, err = io.CopyBuffer(w, r, buf)
+ copyBufferPool.Put(bufp)
+
return obj, err
}
+var copyBufferPool = sync.Pool{
+ New: func() interface{} {
+ b := make([]byte, 32*1024)
+ return &b
+ },
+}
+
// Get returns the object with the given hash, by searching for it in
// the packfile.
func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
@@ -525,14 +537,21 @@ func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, pl
return plumbing.ZeroHash, plumbing.ZeroHash, -1
}
+// HashesWithPrefix returns all objects with a hash that starts with a prefix by searching for
+// them in the packfile and the git object directories.
func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error) {
hashes, err := s.dir.ObjectsWithPrefix(prefix)
if err != nil {
return nil, err
}
+ seen := hashListAsMap(hashes)
+
// TODO: This could be faster with some idxfile changes,
// or diving into the packfile.
+ if err := s.requireIndex(); err != nil {
+ return nil, err
+ }
for _, index := range s.index {
ei, err := index.Entries()
if err != nil {
@@ -546,6 +565,9 @@ func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error)
return nil, err
}
if bytes.HasPrefix(e.Hash[:], prefix) {
+ if _, ok := seen[e.Hash]; ok {
+ continue
+ }
hashes = append(hashes, e.Hash)
}
}
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index 19a7914..251077a 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -4,7 +4,6 @@ import (
"encoding/hex"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -406,6 +405,21 @@ func (s *FsSuite) TestHashesWithPrefix(c *C) {
c.Assert(hashes[0].String(), Equals, "f3dfe29d268303fc6e1bbce268605fc99573406e")
}
+func (s *FsSuite) TestHashesWithPrefixFromPackfile(c *C) {
+ // Same setup as TestGetFromPackfile
+ fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
+ fs := f.DotGit()
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
+
+ expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+ // Only pass the first 8 bytes
+ hashes, err := o.HashesWithPrefix(expected[:8])
+ c.Assert(err, IsNil)
+ c.Assert(hashes, HasLen, 1)
+ c.Assert(hashes[0], Equals, expected)
+ })
+}
+
func BenchmarkPackfileIter(b *testing.B) {
defer fixtures.Clean()
@@ -495,7 +509,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) {
b.Fatal(err)
}
- if _, err := ioutil.ReadAll(r); err != nil {
+ if _, err := io.ReadAll(r); err != nil {
b.Fatal(err)
}
diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go
index afb600c..ac48fdf 100644
--- a/storage/filesystem/shallow.go
+++ b/storage/filesystem/shallow.go
@@ -34,7 +34,7 @@ func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
return err
}
-// Shallow return the shallow commits reading from shallo file from .git
+// Shallow returns the shallow commits reading from shallo file from .git
func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) {
f, err := s.dir.Shallow()
if f == nil || err != nil {
diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go
index 2c00e75..ee67fc7 100644
--- a/storage/test/storage_suite.go
+++ b/storage/test/storage_suite.go
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
@@ -502,12 +501,12 @@ func objectEquals(a plumbing.EncodedObject, b plumbing.EncodedObject) error {
return fmt.Errorf("can't get reader on b: %q", err)
}
- ca, err := ioutil.ReadAll(ra)
+ ca, err := io.ReadAll(ra)
if err != nil {
return fmt.Errorf("error reading a: %q", err)
}
- cb, err := ioutil.ReadAll(rb)
+ cb, err := io.ReadAll(rb)
if err != nil {
return fmt.Errorf("error reading b: %q", err)
}
generated by cgit (git 2.34.1) at 2024-11-18 13:05:55 +0000