Merge branch 'master' into jc/commit-ammend

7 files changed, 193 insertions, 65 deletions

diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index 6c386f7..e02e6dd 100644
--- a/storage/filesystem/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	stdioutil "io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -16,6 +15,7 @@ import (
 
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/hash"
 	"github.com/go-git/go-git/v5/storage"
 	"github.com/go-git/go-git/v5/utils/ioutil"
 
@@ -552,8 +552,8 @@ func (d *DotGit) hasPack(h plumbing.Hash) error {
 }
 
 func (d *DotGit) objectPath(h plumbing.Hash) string {
-	hash := h.String()
-	return d.fs.Join(objectsPath, hash[0:2], hash[2:40])
+	hex := h.String()
+	return d.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize])
 }
 
 // incomingObjectPath is intended to add support for a git pre-receive hook
@@ -563,15 +563,16 @@ func (d *DotGit) objectPath(h plumbing.Hash) string {
 //
 // More on git hooks found here : https://git-scm.com/docs/githooks
 // More on 'quarantine'/incoming directory here:
-//     https://git-scm.com/docs/git-receive-pack
+//
+//	https://git-scm.com/docs/git-receive-pack
 func (d *DotGit) incomingObjectPath(h plumbing.Hash) string {
 	hString := h.String()
 
 	if d.incomingDirName == "" {
-		return d.fs.Join(objectsPath, hString[0:2], hString[2:40])
+		return d.fs.Join(objectsPath, hString[0:2], hString[2:hash.HexSize])
 	}
 
-	return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40])
+	return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:hash.HexSize])
 }
 
 // hasIncomingObjects searches for an incoming directory and keeps its name
@@ -581,7 +582,9 @@ func (d *DotGit) hasIncomingObjects() bool {
 		directoryContents, err := d.fs.ReadDir(objectsPath)
 		if err == nil {
 			for _, file := range directoryContents {
-				if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() {
+				if file.IsDir() && (strings.HasPrefix(file.Name(), "tmp_objdir-incoming-") ||
+					// Before Git 2.35 incoming commits directory had another prefix
+					strings.HasPrefix(file.Name(), "incoming-")) {
 					d.incomingDirName = file.Name()
 				}
 			}
@@ -645,7 +648,7 @@ func (d *DotGit) ObjectDelete(h plumbing.Hash) error {
 }
 
 func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) {
-	b, err := stdioutil.ReadAll(rd)
+	b, err := io.ReadAll(rd)
 	if err != nil {
 		return nil, err
 	}
@@ -716,48 +719,56 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) {
 	return d.packedRef(name)
 }
 
-func (d *DotGit) findPackedRefsInFile(f billy.File) ([]*plumbing.Reference, error) {
+func (d *DotGit) findPackedRefsInFile(f billy.File, recv refsRecv) error {
 	s := bufio.NewScanner(f)
-	var refs []*plumbing.Reference
 	for s.Scan() {
 		ref, err := d.processLine(s.Text())
 		if err != nil {
-			return nil, err
+			return err
 		}
 
-		if ref != nil {
-			refs = append(refs, ref)
+		if !recv(ref) {
+			// skip parse
+			return nil
 		}
 	}
-
-	return refs, s.Err()
+	if err := s.Err(); err != nil {
+		return err
+	}
+	return nil
 }
 
-func (d *DotGit) findPackedRefs() (r []*plumbing.Reference, err error) {
+// refsRecv: returning true means that the reference continues to be resolved, otherwise it is stopped, which will speed up the lookup of a single reference.
+type refsRecv func(*plumbing.Reference) bool
+
+func (d *DotGit) findPackedRefs(recv refsRecv) error {
 	f, err := d.fs.Open(packedRefsPath)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil
+			return nil
 		}
-		return nil, err
+		return err
 	}
 
 	defer ioutil.CheckClose(f, &err)
-	return d.findPackedRefsInFile(f)
+	return d.findPackedRefsInFile(f, recv)
 }
 
 func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) {
-	refs, err := d.findPackedRefs()
-	if err != nil {
+	var ref *plumbing.Reference
+	if err := d.findPackedRefs(func(r *plumbing.Reference) bool {
+		if r != nil && r.Name() == name {
+			ref = r
+			// ref found
+			return false
+		}
+		return true
+	}); err != nil {
 		return nil, err
 	}
-
-	for _, ref := range refs {
-		if ref.Name() == name {
-			return ref, nil
-		}
+	if ref != nil {
+		return ref, nil
 	}
-
 	return nil, plumbing.ErrReferenceNotFound
 }
 
@@ -777,34 +788,22 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error {
 	return d.rewritePackedRefsWithoutRef(name)
 }
 
-func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) {
-	packedRefs, err := d.findPackedRefs()
-	if err != nil {
-		return err
-	}
-
-	for _, ref := range packedRefs {
-		if !seen[ref.Name()] {
-			*refs = append(*refs, ref)
-			seen[ref.Name()] = true
+func refsRecvFunc(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) refsRecv {
+	return func(r *plumbing.Reference) bool {
+		if r != nil && !seen[r.Name()] {
+			*refs = append(*refs, r)
+			seen[r.Name()] = true
 		}
+		return true
 	}
-	return nil
 }
 
-func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) {
-	packedRefs, err := d.findPackedRefsInFile(f)
-	if err != nil {
-		return err
-	}
+func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference, seen map[plumbing.ReferenceName]bool) (err error) {
+	return d.findPackedRefs(refsRecvFunc(refs, seen))
+}
 
-	for _, ref := range packedRefs {
-		if !seen[ref.Name()] {
-			*refs = append(*refs, ref)
-			seen[ref.Name()] = true
-		}
-	}
-	return nil
+func (d *DotGit) addRefsFromPackedRefsFile(refs *[]*plumbing.Reference, f billy.File, seen map[plumbing.ReferenceName]bool) (err error) {
+	return d.findPackedRefsInFile(f, refsRecvFunc(refs, seen))
 }
 
 func (d *DotGit) openAndLockPackedRefs(doCreate bool) (
@@ -943,6 +942,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
 	files, err := d.fs.ReadDir(d.fs.Join(relPath...))
 	if err != nil {
 		if os.IsNotExist(err) {
+			// a race happened, and our directory is gone now
 			return nil
 		}
 
@@ -960,6 +960,10 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
 		}
 
 		ref, err := d.readReferenceFile(".", strings.Join(newRelPath, "/"))
+		if os.IsNotExist(err) {
+			// a race happened, and our file is gone now
+			continue
+		}
 		if err != nil {
 			return err
 		}
diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index a8f0eb7..1b6c113 100644
--- a/storage/filesystem/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"encoding/hex"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -374,7 +373,7 @@ func (s *SuiteDotGit) TestConfigWriteAndConfig(c *C) {
 	f, err = dir.Config()
 	c.Assert(err, IsNil)
 
-	cnt, err := ioutil.ReadAll(f)
+	cnt, err := io.ReadAll(f)
 	c.Assert(err, IsNil)
 
 	c.Assert(string(cnt), Equals, "foo")
@@ -404,7 +403,7 @@ func (s *SuiteDotGit) TestIndexWriteAndIndex(c *C) {
 	f, err = dir.Index()
 	c.Assert(err, IsNil)
 
-	cnt, err := ioutil.ReadAll(f)
+	cnt, err := io.ReadAll(f)
 	c.Assert(err, IsNil)
 
 	c.Assert(string(cnt), Equals, "foo")
@@ -434,7 +433,7 @@ func (s *SuiteDotGit) TestShallowWriteAndShallow(c *C) {
 	f, err = dir.Shallow()
 	c.Assert(err, IsNil)
 
-	cnt, err := ioutil.ReadAll(f)
+	cnt, err := io.ReadAll(f)
 	c.Assert(err, IsNil)
 
 	c.Assert(string(cnt), Equals, "foo")
@@ -649,6 +648,27 @@ func (s *SuiteDotGit) TestObject(c *C) {
 		Equals, true,
 	)
 	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+	incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
+	incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+	fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+	fs.Create(incomingFilePath)
+
+	_, err = dir.Object(plumbing.NewHash(incomingHash))
+	c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestPreGit235Object(c *C) {
+	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+	dir := New(fs)
+
+	hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+	file, err := dir.Object(hash)
+	c.Assert(err, IsNil)
+	c.Assert(strings.HasSuffix(
+		file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")),
+		Equals, true,
+	)
+	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
 	incomingDirPath := fs.Join("objects", "incoming-123456")
 	incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
 	fs.MkdirAll(incomingDirPath, os.FileMode(0755))
@@ -666,7 +686,7 @@ func (s *SuiteDotGit) TestObjectStat(c *C) {
 	_, err := dir.ObjectStat(hash)
 	c.Assert(err, IsNil)
 	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
-	incomingDirPath := fs.Join("objects", "incoming-123456")
+	incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
 	incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
 	fs.MkdirAll(incomingDirPath, os.FileMode(0755))
 	fs.Create(incomingFilePath)
@@ -684,7 +704,7 @@ func (s *SuiteDotGit) TestObjectDelete(c *C) {
 	c.Assert(err, IsNil)
 
 	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
-	incomingDirPath := fs.Join("objects", "incoming-123456")
+	incomingDirPath := fs.Join("objects", "tmp_objdir-incoming-123456")
 	incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2])
 	incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40])
 
@@ -864,3 +884,71 @@ func (s *SuiteDotGit) TestIncBytes(c *C) {
 		c.Assert(overflow, Equals, test.overflow)
 	}
 }
+
+// this filesystem wrapper returns os.ErrNotExist if the file matches
+// the provided paths list
+type notExistsFS struct {
+	billy.Filesystem
+
+	paths []string
+}
+
+func (f *notExistsFS) matches(path string) bool {
+	p := filepath.ToSlash(path)
+	for _, n := range f.paths {
+		if p == n {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *notExistsFS) Open(filename string) (billy.File, error) {
+	if f.matches(filename) {
+		return nil, os.ErrNotExist
+	}
+
+	return f.Filesystem.Open(filename)
+}
+
+func (f *notExistsFS) ReadDir(path string) ([]os.FileInfo, error) {
+	if f.matches(path) {
+		return nil, os.ErrNotExist
+	}
+
+	return f.Filesystem.ReadDir(path)
+}
+
+func (s *SuiteDotGit) TestDeletedRefs(c *C) {
+	fs, clean := s.TemporalFilesystem()
+	defer clean()
+
+	dir := New(&notExistsFS{
+		Filesystem: fs,
+		paths: []string{
+			"refs/heads/bar",
+			"refs/heads/baz",
+		},
+	})
+
+	err := dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/foo",
+		"e8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+	err = dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/bar",
+		"a8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+	err = dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/baz/baz",
+		"a8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+
+	refs, err := dir.Refs()
+	c.Assert(err, IsNil)
+	c.Assert(refs, HasLen, 1)
+	c.Assert(refs[0].Name(), Equals, plumbing.ReferenceName("refs/heads/foo"))
+}
diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go
index e2ede93..849b7a1 100644
--- a/storage/filesystem/dotgit/writers.go
+++ b/storage/filesystem/dotgit/writers.go
@@ -9,6 +9,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/format/idxfile"
 	"github.com/go-git/go-git/v5/plumbing/format/objfile"
 	"github.com/go-git/go-git/v5/plumbing/format/packfile"
+	"github.com/go-git/go-git/v5/plumbing/hash"
 
 	"github.com/go-git/go-billy/v5"
 )
@@ -277,8 +278,8 @@ func (w *ObjectWriter) Close() error {
 }
 
 func (w *ObjectWriter) save() error {
-	hash := w.Hash().String()
-	file := w.fs.Join(objectsPath, hash[0:2], hash[2:40])
+	hex := w.Hash().String()
+	file := w.fs.Join(objectsPath, hex[0:2], hex[2:hash.HexSize])
 
 	return w.fs.Rename(w.f.Name(), file)
 }
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 5c91bcd..846a7b8 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/go-git/go-git/v5/plumbing"
@@ -419,10 +420,21 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb
 
 	s.objectCache.Put(obj)
 
-	_, err = io.Copy(w, r)
+	bufp := copyBufferPool.Get().(*[]byte)
+	buf := *bufp
+	_, err = io.CopyBuffer(w, r, buf)
+	copyBufferPool.Put(bufp)
+
 	return obj, err
 }
 
+var copyBufferPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, 32*1024)
+		return &b
+	},
+}
+
 // Get returns the object with the given hash, by searching for it in
 // the packfile.
 func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
@@ -525,14 +537,21 @@ func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, pl
 	return plumbing.ZeroHash, plumbing.ZeroHash, -1
 }
 
+// HashesWithPrefix returns all objects with a hash that starts with a prefix by searching for
+// them in the packfile and the git object directories.
 func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error) {
 	hashes, err := s.dir.ObjectsWithPrefix(prefix)
 	if err != nil {
 		return nil, err
 	}
 
+	seen := hashListAsMap(hashes)
+
 	// TODO: This could be faster with some idxfile changes,
 	// or diving into the packfile.
+	if err := s.requireIndex(); err != nil {
+		return nil, err
+	}
 	for _, index := range s.index {
 		ei, err := index.Entries()
 		if err != nil {
@@ -546,6 +565,9 @@ func (s *ObjectStorage) HashesWithPrefix(prefix []byte) ([]plumbing.Hash, error)
 				return nil, err
 			}
 			if bytes.HasPrefix(e.Hash[:], prefix) {
+				if _, ok := seen[e.Hash]; ok {
+					continue
+				}
 				hashes = append(hashes, e.Hash)
 			}
 		}
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index 19a7914..251077a 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -406,6 +405,21 @@ func (s *FsSuite) TestHashesWithPrefix(c *C) {
 	c.Assert(hashes[0].String(), Equals, "f3dfe29d268303fc6e1bbce268605fc99573406e")
 }
 
+func (s *FsSuite) TestHashesWithPrefixFromPackfile(c *C) {
+	// Same setup as TestGetFromPackfile
+	fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
+		fs := f.DotGit()
+		o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
+
+		expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+		// Only pass the first 8 bytes
+		hashes, err := o.HashesWithPrefix(expected[:8])
+		c.Assert(err, IsNil)
+		c.Assert(hashes, HasLen, 1)
+		c.Assert(hashes[0], Equals, expected)
+	})
+}
+
 func BenchmarkPackfileIter(b *testing.B) {
 	defer fixtures.Clean()
 
@@ -495,7 +509,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) {
 								b.Fatal(err)
 							}
 
-							if _, err := ioutil.ReadAll(r); err != nil {
+							if _, err := io.ReadAll(r); err != nil {
 								b.Fatal(err)
 							}
 
diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go
index afb600c..ac48fdf 100644
--- a/storage/filesystem/shallow.go
+++ b/storage/filesystem/shallow.go
@@ -34,7 +34,7 @@ func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error {
 	return err
 }
 
-// Shallow return the shallow commits reading from shallo file from .git
+// Shallow returns the shallow commits reading from shallo file from .git
 func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) {
 	f, err := s.dir.Shallow()
 	if f == nil || err != nil {
diff --git a/storage/test/storage_suite.go b/storage/test/storage_suite.go
index 2c00e75..ee67fc7 100644
--- a/storage/test/storage_suite.go
+++ b/storage/test/storage_suite.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -502,12 +501,12 @@ func objectEquals(a plumbing.EncodedObject, b plumbing.EncodedObject) error {
 		return fmt.Errorf("can't get reader on b: %q", err)
 	}
 
-	ca, err := ioutil.ReadAll(ra)
+	ca, err := io.ReadAll(ra)
 	if err != nil {
 		return fmt.Errorf("error reading a: %q", err)
 	}
 
-	cb, err := ioutil.ReadAll(rb)
+	cb, err := io.ReadAll(rb)
 	if err != nil {
 		return fmt.Errorf("error reading b: %q", err)
 	}