aboutsummaryrefslogtreecommitdiffstats
path: root/storage/filesystem
diff options
context:
space:
mode:
Diffstat (limited to 'storage/filesystem')
-rw-r--r--storage/filesystem/config.go2
-rw-r--r--storage/filesystem/config_test.go2
-rw-r--r--storage/filesystem/dotgit/dotgit.go (renamed from storage/filesystem/internal/dotgit/dotgit.go)306
-rw-r--r--storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go81
-rw-r--r--storage/filesystem/dotgit/dotgit_setref.go (renamed from storage/filesystem/internal/dotgit/dotgit_setref.go)0
-rw-r--r--storage/filesystem/dotgit/dotgit_setref_norwfs.go (renamed from storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go)0
-rw-r--r--storage/filesystem/dotgit/dotgit_test.go (renamed from storage/filesystem/internal/dotgit/dotgit_test.go)114
-rw-r--r--storage/filesystem/dotgit/writers.go (renamed from storage/filesystem/internal/dotgit/writers.go)30
-rw-r--r--storage/filesystem/dotgit/writers_test.go (renamed from storage/filesystem/internal/dotgit/writers_test.go)3
-rw-r--r--storage/filesystem/index.go2
-rw-r--r--storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go17
-rw-r--r--storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go34
-rw-r--r--storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go42
-rw-r--r--storage/filesystem/module.go5
-rw-r--r--storage/filesystem/object.go227
-rw-r--r--storage/filesystem/object_test.go270
-rw-r--r--storage/filesystem/reference.go2
-rw-r--r--storage/filesystem/shallow.go2
-rw-r--r--storage/filesystem/storage.go40
-rw-r--r--storage/filesystem/storage_test.go27
20 files changed, 966 insertions, 240 deletions
diff --git a/storage/filesystem/config.go b/storage/filesystem/config.go
index 85feaf0..be812e4 100644
--- a/storage/filesystem/config.go
+++ b/storage/filesystem/config.go
@@ -5,7 +5,7 @@ import (
"os"
"gopkg.in/src-d/go-git.v4/config"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
)
diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go
index cc03119..71c947d 100644
--- a/storage/filesystem/config_test.go
+++ b/storage/filesystem/config_test.go
@@ -5,7 +5,7 @@ import (
"os"
"gopkg.in/src-d/go-git.v4/config"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
. "gopkg.in/check.v1"
"gopkg.in/src-d/go-billy.v4/osfs"
diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index 52b621c..a58c248 100644
--- a/storage/filesystem/internal/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -57,17 +57,48 @@ var (
ErrSymRefTargetNotFound = errors.New("symbolic reference target not found")
)
+// Options holds configuration for the storage.
+type Options struct {
+ // ExclusiveAccess means that the filesystem is not modified externally
+ // while the repo is open.
+ ExclusiveAccess bool
+ // KeepDescriptors makes the file descriptors to be reused but they will
+ // need to be manually closed calling Close().
+ KeepDescriptors bool
+}
+
// The DotGit type represents a local git repository on disk. This
// type is not zero-value-safe, use the New function to initialize it.
type DotGit struct {
- fs billy.Filesystem
+ options Options
+ fs billy.Filesystem
+
+ // incoming object directory information
+ incomingChecked bool
+ incomingDirName string
+
+ objectList []plumbing.Hash
+ objectMap map[plumbing.Hash]struct{}
+ packList []plumbing.Hash
+ packMap map[plumbing.Hash]struct{}
+
+ files map[string]billy.File
}
// New returns a DotGit value ready to be used. The path argument must
// be the absolute path of a git repository directory (e.g.
// "/foo/bar/.git").
func New(fs billy.Filesystem) *DotGit {
- return &DotGit{fs: fs}
+ return NewWithOptions(fs, Options{})
+}
+
+// NewWithOptions sets non default configuration options.
+// See New for complete help.
+func NewWithOptions(fs billy.Filesystem, o Options) *DotGit {
+ return &DotGit{
+ options: o,
+ fs: fs,
+ }
}
// Initialize creates all the folder scaffolding.
@@ -97,6 +128,28 @@ func (d *DotGit) Initialize() error {
return nil
}
+// Close closes all opened files.
+func (d *DotGit) Close() error {
+ var firstError error
+ if d.files != nil {
+ for _, f := range d.files {
+ err := f.Close()
+ if err != nil && firstError == nil {
+ firstError = err
+ continue
+ }
+ }
+
+ d.files = nil
+ }
+
+ if firstError != nil {
+ return firstError
+ }
+
+ return nil
+}
+
// ConfigWriter returns a file pointer for write to the config file
func (d *DotGit) ConfigWriter() (billy.File, error) {
return d.fs.Create(configPath)
@@ -139,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) {
// NewObjectPack return a writer for a new packfile, it saves the packfile to
// disk and also generates and save the index for the given packfile.
func (d *DotGit) NewObjectPack() (*PackWriter, error) {
+ d.cleanPackList()
return newPackWrite(d.fs)
}
// ObjectPacks returns the list of availables packfiles
func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) {
+ if !d.options.ExclusiveAccess {
+ return d.objectPacks()
+ }
+
+ err := d.genPackList()
+ if err != nil {
+ return nil, err
+ }
+
+ return d.packList, nil
+}
+
+func (d *DotGit) objectPacks() ([]plumbing.Hash, error) {
packDir := d.fs.Join(objectsPath, packPath)
files, err := d.fs.ReadDir(packDir)
if err != nil {
@@ -177,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string {
}
func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) {
- pack, err := d.fs.Open(d.objectPackPath(hash, extension))
+ if d.files == nil {
+ d.files = make(map[string]billy.File)
+ }
+
+ err := d.hasPack(hash)
+ if err != nil {
+ return nil, err
+ }
+
+ path := d.objectPackPath(hash, extension)
+ f, ok := d.files[path]
+ if ok {
+ return f, nil
+ }
+
+ pack, err := d.fs.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrPackfileNotFound
@@ -186,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
return nil, err
}
+ if d.options.KeepDescriptors && extension == "pack" {
+ d.files[path] = pack
+ }
+
return pack, nil
}
// ObjectPack returns a fs.File of the given packfile
func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) {
+ err := d.hasPack(hash)
+ if err != nil {
+ return nil, err
+ }
+
return d.objectPackOpen(hash, `pack`)
}
// ObjectPackIdx returns a fs.File of the index file for a given packfile
func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) {
+ err := d.hasPack(hash)
+ if err != nil {
+ return nil, err
+ }
+
return d.objectPackOpen(hash, `idx`)
}
func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error {
+ d.cleanPackList()
+
path := d.objectPackPath(hash, `pack`)
if !t.IsZero() {
fi, err := d.fs.Stat(path)
@@ -220,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er
// NewObject return a writer for a new object file.
func (d *DotGit) NewObject() (*ObjectWriter, error) {
+ d.cleanObjectList()
+
return newObjectWriter(d.fs)
}
// Objects returns a slice with the hashes of objects found under the
// .git/objects/ directory.
func (d *DotGit) Objects() ([]plumbing.Hash, error) {
+ if d.options.ExclusiveAccess {
+ err := d.genObjectList()
+ if err != nil {
+ return nil, err
+ }
+
+ return d.objectList, nil
+ }
+
var objects []plumbing.Hash
err := d.ForEachObjectHash(func(hash plumbing.Hash) error {
objects = append(objects, hash)
@@ -237,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) {
return objects, nil
}
-// Objects returns a slice with the hashes of objects found under the
-// .git/objects/ directory.
+// ForEachObjectHash iterates over the hashes of objects found under the
+// .git/objects/ directory and executes the provided function.
func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error {
+ if !d.options.ExclusiveAccess {
+ return d.forEachObjectHash(fun)
+ }
+
+ err := d.genObjectList()
+ if err != nil {
+ return err
+ }
+
+ for _, h := range d.objectList {
+ err := fun(h)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error {
files, err := d.fs.ReadDir(objectsPath)
if err != nil {
if os.IsNotExist(err) {
@@ -274,24 +403,178 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error {
return nil
}
+func (d *DotGit) cleanObjectList() {
+ d.objectMap = nil
+ d.objectList = nil
+}
+
+func (d *DotGit) genObjectList() error {
+ if d.objectMap != nil {
+ return nil
+ }
+
+ d.objectMap = make(map[plumbing.Hash]struct{})
+ return d.forEachObjectHash(func(h plumbing.Hash) error {
+ d.objectList = append(d.objectList, h)
+ d.objectMap[h] = struct{}{}
+
+ return nil
+ })
+}
+
+func (d *DotGit) hasObject(h plumbing.Hash) error {
+ if !d.options.ExclusiveAccess {
+ return nil
+ }
+
+ err := d.genObjectList()
+ if err != nil {
+ return err
+ }
+
+ _, ok := d.objectMap[h]
+ if !ok {
+ return plumbing.ErrObjectNotFound
+ }
+
+ return nil
+}
+
+func (d *DotGit) cleanPackList() {
+ d.packMap = nil
+ d.packList = nil
+}
+
+func (d *DotGit) genPackList() error {
+ if d.packMap != nil {
+ return nil
+ }
+
+ op, err := d.objectPacks()
+ if err != nil {
+ return err
+ }
+
+ d.packMap = make(map[plumbing.Hash]struct{})
+ d.packList = nil
+
+ for _, h := range op {
+ d.packList = append(d.packList, h)
+ d.packMap[h] = struct{}{}
+ }
+
+ return nil
+}
+
+func (d *DotGit) hasPack(h plumbing.Hash) error {
+ if !d.options.ExclusiveAccess {
+ return nil
+ }
+
+ err := d.genPackList()
+ if err != nil {
+ return err
+ }
+
+ _, ok := d.packMap[h]
+ if !ok {
+ return ErrPackfileNotFound
+ }
+
+ return nil
+}
+
func (d *DotGit) objectPath(h plumbing.Hash) string {
hash := h.String()
return d.fs.Join(objectsPath, hash[0:2], hash[2:40])
}
+// incomingObjectPath is intended to add support for a git pre-receive hook
+// to be written it adds support for go-git to find objects in an "incoming"
+// directory, so that the library can be used to write a pre-receive hook
+// that deals with the incoming objects.
+//
+// More on git hooks found here : https://git-scm.com/docs/githooks
+// More on 'quarantine'/incoming directory here:
+// https://git-scm.com/docs/git-receive-pack
+func (d *DotGit) incomingObjectPath(h plumbing.Hash) string {
+ hString := h.String()
+
+ if d.incomingDirName == "" {
+ return d.fs.Join(objectsPath, hString[0:2], hString[2:40])
+ }
+
+ return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40])
+}
+
+// hasIncomingObjects searches for an incoming directory and keeps its name
+// so it doesn't have to be found each time an object is accessed.
+func (d *DotGit) hasIncomingObjects() bool {
+ if !d.incomingChecked {
+ directoryContents, err := d.fs.ReadDir(objectsPath)
+ if err == nil {
+ for _, file := range directoryContents {
+ if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() {
+ d.incomingDirName = file.Name()
+ }
+ }
+ }
+
+ d.incomingChecked = true
+ }
+
+ return d.incomingDirName != ""
+}
+
// Object returns a fs.File pointing the object file, if exists
func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) {
- return d.fs.Open(d.objectPath(h))
+ err := d.hasObject(h)
+ if err != nil {
+ return nil, err
+ }
+
+ obj1, err1 := d.fs.Open(d.objectPath(h))
+ if os.IsNotExist(err1) && d.hasIncomingObjects() {
+ obj2, err2 := d.fs.Open(d.incomingObjectPath(h))
+ if err2 != nil {
+ return obj1, err1
+ }
+ return obj2, err2
+ }
+ return obj1, err1
}
// ObjectStat returns a os.FileInfo pointing the object file, if exists
func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) {
- return d.fs.Stat(d.objectPath(h))
+ err := d.hasObject(h)
+ if err != nil {
+ return nil, err
+ }
+
+ obj1, err1 := d.fs.Stat(d.objectPath(h))
+ if os.IsNotExist(err1) && d.hasIncomingObjects() {
+ obj2, err2 := d.fs.Stat(d.incomingObjectPath(h))
+ if err2 != nil {
+ return obj1, err1
+ }
+ return obj2, err2
+ }
+ return obj1, err1
}
// ObjectDelete removes the object file, if exists
func (d *DotGit) ObjectDelete(h plumbing.Hash) error {
- return d.fs.Remove(d.objectPath(h))
+ d.cleanObjectList()
+
+ err1 := d.fs.Remove(d.objectPath(h))
+ if os.IsNotExist(err1) && d.hasIncomingObjects() {
+ err2 := d.fs.Remove(d.incomingObjectPath(h))
+ if err2 != nil {
+ return err1
+ }
+ return err2
+ }
+ return err1
}
func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) {
@@ -469,7 +752,7 @@ func (d *DotGit) openAndLockPackedRefs(doCreate bool) (
// File mode is retrieved from a constant defined in the target specific
// files (dotgit_rewrite_packed_refs_*). Some modes are not available
// in all filesystems.
- openFlags := openAndLockPackedRefsMode
+ openFlags := d.openAndLockPackedRefsMode()
if doCreate {
openFlags |= os.O_CREATE
}
@@ -784,6 +1067,11 @@ func (d *DotGit) Alternates() ([]*DotGit, error) {
return alternates, nil
}
+// Fs returns the underlying filesystem of the DotGit folder.
+func (d *DotGit) Fs() billy.Filesystem {
+ return d.fs
+}
+
func isHex(s string) bool {
for _, b := range []byte(s) {
if isNum(b) {
diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go
new file mode 100644
index 0000000..7f1c02c
--- /dev/null
+++ b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go
@@ -0,0 +1,81 @@
+package dotgit
+
+import (
+ "io"
+ "os"
+ "runtime"
+
+ "gopkg.in/src-d/go-billy.v4"
+ "gopkg.in/src-d/go-git.v4/utils/ioutil"
+)
+
+func (d *DotGit) openAndLockPackedRefsMode() int {
+ if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) {
+ return os.O_RDWR
+ }
+
+ return os.O_RDONLY
+}
+
+func (d *DotGit) rewritePackedRefsWhileLocked(
+ tmp billy.File, pr billy.File) error {
+ // Try plain rename. If we aren't using the bare Windows filesystem as the
+ // storage layer, we might be able to get away with a rename over a locked
+ // file.
+ err := d.fs.Rename(tmp.Name(), pr.Name())
+ if err == nil {
+ return nil
+ }
+
+ // If we are in a filesystem that does not support rename (e.g. sivafs)
+ // a full copy is done.
+ if err == billy.ErrNotSupported {
+ return d.copyNewFile(tmp, pr)
+ }
+
+ if runtime.GOOS != "windows" {
+ return err
+ }
+
+ // Otherwise, Windows doesn't let us rename over a locked file, so
+ // we have to do a straight copy. Unfortunately this could result
+ // in a partially-written file if the process fails before the
+ // copy completes.
+ return d.copyToExistingFile(tmp, pr)
+}
+
+func (d *DotGit) copyToExistingFile(tmp, pr billy.File) error {
+ _, err := pr.Seek(0, io.SeekStart)
+ if err != nil {
+ return err
+ }
+ err = pr.Truncate(0)
+ if err != nil {
+ return err
+ }
+ _, err = tmp.Seek(0, io.SeekStart)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(pr, tmp)
+
+ return err
+}
+
+func (d *DotGit) copyNewFile(tmp billy.File, pr billy.File) (err error) {
+ prWrite, err := d.fs.Create(pr.Name())
+ if err != nil {
+ return err
+ }
+
+ defer ioutil.CheckClose(prWrite, &err)
+
+ _, err = tmp.Seek(0, io.SeekStart)
+ if err != nil {
+ return err
+ }
+
+ _, err = io.Copy(prWrite, tmp)
+
+ return err
+}
diff --git a/storage/filesystem/internal/dotgit/dotgit_setref.go b/storage/filesystem/dotgit/dotgit_setref.go
index d27c1a3..d27c1a3 100644
--- a/storage/filesystem/internal/dotgit/dotgit_setref.go
+++ b/storage/filesystem/dotgit/dotgit_setref.go
diff --git a/storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go b/storage/filesystem/dotgit/dotgit_setref_norwfs.go
index 5695bd3..5695bd3 100644
--- a/storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go
+++ b/storage/filesystem/dotgit/dotgit_setref_norwfs.go
diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index 7733eef..308c6b7 100644
--- a/storage/filesystem/internal/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -9,6 +9,7 @@ import (
"strings"
"testing"
+ "gopkg.in/src-d/go-billy.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
. "gopkg.in/check.v1"
@@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) {
fs := f.DotGit()
dir := New(fs)
+ testObjectPacks(c, fs, dir, f)
+}
+
+func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) {
+ f := fixtures.Basic().ByTag(".git").One()
+ fs := f.DotGit()
+ dir := NewWithOptions(fs, Options{ExclusiveAccess: true})
+
+ testObjectPacks(c, fs, dir, f)
+}
+
+func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) {
hashes, err := dir.ObjectPacks()
c.Assert(err, IsNil)
c.Assert(hashes, HasLen, 1)
@@ -452,6 +465,45 @@ func (s *SuiteDotGit) TestObjectPack(c *C) {
c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
}
+func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) {
+ f := fixtures.Basic().ByTag(".git").One()
+ fs := f.DotGit()
+ dir := NewWithOptions(fs, Options{KeepDescriptors: true})
+
+ pack, err := dir.ObjectPack(f.PackfileHash)
+ c.Assert(err, IsNil)
+ c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
+
+ // Move to an specific offset
+ pack.Seek(42, os.SEEK_SET)
+
+ pack2, err := dir.ObjectPack(f.PackfileHash)
+ c.Assert(err, IsNil)
+
+ // If the file is the same the offset should be the same
+ offset, err := pack2.Seek(0, os.SEEK_CUR)
+ c.Assert(err, IsNil)
+ c.Assert(offset, Equals, int64(42))
+
+ err = dir.Close()
+ c.Assert(err, IsNil)
+
+ pack2, err = dir.ObjectPack(f.PackfileHash)
+ c.Assert(err, IsNil)
+
+ // If the file is opened again its offset should be 0
+ offset, err = pack2.Seek(0, os.SEEK_CUR)
+ c.Assert(err, IsNil)
+ c.Assert(offset, Equals, int64(0))
+
+ err = pack2.Close()
+ c.Assert(err, IsNil)
+
+ err = dir.Close()
+ c.Assert(err, NotNil)
+
+}
+
func (s *SuiteDotGit) TestObjectPackIdx(c *C) {
f := fixtures.Basic().ByTag(".git").One()
fs := f.DotGit()
@@ -506,6 +558,17 @@ func (s *SuiteDotGit) TestObjects(c *C) {
fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
dir := New(fs)
+ testObjects(c, fs, dir)
+}
+
+func (s *SuiteDotGit) TestObjectsExclusive(c *C) {
+ fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+ dir := NewWithOptions(fs, Options{ExclusiveAccess: true})
+
+ testObjects(c, fs, dir)
+}
+
+func testObjects(c *C, fs billy.Filesystem, dir *DotGit) {
hashes, err := dir.Objects()
c.Assert(err, IsNil)
c.Assert(hashes, HasLen, 187)
@@ -537,6 +600,57 @@ func (s *SuiteDotGit) TestObject(c *C) {
file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")),
Equals, true,
)
+ incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+ incomingDirPath := fs.Join("objects", "incoming-123456")
+ incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+ fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+ fs.Create(incomingFilePath)
+
+ file, err = dir.Object(plumbing.NewHash(incomingHash))
+ c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestObjectStat(c *C) {
+ fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+ dir := New(fs)
+
+ hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+ _, err := dir.ObjectStat(hash)
+ c.Assert(err, IsNil)
+ incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+ incomingDirPath := fs.Join("objects", "incoming-123456")
+ incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+ fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+ fs.Create(incomingFilePath)
+
+ _, err = dir.ObjectStat(plumbing.NewHash(incomingHash))
+ c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestObjectDelete(c *C) {
+ fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+ dir := New(fs)
+
+ hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+ err := dir.ObjectDelete(hash)
+ c.Assert(err, IsNil)
+
+ incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+ incomingDirPath := fs.Join("objects", "incoming-123456")
+ incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2])
+ incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40])
+
+ err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755))
+ c.Assert(err, IsNil)
+
+ f, err := fs.Create(incomingFilePath)
+ c.Assert(err, IsNil)
+
+ err = f.Close()
+ c.Assert(err, IsNil)
+
+ err = dir.ObjectDelete(plumbing.NewHash(incomingHash))
+ c.Assert(err, IsNil)
}
func (s *SuiteDotGit) TestObjectNotFound(c *C) {
diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/dotgit/writers.go
index c2b420f..93d2d8c 100644
--- a/storage/filesystem/internal/dotgit/writers.go
+++ b/storage/filesystem/dotgit/writers.go
@@ -20,13 +20,14 @@ import (
// is renamed/moved (depends on the Filesystem implementation) to the final
// location, if the PackWriter is not used, nothing is written
type PackWriter struct {
- Notify func(plumbing.Hash, *packfile.Index)
+ Notify func(plumbing.Hash, *idxfile.Writer)
fs billy.Filesystem
fr, fw billy.File
synced *syncedReader
checksum plumbing.Hash
- index *packfile.Index
+ parser *packfile.Parser
+ writer *idxfile.Writer
result chan error
}
@@ -55,20 +56,21 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) {
func (w *PackWriter) buildIndex() {
s := packfile.NewScanner(w.synced)
- d, err := packfile.NewDecoder(s, nil)
+ w.writer = new(idxfile.Writer)
+ var err error
+ w.parser, err = packfile.NewParser(s, w.writer)
if err != nil {
w.result <- err
return
}
- checksum, err := d.Decode()
+ checksum, err := w.parser.Parse()
if err != nil {
w.result <- err
return
}
w.checksum = checksum
- w.index = d.Index()
w.result <- err
}
@@ -92,8 +94,8 @@ func (w *PackWriter) Write(p []byte) (int, error) {
// was written, the tempfiles are deleted without writing a packfile.
func (w *PackWriter) Close() error {
defer func() {
- if w.Notify != nil && w.index != nil && w.index.Size() > 0 {
- w.Notify(w.checksum, w.index)
+ if w.Notify != nil && w.writer != nil && w.writer.Finished() {
+ w.Notify(w.checksum, w.writer)
}
close(w.result)
@@ -115,7 +117,7 @@ func (w *PackWriter) Close() error {
return err
}
- if w.index == nil || w.index.Size() == 0 {
+ if w.writer == nil || !w.writer.Finished() {
return w.clean()
}
@@ -145,11 +147,13 @@ func (w *PackWriter) save() error {
}
func (w *PackWriter) encodeIdx(writer io.Writer) error {
- idx := w.index.ToIdxFile()
- idx.PackfileChecksum = w.checksum
- idx.Version = idxfile.VersionSupported
+ idx, err := w.writer.Index()
+ if err != nil {
+ return err
+ }
+
e := idxfile.NewEncoder(writer)
- _, err := e.Encode(idx)
+ _, err = e.Encode(idx)
return err
}
@@ -209,7 +213,6 @@ func (s *syncedReader) isBlocked() bool {
func (s *syncedReader) wake() {
if s.isBlocked() {
- // fmt.Println("wake")
atomic.StoreUint32(&s.blocked, 0)
s.news <- true
}
@@ -220,7 +223,6 @@ func (s *syncedReader) sleep() {
written := atomic.LoadUint64(&s.written)
if read >= written {
atomic.StoreUint32(&s.blocked, 1)
- // fmt.Println("sleep", read, written)
<-s.news
}
diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/dotgit/writers_test.go
index bf00762..5a5f7b4 100644
--- a/storage/filesystem/internal/dotgit/writers_test.go
+++ b/storage/filesystem/dotgit/writers_test.go
@@ -9,6 +9,7 @@ import (
"strconv"
"gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
. "gopkg.in/check.v1"
@@ -148,7 +149,7 @@ func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) {
w, err := newPackWrite(fs)
c.Assert(err, IsNil)
- w.Notify = func(h plumbing.Hash, idx *packfile.Index) {
+ w.Notify = func(h plumbing.Hash, idx *idxfile.Writer) {
c.Fatal("unexpected call to PackWriter.Notify")
}
diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go
index 092edec..2ebf57e 100644
--- a/storage/filesystem/index.go
+++ b/storage/filesystem/index.go
@@ -4,7 +4,7 @@ import (
"os"
"gopkg.in/src-d/go-git.v4/plumbing/format/index"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
)
diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go
deleted file mode 100644
index c760793..0000000
--- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// +build !windows,!norwfs
-
-package dotgit
-
-import (
- "os"
-
- "gopkg.in/src-d/go-billy.v4"
-)
-
-const openAndLockPackedRefsMode = os.O_RDWR
-
-func (d *DotGit) rewritePackedRefsWhileLocked(
- tmp billy.File, pr billy.File) error {
- // On non-Windows platforms, we can have atomic rename.
- return d.fs.Rename(tmp.Name(), pr.Name())
-}
diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go
deleted file mode 100644
index 6e43b42..0000000
--- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// +build norwfs
-
-package dotgit
-
-import (
- "io"
- "os"
-
- "gopkg.in/src-d/go-billy.v4"
-)
-
-const openAndLockPackedRefsMode = os.O_RDONLY
-
-// Instead of renaming that can not be supported in simpler filesystems
-// a full copy is done.
-func (d *DotGit) rewritePackedRefsWhileLocked(
- tmp billy.File, pr billy.File) error {
-
- prWrite, err := d.fs.Create(pr.Name())
- if err != nil {
- return err
- }
-
- defer prWrite.Close()
-
- _, err = tmp.Seek(0, io.SeekStart)
- if err != nil {
- return err
- }
-
- _, err = io.Copy(prWrite, tmp)
-
- return err
-}
diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go
deleted file mode 100644
index 897d2c9..0000000
--- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go
+++ /dev/null
@@ -1,42 +0,0 @@
-// +build windows,!norwfs
-
-package dotgit
-
-import (
- "io"
- "os"
-
- "gopkg.in/src-d/go-billy.v4"
-)
-
-const openAndLockPackedRefsMode = os.O_RDWR
-
-func (d *DotGit) rewritePackedRefsWhileLocked(
- tmp billy.File, pr billy.File) error {
- // If we aren't using the bare Windows filesystem as the storage
- // layer, we might be able to get away with a rename over a locked
- // file.
- err := d.fs.Rename(tmp.Name(), pr.Name())
- if err == nil {
- return nil
- }
-
- // Otherwise, Windows doesn't let us rename over a locked file, so
- // we have to do a straight copy. Unfortunately this could result
- // in a partially-written file if the process fails before the
- // copy completes.
- _, err = pr.Seek(0, io.SeekStart)
- if err != nil {
- return err
- }
- err = pr.Truncate(0)
- if err != nil {
- return err
- }
- _, err = tmp.Seek(0, io.SeekStart)
- if err != nil {
- return err
- }
- _, err = io.Copy(pr, tmp)
- return err
-}
diff --git a/storage/filesystem/module.go b/storage/filesystem/module.go
index 6f3de3f..9272206 100644
--- a/storage/filesystem/module.go
+++ b/storage/filesystem/module.go
@@ -1,8 +1,9 @@
package filesystem
import (
+ "gopkg.in/src-d/go-git.v4/plumbing/cache"
"gopkg.in/src-d/go-git.v4/storage"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
)
type ModuleStorage struct {
@@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) {
return nil, err
}
- return NewStorage(fs)
+ return NewStorage(fs, cache.NewObjectLRUDefault()), nil
}
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 26190fd..68bd140 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -11,28 +11,34 @@ import (
"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
"gopkg.in/src-d/go-git.v4/plumbing/storer"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
- "gopkg.in/src-d/go-git.v4/storage/memory"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
"gopkg.in/src-d/go-billy.v4"
)
type ObjectStorage struct {
+ options Options
+
// deltaBaseCache is an object cache uses to cache delta's bases when
deltaBaseCache cache.Object
dir *dotgit.DotGit
- index map[plumbing.Hash]*packfile.Index
+ index map[plumbing.Hash]idxfile.Index
+}
+
+// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache.
+func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage {
+ return NewObjectStorageWithOptions(dir, cache, Options{})
}
-func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) {
- s := ObjectStorage{
- deltaBaseCache: cache.NewObjectLRUDefault(),
+// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options
+func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage {
+ return &ObjectStorage{
+ options: ops,
+ deltaBaseCache: cache,
dir: dir,
}
-
- return s, nil
}
func (s *ObjectStorage) requireIndex() error {
@@ -40,7 +46,7 @@ func (s *ObjectStorage) requireIndex() error {
return nil
}
- s.index = make(map[plumbing.Hash]*packfile.Index)
+ s.index = make(map[plumbing.Hash]idxfile.Index)
packs, err := s.dir.ObjectPacks()
if err != nil {
return err
@@ -62,13 +68,14 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) {
}
defer ioutil.CheckClose(f, &err)
- idxf := idxfile.NewIdxfile()
+
+ idxf := idxfile.NewMemoryIndex()
d := idxfile.NewDecoder(f)
if err = d.Decode(idxf); err != nil {
return err
}
- s.index[h] = packfile.NewIndexFromIdxFile(idxf)
+ s.index[h] = idxf
return err
}
@@ -86,8 +93,11 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) {
return nil, err
}
- w.Notify = func(h plumbing.Hash, idx *packfile.Index) {
- s.index[h] = idx
+ w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) {
+ index, err := writer.Index()
+ if err == nil {
+ s.index[h] = index
+ }
}
return w, nil
@@ -166,10 +176,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p
// Create a new object storage with the DotGit(s) and check for the
// required hash object. Skip when not found.
for _, dg := range dotgits {
- o, oe := newObjectStorage(dg)
- if oe != nil {
- continue
- }
+ o := NewObjectStorage(dg, s.deltaBaseCache)
enobj, enerr := o.EncodedObject(t, h)
if enerr != nil {
continue
@@ -265,7 +272,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
return nil, err
}
- defer ioutil.CheckClose(f, &err)
+ if !s.options.KeepDescriptors {
+ defer ioutil.CheckClose(f, &err)
+ }
idx := s.index[pack]
if canBeDelta {
@@ -277,30 +286,37 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
func (s *ObjectStorage) decodeObjectAt(
f billy.File,
- idx *packfile.Index,
- offset int64) (plumbing.EncodedObject, error) {
- if _, err := f.Seek(0, io.SeekStart); err != nil {
- return nil, err
+ idx idxfile.Index,
+ offset int64,
+) (plumbing.EncodedObject, error) {
+ hash, err := idx.FindHash(offset)
+ if err == nil {
+ obj, ok := s.deltaBaseCache.Get(hash)
+ if ok {
+ return obj, nil
+ }
}
- p := packfile.NewScanner(f)
-
- d, err := packfile.NewDecoderWithCache(p, memory.NewStorage(),
- s.deltaBaseCache)
- if err != nil {
+ if err != nil && err != plumbing.ErrObjectNotFound {
return nil, err
}
- d.SetIndex(idx)
- obj, err := d.DecodeObjectAt(offset)
- return obj, err
+ var p *packfile.Packfile
+ if s.deltaBaseCache != nil {
+ p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache)
+ } else {
+ p = packfile.NewPackfile(idx, s.dir.Fs(), f)
+ }
+
+ return p.GetByOffset(offset)
}
func (s *ObjectStorage) decodeDeltaObjectAt(
f billy.File,
- idx *packfile.Index,
+ idx idxfile.Index,
offset int64,
- hash plumbing.Hash) (plumbing.EncodedObject, error) {
+ hash plumbing.Hash,
+) (plumbing.EncodedObject, error) {
if _, err := f.Seek(0, io.SeekStart); err != nil {
return nil, err
}
@@ -323,12 +339,10 @@ func (s *ObjectStorage) decodeDeltaObjectAt(
case plumbing.REFDeltaObject:
base = header.Reference
case plumbing.OFSDeltaObject:
- e, ok := idx.LookupOffset(uint64(header.OffsetReference))
- if !ok {
- return nil, plumbing.ErrObjectNotFound
+ base, err = idx.FindHash(header.OffsetReference)
+ if err != nil {
+ return nil, err
}
-
- base = e.Hash
default:
return s.decodeObjectAt(f, idx, offset)
}
@@ -349,8 +363,9 @@ func (s *ObjectStorage) decodeDeltaObjectAt(
func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) {
for packfile, index := range s.index {
- if e, ok := index.LookupHash(h); ok {
- return packfile, e.Hash, int64(e.Offset)
+ offset, err := index.FindOffset(h)
+ if err == nil {
+ return packfile, h, offset
}
}
@@ -381,7 +396,10 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode
return storer.NewMultiEncodedObjectIter(iters), nil
}
-func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]struct{}) (storer.EncodedObjectIter, error) {
+func (s *ObjectStorage) buildPackfileIters(
+ t plumbing.ObjectType,
+ seen map[plumbing.Hash]struct{},
+) (storer.EncodedObjectIter, error) {
if err := s.requireIndex(); err != nil {
return nil, err
}
@@ -397,11 +415,19 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb
if err != nil {
return nil, err
}
- return newPackfileIter(pack, t, seen, s.index[h], s.deltaBaseCache)
+ return newPackfileIter(
+ s.dir.Fs(), pack, t, seen, s.index[h],
+ s.deltaBaseCache, s.options.KeepDescriptors,
+ )
},
}, nil
}
+// Close closes all opened files.
+func (s *ObjectStorage) Close() error {
+ return s.dir.Close()
+}
+
type lazyPackfilesIter struct {
hashes []plumbing.Hash
open func(h plumbing.Hash) (storer.EncodedObjectIter, error)
@@ -450,76 +476,104 @@ func (it *lazyPackfilesIter) Close() {
}
type packfileIter struct {
- f billy.File
- d *packfile.Decoder
- t plumbing.ObjectType
+ pack billy.File
+ iter storer.EncodedObjectIter
+ seen map[plumbing.Hash]struct{}
- seen map[plumbing.Hash]struct{}
- position uint32
- total uint32
+ // tells whether the pack file should be left open after iteration or not
+ keepPack bool
}
-func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
- return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil)
-}
+// NewPackfileIter returns a new EncodedObjectIter for the provided packfile
+// and object type. Packfile and index file will be closed after they're
+// used. If keepPack is true the packfile won't be closed after the iteration
+// finished.
+func NewPackfileIter(
+ fs billy.Filesystem,
+ f billy.File,
+ idxFile billy.File,
+ t plumbing.ObjectType,
+ keepPack bool,
+) (storer.EncodedObjectIter, error) {
+ idx := idxfile.NewMemoryIndex()
+ if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil {
+ return nil, err
+ }
-func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{},
- index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) {
- s := packfile.NewScanner(f)
- _, total, err := s.Header()
- if err != nil {
+ if err := idxFile.Close(); err != nil {
return nil, err
}
- d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t, cache)
+ seen := make(map[plumbing.Hash]struct{})
+ return newPackfileIter(fs, f, t, seen, idx, nil, keepPack)
+}
+
+func newPackfileIter(
+ fs billy.Filesystem,
+ f billy.File,
+ t plumbing.ObjectType,
+ seen map[plumbing.Hash]struct{},
+ index idxfile.Index,
+ cache cache.Object,
+ keepPack bool,
+) (storer.EncodedObjectIter, error) {
+ var p *packfile.Packfile
+ if cache != nil {
+ p = packfile.NewPackfileWithCache(index, fs, f, cache)
+ } else {
+ p = packfile.NewPackfile(index, fs, f)
+ }
+
+ iter, err := p.GetByType(t)
if err != nil {
return nil, err
}
- d.SetIndex(index)
-
return &packfileIter{
- f: f,
- d: d,
- t: t,
-
- total: total,
- seen: seen,
+ pack: f,
+ iter: iter,
+ seen: seen,
+ keepPack: keepPack,
}, nil
}
func (iter *packfileIter) Next() (plumbing.EncodedObject, error) {
for {
- if iter.position >= iter.total {
- return nil, io.EOF
- }
-
- obj, err := iter.d.DecodeObject()
+ obj, err := iter.iter.Next()
if err != nil {
return nil, err
}
- iter.position++
- if obj == nil {
- continue
- }
-
if _, ok := iter.seen[obj.Hash()]; ok {
- return iter.Next()
+ continue
}
return obj, nil
}
}
-// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error {
- return nil
+ for {
+ o, err := iter.Next()
+ if err != nil {
+ if err == io.EOF {
+ iter.Close()
+ return nil
+ }
+ return err
+ }
+
+ if err := cb(o); err != nil {
+ return err
+ }
+ }
}
func (iter *packfileIter) Close() {
- iter.f.Close()
- iter.d.Close()
+ iter.iter.Close()
+ if !iter.keepPack {
+ _ = iter.pack.Close()
+ }
}
type objectsIter struct {
@@ -547,9 +601,20 @@ func (iter *objectsIter) Next() (plumbing.EncodedObject, error) {
return obj, err
}
-// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error {
- return nil
+ for {
+ o, err := iter.Next()
+ if err != nil {
+ if err == io.EOF {
+ return nil
+ }
+ return err
+ }
+
+ if err := cb(o); err != nil {
+ return err
+ }
+ }
}
func (iter *objectsIter) Close() {
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index de8f2b2..407abf2 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -1,8 +1,13 @@
package filesystem
import (
+ "io/ioutil"
+ "os"
+ "testing"
+
"gopkg.in/src-d/go-git.v4/plumbing"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/plumbing/cache"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
. "gopkg.in/check.v1"
"gopkg.in/src-d/go-git-fixtures.v3"
@@ -10,22 +15,20 @@ import (
type FsSuite struct {
fixtures.Suite
- Types []plumbing.ObjectType
}
-var _ = Suite(&FsSuite{
- Types: []plumbing.ObjectType{
- plumbing.CommitObject,
- plumbing.TagObject,
- plumbing.TreeObject,
- plumbing.BlobObject,
- },
-})
+var objectTypes = []plumbing.ObjectType{
+ plumbing.CommitObject,
+ plumbing.TagObject,
+ plumbing.TreeObject,
+ plumbing.BlobObject,
+}
+
+var _ = Suite(&FsSuite{})
func (s *FsSuite) TestGetFromObjectFile(c *C) {
fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
- o, err := newObjectStorage(dotgit.New(fs))
- c.Assert(err, IsNil)
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e")
obj, err := o.EncodedObject(plumbing.AnyObject, expected)
@@ -36,20 +39,53 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) {
func (s *FsSuite) TestGetFromPackfile(c *C) {
fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
fs := f.DotGit()
- o, err := newObjectStorage(dotgit.New(fs))
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
+
+ expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+ obj, err := o.EncodedObject(plumbing.AnyObject, expected)
c.Assert(err, IsNil)
+ c.Assert(obj.Hash(), Equals, expected)
+ })
+}
+
+func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) {
+ fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) {
+ fs := f.DotGit()
+ dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true})
+ o := NewObjectStorageWithOptions(dg, cache.NewObjectLRUDefault(), Options{KeepDescriptors: true})
expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
obj, err := o.EncodedObject(plumbing.AnyObject, expected)
c.Assert(err, IsNil)
c.Assert(obj.Hash(), Equals, expected)
+
+ packfiles, err := dg.ObjectPacks()
+ c.Assert(err, IsNil)
+
+ pack1, err := dg.ObjectPack(packfiles[0])
+ c.Assert(err, IsNil)
+
+ pack1.Seek(42, os.SEEK_SET)
+
+ err = o.Close()
+ c.Assert(err, IsNil)
+
+ pack2, err := dg.ObjectPack(packfiles[0])
+ c.Assert(err, IsNil)
+
+ offset, err := pack2.Seek(0, os.SEEK_CUR)
+ c.Assert(err, IsNil)
+ c.Assert(offset, Equals, int64(0))
+
+ err = o.Close()
+ c.Assert(err, IsNil)
+
})
}
func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) {
fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit()
- o, err := newObjectStorage(dotgit.New(fs))
- c.Assert(err, IsNil)
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3")
obj, err := o.getFromPackfile(expected, false)
@@ -65,8 +101,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) {
func (s *FsSuite) TestIter(c *C) {
fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
fs := f.DotGit()
- o, err := newObjectStorage(dotgit.New(fs))
- c.Assert(err, IsNil)
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
iter, err := o.IterEncodedObjects(plumbing.AnyObject)
c.Assert(err, IsNil)
@@ -84,10 +119,9 @@ func (s *FsSuite) TestIter(c *C) {
func (s *FsSuite) TestIterWithType(c *C) {
fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
- for _, t := range s.Types {
+ for _, t := range objectTypes {
fs := f.DotGit()
- o, err := newObjectStorage(dotgit.New(fs))
- c.Assert(err, IsNil)
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
iter, err := o.IterEncodedObjects(t)
c.Assert(err, IsNil)
@@ -108,23 +142,215 @@ func (s *FsSuite) TestPackfileIter(c *C) {
fs := f.DotGit()
dg := dotgit.New(fs)
- for _, t := range s.Types {
+ for _, t := range objectTypes {
+ ph, err := dg.ObjectPacks()
+ c.Assert(err, IsNil)
+
+ for _, h := range ph {
+ f, err := dg.ObjectPack(h)
+ c.Assert(err, IsNil)
+
+ idxf, err := dg.ObjectPackIdx(h)
+ c.Assert(err, IsNil)
+
+ iter, err := NewPackfileIter(fs, f, idxf, t, false)
+ c.Assert(err, IsNil)
+
+ err = iter.ForEach(func(o plumbing.EncodedObject) error {
+ c.Assert(o.Type(), Equals, t)
+ return nil
+ })
+ c.Assert(err, IsNil)
+ }
+ }
+ })
+}
+
+func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) {
+ fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
+ fs := f.DotGit()
+ ops := dotgit.Options{KeepDescriptors: true}
+ dg := dotgit.NewWithOptions(fs, ops)
+
+ for _, t := range objectTypes {
ph, err := dg.ObjectPacks()
c.Assert(err, IsNil)
for _, h := range ph {
f, err := dg.ObjectPack(h)
c.Assert(err, IsNil)
- iter, err := NewPackfileIter(f, t)
+
+ idxf, err := dg.ObjectPackIdx(h)
c.Assert(err, IsNil)
+
+ iter, err := NewPackfileIter(fs, f, idxf, t, true)
+ c.Assert(err, IsNil)
+
err = iter.ForEach(func(o plumbing.EncodedObject) error {
c.Assert(o.Type(), Equals, t)
return nil
})
+ c.Assert(err, IsNil)
+ // test twice to check that packfiles are not closed
+ err = iter.ForEach(func(o plumbing.EncodedObject) error {
+ c.Assert(o.Type(), Equals, t)
+ return nil
+ })
c.Assert(err, IsNil)
}
}
})
+}
+func BenchmarkPackfileIter(b *testing.B) {
+ if err := fixtures.Init(); err != nil {
+ b.Fatal(err)
+ }
+
+ defer func() {
+ if err := fixtures.Clean(); err != nil {
+ b.Fatal(err)
+ }
+ }()
+
+ for _, f := range fixtures.ByTag(".git") {
+ b.Run(f.URL, func(b *testing.B) {
+ fs := f.DotGit()
+ dg := dotgit.New(fs)
+
+ for i := 0; i < b.N; i++ {
+ for _, t := range objectTypes {
+ ph, err := dg.ObjectPacks()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ for _, h := range ph {
+ f, err := dg.ObjectPack(h)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ idxf, err := dg.ObjectPackIdx(h)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ iter, err := NewPackfileIter(fs, f, idxf, t, false)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ err = iter.ForEach(func(o plumbing.EncodedObject) error {
+ if o.Type() != t {
+ b.Errorf("expecting %s, got %s", t, o.Type())
+ }
+ return nil
+ })
+
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ }
+ }
+ })
+ }
+}
+
+func BenchmarkPackfileIterReadContent(b *testing.B) {
+ if err := fixtures.Init(); err != nil {
+ b.Fatal(err)
+ }
+
+ defer func() {
+ if err := fixtures.Clean(); err != nil {
+ b.Fatal(err)
+ }
+ }()
+
+ for _, f := range fixtures.ByTag(".git") {
+ b.Run(f.URL, func(b *testing.B) {
+ fs := f.DotGit()
+ dg := dotgit.New(fs)
+
+ for i := 0; i < b.N; i++ {
+ for _, t := range objectTypes {
+ ph, err := dg.ObjectPacks()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ for _, h := range ph {
+ f, err := dg.ObjectPack(h)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ idxf, err := dg.ObjectPackIdx(h)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ iter, err := NewPackfileIter(fs, f, idxf, t, false)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ err = iter.ForEach(func(o plumbing.EncodedObject) error {
+ if o.Type() != t {
+ b.Errorf("expecting %s, got %s", t, o.Type())
+ }
+
+ r, err := o.Reader()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ if _, err := ioutil.ReadAll(r); err != nil {
+ b.Fatal(err)
+ }
+
+ return r.Close()
+ })
+
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ }
+ }
+ })
+ }
+}
+
+func BenchmarkGetObjectFromPackfile(b *testing.B) {
+ if err := fixtures.Init(); err != nil {
+ b.Fatal(err)
+ }
+
+ defer func() {
+ if err := fixtures.Clean(); err != nil {
+ b.Fatal(err)
+ }
+ }()
+
+ for _, f := range fixtures.Basic() {
+ b.Run(f.URL, func(b *testing.B) {
+ fs := f.DotGit()
+ o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())
+ for i := 0; i < b.N; i++ {
+ expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+ obj, err := o.EncodedObject(plumbing.AnyObject, expected)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ if obj.Hash() != expected {
+ b.Errorf("expecting %s, got %s", expected, obj.Hash())
+ }
+ }
+ })
+ }
}
diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go
index 7313f05..a891b83 100644
--- a/storage/filesystem/reference.go
+++ b/storage/filesystem/reference.go
@@ -3,7 +3,7 @@ package filesystem
import (
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/storer"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
)
type ReferenceStorage struct {
diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go
index 173767c..502d406 100644
--- a/storage/filesystem/shallow.go
+++ b/storage/filesystem/shallow.go
@@ -5,7 +5,7 @@ import (
"fmt"
"gopkg.in/src-d/go-git.v4/plumbing"
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
"gopkg.in/src-d/go-git.v4/utils/ioutil"
)
diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go
index 82b137c..14a772a 100644
--- a/storage/filesystem/storage.go
+++ b/storage/filesystem/storage.go
@@ -2,7 +2,8 @@
package filesystem
import (
- "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
+ "gopkg.in/src-d/go-git.v4/plumbing/cache"
+ "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
"gopkg.in/src-d/go-billy.v4"
)
@@ -22,25 +23,45 @@ type Storage struct {
ModuleStorage
}
-// NewStorage returns a new Storage backed by a given `fs.Filesystem`
-func NewStorage(fs billy.Filesystem) (*Storage, error) {
- dir := dotgit.New(fs)
- o, err := newObjectStorage(dir)
- if err != nil {
- return nil, err
+// Options holds configuration for the storage.
+type Options struct {
+ // ExclusiveAccess means that the filesystem is not modified externally
+ // while the repo is open.
+ ExclusiveAccess bool
+ // KeepDescriptors makes the file descriptors to be reused but they will
+ // need to be manually closed calling Close().
+ KeepDescriptors bool
+}
+
+// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache.
+func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage {
+ return NewStorageWithOptions(fs, cache, Options{})
+}
+
+// NewStorageWithOptions returns a new Storage with extra options,
+// backed by a given `fs.Filesystem` and cache.
+func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage {
+ dirOps := dotgit.Options{
+ ExclusiveAccess: ops.ExclusiveAccess,
+ KeepDescriptors: ops.KeepDescriptors,
}
+ dir := dotgit.NewWithOptions(fs, dirOps)
return &Storage{
fs: fs,
dir: dir,
- ObjectStorage: o,
+ ObjectStorage: ObjectStorage{
+ options: ops,
+ deltaBaseCache: cache,
+ dir: dir,
+ },
ReferenceStorage: ReferenceStorage{dir: dir},
IndexStorage: IndexStorage{dir: dir},
ShallowStorage: ShallowStorage{dir: dir},
ConfigStorage: ConfigStorage{dir: dir},
ModuleStorage: ModuleStorage{dir: dir},
- }, nil
+ }
}
// Filesystem returns the underlying filesystem
@@ -48,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem {
return s.fs
}
+// Init initializes .git directory
func (s *Storage) Init() error {
return s.dir.Initialize()
}
diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go
index 4d9ba6f..6fa0d90 100644
--- a/storage/filesystem/storage_test.go
+++ b/storage/filesystem/storage_test.go
@@ -4,6 +4,7 @@ import (
"io/ioutil"
"testing"
+ "gopkg.in/src-d/go-git.v4/plumbing/cache"
"gopkg.in/src-d/go-git.v4/plumbing/storer"
"gopkg.in/src-d/go-git.v4/storage/test"
@@ -23,9 +24,12 @@ var _ = Suite(&StorageSuite{})
func (s *StorageSuite) SetUpTest(c *C) {
s.dir = c.MkDir()
- storage, err := NewStorage(osfs.New(s.dir))
- c.Assert(err, IsNil)
+ storage := NewStorage(osfs.New(s.dir), cache.NewObjectLRUDefault())
+
+ setUpTest(s, c, storage)
+}
+func setUpTest(s *StorageSuite, c *C, storage *Storage) {
// ensure that right interfaces are implemented
var _ storer.EncodedObjectStorer = storage
var _ storer.IndexStorer = storage
@@ -40,8 +44,7 @@ func (s *StorageSuite) SetUpTest(c *C) {
func (s *StorageSuite) TestFilesystem(c *C) {
fs := memfs.New()
- storage, err := NewStorage(fs)
- c.Assert(err, IsNil)
+ storage := NewStorage(fs, cache.NewObjectLRUDefault())
c.Assert(storage.Filesystem(), Equals, fs)
}
@@ -51,3 +54,19 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) {
c.Assert(err, IsNil)
c.Assert(fis, HasLen, 0)
}
+
+type StorageExclusiveSuite struct {
+ StorageSuite
+}
+
+var _ = Suite(&StorageExclusiveSuite{})
+
+func (s *StorageExclusiveSuite) SetUpTest(c *C) {
+ s.dir = c.MkDir()
+ storage := NewStorageWithOptions(
+ osfs.New(s.dir),
+ cache.NewObjectLRUDefault(),
+ Options{ExclusiveAccess: true})
+
+ setUpTest(&s.StorageSuite, c, storage)
+}