From ae5501623169ec981091a5f1cfb56ab8e7688031 Mon Sep 17 00:00:00 2001 From: noxora Date: Tue, 14 Aug 2018 14:02:26 -0500 Subject: added hook support Signed-off-by: noxora trying a possible fix to the delete test Signed-off-by: noxora still trying to fix this test Signed-off-by: noxora fixes did not work, seems to be a windows env problem Signed-off-by: noxora --- storage/filesystem/dotgit/dotgit.go | 53 ++++++++++++++++++++++++++++++-- storage/filesystem/dotgit/dotgit_test.go | 49 ++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 4 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index dc12f23..d16a77d 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -279,19 +279,66 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } +//incomingObjectPath is intended to add support for a git pre-recieve hook to be written +//it adds support for go-git to find objects in an "incoming" directory, so that the library +//can be used to write a pre-recieve hook that deals with the incoming objects. +//More on git hooks found here : https://git-scm.com/docs/githooks +//More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack +func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { + hString := h.String() + directoryContents, err := d.fs.ReadDir(objectsPath) + if err != nil { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + var incomingDirName string + for _, file := range directoryContents { + if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + incomingDirName = file.Name() + } + } + if incomingDirName == "" { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + return d.fs.Join(objectsPath, incomingDirName, hString[0:2], hString[2:40]) +} + // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { - return d.fs.Open(d.objectPath(h)) + obj1, err1 := d.fs.Open(d.objectPath(h)) + if os.IsNotExist(err1) { + obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { - return d.fs.Stat(d.objectPath(h)) + obj1, err1 := d.fs.Stat(d.objectPath(h)) + if os.IsNotExist(err1) { + obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { - return d.fs.Remove(d.objectPath(h)) + err1 := d.fs.Remove(d.objectPath(h)) + if os.IsNotExist(err1) { + err2 := d.fs.Remove(d.incomingObjectPath(h)) + if err2 != nil { + return err1 + } + return err2 + } + return err1 } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 7733eef..ce30500 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,11 +9,11 @@ import ( "strings" "testing" + fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" - "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -537,6 +537,53 @@ func (s *SuiteDotGit) TestObject(c *C) { file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")), Equals, true, ) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + file, err = dir.Object(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectStat(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + _, err := dir.ObjectStat(hash) + c.Assert(err, IsNil) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + _, err = dir.ObjectStat(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectDelete(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + err := dir.ObjectDelete(hash) + c.Assert(err, IsNil) + //incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + //incomingDirPath := fs.Join("objects", "incoming-123456") + //incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + //incomingFilePath := fs.Join(incomingDirPath, incomingHash[2:40]) + //err = fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + //c.Assert(err, IsNil) + //err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + //c.Assert(err, IsNil) + //_, err = fs.Create(incomingFilePath) + //c.Assert(err, IsNil) + + //err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + //c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { -- cgit From 8cf7edbc99282245bc8803a322dbf499ab77575d Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Fri, 17 Aug 2018 12:18:06 +0200 Subject: dotgit: fix object delete test Signed-off-by: Santiago M. Mola --- storage/filesystem/dotgit/dotgit_test.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index ce30500..64c2aee 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,11 +9,11 @@ import ( "strings" "testing" - fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -571,19 +571,23 @@ func (s *SuiteDotGit) TestObjectDelete(c *C) { hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") err := dir.ObjectDelete(hash) c.Assert(err, IsNil) - //incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash - //incomingDirPath := fs.Join("objects", "incoming-123456") - //incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) - //incomingFilePath := fs.Join(incomingDirPath, incomingHash[2:40]) - //err = fs.MkdirAll(incomingDirPath, os.FileMode(0755)) - //c.Assert(err, IsNil) - //err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) - //c.Assert(err, IsNil) - //_, err = fs.Create(incomingFilePath) - //c.Assert(err, IsNil) - - //err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) - //c.Assert(err, IsNil) + + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40]) + + err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + c.Assert(err, IsNil) + + f, err := fs.Create(incomingFilePath) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { -- cgit From c7a4011d78a00bd93a2f82a39bb67c2dda5453f5 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Sat, 25 Aug 2018 19:17:45 +0200 Subject: storage/dotgit: search for incoming dir only once Search for incoming object directory was done once each time objects were accessed. This means a ReadDir of the objects path that is expensive. Now incoming directory is searched the first time an object is accessed and its name kept in DotGit to be reused. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 41 +++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index d0a14ae..addb64c 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -61,6 +61,10 @@ var ( // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string } // New returns a DotGit value ready to be used. The path argument must @@ -286,26 +290,37 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { //More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() - directoryContents, err := d.fs.ReadDir(objectsPath) - if err != nil { + + if d.incomingDirName == "" { return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) } - var incomingDirName string - for _, file := range directoryContents { - if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { - incomingDirName = file.Name() + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + d.incomingDirName = file.Name() + } + } } + + d.incomingChecked = true } - if incomingDirName == "" { - return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) - } - return d.fs.Join(objectsPath, incomingDirName, hString[0:2], hString[2:40]) + + return d.incomingDirName != "" } // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { obj1, err1 := d.fs.Open(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -318,7 +333,7 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { obj1, err1 := d.fs.Stat(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -331,7 +346,7 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { err1 := d.fs.Remove(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) if err2 != nil { return err1 -- cgit From b1da90b0dde34b521cb252bc28c59e4ffd840d1d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 27 Aug 2018 14:45:24 +0200 Subject: storage/dotgit: use HasPrefix instead of Split Also reformatted function comment and fixed some typos. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index addb64c..df4f756 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -283,11 +283,14 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } -//incomingObjectPath is intended to add support for a git pre-recieve hook to be written -//it adds support for go-git to find objects in an "incoming" directory, so that the library -//can be used to write a pre-recieve hook that deals with the incoming objects. -//More on git hooks found here : https://git-scm.com/docs/githooks -//More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() @@ -305,7 +308,7 @@ func (d *DotGit) hasIncomingObjects() bool { directoryContents, err := d.fs.ReadDir(objectsPath) if err == nil { for _, file := range directoryContents { - if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { d.incomingDirName = file.Name() } } -- cgit From 1e1a7d0623459807d6f1e871492147f971f7540c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 15:29:51 +0200 Subject: git: add Static option to PlainOpen Also adds Static configuration to Storage and DotGit. This option means that the git repository is not expected to be modified while open and enables some optimizations. Each time a file is accessed the storer tries to open an object file for the requested hash. When this is done for a lot of objects it is expensive. With Static option a list of object files is generated the first time an object is accessed and used to check if exists instead of using system calls. A similar optimization is done for packfiles. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 182 +++++++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 3 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index df4f756..2048ddc 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -60,18 +60,39 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { + DotGitOptions fs billy.Filesystem // incoming object directory information incomingChecked bool incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} +} + +// DotGitOptions holds configuration options for new DotGit objects. +type DotGitOptions struct { + // Static means that the filesystem won't be changed while the repo is open. + Static bool } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, DotGitOptions{}) +} + +// NewWithOptions creates a new DotGit and sets non default configuration +// options. See New for complete help. +func NewWithOptions(fs billy.Filesystem, o DotGitOptions) *DotGit { + return &DotGit{ + DotGitOptions: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -143,11 +164,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.Static { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -181,6 +216,11 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + pack, err := d.fs.Open(d.objectPackPath(hash, extension)) if err != nil { if os.IsNotExist(err) { @@ -195,15 +235,27 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -224,12 +276,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.Static { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -241,9 +304,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided . func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.Static { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -278,6 +361,87 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.Static { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.Static { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) @@ -322,6 +486,11 @@ func (d *DotGit) hasIncomingObjects() bool { // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Open(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) @@ -335,6 +504,11 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Stat(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) @@ -348,6 +522,8 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { + d.cleanObjectList() + err1 := d.fs.Remove(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) -- cgit From 82945e31dd8bce5fc51d4fd16d696a6d326e5f44 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 18:33:37 +0200 Subject: git, storer: use a common storer.Options for storer and PlainOpen Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 2048ddc..41e5c75 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -14,6 +14,7 @@ import ( "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -60,7 +61,7 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - DotGitOptions + storer.Options fs billy.Filesystem // incoming object directory information @@ -73,25 +74,19 @@ type DotGit struct { packMap map[plumbing.Hash]struct{} } -// DotGitOptions holds configuration options for new DotGit objects. -type DotGitOptions struct { - // Static means that the filesystem won't be changed while the repo is open. - Static bool -} - // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return NewWithOptions(fs, DotGitOptions{}) + return NewWithOptions(fs, storer.Options{}) } // NewWithOptions creates a new DotGit and sets non default configuration // options. See New for complete help. -func NewWithOptions(fs billy.Filesystem, o DotGitOptions) *DotGit { +func NewWithOptions(fs billy.Filesystem, o storer.Options) *DotGit { return &DotGit{ - DotGitOptions: o, - fs: fs, + Options: o, + fs: fs, } } -- cgit From d7e6cf5b73947108d0c16b9c04b38891de47ef5d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 18:35:39 +0200 Subject: dotgit: fix typo in comment Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 41e5c75..c42ed88 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -300,7 +300,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { } // ForEachObjectHash iterates over the hashes of objects found under the -// .git/objects/ directory and executes the provided . +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { if !d.Static { return d.forEachObjectHash(fun) -- cgit From cf626677508238893c7c88c3c786a02f17afcc4c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 31 Aug 2018 14:56:23 +0200 Subject: plumbing/storer: rename Static option to ExclusiveAccess Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index c42ed88..7626078 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -165,7 +165,7 @@ func (d *DotGit) NewObjectPack() (*PackWriter, error) { // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { - if !d.Static { + if !d.ExclusiveAccess { return d.objectPacks() } @@ -279,7 +279,7 @@ func (d *DotGit) NewObject() (*ObjectWriter, error) { // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { - if d.Static { + if d.ExclusiveAccess { err := d.genObjectList() if err != nil { return nil, err @@ -302,7 +302,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { // ForEachObjectHash iterates over the hashes of objects found under the // .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { - if !d.Static { + if !d.ExclusiveAccess { return d.forEachObjectHash(fun) } @@ -376,7 +376,7 @@ func (d *DotGit) genObjectList() error { } func (d *DotGit) hasObject(h plumbing.Hash) error { - if !d.Static { + if !d.ExclusiveAccess { return nil } @@ -420,7 +420,7 @@ func (d *DotGit) genPackList() error { } func (d *DotGit) hasPack(h plumbing.Hash) error { - if !d.Static { + if !d.ExclusiveAccess { return nil } -- cgit From 874f669becc25489081306bbbcbbc27b970f6295 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 19:40:22 +0200 Subject: storage/filesystem: move Options to filesytem and dotgit Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 7626078..00dd2a4 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -14,7 +14,6 @@ import ( "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -58,11 +57,18 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - storer.Options - fs billy.Filesystem + options Options + fs billy.Filesystem // incoming object directory information incomingChecked bool @@ -78,14 +84,14 @@ type DotGit struct { // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return NewWithOptions(fs, storer.Options{}) + return NewWithOptions(fs, Options{}) } // NewWithOptions creates a new DotGit and sets non default configuration // options. See New for complete help. -func NewWithOptions(fs billy.Filesystem, o storer.Options) *DotGit { +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { return &DotGit{ - Options: o, + options: o, fs: fs, } } @@ -165,7 +171,7 @@ func (d *DotGit) NewObjectPack() (*PackWriter, error) { // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return d.objectPacks() } @@ -279,7 +285,7 @@ func (d *DotGit) NewObject() (*ObjectWriter, error) { // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { - if d.ExclusiveAccess { + if d.options.ExclusiveAccess { err := d.genObjectList() if err != nil { return nil, err @@ -302,7 +308,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { // ForEachObjectHash iterates over the hashes of objects found under the // .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return d.forEachObjectHash(fun) } @@ -376,7 +382,7 @@ func (d *DotGit) genObjectList() error { } func (d *DotGit) hasObject(h plumbing.Hash) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return nil } @@ -420,7 +426,7 @@ func (d *DotGit) genPackList() error { } func (d *DotGit) hasPack(h plumbing.Hash) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return nil } -- cgit From 659ec443b4a975e3adf78f24e59ad69d210d2c0b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 20:02:25 +0200 Subject: storage/dotgit: add ExclusiveAccess tests in dotgit This functionality was already tested in storage/filesystem. The coverage tool only takes into account files from the same package of the test. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 64c2aee..c34543e 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" @@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) { fs := f.DotGit() dir := New(fs) + testObjectPacks(c, fs, dir, f) +} + +func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjectPacks(c, fs, dir, f) +} + +func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) { hashes, err := dir.ObjectPacks() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 1) @@ -506,6 +519,17 @@ func (s *SuiteDotGit) TestObjects(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() dir := New(fs) + testObjects(c, fs, dir) +} + +func (s *SuiteDotGit) TestObjectsExclusive(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjects(c, fs, dir) +} + +func testObjects(c *C, fs billy.Filesystem, dir *DotGit) { hashes, err := dir.Objects() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 187) -- cgit From 6384ab93a2dbac9045ee19099455cbcfe82d0201 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 20:28:40 +0200 Subject: storage/dotgit: add KeepDescriptors option This option maintains packfile file descriptors opened after reading objects from them. It improves performance as it does not have to be opening packfiles each time an object is needed. Also adds Close to EncodedObjectStorer to close all the files manualy. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 43 +++++++++++++++++++++++++++++++- storage/filesystem/dotgit/dotgit_test.go | 28 +++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 00dd2a4..df5cd10 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -62,6 +62,9 @@ type Options struct { // ExclusiveAccess means that the filesystem is not modified externally // while the repo is open. ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool } // The DotGit type represents a local git repository on disk. This @@ -78,6 +81,8 @@ type DotGit struct { objectMap map[plumbing.Hash]struct{} packList []plumbing.Hash packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must @@ -123,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -217,12 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { + if d.files == nil { + d.files = make(map[string]billy.File) + } + err := d.hasPack(hash) if err != nil { return nil, err } - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -231,6 +268,10 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index c34543e..50f8e64 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -465,6 +465,34 @@ func (s *SuiteDotGit) TestObjectPack(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") } +func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{KeepDescriptors: true}) + + pack, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + + pack2, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(pack, Equals, pack2) + + err = dir.Close() + c.Assert(err, IsNil) + + pack2, err = dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(pack, Not(Equals), pack2) + + err = pack2.Close() + c.Assert(err, IsNil) + + err = dir.Close() + c.Assert(err, NotNil) + +} + func (s *SuiteDotGit) TestObjectPackIdx(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() -- cgit From 8176f084d861891d1846a2d46bf669d0d3463ebd Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 6 Sep 2018 19:09:02 +0200 Subject: storage/filesystem: compare files using offset in test Using equals to compare files it uses diff to do so. This can potentially consume lots of ram. Changed the comparison to use file offsets. If the descriptor is reused the offset is maintained. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 50f8e64..308c6b7 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -474,16 +474,27 @@ func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { c.Assert(err, IsNil) c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + // Move to an specific offset + pack.Seek(42, os.SEEK_SET) + pack2, err := dir.ObjectPack(f.PackfileHash) c.Assert(err, IsNil) - c.Assert(pack, Equals, pack2) + + // If the file is the same the offset should be the same + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) err = dir.Close() c.Assert(err, IsNil) pack2, err = dir.ObjectPack(f.PackfileHash) c.Assert(err, IsNil) - c.Assert(pack, Not(Equals), pack2) + + // If the file is opened again its offset should be 0 + offset, err = pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) err = pack2.Close() c.Assert(err, IsNil) -- cgit From 8f6b3127c1ff7661113fff2662416c328971a285 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Fri, 7 Sep 2018 09:27:35 +0200 Subject: Expose Storage cache. Signed-off-by: kuba-- --- storage/filesystem/dotgit/dotgit.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'storage/filesystem/dotgit') diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index df5cd10..a58c248 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -92,8 +92,8 @@ func New(fs billy.Filesystem) *DotGit { return NewWithOptions(fs, Options{}) } -// NewWithOptions creates a new DotGit and sets non default configuration -// options. See New for complete help. +// NewWithOptions sets non default configuration options. +// See New for complete help. func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { return &DotGit{ options: o, -- cgit