diff options
Diffstat (limited to 'storage')
21 files changed, 1064 insertions, 278 deletions
diff --git a/storage/filesystem/config.go b/storage/filesystem/config.go index a2cc173..be812e4 100644 --- a/storage/filesystem/config.go +++ b/storage/filesystem/config.go @@ -5,7 +5,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/config" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -13,7 +13,7 @@ type ConfigStorage struct { dir *dotgit.DotGit } -func (c *ConfigStorage) Config() (*config.Config, error) { +func (c *ConfigStorage) Config() (conf *config.Config, err error) { cfg := config.NewConfig() f, err := c.dir.Config() @@ -32,15 +32,15 @@ func (c *ConfigStorage) Config() (*config.Config, error) { return nil, err } - if err := cfg.Unmarshal(b); err != nil { + if err = cfg.Unmarshal(b); err != nil { return nil, err } return cfg, err } -func (c *ConfigStorage) SetConfig(cfg *config.Config) error { - if err := cfg.Validate(); err != nil { +func (c *ConfigStorage) SetConfig(cfg *config.Config) (err error) { + if err = cfg.Validate(); err != nil { return err } diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go index cc03119..71c947d 100644 --- a/storage/filesystem/config_test.go +++ b/storage/filesystem/config_test.go @@ -5,7 +5,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/config" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index fac7aec..a58c248 100644 --- a/storage/filesystem/internal/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -57,17 +57,48 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem + options Options + fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, Options{}) +} + +// NewWithOptions sets non default configuration options. +// See New for complete help. +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { + return &DotGit{ + options: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -97,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -139,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.options.ExclusiveAccess { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -162,8 +229,11 @@ func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { n := f.Name() h := plumbing.NewHash(n[5 : len(n)-5]) //pack-(hash).pack + if h.IsZero() { + // Ignore files with badly-formatted names. + continue + } packs = append(packs, h) - } return packs, nil @@ -174,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + if d.files == nil { + d.files = make(map[string]billy.File) + } + + err := d.hasPack(hash) + if err != nil { + return nil, err + } + + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -183,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -217,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.options.ExclusiveAccess { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -234,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.options.ExclusiveAccess { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -255,7 +387,12 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { } for _, o := range d { - err = fun(plumbing.NewHash(base + o.Name())) + h := plumbing.NewHash(base + o.Name()) + if h.IsZero() { + // Ignore files with badly-formatted names. + continue + } + err = fun(h) if err != nil { return err } @@ -266,24 +403,178 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack +func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { + hString := h.String() + + if d.incomingDirName == "" { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { + d.incomingDirName = file.Name() + } + } + } + + d.incomingChecked = true + } + + return d.incomingDirName != "" +} + // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { - return d.fs.Open(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Open(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { - return d.fs.Stat(d.objectPath(h)) + err := d.hasObject(h) + if err != nil { + return nil, err + } + + obj1, err1 := d.fs.Stat(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { - return d.fs.Remove(d.objectPath(h)) + d.cleanObjectList() + + err1 := d.fs.Remove(d.objectPath(h)) + if os.IsNotExist(err1) && d.hasIncomingObjects() { + err2 := d.fs.Remove(d.incomingObjectPath(h)) + if err2 != nil { + return err1 + } + return err2 + } + return err1 } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { @@ -375,7 +666,7 @@ func (d *DotGit) findPackedRefsInFile(f billy.File) ([]*plumbing.Reference, erro return refs, s.Err() } -func (d *DotGit) findPackedRefs() ([]*plumbing.Reference, error) { +func (d *DotGit) findPackedRefs() (r []*plumbing.Reference, err error) { f, err := d.fs.Open(packedRefsPath) if err != nil { if os.IsNotExist(err) { @@ -461,7 +752,7 @@ func (d *DotGit) openAndLockPackedRefs(doCreate bool) ( // File mode is retrieved from a constant defined in the target specific // files (dotgit_rewrite_packed_refs_*). Some modes are not available // in all filesystems. - openFlags := openAndLockPackedRefsMode + openFlags := d.openAndLockPackedRefsMode() if doCreate { openFlags |= os.O_CREATE } @@ -676,7 +967,7 @@ func (d *DotGit) PackRefs() (err error) { // Gather all refs using addRefsFromRefDir and addRefsFromPackedRefs. var refs []*plumbing.Reference seen := make(map[plumbing.ReferenceName]bool) - if err := d.addRefsFromRefDir(&refs, seen); err != nil { + if err = d.addRefsFromRefDir(&refs, seen); err != nil { return err } if len(refs) == 0 { @@ -684,7 +975,7 @@ func (d *DotGit) PackRefs() (err error) { return nil } numLooseRefs := len(refs) - if err := d.addRefsFromPackedRefsFile(&refs, f, seen); err != nil { + if err = d.addRefsFromPackedRefsFile(&refs, f, seen); err != nil { return err } @@ -701,7 +992,7 @@ func (d *DotGit) PackRefs() (err error) { w := bufio.NewWriter(tmp) for _, ref := range refs { - _, err := w.WriteString(ref.String() + "\n") + _, err = w.WriteString(ref.String() + "\n") if err != nil { return err } @@ -776,6 +1067,11 @@ func (d *DotGit) Alternates() ([]*DotGit, error) { return alternates, nil } +// Fs returns the underlying filesystem of the DotGit folder. +func (d *DotGit) Fs() billy.Filesystem { + return d.fs +} + func isHex(s string) bool { for _, b := range []byte(s) { if isNum(b) { @@ -798,5 +1094,3 @@ func isNum(b byte) bool { func isHexAlpha(b byte) bool { return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F' } - -type refCache map[plumbing.ReferenceName]*plumbing.Reference diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go new file mode 100644 index 0000000..7f1c02c --- /dev/null +++ b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go @@ -0,0 +1,81 @@ +package dotgit + +import ( + "io" + "os" + "runtime" + + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +func (d *DotGit) openAndLockPackedRefsMode() int { + if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) { + return os.O_RDWR + } + + return os.O_RDONLY +} + +func (d *DotGit) rewritePackedRefsWhileLocked( + tmp billy.File, pr billy.File) error { + // Try plain rename. If we aren't using the bare Windows filesystem as the + // storage layer, we might be able to get away with a rename over a locked + // file. + err := d.fs.Rename(tmp.Name(), pr.Name()) + if err == nil { + return nil + } + + // If we are in a filesystem that does not support rename (e.g. sivafs) + // a full copy is done. + if err == billy.ErrNotSupported { + return d.copyNewFile(tmp, pr) + } + + if runtime.GOOS != "windows" { + return err + } + + // Otherwise, Windows doesn't let us rename over a locked file, so + // we have to do a straight copy. Unfortunately this could result + // in a partially-written file if the process fails before the + // copy completes. + return d.copyToExistingFile(tmp, pr) +} + +func (d *DotGit) copyToExistingFile(tmp, pr billy.File) error { + _, err := pr.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = pr.Truncate(0) + if err != nil { + return err + } + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = io.Copy(pr, tmp) + + return err +} + +func (d *DotGit) copyNewFile(tmp billy.File, pr billy.File) (err error) { + prWrite, err := d.fs.Create(pr.Name()) + if err != nil { + return err + } + + defer ioutil.CheckClose(prWrite, &err) + + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + + _, err = io.Copy(prWrite, tmp) + + return err +} diff --git a/storage/filesystem/internal/dotgit/dotgit_setref.go b/storage/filesystem/dotgit/dotgit_setref.go index c732c9f..d27c1a3 100644 --- a/storage/filesystem/internal/dotgit/dotgit_setref.go +++ b/storage/filesystem/dotgit/dotgit_setref.go @@ -9,7 +9,7 @@ import ( "gopkg.in/src-d/go-git.v4/utils/ioutil" ) -func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) error { +func (d *DotGit) setRef(fileName, content string, old *plumbing.Reference) (err error) { // If we are not checking an old ref, just truncate the file. mode := os.O_RDWR | os.O_CREATE if old == nil { diff --git a/storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go b/storage/filesystem/dotgit/dotgit_setref_norwfs.go index 5695bd3..5695bd3 100644 --- a/storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go +++ b/storage/filesystem/dotgit/dotgit_setref_norwfs.go diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 2c43295..308c6b7 100644 --- a/storage/filesystem/internal/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" @@ -151,6 +152,7 @@ func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) { } func BenchmarkRefMultipleTimes(b *testing.B) { + fixtures.Init() fs := fixtures.Basic().ByTag(".git").One().DotGit() refname := plumbing.ReferenceName("refs/remotes/origin/branch") @@ -418,15 +420,39 @@ func findReference(refs []*plumbing.Reference, name string) *plumbing.Reference return nil } -func (s *SuiteDotGit) TestObjectsPack(c *C) { +func (s *SuiteDotGit) TestObjectPacks(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() dir := New(fs) + testObjectPacks(c, fs, dir, f) +} + +func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjectPacks(c, fs, dir, f) +} + +func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) { hashes, err := dir.ObjectPacks() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 1) c.Assert(hashes[0], Equals, f.PackfileHash) + + // Make sure that a random file in the pack directory doesn't + // break everything. + badFile, err := fs.Create("objects/pack/OOPS_THIS_IS_NOT_RIGHT.pack") + c.Assert(err, IsNil) + err = badFile.Close() + c.Assert(err, IsNil) + + hashes2, err := dir.ObjectPacks() + c.Assert(err, IsNil) + c.Assert(hashes2, HasLen, 1) + c.Assert(hashes[0], Equals, hashes2[0]) } func (s *SuiteDotGit) TestObjectPack(c *C) { @@ -439,6 +465,45 @@ func (s *SuiteDotGit) TestObjectPack(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") } +func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{KeepDescriptors: true}) + + pack, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + + // Move to an specific offset + pack.Seek(42, os.SEEK_SET) + + pack2, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is the same the offset should be the same + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) + + err = dir.Close() + c.Assert(err, IsNil) + + pack2, err = dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is opened again its offset should be 0 + offset, err = pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = pack2.Close() + c.Assert(err, IsNil) + + err = dir.Close() + c.Assert(err, NotNil) + +} + func (s *SuiteDotGit) TestObjectPackIdx(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() @@ -493,6 +558,17 @@ func (s *SuiteDotGit) TestObjects(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() dir := New(fs) + testObjects(c, fs, dir) +} + +func (s *SuiteDotGit) TestObjectsExclusive(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjects(c, fs, dir) +} + +func testObjects(c *C, fs billy.Filesystem, dir *DotGit) { hashes, err := dir.Objects() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 187) @@ -524,6 +600,57 @@ func (s *SuiteDotGit) TestObject(c *C) { file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")), Equals, true, ) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + file, err = dir.Object(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectStat(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + _, err := dir.ObjectStat(hash) + c.Assert(err, IsNil) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + _, err = dir.ObjectStat(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectDelete(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + err := dir.ObjectDelete(hash) + c.Assert(err, IsNil) + + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40]) + + err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + c.Assert(err, IsNil) + + f, err := fs.Create(incomingFilePath) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index c2b420f..93d2d8c 100644 --- a/storage/filesystem/internal/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -20,13 +20,14 @@ import ( // is renamed/moved (depends on the Filesystem implementation) to the final // location, if the PackWriter is not used, nothing is written type PackWriter struct { - Notify func(plumbing.Hash, *packfile.Index) + Notify func(plumbing.Hash, *idxfile.Writer) fs billy.Filesystem fr, fw billy.File synced *syncedReader checksum plumbing.Hash - index *packfile.Index + parser *packfile.Parser + writer *idxfile.Writer result chan error } @@ -55,20 +56,21 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) - d, err := packfile.NewDecoder(s, nil) + w.writer = new(idxfile.Writer) + var err error + w.parser, err = packfile.NewParser(s, w.writer) if err != nil { w.result <- err return } - checksum, err := d.Decode() + checksum, err := w.parser.Parse() if err != nil { w.result <- err return } w.checksum = checksum - w.index = d.Index() w.result <- err } @@ -92,8 +94,8 @@ func (w *PackWriter) Write(p []byte) (int, error) { // was written, the tempfiles are deleted without writing a packfile. func (w *PackWriter) Close() error { defer func() { - if w.Notify != nil && w.index != nil && w.index.Size() > 0 { - w.Notify(w.checksum, w.index) + if w.Notify != nil && w.writer != nil && w.writer.Finished() { + w.Notify(w.checksum, w.writer) } close(w.result) @@ -115,7 +117,7 @@ func (w *PackWriter) Close() error { return err } - if w.index == nil || w.index.Size() == 0 { + if w.writer == nil || !w.writer.Finished() { return w.clean() } @@ -145,11 +147,13 @@ func (w *PackWriter) save() error { } func (w *PackWriter) encodeIdx(writer io.Writer) error { - idx := w.index.ToIdxFile() - idx.PackfileChecksum = w.checksum - idx.Version = idxfile.VersionSupported + idx, err := w.writer.Index() + if err != nil { + return err + } + e := idxfile.NewEncoder(writer) - _, err := e.Encode(idx) + _, err = e.Encode(idx) return err } @@ -209,7 +213,6 @@ func (s *syncedReader) isBlocked() bool { func (s *syncedReader) wake() { if s.isBlocked() { - // fmt.Println("wake") atomic.StoreUint32(&s.blocked, 0) s.news <- true } @@ -220,7 +223,6 @@ func (s *syncedReader) sleep() { written := atomic.LoadUint64(&s.written) if read >= written { atomic.StoreUint32(&s.blocked, 1) - // fmt.Println("sleep", read, written) <-s.news } diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/dotgit/writers_test.go index bf00762..5a5f7b4 100644 --- a/storage/filesystem/internal/dotgit/writers_test.go +++ b/storage/filesystem/dotgit/writers_test.go @@ -9,6 +9,7 @@ import ( "strconv" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" . "gopkg.in/check.v1" @@ -148,7 +149,7 @@ func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) { w, err := newPackWrite(fs) c.Assert(err, IsNil) - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { + w.Notify = func(h plumbing.Hash, idx *idxfile.Writer) { c.Fatal("unexpected call to PackWriter.Notify") } diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go index 14ab09a..2ebf57e 100644 --- a/storage/filesystem/index.go +++ b/storage/filesystem/index.go @@ -4,7 +4,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/plumbing/format/index" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -12,7 +12,7 @@ type IndexStorage struct { dir *dotgit.DotGit } -func (s *IndexStorage) SetIndex(idx *index.Index) error { +func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { f, err := s.dir.IndexWriter() if err != nil { return err @@ -25,7 +25,7 @@ func (s *IndexStorage) SetIndex(idx *index.Index) error { return err } -func (s *IndexStorage) Index() (*index.Index, error) { +func (s *IndexStorage) Index() (i *index.Index, err error) { idx := &index.Index{ Version: 2, } diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go deleted file mode 100644 index c760793..0000000 --- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build !windows,!norwfs - -package dotgit - -import ( - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // On non-Windows platforms, we can have atomic rename. - return d.fs.Rename(tmp.Name(), pr.Name()) -} diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go deleted file mode 100644 index 6e43b42..0000000 --- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDONLY - -// Instead of renaming that can not be supported in simpler filesystems -// a full copy is done. -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - - prWrite, err := d.fs.Create(pr.Name()) - if err != nil { - return err - } - - defer prWrite.Close() - - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - - _, err = io.Copy(prWrite, tmp) - - return err -} diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go b/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go deleted file mode 100644 index 897d2c9..0000000 --- a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go +++ /dev/null @@ -1,42 +0,0 @@ -// +build windows,!norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // If we aren't using the bare Windows filesystem as the storage - // layer, we might be able to get away with a rename over a locked - // file. - err := d.fs.Rename(tmp.Name(), pr.Name()) - if err == nil { - return nil - } - - // Otherwise, Windows doesn't let us rename over a locked file, so - // we have to do a straight copy. Unfortunately this could result - // in a partially-written file if the process fails before the - // copy completes. - _, err = pr.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = pr.Truncate(0) - if err != nil { - return err - } - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - _, err = io.Copy(pr, tmp) - return err -} diff --git a/storage/filesystem/module.go b/storage/filesystem/module.go index 6f3de3f..9272206 100644 --- a/storage/filesystem/module.go +++ b/storage/filesystem/module.go @@ -1,8 +1,9 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) type ModuleStorage struct { @@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) { return nil, err } - return NewStorage(fs) + return NewStorage(fs, cache.NewObjectLRUDefault()), nil } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 3ec7304..68bd140 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -11,28 +11,34 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/objfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" - "gopkg.in/src-d/go-git.v4/storage/memory" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" ) type ObjectStorage struct { + options Options + // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object dir *dotgit.DotGit - index map[plumbing.Hash]*packfile.Index + index map[plumbing.Hash]idxfile.Index +} + +// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. +func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, cache, Options{}) } -func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { - s := ObjectStorage{ - deltaBaseCache: cache.NewObjectLRUDefault(), +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options +func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { + return &ObjectStorage{ + options: ops, + deltaBaseCache: cache, dir: dir, } - - return s, nil } func (s *ObjectStorage) requireIndex() error { @@ -40,7 +46,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index) + s.index = make(map[plumbing.Hash]idxfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -55,20 +61,21 @@ func (s *ObjectStorage) requireIndex() error { return nil } -func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) error { +func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { f, err := s.dir.ObjectPackIdx(h) if err != nil { return err } defer ioutil.CheckClose(f, &err) - idxf := idxfile.NewIdxfile() + + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { return err } - s.index[h] = packfile.NewIndexFromIdxFile(idxf) + s.index[h] = idxf return err } @@ -86,15 +93,18 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { return nil, err } - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { - s.index[h] = idx + w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { + index, err := writer.Index() + if err == nil { + s.index[h] = index + } } return w, nil } // SetEncodedObject adds a new object to the storage. -func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (plumbing.Hash, error) { +func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (h plumbing.Hash, err error) { if o.Type() == plumbing.OFSDeltaObject || o.Type() == plumbing.REFDeltaObject { return plumbing.ZeroHash, plumbing.ErrInvalidType } @@ -113,11 +123,11 @@ func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (plumbing.Has defer ioutil.CheckClose(or, &err) - if err := ow.WriteHeader(o.Type(), o.Size()); err != nil { + if err = ow.WriteHeader(o.Type(), o.Size()); err != nil { return plumbing.ZeroHash, err } - if _, err := io.Copy(ow, or); err != nil { + if _, err = io.Copy(ow, or); err != nil { return plumbing.ZeroHash, err } @@ -166,10 +176,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := newObjectStorage(dg) - if oe != nil { - continue - } + o := NewObjectStorage(dg, s.deltaBaseCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -265,7 +272,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( return nil, err } - defer ioutil.CheckClose(f, &err) + if !s.options.KeepDescriptors { + defer ioutil.CheckClose(f, &err) + } idx := s.index[pack] if canBeDelta { @@ -277,30 +286,37 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, - idx *packfile.Index, - offset int64) (plumbing.EncodedObject, error) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, err + idx idxfile.Index, + offset int64, +) (plumbing.EncodedObject, error) { + hash, err := idx.FindHash(offset) + if err == nil { + obj, ok := s.deltaBaseCache.Get(hash) + if ok { + return obj, nil + } } - p := packfile.NewScanner(f) - - d, err := packfile.NewDecoderWithCache(p, memory.NewStorage(), - s.deltaBaseCache) - if err != nil { + if err != nil && err != plumbing.ErrObjectNotFound { return nil, err } - d.SetIndex(idx) - obj, err := d.DecodeObjectAt(offset) - return obj, err + var p *packfile.Packfile + if s.deltaBaseCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + } else { + p = packfile.NewPackfile(idx, s.dir.Fs(), f) + } + + return p.GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64, - hash plumbing.Hash) (plumbing.EncodedObject, error) { + hash plumbing.Hash, +) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err } @@ -323,12 +339,10 @@ func (s *ObjectStorage) decodeDeltaObjectAt( case plumbing.REFDeltaObject: base = header.Reference case plumbing.OFSDeltaObject: - e, ok := idx.LookupOffset(uint64(header.OffsetReference)) - if !ok { - return nil, plumbing.ErrObjectNotFound + base, err = idx.FindHash(header.OffsetReference) + if err != nil { + return nil, err } - - base = e.Hash default: return s.decodeObjectAt(f, idx, offset) } @@ -349,8 +363,9 @@ func (s *ObjectStorage) decodeDeltaObjectAt( func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { for packfile, index := range s.index { - if e, ok := index.LookupHash(h); ok { - return packfile, e.Hash, int64(e.Offset) + offset, err := index.FindOffset(h) + if err == nil { + return packfile, h, offset } } @@ -365,7 +380,7 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return nil, err } - seen := make(map[plumbing.Hash]bool) + seen := make(map[plumbing.Hash]struct{}) var iters []storer.EncodedObjectIter if len(objects) != 0 { iters = append(iters, &objectsIter{s: s, t: t, h: objects}) @@ -377,11 +392,14 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return nil, err } - iters = append(iters, packi...) + iters = append(iters, packi) return storer.NewMultiEncodedObjectIter(iters), nil } -func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]bool) ([]storer.EncodedObjectIter, error) { +func (s *ObjectStorage) buildPackfileIters( + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, +) (storer.EncodedObjectIter, error) { if err := s.requireIndex(); err != nil { return nil, err } @@ -390,96 +408,172 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb if err != nil { return nil, err } + return &lazyPackfilesIter{ + hashes: packs, + open: func(h plumbing.Hash) (storer.EncodedObjectIter, error) { + pack, err := s.dir.ObjectPack(h) + if err != nil { + return nil, err + } + return newPackfileIter( + s.dir.Fs(), pack, t, seen, s.index[h], + s.deltaBaseCache, s.options.KeepDescriptors, + ) + }, + }, nil +} - var iters []storer.EncodedObjectIter - for _, h := range packs { - pack, err := s.dir.ObjectPack(h) - if err != nil { - return nil, err - } +// Close closes all opened files. +func (s *ObjectStorage) Close() error { + return s.dir.Close() +} - iter, err := newPackfileIter(pack, t, seen, s.index[h], s.deltaBaseCache) - if err != nil { +type lazyPackfilesIter struct { + hashes []plumbing.Hash + open func(h plumbing.Hash) (storer.EncodedObjectIter, error) + cur storer.EncodedObjectIter +} + +func (it *lazyPackfilesIter) Next() (plumbing.EncodedObject, error) { + for { + if it.cur == nil { + if len(it.hashes) == 0 { + return nil, io.EOF + } + h := it.hashes[0] + it.hashes = it.hashes[1:] + + sub, err := it.open(h) + if err == io.EOF { + continue + } else if err != nil { + return nil, err + } + it.cur = sub + } + ob, err := it.cur.Next() + if err == io.EOF { + it.cur.Close() + it.cur = nil + continue + } else if err != nil { return nil, err } - - iters = append(iters, iter) + return ob, nil } +} + +func (it *lazyPackfilesIter) ForEach(cb func(plumbing.EncodedObject) error) error { + return storer.ForEachIterator(it, cb) +} - return iters, nil +func (it *lazyPackfilesIter) Close() { + if it.cur != nil { + it.cur.Close() + it.cur = nil + } + it.hashes = nil } type packfileIter struct { - f billy.File - d *packfile.Decoder - t plumbing.ObjectType + pack billy.File + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} - seen map[plumbing.Hash]bool - position uint32 - total uint32 + // tells whether the pack file should be left open after iteration or not + keepPack bool } -func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) { - return newPackfileIter(f, t, make(map[plumbing.Hash]bool), nil, nil) -} +// NewPackfileIter returns a new EncodedObjectIter for the provided packfile +// and object type. Packfile and index file will be closed after they're +// used. If keepPack is true the packfile won't be closed after the iteration +// finished. +func NewPackfileIter( + fs billy.Filesystem, + f billy.File, + idxFile billy.File, + t plumbing.ObjectType, + keepPack bool, +) (storer.EncodedObjectIter, error) { + idx := idxfile.NewMemoryIndex() + if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { + return nil, err + } -func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]bool, - index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) { - s := packfile.NewScanner(f) - _, total, err := s.Header() - if err != nil { + if err := idxFile.Close(); err != nil { return nil, err } - d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t, cache) + seen := make(map[plumbing.Hash]struct{}) + return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) +} + +func newPackfileIter( + fs billy.Filesystem, + f billy.File, + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, + index idxfile.Index, + cache cache.Object, + keepPack bool, +) (storer.EncodedObjectIter, error) { + var p *packfile.Packfile + if cache != nil { + p = packfile.NewPackfileWithCache(index, fs, f, cache) + } else { + p = packfile.NewPackfile(index, fs, f) + } + + iter, err := p.GetByType(t) if err != nil { return nil, err } - d.SetIndex(index) - return &packfileIter{ - f: f, - d: d, - t: t, - - total: total, - seen: seen, + pack: f, + iter: iter, + seen: seen, + keepPack: keepPack, }, nil } func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { for { - if iter.position >= iter.total { - return nil, io.EOF - } - - obj, err := iter.d.DecodeObject() + obj, err := iter.iter.Next() if err != nil { return nil, err } - iter.position++ - if obj == nil { + if _, ok := iter.seen[obj.Hash()]; ok { continue } - if iter.seen[obj.Hash()] { - return iter.Next() - } - return obj, nil } } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + iter.Close() + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *packfileIter) Close() { - iter.f.Close() - iter.d.Close() + iter.iter.Close() + if !iter.keepPack { + _ = iter.pack.Close() + } } type objectsIter struct { @@ -507,21 +601,31 @@ func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { return obj, err } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *objectsIter) Close() { iter.h = []plumbing.Hash{} } -func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]bool { - m := make(map[plumbing.Hash]bool, len(l)) +func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]struct{} { + m := make(map[plumbing.Hash]struct{}, len(l)) for _, h := range l { - m[h] = true + m[h] = struct{}{} } - return m } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index de8f2b2..407abf2 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,8 +1,13 @@ package filesystem import ( + "io/ioutil" + "os" + "testing" + "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -10,22 +15,20 @@ import ( type FsSuite struct { fixtures.Suite - Types []plumbing.ObjectType } -var _ = Suite(&FsSuite{ - Types: []plumbing.ObjectType{ - plumbing.CommitObject, - plumbing.TagObject, - plumbing.TreeObject, - plumbing.BlobObject, - }, -}) +var objectTypes = []plumbing.ObjectType{ + plumbing.CommitObject, + plumbing.TagObject, + plumbing.TreeObject, + plumbing.BlobObject, +} + +var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() - o, err := newObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -36,20 +39,53 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) { func (s *FsSuite) TestGetFromPackfile(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) + + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + }) +} + +func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { + fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true}) + o := NewObjectStorageWithOptions(dg, cache.NewObjectLRUDefault(), Options{KeepDescriptors: true}) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) c.Assert(obj.Hash(), Equals, expected) + + packfiles, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + pack1, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + pack1.Seek(42, os.SEEK_SET) + + err = o.Close() + c.Assert(err, IsNil) + + pack2, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = o.Close() + c.Assert(err, IsNil) + }) } func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() - o, err := newObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") obj, err := o.getFromPackfile(expected, false) @@ -65,8 +101,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -84,10 +119,9 @@ func (s *FsSuite) TestIter(c *C) { func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { - for _, t := range s.Types { + for _, t := range objectTypes { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(t) c.Assert(err, IsNil) @@ -108,23 +142,215 @@ func (s *FsSuite) TestPackfileIter(c *C) { fs := f.DotGit() dg := dotgit.New(fs) - for _, t := range s.Types { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + for _, h := range ph { + f, err := dg.ObjectPack(h) + c.Assert(err, IsNil) + + idxf, err := dg.ObjectPackIdx(h) + c.Assert(err, IsNil) + + iter, err := NewPackfileIter(fs, f, idxf, t, false) + c.Assert(err, IsNil) + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + c.Assert(o.Type(), Equals, t) + return nil + }) + c.Assert(err, IsNil) + } + } + }) +} + +func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) { + fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + ops := dotgit.Options{KeepDescriptors: true} + dg := dotgit.NewWithOptions(fs, ops) + + for _, t := range objectTypes { ph, err := dg.ObjectPacks() c.Assert(err, IsNil) for _, h := range ph { f, err := dg.ObjectPack(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(f, t) + + idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) + + iter, err := NewPackfileIter(fs, f, idxf, t, true) + c.Assert(err, IsNil) + err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) return nil }) + c.Assert(err, IsNil) + // test twice to check that packfiles are not closed + err = iter.ForEach(func(o plumbing.EncodedObject) error { + c.Assert(o.Type(), Equals, t) + return nil + }) c.Assert(err, IsNil) } } }) +} +func BenchmarkPackfileIter(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(fs, f, idxf, t, false) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + return nil + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} + +func BenchmarkPackfileIterReadContent(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(fs, f, idxf, t, false) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + + r, err := o.Reader() + if err != nil { + b.Fatal(err) + } + + if _, err := ioutil.ReadAll(r); err != nil { + b.Fatal(err) + } + + return r.Close() + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} + +func BenchmarkGetObjectFromPackfile(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.Basic() { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) + for i := 0; i < b.N; i++ { + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) + if err != nil { + b.Fatal(err) + } + + if obj.Hash() != expected { + b.Errorf("expecting %s, got %s", expected, obj.Hash()) + } + } + }) + } } diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go index 7313f05..a891b83 100644 --- a/storage/filesystem/reference.go +++ b/storage/filesystem/reference.go @@ -3,7 +3,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) type ReferenceStorage struct { diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go index 394e6ed..502d406 100644 --- a/storage/filesystem/shallow.go +++ b/storage/filesystem/shallow.go @@ -5,7 +5,7 @@ import ( "fmt" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -26,7 +26,7 @@ func (s *ShallowStorage) SetShallow(commits []plumbing.Hash) error { defer ioutil.CheckClose(f, &err) for _, h := range commits { - if _, err := fmt.Fprintf(f, "%s\n", h); err != err { + if _, err := fmt.Fprintf(f, "%s\n", h); err != nil { return err } } @@ -41,6 +41,8 @@ func (s *ShallowStorage) Shallow() ([]plumbing.Hash, error) { return nil, err } + defer ioutil.CheckClose(f, &err) + var hash []plumbing.Hash scn := bufio.NewScanner(f) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 82b137c..14a772a 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,7 +2,8 @@ package filesystem import ( - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" ) @@ -22,25 +23,45 @@ type Storage struct { ModuleStorage } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) - o, err := newObjectStorage(dir) - if err != nil { - return nil, err +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) +} + +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } + dir := dotgit.NewWithOptions(fs, dirOps) return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -48,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 4d9ba6f..6fa0d90 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/test" @@ -23,9 +24,12 @@ var _ = Suite(&StorageSuite{}) func (s *StorageSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorage(osfs.New(s.dir)) - c.Assert(err, IsNil) + storage := NewStorage(osfs.New(s.dir), cache.NewObjectLRUDefault()) + + setUpTest(s, c, storage) +} +func setUpTest(s *StorageSuite, c *C, storage *Storage) { // ensure that right interfaces are implemented var _ storer.EncodedObjectStorer = storage var _ storer.IndexStorer = storage @@ -40,8 +44,7 @@ func (s *StorageSuite) SetUpTest(c *C) { func (s *StorageSuite) TestFilesystem(c *C) { fs := memfs.New() - storage, err := NewStorage(fs) - c.Assert(err, IsNil) + storage := NewStorage(fs, cache.NewObjectLRUDefault()) c.Assert(storage.Filesystem(), Equals, fs) } @@ -51,3 +54,19 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) { c.Assert(err, IsNil) c.Assert(fis, HasLen, 0) } + +type StorageExclusiveSuite struct { + StorageSuite +} + +var _ = Suite(&StorageExclusiveSuite{}) + +func (s *StorageExclusiveSuite) SetUpTest(c *C) { + s.dir = c.MkDir() + storage := NewStorageWithOptions( + osfs.New(s.dir), + cache.NewObjectLRUDefault(), + Options{ExclusiveAccess: true}) + + setUpTest(&s.StorageSuite, c, storage) +} diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 0f66f1e..2e32509 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -165,7 +165,7 @@ func (o *ObjectStorage) Begin() storer.Transaction { } func (o *ObjectStorage) ForEachObjectHash(fun func(plumbing.Hash) error) error { - for h, _ := range o.Objects { + for h := range o.Objects { err := fun(h) if err != nil { if err == storer.ErrStop { |