Merge branch 'master' of github.com:src-d/go-git into f-add-tagging-support

2 files changed, 405 insertions, 8 deletions

diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index af07eb5..a58c248 100644
--- a/storage/filesystem/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -57,17 +57,48 @@ var (
 	ErrSymRefTargetNotFound = errors.New("symbolic reference target not found")
 )
 
+// Options holds configuration for the storage.
+type Options struct {
+	// ExclusiveAccess means that the filesystem is not modified externally
+	// while the repo is open.
+	ExclusiveAccess bool
+	// KeepDescriptors makes the file descriptors to be reused but they will
+	// need to be manually closed calling Close().
+	KeepDescriptors bool
+}
+
 // The DotGit type represents a local git repository on disk. This
 // type is not zero-value-safe, use the New function to initialize it.
 type DotGit struct {
-	fs billy.Filesystem
+	options Options
+	fs      billy.Filesystem
+
+	// incoming object directory information
+	incomingChecked bool
+	incomingDirName string
+
+	objectList []plumbing.Hash
+	objectMap  map[plumbing.Hash]struct{}
+	packList   []plumbing.Hash
+	packMap    map[plumbing.Hash]struct{}
+
+	files map[string]billy.File
 }
 
 // New returns a DotGit value ready to be used. The path argument must
 // be the absolute path of a git repository directory (e.g.
 // "/foo/bar/.git").
 func New(fs billy.Filesystem) *DotGit {
-	return &DotGit{fs: fs}
+	return NewWithOptions(fs, Options{})
+}
+
+// NewWithOptions sets non default configuration options.
+// See New for complete help.
+func NewWithOptions(fs billy.Filesystem, o Options) *DotGit {
+	return &DotGit{
+		options: o,
+		fs:      fs,
+	}
 }
 
 // Initialize creates all the folder scaffolding.
@@ -97,6 +128,28 @@ func (d *DotGit) Initialize() error {
 	return nil
 }
 
+// Close closes all opened files.
+func (d *DotGit) Close() error {
+	var firstError error
+	if d.files != nil {
+		for _, f := range d.files {
+			err := f.Close()
+			if err != nil && firstError == nil {
+				firstError = err
+				continue
+			}
+		}
+
+		d.files = nil
+	}
+
+	if firstError != nil {
+		return firstError
+	}
+
+	return nil
+}
+
 // ConfigWriter returns a file pointer for write to the config file
 func (d *DotGit) ConfigWriter() (billy.File, error) {
 	return d.fs.Create(configPath)
@@ -139,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) {
 // NewObjectPack return a writer for a new packfile, it saves the packfile to
 // disk and also generates and save the index for the given packfile.
 func (d *DotGit) NewObjectPack() (*PackWriter, error) {
+	d.cleanPackList()
 	return newPackWrite(d.fs)
 }
 
 // ObjectPacks returns the list of availables packfiles
 func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) {
+	if !d.options.ExclusiveAccess {
+		return d.objectPacks()
+	}
+
+	err := d.genPackList()
+	if err != nil {
+		return nil, err
+	}
+
+	return d.packList, nil
+}
+
+func (d *DotGit) objectPacks() ([]plumbing.Hash, error) {
 	packDir := d.fs.Join(objectsPath, packPath)
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
@@ -177,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string {
 }
 
 func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) {
-	pack, err := d.fs.Open(d.objectPackPath(hash, extension))
+	if d.files == nil {
+		d.files = make(map[string]billy.File)
+	}
+
+	err := d.hasPack(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	path := d.objectPackPath(hash, extension)
+	f, ok := d.files[path]
+	if ok {
+		return f, nil
+	}
+
+	pack, err := d.fs.Open(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, ErrPackfileNotFound
@@ -186,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
 		return nil, err
 	}
 
+	if d.options.KeepDescriptors && extension == "pack" {
+		d.files[path] = pack
+	}
+
 	return pack, nil
 }
 
 // ObjectPack returns a fs.File of the given packfile
 func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) {
+	err := d.hasPack(hash)
+	if err != nil {
+		return nil, err
+	}
+
 	return d.objectPackOpen(hash, `pack`)
 }
 
 // ObjectPackIdx returns a fs.File of the index file for a given packfile
 func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) {
+	err := d.hasPack(hash)
+	if err != nil {
+		return nil, err
+	}
+
 	return d.objectPackOpen(hash, `idx`)
 }
 
 func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error {
+	d.cleanPackList()
+
 	path := d.objectPackPath(hash, `pack`)
 	if !t.IsZero() {
 		fi, err := d.fs.Stat(path)
@@ -220,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er
 
 // NewObject return a writer for a new object file.
 func (d *DotGit) NewObject() (*ObjectWriter, error) {
+	d.cleanObjectList()
+
 	return newObjectWriter(d.fs)
 }
 
 // Objects returns a slice with the hashes of objects found under the
 // .git/objects/ directory.
 func (d *DotGit) Objects() ([]plumbing.Hash, error) {
+	if d.options.ExclusiveAccess {
+		err := d.genObjectList()
+		if err != nil {
+			return nil, err
+		}
+
+		return d.objectList, nil
+	}
+
 	var objects []plumbing.Hash
 	err := d.ForEachObjectHash(func(hash plumbing.Hash) error {
 		objects = append(objects, hash)
@@ -237,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) {
 	return objects, nil
 }
 
-// Objects returns a slice with the hashes of objects found under the
-// .git/objects/ directory.
+// ForEachObjectHash iterates over the hashes of objects found under the
+// .git/objects/ directory and executes the provided function.
 func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error {
+	if !d.options.ExclusiveAccess {
+		return d.forEachObjectHash(fun)
+	}
+
+	err := d.genObjectList()
+	if err != nil {
+		return err
+	}
+
+	for _, h := range d.objectList {
+		err := fun(h)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error {
 	files, err := d.fs.ReadDir(objectsPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -274,24 +403,178 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error {
 	return nil
 }
 
+func (d *DotGit) cleanObjectList() {
+	d.objectMap = nil
+	d.objectList = nil
+}
+
+func (d *DotGit) genObjectList() error {
+	if d.objectMap != nil {
+		return nil
+	}
+
+	d.objectMap = make(map[plumbing.Hash]struct{})
+	return d.forEachObjectHash(func(h plumbing.Hash) error {
+		d.objectList = append(d.objectList, h)
+		d.objectMap[h] = struct{}{}
+
+		return nil
+	})
+}
+
+func (d *DotGit) hasObject(h plumbing.Hash) error {
+	if !d.options.ExclusiveAccess {
+		return nil
+	}
+
+	err := d.genObjectList()
+	if err != nil {
+		return err
+	}
+
+	_, ok := d.objectMap[h]
+	if !ok {
+		return plumbing.ErrObjectNotFound
+	}
+
+	return nil
+}
+
+func (d *DotGit) cleanPackList() {
+	d.packMap = nil
+	d.packList = nil
+}
+
+func (d *DotGit) genPackList() error {
+	if d.packMap != nil {
+		return nil
+	}
+
+	op, err := d.objectPacks()
+	if err != nil {
+		return err
+	}
+
+	d.packMap = make(map[plumbing.Hash]struct{})
+	d.packList = nil
+
+	for _, h := range op {
+		d.packList = append(d.packList, h)
+		d.packMap[h] = struct{}{}
+	}
+
+	return nil
+}
+
+func (d *DotGit) hasPack(h plumbing.Hash) error {
+	if !d.options.ExclusiveAccess {
+		return nil
+	}
+
+	err := d.genPackList()
+	if err != nil {
+		return err
+	}
+
+	_, ok := d.packMap[h]
+	if !ok {
+		return ErrPackfileNotFound
+	}
+
+	return nil
+}
+
 func (d *DotGit) objectPath(h plumbing.Hash) string {
 	hash := h.String()
 	return d.fs.Join(objectsPath, hash[0:2], hash[2:40])
 }
 
+// incomingObjectPath is intended to add support for a git pre-receive hook
+// to be written it adds support for go-git to find objects in an "incoming"
+// directory, so that the library can be used to write a pre-receive hook
+// that deals with the incoming objects.
+//
+// More on git hooks found here : https://git-scm.com/docs/githooks
+// More on 'quarantine'/incoming directory here:
+//     https://git-scm.com/docs/git-receive-pack
+func (d *DotGit) incomingObjectPath(h plumbing.Hash) string {
+	hString := h.String()
+
+	if d.incomingDirName == "" {
+		return d.fs.Join(objectsPath, hString[0:2], hString[2:40])
+	}
+
+	return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40])
+}
+
+// hasIncomingObjects searches for an incoming directory and keeps its name
+// so it doesn't have to be found each time an object is accessed.
+func (d *DotGit) hasIncomingObjects() bool {
+	if !d.incomingChecked {
+		directoryContents, err := d.fs.ReadDir(objectsPath)
+		if err == nil {
+			for _, file := range directoryContents {
+				if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() {
+					d.incomingDirName = file.Name()
+				}
+			}
+		}
+
+		d.incomingChecked = true
+	}
+
+	return d.incomingDirName != ""
+}
+
 // Object returns a fs.File pointing the object file, if exists
 func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) {
-	return d.fs.Open(d.objectPath(h))
+	err := d.hasObject(h)
+	if err != nil {
+		return nil, err
+	}
+
+	obj1, err1 := d.fs.Open(d.objectPath(h))
+	if os.IsNotExist(err1) && d.hasIncomingObjects() {
+		obj2, err2 := d.fs.Open(d.incomingObjectPath(h))
+		if err2 != nil {
+			return obj1, err1
+		}
+		return obj2, err2
+	}
+	return obj1, err1
 }
 
 // ObjectStat returns a os.FileInfo pointing the object file, if exists
 func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) {
-	return d.fs.Stat(d.objectPath(h))
+	err := d.hasObject(h)
+	if err != nil {
+		return nil, err
+	}
+
+	obj1, err1 := d.fs.Stat(d.objectPath(h))
+	if os.IsNotExist(err1) && d.hasIncomingObjects() {
+		obj2, err2 := d.fs.Stat(d.incomingObjectPath(h))
+		if err2 != nil {
+			return obj1, err1
+		}
+		return obj2, err2
+	}
+	return obj1, err1
 }
 
 // ObjectDelete removes the object file, if exists
 func (d *DotGit) ObjectDelete(h plumbing.Hash) error {
-	return d.fs.Remove(d.objectPath(h))
+	d.cleanObjectList()
+
+	err1 := d.fs.Remove(d.objectPath(h))
+	if os.IsNotExist(err1) && d.hasIncomingObjects() {
+		err2 := d.fs.Remove(d.incomingObjectPath(h))
+		if err2 != nil {
+			return err1
+		}
+		return err2
+	}
+	return err1
 }
 
 func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) {
diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index 7733eef..308c6b7 100644
--- a/storage/filesystem/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 	"testing"
 
+	"gopkg.in/src-d/go-billy.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 
 	. "gopkg.in/check.v1"
@@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) {
 	fs := f.DotGit()
 	dir := New(fs)
 
+	testObjectPacks(c, fs, dir, f)
+}
+
+func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) {
+	f := fixtures.Basic().ByTag(".git").One()
+	fs := f.DotGit()
+	dir := NewWithOptions(fs, Options{ExclusiveAccess: true})
+
+	testObjectPacks(c, fs, dir, f)
+}
+
+func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) {
 	hashes, err := dir.ObjectPacks()
 	c.Assert(err, IsNil)
 	c.Assert(hashes, HasLen, 1)
@@ -452,6 +465,45 @@ func (s *SuiteDotGit) TestObjectPack(c *C) {
 	c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
 }
 
+func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) {
+	f := fixtures.Basic().ByTag(".git").One()
+	fs := f.DotGit()
+	dir := NewWithOptions(fs, Options{KeepDescriptors: true})
+
+	pack, err := dir.ObjectPack(f.PackfileHash)
+	c.Assert(err, IsNil)
+	c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
+
+	// Move to an specific offset
+	pack.Seek(42, os.SEEK_SET)
+
+	pack2, err := dir.ObjectPack(f.PackfileHash)
+	c.Assert(err, IsNil)
+
+	// If the file is the same the offset should be the same
+	offset, err := pack2.Seek(0, os.SEEK_CUR)
+	c.Assert(err, IsNil)
+	c.Assert(offset, Equals, int64(42))
+
+	err = dir.Close()
+	c.Assert(err, IsNil)
+
+	pack2, err = dir.ObjectPack(f.PackfileHash)
+	c.Assert(err, IsNil)
+
+	// If the file is opened again its offset should be 0
+	offset, err = pack2.Seek(0, os.SEEK_CUR)
+	c.Assert(err, IsNil)
+	c.Assert(offset, Equals, int64(0))
+
+	err = pack2.Close()
+	c.Assert(err, IsNil)
+
+	err = dir.Close()
+	c.Assert(err, NotNil)
+
+}
+
 func (s *SuiteDotGit) TestObjectPackIdx(c *C) {
 	f := fixtures.Basic().ByTag(".git").One()
 	fs := f.DotGit()
@@ -506,6 +558,17 @@ func (s *SuiteDotGit) TestObjects(c *C) {
 	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
 	dir := New(fs)
 
+	testObjects(c, fs, dir)
+}
+
+func (s *SuiteDotGit) TestObjectsExclusive(c *C) {
+	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+	dir := NewWithOptions(fs, Options{ExclusiveAccess: true})
+
+	testObjects(c, fs, dir)
+}
+
+func testObjects(c *C, fs billy.Filesystem, dir *DotGit) {
 	hashes, err := dir.Objects()
 	c.Assert(err, IsNil)
 	c.Assert(hashes, HasLen, 187)
@@ -537,6 +600,57 @@ func (s *SuiteDotGit) TestObject(c *C) {
 		file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")),
 		Equals, true,
 	)
+	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+	incomingDirPath := fs.Join("objects", "incoming-123456")
+	incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+	fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+	fs.Create(incomingFilePath)
+
+	file, err = dir.Object(plumbing.NewHash(incomingHash))
+	c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestObjectStat(c *C) {
+	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+	dir := New(fs)
+
+	hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+	_, err := dir.ObjectStat(hash)
+	c.Assert(err, IsNil)
+	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+	incomingDirPath := fs.Join("objects", "incoming-123456")
+	incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40])
+	fs.MkdirAll(incomingDirPath, os.FileMode(0755))
+	fs.Create(incomingFilePath)
+
+	_, err = dir.ObjectStat(plumbing.NewHash(incomingHash))
+	c.Assert(err, IsNil)
+}
+
+func (s *SuiteDotGit) TestObjectDelete(c *C) {
+	fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
+	dir := New(fs)
+
+	hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e")
+	err := dir.ObjectDelete(hash)
+	c.Assert(err, IsNil)
+
+	incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash
+	incomingDirPath := fs.Join("objects", "incoming-123456")
+	incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2])
+	incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40])
+
+	err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755))
+	c.Assert(err, IsNil)
+
+	f, err := fs.Create(incomingFilePath)
+	c.Assert(err, IsNil)
+
+	err = f.Close()
+	c.Assert(err, IsNil)
+
+	err = dir.ObjectDelete(plumbing.NewHash(incomingHash))
+	c.Assert(err, IsNil)
 }
 
 func (s *SuiteDotGit) TestObjectNotFound(c *C) {