Support non packed objects (#68)

* Support non packed git objects

* Support non packed git objects for Iterator

* Fix error handling from Writer() in FillObject()

* Fix format in func (r *Reader) FillObject(obj core.Object) error

* Fix to return d.addRefsFromPackedRefs() error

And if packed-refs dosen't exist not to return error in d.addRefsFromPackedRefs

* Remove debug code

* Add GoDoc for func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Add GoDoc for func (r *Reader) FillObject(obj core.Object) error

* Add GoDoc for func (d *DotGit) Objectfiles() (fs.FS, []core.Hash, error)

* Fix format in func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Rename value dotGitobjcts -> objsDir

* Change regexp.Compile -> regexp.MustCompile

* Move regexp to variable initialization

* Rename regexp value to be more coherent

* Fix object directory name and object file name to correct character

* Faster Objectfiles func

* Add test for FillObject

* Add GoDoc for func (s *ObjectStorage) Get(h core.Hash) (core.Object, error)

* defer Close()

* Return name values for defer function overwrite the error value.

* Fix error handling in func (s *ObjectStorage) Get()

Return error that gets error except for ErrObjfileNotFound from getFromUnpacked()

* Rename getFromObject -> getFromUnpacked

* Add test for func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Add test for func (d *DotGit) Objectfiles() (fs.FS, []core.Hash, error)

* Faster check git object name

* Faster dotgit_test.go

* Fix Godoc for Objectfiles func

* Refactor variable name in Objectfiles func

* Fix GoDoc for objectfile func

* Fix TestObjectfile func and TestObjectfiles func

* Rename fixobj -> fixObj in Test Objectfile func

* Fix test compare method

* Refactor Get func in object.go

* Refactor getFromUnpacked func in object.go

* Fix GoDoc for ErrObjfileNotFound

* Fix TestObjectfiles for not guarantee the slice order

* Change error no such file or directory to target file not found

* Change spec func (s *ObjectStorage) Get(h core.Hash) (core.Object, error)

return core.ErrObjectNotFound, if index pointer is nil.

* Add space

* storage: Add object type hint parameter to ObjectStorage.getFromUnpacked

4 files changed, 279 insertions, 8 deletions

diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go
index 448f6a2..75c98ff 100644
--- a/storage/filesystem/internal/dotgit/dotgit.go
+++ b/storage/filesystem/internal/dotgit/dotgit.go
@@ -22,6 +22,8 @@ var (
 	ErrIdxNotFound = errors.New("idx file not found")
 	// ErrPackfileNotFound is returned by Packfile when the packfile is not found
 	ErrPackfileNotFound = errors.New("packfile not found")
+	// ErrObjfileNotFound is returned by Objectfile when the objectffile is not found
+	ErrObjfileNotFound = errors.New("object file not found")
 	// ErrConfigNotFound is returned by Config when the config is not found
 	ErrConfigNotFound = errors.New("config file not found")
 )
@@ -74,6 +76,10 @@ func (d *DotGit) Packfile() (fs.FS, string, error) {
 	packDir := d.fs.Join(d.path, "objects", "pack")
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrPackfileNotFound
+		}
+
 		return nil, "", err
 	}
 
@@ -93,6 +99,10 @@ func (d *DotGit) Idxfile() (fs.FS, string, error) {
 	packDir := d.fs.Join(d.path, "objects", "pack")
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrIdxNotFound
+		}
+
 		return nil, "", err
 	}
 
@@ -118,3 +128,75 @@ func (d *DotGit) Config() (fs.FS, string, error) {
 
 	return d.fs, configFile, nil
 }
+
+// Objectfiles returns a slice with the hashes of objects found under the
+// .git/objects/ directory.
+func (dg *DotGit) Objectfiles() (fs.FS, []core.Hash, error) {
+	objsDir := dg.fs.Join(dg.path, "objects")
+
+	files, err := dg.fs.ReadDir(objsDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil, ErrObjfileNotFound
+		}
+
+		return nil, nil, err
+	}
+
+	var objects []core.Hash
+	for _, f := range files {
+		if f.IsDir() && len(f.Name()) == 2 && isHex(f.Name()) {
+			objDir := f.Name()
+			d, err := dg.fs.ReadDir(dg.fs.Join(objsDir, objDir))
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for _, o := range d {
+				objects = append(objects, core.NewHash(objDir+o.Name()))
+			}
+		}
+	}
+
+	return dg.fs, objects, nil
+}
+
+func isHex(s string) bool {
+	for _, b := range []byte(s) {
+		if isNum(b) {
+			continue
+		}
+		if isHexAlpha(b) {
+			continue
+		}
+
+		return false
+	}
+
+	return true
+}
+
+func isNum(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
+func isHexAlpha(b byte) bool {
+	return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
+}
+
+// Objectfile returns the path of the object file for a given hash
+// *if the file exists*, otherwise returns an ErrObjfileNotFound error.
+func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error) {
+	hash := h.String()
+	objFile := d.fs.Join(d.path, "objects", hash[0:2], hash[2:40])
+
+	if _, err := d.fs.Stat(objFile); err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrObjfileNotFound
+		}
+
+		return nil, "", err
+	}
+
+	return d.fs, objFile, nil
+}
diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go
index 7c39c87..954eef1 100644
--- a/storage/filesystem/internal/dotgit/dotgit_test.go
+++ b/storage/filesystem/internal/dotgit/dotgit_test.go
@@ -22,6 +22,7 @@ var initFixtures = [...]struct {
 	capabilities [][2]string
 	packfile     string
 	idxfile      string
+	objectfiles  []fixtureObject
 }{
 	{
 		name: "spinnaker",
@@ -37,7 +38,47 @@ var initFixtures = [...]struct {
 	}, {
 		name: "empty",
 		tgz:  "fixtures/empty-gitdir.tgz",
+	}, {
+		name: "unpacked",
+		tgz:  "fixtures/unpacked-objects-no-packfile-no-idx.tgz",
+		objectfiles: []fixtureObject{
+			fixtureObject{
+				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+			},
+			fixtureObject{
+				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
+				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
+			},
+			fixtureObject{
+				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+			},
+		},
 	},
+	{
+		name: "unpacked-dummy",
+		tgz:  "fixtures/unpacked-objects-exist-one-dummy-object-no-packfile-no-idx.tgz",
+		objectfiles: []fixtureObject{
+			fixtureObject{
+				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+			},
+			fixtureObject{
+				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
+				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
+			},
+			fixtureObject{
+				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+			},
+		},
+	},
+}
+
+type fixtureObject struct {
+	path string
+	hash string
 }
 
 type fixture struct {
@@ -47,6 +88,7 @@ type fixture struct {
 	capabilities *common.Capabilities // expected capabilities
 	packfile     string               // path of the packfile
 	idxfile      string               // path of the idxfile
+	objectfiles  []fixtureObject      // path and hash of the object files
 }
 
 type SuiteDotGit struct {
@@ -77,6 +119,7 @@ func (s *SuiteDotGit) SetUpSuite(c *C) {
 
 		f.packfile = init.packfile
 		f.idxfile = init.idxfile
+		f.objectfiles = init.objectfiles
 
 		s.fixtures[init.name] = f
 	}
@@ -193,11 +236,11 @@ func (s *SuiteDotGit) TestPackfile(c *C) {
 		}, {
 			fixture: "empty",
 			fn:      packfile,
-			err:     ".* no such file or directory",
+			err:     "packfile not found",
 		}, {
 			fixture: "empty",
 			fn:      idxfile,
-			err:     ".* no such file or directory",
+			err:     "idx file not found",
 		}, {
 			fixture: "no-packfile-no-idx",
 			fn:      packfile,
@@ -224,9 +267,87 @@ func (s *SuiteDotGit) TestPackfile(c *C) {
 	}
 }
 
+func (s *SuiteDotGit) TestObjectfiles(c *C) {
+	for _, test := range [...]struct {
+		fixture string
+		err     error
+	}{
+		{
+			fixture: "unpacked",
+		},
+		{
+			fixture: "unpacked-dummy",
+		}, {
+			fixture: "empty",
+			err:     ErrObjfileNotFound,
+		}, {
+			fixture: "no-packfile-no-idx",
+		},
+	} {
+		com := Commentf("fixture = %s", test.fixture)
+
+		fix, dir := s.newFixtureDir(c, test.fixture)
+
+		_, hashes, err := dir.Objectfiles()
+
+		if test.err != nil {
+			c.Assert(err, Equals, test.err, com)
+		} else {
+			c.Assert(err, IsNil, com)
+			c.Assert(len(hashes), Equals, len(fix.objectfiles), com)
+
+			for _, hash := range hashes {
+				c.Assert(containsObject(fix.objectfiles, hash), Equals, true, com)
+			}
+		}
+	}
+}
+
+func (s *SuiteDotGit) TestObjectfile(c *C) {
+	for _, test := range [...]struct {
+		fixture string
+		err     error
+	}{
+		{
+			fixture: "unpacked",
+		}, {
+			fixture: "empty",
+			err:     ErrObjfileNotFound,
+		}, {
+			fixture: "no-packfile-no-idx",
+			err:     ErrObjfileNotFound,
+		},
+	} {
+		com := Commentf("fixture = %s", test.fixture)
+
+		fix, dir := s.newFixtureDir(c, test.fixture)
+
+		for _, fixObj := range fix.objectfiles {
+			_, path, err := dir.Objectfile(core.NewHash(fixObj.hash))
+
+			if test.err != nil {
+				c.Assert(err, Equals, test.err, com)
+			} else {
+				c.Assert(err, IsNil, com)
+				c.Assert(strings.HasSuffix(path, fixObj.path),
+					Equals, true, com)
+			}
+		}
+	}
+}
+
 type getPathFn func(*DotGit) (fs.FS, string, error)
 
 func noExt(path string) string {
 	ext := filepath.Ext(path)
 	return path[0 : len(path)-len(ext)]
 }
+
+func containsObject(objs []fixtureObject, hash core.Hash) bool {
+	for _, o := range objs {
+		if strings.ToLower(o.hash) == strings.ToLower(hash.String()) {
+			return true
+		}
+	}
+	return false
+}
diff --git a/storage/filesystem/internal/dotgit/refs.go b/storage/filesystem/internal/dotgit/refs.go
index 894732f..c32a7e5 100644
--- a/storage/filesystem/internal/dotgit/refs.go
+++ b/storage/filesystem/internal/dotgit/refs.go
@@ -32,10 +32,9 @@ func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) {
 	path := d.fs.Join(d.path, packedRefsPath)
 	f, err := d.fs.Open(path)
 	if err != nil {
-		if err == os.ErrNotExist {
+		if os.IsNotExist(err) {
 			return nil
 		}
-
 		return err
 	}
 
@@ -44,7 +43,6 @@ func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) {
 			err = errClose
 		}
 	}()
-
 	s := bufio.NewScanner(f)
 	for s.Scan() {
 		ref, err := d.processLine(s.Text())
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index 1b1ce9d..3888fd8 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"gopkg.in/src-d/go-git.v4/core"
+	"gopkg.in/src-d/go-git.v4/formats/objfile"
 	"gopkg.in/src-d/go-git.v4/formats/packfile"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index"
@@ -37,8 +38,62 @@ func (s *ObjectStorage) Set(core.Object) (core.Hash, error) {
 }
 
 // Get returns the object with the given hash, by searching for it in
-// the packfile.
+// the packfile and the git object directories.
 func (s *ObjectStorage) Get(t core.ObjectType, h core.Hash) (core.Object, error) {
+	obj, err := s.getFromUnpacked(t, h)
+	if err == dotgit.ErrObjfileNotFound {
+		if s.index == nil {
+			return nil, core.ErrObjectNotFound
+		}
+		return s.getFromPackfile(t, h)
+	}
+
+	return obj, err
+}
+
+func (s *ObjectStorage) getFromUnpacked(t core.ObjectType, h core.Hash) (obj core.Object, err error) {
+	fs, path, err := s.dir.Objectfile(h)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := fs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		errClose := f.Close()
+		if err == nil {
+			err = errClose
+		}
+	}()
+
+	obj = s.NewObject()
+	objReader, err := objfile.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		errClose := objReader.Close()
+		if err == nil {
+			err = errClose
+		}
+	}()
+
+	err = objReader.FillObject(obj)
+	if err != nil {
+		return nil, err
+	}
+	if core.AnyObject != t && obj.Type() != t {
+		return nil, core.ErrObjectNotFound
+	}
+	return obj, nil
+}
+
+// Get returns the object with the given hash, by searching for it in
+// the packfile.
+func (s *ObjectStorage) getFromPackfile(t core.ObjectType, h core.Hash) (obj core.Object, err error) {
 	offset, err := s.index.Get(h)
 	if err != nil {
 		return nil, err
@@ -70,7 +125,7 @@ func (s *ObjectStorage) Get(t core.ObjectType, h core.Hash) (core.Object, error)
 	r.HashToOffset = map[core.Hash]int64(s.index)
 	p := packfile.NewParser(r)
 
-	obj := s.NewObject()
+	obj = s.NewObject()
 	err = p.FillObject(obj)
 	if err != nil {
 		return nil, err
@@ -86,8 +141,23 @@ func (s *ObjectStorage) Get(t core.ObjectType, h core.Hash) (core.Object, error)
 func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	var objects []core.Object
 
+	_, hashes, err := s.dir.Objectfiles()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, hash := range hashes {
+		object, err := s.getFromUnpacked(core.AnyObject, hash)
+		if err != nil {
+			return nil, err
+		}
+		if object.Type() == t {
+			objects = append(objects, object)
+		}
+	}
+
 	for hash := range s.index {
-		object, err := s.Get(core.AnyObject, hash)
+		object, err := s.getFromPackfile(core.AnyObject, hash)
 		if err != nil {
 			return nil, err
 		}