storage: filesystem idx generation (wip)

9 files changed, 341 insertions, 427 deletions

diff --git a/storage/filesystem/config.go b/storage/filesystem/config.go
index c83a59e..c91ba58 100644
--- a/storage/filesystem/config.go
+++ b/storage/filesystem/config.go
@@ -52,18 +52,15 @@ func (c *ConfigStorage) DeleteRemote(name string) error {
 }
 
 func (c *ConfigStorage) read() (*ConfigFile, error) {
-	fs, path, err := c.dir.Config()
+	f, err := c.dir.Config()
 	if err != nil {
 		return nil, err
 	}
 
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
+	defer f.Close()
 
-	f := &ConfigFile{}
-	return f, f.Decode(r)
+	config := &ConfigFile{}
+	return config, config.Decode(f)
 }
 
 type ConfigFile struct {
diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go
index 75c98ff..71af7a0 100644
--- a/storage/filesystem/internal/dotgit/dotgit.go
+++ b/storage/filesystem/internal/dotgit/dotgit.go
@@ -1,11 +1,17 @@
+// https://github.com/git/git/blob/master/Documentation/gitrepository-layout.txt
 package dotgit
 
 import (
+	"crypto/sha1"
 	"errors"
+	"fmt"
+	"io"
 	"os"
 	"strings"
+	"time"
 
 	"gopkg.in/src-d/go-git.v4/core"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index"
 	"gopkg.in/src-d/go-git.v4/utils/fs"
 )
 
@@ -13,6 +19,12 @@ const (
 	suffix         = ".git"
 	packedRefsPath = "packed-refs"
 	configPath     = "config"
+
+	objectsPath = "objects"
+	packPath    = "pack"
+
+	packExt = ".pack"
+	idxExt  = ".idx"
 )
 
 var (
@@ -31,23 +43,19 @@ var (
 // The DotGit type represents a local git repository on disk. This
 // type is not zero-value-safe, use the New function to initialize it.
 type DotGit struct {
-	fs   fs.FS
-	path string
+	fs fs.Filesystem
 }
 
 // New returns a DotGit value ready to be used. The path argument must
 // be the absolute path of a git repository directory (e.g.
 // "/foo/bar/.git").
-func New(fs fs.FS, path string) (*DotGit, error) {
-	d := &DotGit{fs: fs, path: path}
-	if _, err := fs.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return nil, ErrNotFound
-		}
-		return nil, err
-	}
+func New(fs fs.Filesystem) *DotGit {
+	return &DotGit{fs: fs}
+}
 
-	return d, nil
+// Config returns the path of the config file
+func (d *DotGit) Config() (fs.File, error) {
+	return d.fs.Open(configPath)
 }
 
 // Refs scans the git directory collecting references, which it returns.
@@ -69,96 +77,97 @@ func (d *DotGit) Refs() ([]*core.Reference, error) {
 	return refs, nil
 }
 
-// Packfile returns the path of the packfile (really, it returns the
-// path of the first file in the "objects/pack/" directory with a
-// ".pack" extension.
-func (d *DotGit) Packfile() (fs.FS, string, error) {
-	packDir := d.fs.Join(d.path, "objects", "pack")
+func (d *DotGit) NewObjectPack() (*PackWriter, error) {
+	return newPackWrite(d.fs)
+}
+
+// ObjectsPacks returns the list of availables packfiles
+func (d *DotGit) ObjectsPacks() ([]fs.FileInfo, error) {
+	packDir := d.fs.Join(objectsPath, packPath)
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrPackfileNotFound
+			return nil, nil
 		}
 
-		return nil, "", err
+		return nil, err
 	}
 
+	var packs []fs.FileInfo
 	for _, f := range files {
-		if strings.HasSuffix(f.Name(), ".pack") {
-			return d.fs, d.fs.Join(packDir, f.Name()), nil
+		if strings.HasSuffix(f.Name(), packExt) {
+			packs = append(packs, f)
 		}
 	}
 
-	return nil, "", ErrPackfileNotFound
+	return packs, nil
 }
 
-// Idxfile returns the path of the idx file (really, it returns the
-// path of the first file in the "objects/pack/" directory with an
-// ".idx" extension.
-func (d *DotGit) Idxfile() (fs.FS, string, error) {
-	packDir := d.fs.Join(d.path, "objects", "pack")
-	files, err := d.fs.ReadDir(packDir)
+// ObjectPack returns the requested packfile and his idx
+func (d *DotGit) ObjectPack(filename string) (pack, idx fs.File, err error) {
+	if !strings.HasSuffix(filename, packExt) {
+		return nil, nil, fmt.Errorf("a .pack file should be provided")
+	}
+
+	pack, err = d.fs.Open(d.fs.Join(objectsPath, packPath, filename))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrIdxNotFound
+			return nil, nil, ErrPackfileNotFound
 		}
 
-		return nil, "", err
+		return
 	}
 
-	for _, f := range files {
-		if strings.HasSuffix(f.Name(), ".idx") {
-			return d.fs, d.fs.Join(packDir, f.Name()), nil
-		}
-	}
-
-	return nil, "", ErrIdxNotFound
-}
-
-// Config returns the path of the config file
-func (d *DotGit) Config() (fs.FS, string, error) {
-	configFile := d.fs.Join(d.path, configPath)
-	if _, err := d.fs.Stat(configFile); err != nil {
+	idxfile := filename[0:len(filename)-len(packExt)] + idxExt
+	idxpath := d.fs.Join(objectsPath, packPath, idxfile)
+	idx, err = d.fs.Open(idxpath)
+	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrNotFound
+			return nil, nil, ErrIdxNotFound
 		}
 
-		return nil, "", err
+		return
 	}
 
-	return d.fs, configFile, nil
+	return
 }
 
-// Objectfiles returns a slice with the hashes of objects found under the
+// Objects returns a slice with the hashes of objects found under the
 // .git/objects/ directory.
-func (dg *DotGit) Objectfiles() (fs.FS, []core.Hash, error) {
-	objsDir := dg.fs.Join(dg.path, "objects")
-
-	files, err := dg.fs.ReadDir(objsDir)
+func (d *DotGit) Objects() ([]core.Hash, error) {
+	files, err := d.fs.ReadDir(objectsPath)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil, ErrObjfileNotFound
+			return nil, nil
 		}
 
-		return nil, nil, err
+		return nil, err
 	}
 
 	var objects []core.Hash
 	for _, f := range files {
 		if f.IsDir() && len(f.Name()) == 2 && isHex(f.Name()) {
-			objDir := f.Name()
-			d, err := dg.fs.ReadDir(dg.fs.Join(objsDir, objDir))
+			base := f.Name()
+			d, err := d.fs.ReadDir(d.fs.Join(objectsPath, base))
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
 			for _, o := range d {
-				objects = append(objects, core.NewHash(objDir+o.Name()))
+				objects = append(objects, core.NewHash(base+o.Name()))
 			}
 		}
 	}
 
-	return dg.fs, objects, nil
+	return objects, nil
+}
+
+// Object return a fs.File poiting the object file, if exists
+func (d *DotGit) Object(h core.Hash) (fs.File, error) {
+	hash := h.String()
+	file := d.fs.Join(objectsPath, hash[0:2], hash[2:40])
+
+	return d.fs.Open(file)
 }
 
 func isHex(s string) bool {
@@ -184,19 +193,78 @@ func isHexAlpha(b byte) bool {
 	return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
 }
 
-// Objectfile returns the path of the object file for a given hash
-// *if the file exists*, otherwise returns an ErrObjfileNotFound error.
-func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error) {
-	hash := h.String()
-	objFile := d.fs.Join(d.path, "objects", hash[0:2], hash[2:40])
+type PackWriter struct {
+	fs         fs.Filesystem
+	file       fs.File
+	writer     io.Writer
+	pipeReader io.ReadCloser
+	pipeWriter io.WriteCloser
+	hash       core.Hash
+	index      index.Index
+	result     chan error
+}
 
-	if _, err := d.fs.Stat(objFile); err != nil {
-		if os.IsNotExist(err) {
-			return nil, "", ErrObjfileNotFound
-		}
+func newPackWrite(fs fs.Filesystem) (*PackWriter, error) {
+	r, w := io.Pipe()
+
+	temp := sha1.Sum([]byte(time.Now().String()))
+	filename := fmt.Sprintf(".%x", temp)
+
+	file, err := fs.Create(fs.Join(objectsPath, packPath, filename))
+	if err != nil {
+		return nil, err
+	}
 
-		return nil, "", err
+	writer := &PackWriter{
+		fs:         fs,
+		file:       file,
+		writer:     io.MultiWriter(w, file),
+		pipeReader: r,
+		pipeWriter: w,
+		result:     make(chan error),
 	}
 
-	return d.fs, objFile, nil
+	go writer.buildIndex()
+	return writer, nil
+}
+
+func (w *PackWriter) buildIndex() {
+	defer w.pipeReader.Close()
+	index, hash, err := index.NewFromPackfileInMemory(w.pipeReader)
+	w.index = index
+	w.hash = hash
+
+	w.result <- err
+}
+
+func (w *PackWriter) Write(p []byte) (int, error) {
+	return w.writer.Write(p)
+}
+
+func (w *PackWriter) Close() error {
+	defer func() {
+		close(w.result)
+	}()
+
+	if err := w.file.Close(); err != nil {
+		return err
+	}
+
+	if err := w.pipeWriter.Close(); err != nil {
+		return err
+	}
+
+	if err := <-w.result; err != nil {
+		return err
+	}
+
+	return w.save()
+}
+
+func (w *PackWriter) save() error {
+	base := w.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s", w.hash))
+
+	//idx, err := w.fs.Create(fmt.Sprintf("%s.idx", base))
+
+	return w.fs.Rename(w.file.Filename(), fmt.Sprintf("%s.pack", base))
 }
diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go
index 954eef1..3f0a0eb 100644
--- a/storage/filesystem/internal/dotgit/dotgit_test.go
+++ b/storage/filesystem/internal/dotgit/dotgit_test.go
@@ -1,12 +1,13 @@
 package dotgit
 
 import (
+	"io"
+	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
-	"gopkg.in/src-d/go-git.v4/clients/common"
 	"gopkg.in/src-d/go-git.v4/core"
 	"gopkg.in/src-d/go-git.v4/utils/fs"
 
@@ -17,21 +18,12 @@ import (
 func Test(t *testing.T) { TestingT(t) }
 
 var initFixtures = [...]struct {
-	name         string
-	tgz          string
-	capabilities [][2]string
-	packfile     string
-	idxfile      string
-	objectfiles  []fixtureObject
+	name string
+	tgz  string
 }{
 	{
 		name: "spinnaker",
 		tgz:  "fixtures/spinnaker-gc.tgz",
-		capabilities: [][2]string{
-			{"symref", "HEAD:refs/heads/master"},
-		},
-		packfile: "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack",
-		idxfile:  "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.idx",
 	}, {
 		name: "no-packfile-no-idx",
 		tgz:  "fixtures/no-packfile-no-idx.tgz",
@@ -41,64 +33,20 @@ var initFixtures = [...]struct {
 	}, {
 		name: "unpacked",
 		tgz:  "fixtures/unpacked-objects-no-packfile-no-idx.tgz",
-		objectfiles: []fixtureObject{
-			fixtureObject{
-				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
-				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
-			},
-			fixtureObject{
-				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
-				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
-			},
-			fixtureObject{
-				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-			},
-		},
-	},
-	{
+	}, {
 		name: "unpacked-dummy",
 		tgz:  "fixtures/unpacked-objects-exist-one-dummy-object-no-packfile-no-idx.tgz",
-		objectfiles: []fixtureObject{
-			fixtureObject{
-				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
-				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
-			},
-			fixtureObject{
-				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
-				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
-			},
-			fixtureObject{
-				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-			},
-		},
 	},
 }
 
-type fixtureObject struct {
-	path string
-	hash string
-}
-
-type fixture struct {
-	installDir   string
-	fs           fs.FS
-	path         string               // repo names to paths of the extracted tgz
-	capabilities *common.Capabilities // expected capabilities
-	packfile     string               // path of the packfile
-	idxfile      string               // path of the idxfile
-	objectfiles  []fixtureObject      // path and hash of the object files
-}
-
 type SuiteDotGit struct {
-	fixtures map[string]fixture
+	fixtures map[string]fs.Filesystem
 }
 
 var _ = Suite(&SuiteDotGit{})
 
 func (s *SuiteDotGit) SetUpSuite(c *C) {
-	s.fixtures = make(map[string]fixture, len(initFixtures))
+	s.fixtures = make(map[string]fs.Filesystem, len(initFixtures))
 
 	for _, init := range initFixtures {
 		com := Commentf("fixture name = %s\n", init.name)
@@ -106,56 +54,21 @@ func (s *SuiteDotGit) SetUpSuite(c *C) {
 		path, err := tgz.Extract(init.tgz)
 		c.Assert(err, IsNil, com)
 
-		f := fixture{}
-
-		f.installDir = path
-		f.fs = fs.NewOS()
-		f.path = f.fs.Join(path, ".git")
-
-		f.capabilities = common.NewCapabilities()
-		for _, pair := range init.capabilities {
-			f.capabilities.Add(pair[0], pair[1])
-		}
-
-		f.packfile = init.packfile
-		f.idxfile = init.idxfile
-		f.objectfiles = init.objectfiles
-
-		s.fixtures[init.name] = f
+		s.fixtures[init.name] = fs.NewOSClient(filepath.Join(path, ".git"))
 	}
 }
 
 func (s *SuiteDotGit) TearDownSuite(c *C) {
-	for n, f := range s.fixtures {
-		err := os.RemoveAll(f.installDir)
-		c.Assert(err, IsNil, Commentf("cannot delete tmp dir for fixture %s: %s\n",
-			n, f.installDir))
-	}
-}
-
-func (s *SuiteDotGit) TestNewErrors(c *C) {
-	for i, test := range [...]struct {
-		input string
-		err   error
-	}{
-		{
-			input: "./tmp/foo",
-			err:   ErrNotFound,
-		}, {
-			input: "./tmp/foo/.git",
-			err:   ErrNotFound,
-		},
-	} {
-		com := Commentf("subtest %d", i)
-
-		_, err := New(fs.NewOS(), test.input)
-		c.Assert(err, Equals, test.err, com)
+	for _, f := range s.fixtures {
+		err := os.RemoveAll(f.Base())
+		c.Assert(err, IsNil)
 	}
 }
 
 func (s *SuiteDotGit) TestRefsFromPackedRefs(c *C) {
-	_, d := s.newFixtureDir(c, "spinnaker")
-	refs, err := d.Refs()
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	refs, err := dir.Refs()
 	c.Assert(err, IsNil)
 
 	ref := findReference(refs, "refs/tags/v0.37.0")
@@ -164,8 +77,9 @@ func (s *SuiteDotGit) TestRefsFromPackedRefs(c *C) {
 
 }
 func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) {
-	_, d := s.newFixtureDir(c, "spinnaker")
-	refs, err := d.Refs()
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	refs, err := dir.Refs()
 	c.Assert(err, IsNil)
 
 	ref := findReference(refs, "refs/remotes/origin/HEAD")
@@ -176,8 +90,9 @@ func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) {
 }
 
 func (s *SuiteDotGit) TestRefsFromHEADFile(c *C) {
-	_, d := s.newFixtureDir(c, "spinnaker")
-	refs, err := d.Refs()
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	refs, err := dir.Refs()
 	c.Assert(err, IsNil)
 
 	ref := findReference(refs, "HEAD")
@@ -187,11 +102,11 @@ func (s *SuiteDotGit) TestRefsFromHEADFile(c *C) {
 }
 
 func (s *SuiteDotGit) TestConfig(c *C) {
-	_, d := s.newFixtureDir(c, "spinnaker")
-	fs, path, err := d.Config()
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	file, err := dir.Config()
 	c.Assert(err, IsNil)
-	c.Assert(fs, NotNil)
-	c.Assert(path, Not(Equals), "")
+	c.Assert(filepath.Base(file.Filename()), Equals, "config")
 }
 
 func findReference(refs []*core.Reference, name string) *core.Reference {
@@ -205,149 +120,130 @@ func findReference(refs []*core.Reference, name string) *core.Reference {
 	return nil
 }
 
-func (s *SuiteDotGit) newFixtureDir(c *C, fixName string) (*fixture, *DotGit) {
+func (s *SuiteDotGit) newFixtureDir(c *C, fixName string) *DotGit {
 	f, ok := s.fixtures[fixName]
 	c.Assert(ok, Equals, true)
 
-	d, err := New(fs.NewOS(), f.path)
+	return New(f)
+}
+
+func (s *SuiteDotGit) TestObjectsPack(c *C) {
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	files, err := dir.ObjectsPacks()
 	c.Assert(err, IsNil)
+	c.Assert(files, HasLen, 1)
+}
+
+func (s *SuiteDotGit) TestObjectsNoPackile(c *C) {
+	dir := s.newFixtureDir(c, "no-packfile-no-idx")
 
-	return &f, d
+	files, err := dir.ObjectsPacks()
+	c.Assert(err, IsNil)
+	c.Assert(files, HasLen, 0)
 }
 
-func (s *SuiteDotGit) TestPackfile(c *C) {
-	packfile := func(d *DotGit) (fs.FS, string, error) {
-		return d.Packfile()
-	}
-	idxfile := func(d *DotGit) (fs.FS, string, error) {
-		return d.Idxfile()
-	}
-	for _, test := range [...]struct {
-		fixture string
-		fn      getPathFn
-		err     string // error regexp
-	}{
-		{
-			fixture: "spinnaker",
-			fn:      packfile,
-		}, {
-			fixture: "spinnaker",
-			fn:      idxfile,
-		}, {
-			fixture: "empty",
-			fn:      packfile,
-			err:     "packfile not found",
-		}, {
-			fixture: "empty",
-			fn:      idxfile,
-			err:     "idx file not found",
-		}, {
-			fixture: "no-packfile-no-idx",
-			fn:      packfile,
-			err:     "packfile not found",
-		}, {
-			fixture: "no-packfile-no-idx",
-			fn:      idxfile,
-			err:     "idx file not found",
-		},
-	} {
-		com := Commentf("fixture = %s", test.fixture)
-
-		fix, dir := s.newFixtureDir(c, test.fixture)
-
-		_, path, err := test.fn(dir)
-
-		if test.err != "" {
-			c.Assert(err, ErrorMatches, test.err, com)
-		} else {
-			c.Assert(err, IsNil, com)
-			c.Assert(strings.HasSuffix(noExt(path), noExt(fix.packfile)),
-				Equals, true, com)
-		}
-	}
+func (s *SuiteDotGit) TestObjectsPackFolderNotExists(c *C) {
+	dir := s.newFixtureDir(c, "empty")
+
+	files, err := dir.ObjectsPacks()
+	c.Assert(err, IsNil)
+	c.Assert(files, HasLen, 0)
 }
 
-func (s *SuiteDotGit) TestObjectfiles(c *C) {
-	for _, test := range [...]struct {
-		fixture string
-		err     error
-	}{
-		{
-			fixture: "unpacked",
-		},
-		{
-			fixture: "unpacked-dummy",
-		}, {
-			fixture: "empty",
-			err:     ErrObjfileNotFound,
-		}, {
-			fixture: "no-packfile-no-idx",
-		},
-	} {
-		com := Commentf("fixture = %s", test.fixture)
-
-		fix, dir := s.newFixtureDir(c, test.fixture)
-
-		_, hashes, err := dir.Objectfiles()
-
-		if test.err != nil {
-			c.Assert(err, Equals, test.err, com)
-		} else {
-			c.Assert(err, IsNil, com)
-			c.Assert(len(hashes), Equals, len(fix.objectfiles), com)
-
-			for _, hash := range hashes {
-				c.Assert(containsObject(fix.objectfiles, hash), Equals, true, com)
-			}
-		}
-	}
+func (s *SuiteDotGit) TestObjectPack(c *C) {
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	filename := "pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack"
+	pack, idx, err := dir.ObjectPack(filename)
+	c.Assert(err, IsNil)
+	c.Assert(filepath.Ext(pack.Filename()), Equals, ".pack")
+	c.Assert(filepath.Ext(idx.Filename()), Equals, ".idx")
 }
 
-func (s *SuiteDotGit) TestObjectfile(c *C) {
-	for _, test := range [...]struct {
-		fixture string
-		err     error
-	}{
-		{
-			fixture: "unpacked",
-		}, {
-			fixture: "empty",
-			err:     ErrObjfileNotFound,
-		}, {
-			fixture: "no-packfile-no-idx",
-			err:     ErrObjfileNotFound,
-		},
-	} {
-		com := Commentf("fixture = %s", test.fixture)
-
-		fix, dir := s.newFixtureDir(c, test.fixture)
-
-		for _, fixObj := range fix.objectfiles {
-			_, path, err := dir.Objectfile(core.NewHash(fixObj.hash))
-
-			if test.err != nil {
-				c.Assert(err, Equals, test.err, com)
-			} else {
-				c.Assert(err, IsNil, com)
-				c.Assert(strings.HasSuffix(path, fixObj.path),
-					Equals, true, com)
-			}
-		}
-	}
+func (s *SuiteDotGit) TestObjectPackNotFound(c *C) {
+	dir := s.newFixtureDir(c, "spinnaker")
+
+	filename := "pack-not-exists.pack"
+	pack, idx, err := dir.ObjectPack(filename)
+	c.Assert(err, Equals, ErrPackfileNotFound)
+	c.Assert(pack, IsNil)
+	c.Assert(idx, IsNil)
 }
 
-type getPathFn func(*DotGit) (fs.FS, string, error)
+func (s *SuiteDotGit) TestObjects(c *C) {
+	dir := s.newFixtureDir(c, "unpacked")
 
-func noExt(path string) string {
-	ext := filepath.Ext(path)
-	return path[0 : len(path)-len(ext)]
+	hashes, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hashes, HasLen, 3)
+	c.Assert(hashes[0].String(), Equals, "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e")
+	c.Assert(hashes[1].String(), Equals, "5efb9bc29c482e023e40e0a2b3b7e49cec842034")
+	c.Assert(hashes[2].String(), Equals, "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391")
 }
 
-func containsObject(objs []fixtureObject, hash core.Hash) bool {
-	for _, o := range objs {
-		if strings.ToLower(o.hash) == strings.ToLower(hash.String()) {
-			return true
-		}
+func (s *SuiteDotGit) TestObjectsWithGarbage(c *C) {
+	dir := s.newFixtureDir(c, "unpacked-dummy")
+
+	hashes, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hashes, HasLen, 3)
+	c.Assert(hashes[0].String(), Equals, "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e")
+	c.Assert(hashes[1].String(), Equals, "5efb9bc29c482e023e40e0a2b3b7e49cec842034")
+	c.Assert(hashes[2].String(), Equals, "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391")
+}
+
+func (s *SuiteDotGit) TestObjectsNoPackage(c *C) {
+	dir := s.newFixtureDir(c, "empty")
+
+	hashes, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hashes, HasLen, 0)
+}
+
+func (s *SuiteDotGit) TestObjectsNoObjects(c *C) {
+	dir := s.newFixtureDir(c, "no-packfile-no-idx")
+
+	hashes, err := dir.Objects()
+	c.Assert(err, IsNil)
+	c.Assert(hashes, HasLen, 0)
+}
+
+func (s *SuiteDotGit) TestObject(c *C) {
+	dir := s.newFixtureDir(c, "unpacked")
+
+	hash := core.NewHash("1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e")
+	file, err := dir.Object(hash)
+	c.Assert(err, IsNil)
+	c.Assert(file.Filename(), Not(Equals), "")
+}
+
+func (s *SuiteDotGit) TestObjectNotFound(c *C) {
+	dir := s.newFixtureDir(c, "unpacked")
+
+	hash := core.NewHash("not-found-object")
+	file, err := dir.Object(hash)
+	c.Assert(err, NotNil)
+	c.Assert(file, IsNil)
+}
+
+func (s *SuiteDotGit) TestNewObjectPack(c *C) {
+	dir, err := ioutil.TempDir("", "example")
+	if err != nil {
+		log.Fatal(err)
 	}
-	return false
+
+	dot := New(fs.NewOSClient(dir))
+
+	r, err := os.Open("../../../../formats/packfile/fixtures/git-fixture.ofs-delta")
+	c.Assert(err, IsNil)
+
+	w, err := dot.NewObjectPack()
+	c.Assert(err, IsNil)
+
+	n, err := io.Copy(w, r)
+	c.Assert(err, IsNil)
+	c.Check(n, Equals, int64(85300))
+
+	c.Assert(w.Close(), IsNil)
 }
diff --git a/storage/filesystem/internal/dotgit/refs.go b/storage/filesystem/internal/dotgit/refs.go
index c32a7e5..ca11f6c 100644
--- a/storage/filesystem/internal/dotgit/refs.go
+++ b/storage/filesystem/internal/dotgit/refs.go
@@ -29,8 +29,7 @@ const (
 )
 
 func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) {
-	path := d.fs.Join(d.path, packedRefsPath)
-	f, err := d.fs.Open(path)
+	f, err := d.fs.Open(packedRefsPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
@@ -80,7 +79,7 @@ func (d *DotGit) addRefsFromRefDir(refs *[]*core.Reference) error {
 }
 
 func (d *DotGit) walkReferencesTree(refs *[]*core.Reference, relPath string) error {
-	files, err := d.fs.ReadDir(d.fs.Join(d.path, relPath))
+	files, err := d.fs.ReadDir(relPath)
 	if err != nil {
 		return err
 	}
@@ -95,7 +94,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*core.Reference, relPath string) err
 			continue
 		}
 
-		ref, err := d.readReferenceFile(d.path, newRelPath)
+		ref, err := d.readReferenceFile(".", newRelPath)
 		if err != nil {
 			return err
 		}
@@ -109,7 +108,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*core.Reference, relPath string) err
 }
 
 func (d *DotGit) addRefFromHEAD(refs *[]*core.Reference) error {
-	ref, err := d.readReferenceFile(d.path, "HEAD")
+	ref, err := d.readReferenceFile(".", "HEAD")
 	if err != nil {
 		return err
 	}
diff --git a/storage/filesystem/internal/index/index.go b/storage/filesystem/internal/index/index.go
index 233dcbd..d33533e 100644
--- a/storage/filesystem/internal/index/index.go
+++ b/storage/filesystem/internal/index/index.go
@@ -34,50 +34,49 @@ func NewFromIdx(r io.Reader) (Index, error) {
 }
 
 // NewFrompackfile returns a new index from a packfile reader.
-func NewFromPackfile(rs io.ReadSeeker) (Index, error) {
+func NewFromPackfile(rs io.ReadSeeker) (Index, core.Hash, error) {
 	s := packfile.NewSeekable(rs)
 	return newFromPackfile(rs, s)
 }
 
-func NewFromPackfileInMemory(rs io.Reader) (Index, error) {
+func NewFromPackfileInMemory(rs io.Reader) (Index, core.Hash, error) {
 	s := packfile.NewStream(rs)
 	return newFromPackfile(rs, s)
 }
 
-func newFromPackfile(r io.Reader, s packfile.ReadRecaller) (Index, error) {
+func newFromPackfile(r io.Reader, s packfile.ReadRecaller) (Index, core.Hash, error) {
 	index := make(Index)
 
 	p := packfile.NewParser(s)
 	count, err := p.ReadHeader()
 	if err != nil {
-		return nil, err
+		return nil, core.ZeroHash, err
 	}
 
 	for i := 0; i < int(count); i++ {
 		offset, err := s.Offset()
 		if err != nil {
-			return nil, err
+			return nil, core.ZeroHash, err
 		}
 
 		obj := &core.MemoryObject{}
 		if err := p.FillObject(obj); err != nil {
-			return nil, err
+			return nil, core.ZeroHash, err
 		}
 
 		err = s.Remember(offset, obj)
 		if err != nil {
-			return nil, err
+			return nil, core.ZeroHash, err
 		}
 
 		if err = index.Set(obj.Hash(), offset); err != nil {
-			return nil, err
+			return nil, core.ZeroHash, err
 		}
 	}
 
 	//The trailer records 20-byte SHA-1 checksum of all of the above.
-	p.ReadHash()
-
-	return index, nil
+	hash, err := p.ReadHash()
+	return index, hash, err
 }
 
 // Get returns the offset that an object has the packfile.
diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go
index fbcc9ae..c66e1ff 100644
--- a/storage/filesystem/object.go
+++ b/storage/filesystem/object.go
@@ -38,52 +38,6 @@ func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
 	return newPackWrite(o, file), nil
 }
 
-type packWriter struct {
-	writer     io.Writer
-	pipeReader io.ReadCloser
-	pipeWriter io.WriteCloser
-	file       io.Writer
-	result     chan error
-}
-
-func newPackWrite(o *ObjectStorage, file io.Writer) io.WriteCloser {
-	r, w := io.Pipe()
-
-	ch := make(chan error)
-	go func(r io.ReadCloser) {
-		defer r.Close()
-		index, err := index.NewFromPackfileInMemory(r)
-		o.index = index
-
-		ch <- err
-	}(r)
-
-	return &packWriter{
-		writer:     io.MultiWriter(w, file),
-		pipeReader: r,
-		pipeWriter: w,
-		file:       file,
-		result:     ch,
-	}
-
-}
-
-func (w *packWriter) Write(p []byte) (int, error) {
-	return w.writer.Write(p)
-}
-
-func (w *packWriter) Close() error {
-	defer func() {
-		close(w.result)
-	}()
-
-	if err := w.pipeWriter.Close(); err != nil {
-		return err
-	}
-
-	return <-w.result
-}
-
 // Set adds a new object to the storage. As this functionality is not
 // yet supported, this method always returns a "not implemented yet"
 // error an zero hash.
@@ -106,12 +60,7 @@ func (s *ObjectStorage) Get(t core.ObjectType, h core.Hash) (core.Object, error)
 }
 
 func (s *ObjectStorage) getFromUnpacked(t core.ObjectType, h core.Hash) (obj core.Object, err error) {
-	fs, path, err := s.dir.Objectfile(h)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := fs.Open(path)
+	f, err := s.dir.Object(h)
 	if err != nil {
 		return nil, err
 	}
@@ -153,16 +102,16 @@ func (s *ObjectStorage) getFromPackfile(t core.ObjectType, h core.Hash) (obj cor
 		return nil, err
 	}
 
-	fs, path, err := s.dir.Packfile()
+	packs, err := s.dir.ObjectsPacks()
 	if err != nil {
 		return nil, err
 	}
 
-	f, err := fs.Open(path)
-	if err != nil {
-		return nil, err
+	if len(packs) == 0 {
+		return nil, nil
 	}
 
+	f, _, err := s.dir.ObjectPack(packs[0].Name())
 	defer func() {
 		errClose := f.Close()
 		if err == nil {
@@ -195,7 +144,7 @@ func (s *ObjectStorage) getFromPackfile(t core.ObjectType, h core.Hash) (obj cor
 func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	var objects []core.Object
 
-	_, hashes, err := s.dir.Objectfiles()
+	hashes, err := s.dir.Objects()
 	if err != nil {
 		return nil, err
 	}
@@ -223,8 +172,17 @@ func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
 	return core.NewObjectSliceIter(objects), nil
 }
 
-func buildIndex(dir *dotgit.DotGit) (index.Index, error) {
-	fs, idxfile, err := dir.Idxfile()
+func buildIndex(fs fs.Filesystem, dir *dotgit.DotGit) (index.Index, error) {
+	packs, err := dir.ObjectsPacks()
+	if err != nil {
+		return nil, err
+	}
+
+	if len(packs) == 0 {
+		return nil, nil
+	}
+
+	_, _, err = dir.ObjectPack(packs[0].Name())
 	if err != nil {
 		if err == dotgit.ErrIdxNotFound {
 			return buildIndexFromPackfile(dir)
@@ -232,16 +190,20 @@ func buildIndex(dir *dotgit.DotGit) (index.Index, error) {
 		return nil, err
 	}
 
-	return buildIndexFromIdxfile(fs, idxfile)
+	return buildIndexFromIdxfile(fs, "")
 }
 
 func buildIndexFromPackfile(dir *dotgit.DotGit) (index.Index, error) {
-	fs, packfile, err := dir.Packfile()
+	packs, err := dir.ObjectsPacks()
 	if err != nil {
 		return nil, err
 	}
 
-	f, err := fs.Open(packfile)
+	if len(packs) == 0 {
+		return nil, nil
+	}
+
+	f, _, err := dir.ObjectPack(packs[0].Name())
 	if err != nil {
 		return nil, err
 	}
@@ -256,7 +218,7 @@ func buildIndexFromPackfile(dir *dotgit.DotGit) (index.Index, error) {
 	return index.NewFromPackfile(f)
 }
 
-func buildIndexFromIdxfile(fs fs.FS, path string) (index.Index, error) {
+func buildIndexFromIdxfile(fs fs.Filesystem, path string) (index.Index, error) {
 	f, err := fs.Open(path)
 	if err != nil {
 		return nil, err
@@ -271,6 +233,7 @@ func buildIndexFromIdxfile(fs fs.FS, path string) (index.Index, error) {
 
 	return index.NewFromIdx(f)
 }
+
 func (o *ObjectStorage) Begin() core.TxObjectStorage {
 	return &TxObjectStorage{}
 }
diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go
index 78ec20c..e9cfa4c 100644
--- a/storage/filesystem/object_test.go
+++ b/storage/filesystem/object_test.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 
@@ -90,9 +91,9 @@ func (s *FsSuite) TestHashNotFound(c *C) {
 
 func (s *FsSuite) newObjectStorage(c *C, fixtureName string) core.ObjectStorage {
 	path := fixture(fixtureName, c)
-	fs := fs.NewOS()
+	fs := fs.NewOSClient(filepath.Join(path, ".git/"))
 
-	store, err := NewStorage(fs, fs.Join(path, ".git/"))
+	store, err := NewStorage(fs)
 	c.Assert(err, IsNil)
 
 	return store.ObjectStorage()
@@ -108,8 +109,8 @@ func (s *FsSuite) TestGetCompareWithMemoryStorage(c *C) {
 		com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)",
 			i, fixId, path)
 
-		fs := fs.NewOS()
-		gitPath := fs.Join(path, ".git/")
+		gitPath := filepath.Join(path, ".git/")
+		fs := fs.NewOSClient(gitPath)
 
 		memSto, err := memStorageFromGitDir(fs, gitPath)
 		c.Assert(err, IsNil, com)
@@ -123,18 +124,19 @@ func (s *FsSuite) TestGetCompareWithMemoryStorage(c *C) {
 	}
 }
 
-func memStorageFromGitDir(fs fs.FS, path string) (core.ObjectStorage, error) {
-	dir, err := dotgit.New(fs, path)
+func memStorageFromGitDir(fs fs.Filesystem, path string) (core.ObjectStorage, error) {
+	dir := dotgit.New(fs)
+	packs, err := dir.ObjectsPacks()
 	if err != nil {
 		return nil, err
 	}
 
-	fs, packfilePath, err := dir.Packfile()
-	if err != nil {
-		return nil, err
+	if len(packs) == 0 {
+		return nil, nil
 	}
 
-	f, err := fs.Open(packfilePath)
+	fmt.Println(packs[0].Name())
+	f, _, err := dir.ObjectPack(packs[0].Name())
 	if err != nil {
 		return nil, err
 	}
@@ -238,8 +240,8 @@ func (s *FsSuite) TestIterCompareWithMemoryStorage(c *C) {
 		com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)",
 			i, fixId, path)
 
-		fs := fs.NewOS()
-		gitPath := fs.Join(path, ".git/")
+		gitPath := filepath.Join(path, ".git/")
+		fs := fs.NewOSClient(gitPath)
 
 		memSto, err := memStorageFromDirPath(fs, gitPath)
 		c.Assert(err, IsNil, com)
@@ -266,23 +268,25 @@ func (s *FsSuite) TestIterCompareWithMemoryStorage(c *C) {
 	}
 }
 
-func memStorageFromDirPath(fs fs.FS, path string) (core.ObjectStorage, error) {
-	dir, err := dotgit.New(fs, path)
+func memStorageFromDirPath(fs fs.Filesystem, path string) (core.ObjectStorage, error) {
+	dir := dotgit.New(fs)
+	packs, err := dir.ObjectsPacks()
 	if err != nil {
+		fmt.Println("errr", err)
 		return nil, err
 	}
 
-	fs, packfilePath, err := dir.Packfile()
-	if err != nil {
-		return nil, err
+	if len(packs) == 0 {
+		return nil, nil
 	}
 
-	sto := memory.NewStorage()
-	f, err := fs.Open(packfilePath)
+	f, _, err := dir.ObjectPack(packs[0].Name())
 	if err != nil {
 		return nil, err
 	}
 
+	sto := memory.NewStorage()
+
 	r := packfile.NewStream(f)
 	d := packfile.NewDecoder(r, sto.ObjectStorage())
 	err = d.Decode()
diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go
index a0cb0b1..b0eaf6f 100644
--- a/storage/filesystem/storage.go
+++ b/storage/filesystem/storage.go
@@ -10,19 +10,15 @@ import (
 
 type Storage struct {
 	dir *dotgit.DotGit
+	fs  fs.Filesystem
 
 	o *ObjectStorage
 	r *ReferenceStorage
 	c *ConfigStorage
 }
 
-func NewStorage(fs fs.FS, path string) (*Storage, error) {
-	dir, err := dotgit.New(fs, path)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Storage{dir: dir}, nil
+func NewStorage(fs fs.Filesystem) (*Storage, error) {
+	return &Storage{dir: dotgit.New(fs), fs: fs}, nil
 }
 
 func (s *Storage) ObjectStorage() core.ObjectStorage {
@@ -31,7 +27,7 @@ func (s *Storage) ObjectStorage() core.ObjectStorage {
 	}
 
 	//TODO: error being ignored
-	i, _ := buildIndex(s.dir)
+	i, _ := buildIndex(s.fs, s.dir)
 	return &ObjectStorage{dir: s.dir, index: i}
 }
 
diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go
index 3cb7dd8..f1f18f9 100644
--- a/storage/filesystem/storage_test.go
+++ b/storage/filesystem/storage_test.go
@@ -4,8 +4,6 @@ import (
 	"testing"
 
 	. "gopkg.in/check.v1"
-	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
-	"gopkg.in/src-d/go-git.v4/utils/fs"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -13,9 +11,3 @@ func Test(t *testing.T) { TestingT(t) }
 type StorageSuite struct{}
 
 var _ = Suite(&StorageSuite{})
-
-func (s *StorageSuite) TestNewErrorNotFound(c *C) {
-	fs := fs.NewOS()
-	_, err := NewStorage(fs, "not_found/.git")
-	c.Assert(err, Equals, dotgit.ErrNotFound)
-}