utils: fs, new memory filesystem (#108)

* utils: fs, new memory filesystem

* utils: fs, renamed os.NewOS to os.New

* utils: fs, memory changes requested by @alcortes

12 files changed, 442 insertions, 12 deletions

diff --git a/fixtures/fixtures.go b/fixtures/fixtures.go
index 7bc82e8..351c28a 100644
--- a/fixtures/fixtures.go
+++ b/fixtures/fixtures.go
@@ -181,7 +181,7 @@ func (f *Fixture) DotGit() fs.Filesystem {
 	}
 
 	folders = append(folders, path)
-	return osfs.NewOS(path)
+	return osfs.New(path)
 }
 
 type Fixtures []*Fixture
diff --git a/remote_test.go b/remote_test.go
index 7aaaac7..37539af 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -101,7 +101,7 @@ func (s *RemoteSuite) TestFetchObjectStorageWriter(c *C) {
 	defer os.RemoveAll(dir) // clean up
 
 	var sto Storage
-	sto, err = filesystem.NewStorage(osfs.NewOS(dir))
+	sto, err = filesystem.NewStorage(osfs.New(dir))
 	c.Assert(err, IsNil)
 
 	r := newRemote(sto, &config.RemoteConfig{Name: "foo", URL: RepositoryFixture})
diff --git a/repository.go b/repository.go
index 2e2e57e..cd94159 100644
--- a/repository.go
+++ b/repository.go
@@ -33,7 +33,7 @@ func NewMemoryRepository() *Repository {
 // based on a fs.OS, if you want to use a custom one you need to use the function
 // NewRepository and build you filesystem.Storage
 func NewFilesystemRepository(path string) (*Repository, error) {
-	s, err := filesystem.NewStorage(osfs.NewOS(path))
+	s, err := filesystem.NewStorage(osfs.New(path))
 	if err != nil {
 		return nil, err
 	}
diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go
index b52e6a1..2eb0ab9 100644
--- a/storage/filesystem/config_test.go
+++ b/storage/filesystem/config_test.go
@@ -25,7 +25,7 @@ func (s *ConfigSuite) SetUpTest(c *C) {
 	tmp, err := ioutil.TempDir("", "go-git-filestystem-config")
 	c.Assert(err, IsNil)
 
-	s.dir = dotgit.New(os.NewOS(tmp))
+	s.dir = dotgit.New(os.New(tmp))
 	s.path = tmp
 }
 
diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go
index a4cce09..a5d25e3 100644
--- a/storage/filesystem/internal/dotgit/dotgit_test.go
+++ b/storage/filesystem/internal/dotgit/dotgit_test.go
@@ -27,7 +27,7 @@ func (s *SuiteDotGit) TestSetRefs(c *C) {
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(tmp)
 
-	fs := osfs.NewOS(tmp)
+	fs := osfs.New(tmp)
 	dir := New(fs)
 
 	err = dir.SetRef(core.NewReferenceFromStrings(
@@ -189,7 +189,7 @@ func (s *SuiteDotGit) TestNewObject(c *C) {
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(tmp)
 
-	fs := osfs.NewOS(tmp)
+	fs := osfs.New(tmp)
 	dir := New(fs)
 	w, err := dir.NewObject()
 	c.Assert(err, IsNil)
diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/internal/dotgit/writers_test.go
index 2dff4e9..23d3a0e 100644
--- a/storage/filesystem/internal/dotgit/writers_test.go
+++ b/storage/filesystem/internal/dotgit/writers_test.go
@@ -24,7 +24,7 @@ func (s *SuiteDotGit) TestNewObjectPack(c *C) {
 
 	defer os.RemoveAll(dir)
 
-	fs := osfs.NewOS(dir)
+	fs := osfs.New(dir)
 	dot := New(fs)
 
 	w, err := dot.NewObjectPack()
diff --git a/utils/binary/read_test.go b/utils/binary/read_test.go
index 6579ffb..ad5ef29 100644
--- a/utils/binary/read_test.go
+++ b/utils/binary/read_test.go
@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	. "gopkg.in/check.v1"
-	"gopkg.in/src-d/go-git.v3/core"
+	"gopkg.in/src-d/go-git.v4/core"
 )
 
 func Test(t *testing.T) { TestingT(t) }
diff --git a/utils/fs/memory/memory.go b/utils/fs/memory/memory.go
new file mode 100644
index 0000000..08a9599
--- /dev/null
+++ b/utils/fs/memory/memory.go
@@ -0,0 +1,354 @@
+package memory
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/src-d/go-git.v4/utils/fs"
+)
+
+const separator = '/'
+
+// Memory a very convenient filesystem based on memory files
+type Memory struct {
+	base      string
+	s         *storage
+	tempCount int
+}
+
+//New returns a new Memory filesystem
+func New() *Memory {
+	return &Memory{
+		base: "/",
+		s:    &storage{make(map[string]*file, 0)},
+	}
+}
+
+func (fs *Memory) Create(filename string) (fs.File, error) {
+	return fs.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0)
+}
+
+func (fs *Memory) Open(filename string) (fs.File, error) {
+	return fs.OpenFile(filename, os.O_RDONLY, 0)
+}
+
+func (fs *Memory) OpenFile(filename string, flag int, perm os.FileMode) (fs.File, error) {
+	fullpath := fs.Join(fs.base, filename)
+	f, ok := fs.s.files[fullpath]
+	if !ok && !isCreate(flag) {
+		return nil, os.ErrNotExist
+	}
+
+	if f == nil {
+		fs.s.files[fullpath] = newFile(fs.base, fullpath, flag)
+		return fs.s.files[fullpath], nil
+	}
+
+	n := newFile(fs.base, fullpath, flag)
+	n.content = f.content
+
+	if isAppend(flag) {
+		n.position = int64(n.content.Len())
+	}
+
+	if isTruncate(flag) {
+		n.content.Truncate()
+	}
+
+	return n, nil
+}
+
+func (fs *Memory) Stat(filename string) (fs.FileInfo, error) {
+	fullpath := fs.Join(fs.base, filename)
+
+	if _, ok := fs.s.files[filename]; ok {
+		return newFileInfo(fs.base, fullpath, fs.s.files[filename].content.Len()), nil
+	}
+
+	info, err := fs.ReadDir(filename)
+	if err == nil && len(info) != 0 {
+		return newFileInfo(fs.base, fullpath, len(info)), nil
+	}
+
+	return nil, os.ErrNotExist
+}
+
+func (fs *Memory) ReadDir(base string) (entries []fs.FileInfo, err error) {
+	base = fs.Join(fs.base, base)
+
+	appendedDirs := make(map[string]bool, 0)
+	for fullpath, f := range fs.s.files {
+		if !strings.HasPrefix(fullpath, base) {
+			continue
+		}
+
+		fullpath, _ = filepath.Rel(base, fullpath)
+		parts := strings.Split(fullpath, string(separator))
+
+		if len(parts) == 1 {
+			entries = append(entries, newFileInfo(fs.base, fullpath, f.content.Len()))
+			continue
+		}
+
+		if _, ok := appendedDirs[parts[0]]; ok {
+			continue
+		}
+
+		entries = append(entries, &fileInfo{name: parts[0], isDir: true})
+		appendedDirs[parts[0]] = true
+	}
+
+	return
+}
+
+var maxTempFiles = 1024 * 4
+
+func (fs *Memory) TempFile(dir, prefix string) (fs.File, error) {
+	var fullpath string
+	for {
+		if fs.tempCount >= maxTempFiles {
+			return nil, errors.New("max. number of tempfiles reached")
+		}
+
+		fullpath = fs.getTempFilename(dir, prefix)
+		if _, ok := fs.s.files[fullpath]; !ok {
+			break
+		}
+	}
+
+	return fs.Create(fullpath)
+}
+
+func (fs *Memory) getTempFilename(dir, prefix string) string {
+	fs.tempCount++
+	filename := fmt.Sprintf("%s_%d_%d", prefix, fs.tempCount, time.Now().UnixNano())
+	return fs.Join(fs.base, dir, filename)
+}
+
+func (fs *Memory) Rename(from, to string) error {
+	from = fs.Join(fs.base, from)
+	to = fs.Join(fs.base, to)
+
+	if _, ok := fs.s.files[from]; !ok {
+		return os.ErrNotExist
+	}
+
+	fs.s.files[to] = fs.s.files[from]
+	fs.s.files[to].BaseFilename = to
+	delete(fs.s.files, from)
+
+	return nil
+}
+
+func (fs *Memory) Remove(filename string) error {
+	fullpath := fs.Join(fs.base, filename)
+	if _, ok := fs.s.files[fullpath]; !ok {
+		return os.ErrNotExist
+	}
+
+	delete(fs.s.files, fullpath)
+	return nil
+}
+
+func (fs *Memory) Join(elem ...string) string {
+	return filepath.Join(elem...)
+}
+
+func (fs *Memory) Dir(path string) fs.Filesystem {
+	return &Memory{
+		base: fs.Join(fs.base, path),
+		s:    fs.s,
+	}
+}
+
+func (fs *Memory) Base() string {
+	return fs.base
+}
+
+type file struct {
+	fs.BaseFile
+
+	content  *content
+	position int64
+	flag     int
+}
+
+func newFile(base, fullpath string, flag int) *file {
+	filename, _ := filepath.Rel(base, fullpath)
+
+	return &file{
+		BaseFile: fs.BaseFile{BaseFilename: filename},
+		content:  &content{},
+		flag:     flag,
+	}
+}
+
+func (f *file) Read(b []byte) (int, error) {
+	if f.IsClosed() {
+		return 0, fs.ErrClosed
+	}
+
+	if !isReadAndWrite(f.flag) && !isReadOnly(f.flag) {
+		return 0, errors.New("read not supported")
+	}
+
+	n, err := f.content.ReadAt(b, f.position)
+	f.position += int64(n)
+
+	return n, err
+}
+
+func (f *file) Seek(offset int64, whence int) (int64, error) {
+	if f.IsClosed() {
+		return 0, fs.ErrClosed
+	}
+
+	switch whence {
+	case io.SeekCurrent:
+		f.position += offset
+	case io.SeekStart:
+		f.position = offset
+	case io.SeekEnd:
+		f.position = int64(f.content.Len()) - offset
+	}
+
+	return f.position, nil
+}
+
+func (f *file) Write(p []byte) (int, error) {
+	if f.IsClosed() {
+		return 0, fs.ErrClosed
+	}
+
+	if !isReadAndWrite(f.flag) && !isWriteOnly(f.flag) {
+		return 0, errors.New("write not supported")
+	}
+
+	n, err := f.content.WriteAt(p, f.position)
+	f.position += int64(n)
+
+	return n, err
+}
+
+func (f *file) Close() error {
+	if f.IsClosed() {
+		return errors.New("file already closed")
+	}
+
+	f.Closed = true
+	return nil
+}
+
+func (f *file) Open() error {
+	f.Closed = false
+	return nil
+}
+
+type fileInfo struct {
+	name  string
+	size  int
+	isDir bool
+}
+
+func newFileInfo(base, fullpath string, size int) *fileInfo {
+	filename, _ := filepath.Rel(base, fullpath)
+
+	return &fileInfo{
+		name: filename,
+		size: size,
+	}
+}
+
+func (fi *fileInfo) Name() string {
+	return fi.name
+}
+
+func (fi *fileInfo) Size() int64 {
+	return int64(fi.size)
+}
+
+func (fi *fileInfo) Mode() os.FileMode {
+	return os.FileMode(0)
+}
+
+func (*fileInfo) ModTime() time.Time {
+	return time.Now()
+}
+
+func (fi *fileInfo) IsDir() bool {
+	return fi.isDir
+}
+
+func (*fileInfo) Sys() interface{} {
+	return nil
+}
+
+type storage struct {
+	files map[string]*file
+}
+
+type content struct {
+	bytes []byte
+}
+
+func (c *content) WriteAt(p []byte, off int64) (int, error) {
+	prev := len(c.bytes)
+	c.bytes = append(c.bytes[:off], p...)
+	if len(c.bytes) < prev {
+		c.bytes = c.bytes[:prev]
+	}
+
+	return len(p), nil
+}
+
+func (c *content) ReadAt(b []byte, off int64) (int, error) {
+	size := int64(len(c.bytes))
+	if off >= size {
+		return 0, io.EOF
+	}
+
+	l := int64(len(b))
+	if off+l > size {
+		l = size - off
+	}
+
+	n := copy(b, c.bytes[off:off+l])
+	return n, nil
+}
+
+func (c *content) Truncate() {
+	c.bytes = make([]byte, 0)
+}
+
+func (c *content) Len() int {
+	return len(c.bytes)
+}
+
+func isCreate(flag int) bool {
+	return flag&os.O_CREATE != 0
+}
+
+func isAppend(flag int) bool {
+	return flag&os.O_APPEND != 0
+}
+
+func isTruncate(flag int) bool {
+	return flag&os.O_TRUNC != 0
+}
+
+func isReadAndWrite(flag int) bool {
+	return flag&os.O_RDWR != 0
+}
+
+func isReadOnly(flag int) bool {
+	return flag == os.O_RDONLY
+}
+
+func isWriteOnly(flag int) bool {
+	return flag&os.O_WRONLY != 0
+}
diff --git a/utils/fs/memory/memory_test.go b/utils/fs/memory/memory_test.go
new file mode 100644
index 0000000..4645fe7
--- /dev/null
+++ b/utils/fs/memory/memory_test.go
@@ -0,0 +1,33 @@
+package memory
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/utils/fs/test"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MemorySuite struct {
+	test.FilesystemSuite
+	path string
+}
+
+var _ = Suite(&MemorySuite{})
+
+func (s *MemorySuite) SetUpTest(c *C) {
+	s.FilesystemSuite.Fs = New()
+}
+
+func (s *MemorySuite) TestTempFileMaxTempFiles(c *C) {
+	for i := 0; i < maxTempFiles; i++ {
+		f, err := s.FilesystemSuite.Fs.TempFile("", "")
+		c.Assert(err, IsNil)
+		c.Assert(f, NotNil)
+	}
+
+	f, err := s.FilesystemSuite.Fs.TempFile("", "")
+	c.Assert(err, NotNil)
+	c.Assert(f, IsNil)
+}
diff --git a/utils/fs/os/os.go b/utils/fs/os/os.go
index 02395d6..3ff825e 100644
--- a/utils/fs/os/os.go
+++ b/utils/fs/os/os.go
@@ -14,8 +14,8 @@ type OS struct {
 	base string
 }
 
-// NewOS returns a new OS filesystem
-func NewOS(baseDir string) *OS {
+// New returns a new OS filesystem
+func New(baseDir string) *OS {
 	return &OS{
 		base: baseDir,
 	}
@@ -139,7 +139,7 @@ func (fs *OS) Join(elem ...string) string {
 // Dir returns a new Filesystem from the same type of fs using as baseDir the
 // given path
 func (fs *OS) Dir(path string) fs.Filesystem {
-	return NewOS(fs.Join(fs.base, path))
+	return New(fs.Join(fs.base, path))
 }
 
 // Base returns the base path of the filesytem
diff --git a/utils/fs/os/os_test.go b/utils/fs/os/os_test.go
index 02a0a6c..756e284 100644
--- a/utils/fs/os/os_test.go
+++ b/utils/fs/os/os_test.go
@@ -21,7 +21,7 @@ var _ = Suite(&OSSuite{})
 
 func (s *OSSuite) SetUpTest(c *C) {
 	s.path, _ = ioutil.TempDir(stdos.TempDir(), "go-git-os-fs-test")
-	s.FilesystemSuite.Fs = os.NewOS(s.path)
+	s.FilesystemSuite.Fs = os.New(s.path)
 }
 func (s *OSSuite) TearDownTest(c *C) {
 	err := stdos.RemoveAll(s.path)
diff --git a/utils/fs/test/fs_suite.go b/utils/fs/test/fs_suite.go
index 4c0fd09..74301f5 100644
--- a/utils/fs/test/fs_suite.go
+++ b/utils/fs/test/fs_suite.go
@@ -2,6 +2,7 @@ package test
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -184,6 +185,32 @@ func (s *FilesystemSuite) testReadClose(c *C, f File, content string) {
 	c.Assert(f.Close(), IsNil)
 }
 
+func (s *FilesystemSuite) TestFileReadSeek(c *C) {
+	f, err := s.Fs.Create("foo")
+	c.Assert(err, IsNil)
+
+	n, err := f.Write([]byte("0123456789abcdefghijklmnopqrstuvwxyz"))
+	c.Assert(err, IsNil)
+	c.Assert(n, Equals, 36)
+
+	p, err := f.Seek(10, io.SeekStart)
+	c.Assert(err, IsNil)
+	c.Assert(int(p), Equals, 10)
+
+	all, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Assert(string(all), Equals, "abcdefghijklmnopqrstuvwxyz")
+	c.Assert(f.Close(), IsNil)
+}
+
+func (s *FilesystemSuite) TestFileCloseTwice(c *C) {
+	f, err := s.Fs.Create("foo")
+	c.Assert(err, IsNil)
+
+	c.Assert(f.Close(), IsNil)
+	c.Assert(f.Close(), NotNil)
+}
+
 func (s *FilesystemSuite) TestReadDirAndDir(c *C) {
 	files := []string{"foo", "bar", "qux/baz", "qux/qux"}
 	for _, name := range files {
@@ -268,6 +295,20 @@ func (s *FilesystemSuite) TestTempFileFullWithPath(c *C) {
 	c.Assert(strings.HasPrefix(f.Filename(), s.Fs.Join("foo", "bar")), Equals, true)
 }
 
+func (s *FilesystemSuite) TestOpenAndWrite(c *C) {
+	f, err := s.Fs.Create("foo")
+	c.Assert(err, IsNil)
+	c.Assert(f.Close(), IsNil)
+
+	foo, err := s.Fs.Open("foo")
+	c.Assert(foo, NotNil)
+	c.Assert(err, IsNil)
+
+	n, err := foo.Write([]byte("foo"))
+	c.Assert(err, NotNil)
+	c.Assert(n, Equals, 0)
+}
+
 func (s *FilesystemSuite) TestOpenAndStat(c *C) {
 	f, err := s.Fs.Create("foo")
 	c.Assert(err, IsNil)
@@ -299,6 +340,8 @@ func (s *FilesystemSuite) TestRemoveNonExisting(c *C) {
 
 func (s *FilesystemSuite) TestRemoveTempFile(c *C) {
 	f, err := s.Fs.TempFile("test-dir", "test-prefix")
+	c.Assert(err, IsNil)
+
 	fn := f.Filename()
 	c.Assert(err, IsNil)
 	c.Assert(f.Close(), IsNil)