From 3a85c05bcf82ac4a6d48165bd644d81622fae80a Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Fri, 4 Nov 2016 12:29:00 +0100 Subject: utils: fs, new memory filesystem (#108) * utils: fs, new memory filesystem * utils: fs, renamed os.NewOS to os.New * utils: fs, memory changes requested by @alcortes --- utils/binary/read_test.go | 2 +- utils/fs/memory/memory.go | 354 +++++++++++++++++++++++++++++++++++++++++ utils/fs/memory/memory_test.go | 33 ++++ utils/fs/os/os.go | 6 +- utils/fs/os/os_test.go | 2 +- utils/fs/test/fs_suite.go | 43 +++++ 6 files changed, 435 insertions(+), 5 deletions(-) create mode 100644 utils/fs/memory/memory.go create mode 100644 utils/fs/memory/memory_test.go (limited to 'utils') diff --git a/utils/binary/read_test.go b/utils/binary/read_test.go index 6579ffb..ad5ef29 100644 --- a/utils/binary/read_test.go +++ b/utils/binary/read_test.go @@ -6,7 +6,7 @@ import ( "testing" . "gopkg.in/check.v1" - "gopkg.in/src-d/go-git.v3/core" + "gopkg.in/src-d/go-git.v4/core" ) func Test(t *testing.T) { TestingT(t) } diff --git a/utils/fs/memory/memory.go b/utils/fs/memory/memory.go new file mode 100644 index 0000000..08a9599 --- /dev/null +++ b/utils/fs/memory/memory.go @@ -0,0 +1,354 @@ +package memory + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/src-d/go-git.v4/utils/fs" +) + +const separator = '/' + +// Memory a very convenient filesystem based on memory files +type Memory struct { + base string + s *storage + tempCount int +} + +//New returns a new Memory filesystem +func New() *Memory { + return &Memory{ + base: "/", + s: &storage{make(map[string]*file, 0)}, + } +} + +func (fs *Memory) Create(filename string) (fs.File, error) { + return fs.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) +} + +func (fs *Memory) Open(filename string) (fs.File, error) { + return fs.OpenFile(filename, os.O_RDONLY, 0) +} + +func (fs *Memory) OpenFile(filename string, flag int, perm os.FileMode) (fs.File, error) { + fullpath := fs.Join(fs.base, filename) + f, ok := fs.s.files[fullpath] + if !ok && !isCreate(flag) { + return nil, os.ErrNotExist + } + + if f == nil { + fs.s.files[fullpath] = newFile(fs.base, fullpath, flag) + return fs.s.files[fullpath], nil + } + + n := newFile(fs.base, fullpath, flag) + n.content = f.content + + if isAppend(flag) { + n.position = int64(n.content.Len()) + } + + if isTruncate(flag) { + n.content.Truncate() + } + + return n, nil +} + +func (fs *Memory) Stat(filename string) (fs.FileInfo, error) { + fullpath := fs.Join(fs.base, filename) + + if _, ok := fs.s.files[filename]; ok { + return newFileInfo(fs.base, fullpath, fs.s.files[filename].content.Len()), nil + } + + info, err := fs.ReadDir(filename) + if err == nil && len(info) != 0 { + return newFileInfo(fs.base, fullpath, len(info)), nil + } + + return nil, os.ErrNotExist +} + +func (fs *Memory) ReadDir(base string) (entries []fs.FileInfo, err error) { + base = fs.Join(fs.base, base) + + appendedDirs := make(map[string]bool, 0) + for fullpath, f := range fs.s.files { + if !strings.HasPrefix(fullpath, base) { + continue + } + + fullpath, _ = filepath.Rel(base, fullpath) + parts := strings.Split(fullpath, string(separator)) + + if len(parts) == 1 { + entries = append(entries, newFileInfo(fs.base, fullpath, f.content.Len())) + continue + } + + if _, ok := appendedDirs[parts[0]]; ok { + continue + } + + entries = append(entries, &fileInfo{name: parts[0], isDir: true}) + appendedDirs[parts[0]] = true + } + + return +} + +var maxTempFiles = 1024 * 4 + +func (fs *Memory) TempFile(dir, prefix string) (fs.File, error) { + var fullpath string + for { + if fs.tempCount >= maxTempFiles { + return nil, errors.New("max. number of tempfiles reached") + } + + fullpath = fs.getTempFilename(dir, prefix) + if _, ok := fs.s.files[fullpath]; !ok { + break + } + } + + return fs.Create(fullpath) +} + +func (fs *Memory) getTempFilename(dir, prefix string) string { + fs.tempCount++ + filename := fmt.Sprintf("%s_%d_%d", prefix, fs.tempCount, time.Now().UnixNano()) + return fs.Join(fs.base, dir, filename) +} + +func (fs *Memory) Rename(from, to string) error { + from = fs.Join(fs.base, from) + to = fs.Join(fs.base, to) + + if _, ok := fs.s.files[from]; !ok { + return os.ErrNotExist + } + + fs.s.files[to] = fs.s.files[from] + fs.s.files[to].BaseFilename = to + delete(fs.s.files, from) + + return nil +} + +func (fs *Memory) Remove(filename string) error { + fullpath := fs.Join(fs.base, filename) + if _, ok := fs.s.files[fullpath]; !ok { + return os.ErrNotExist + } + + delete(fs.s.files, fullpath) + return nil +} + +func (fs *Memory) Join(elem ...string) string { + return filepath.Join(elem...) +} + +func (fs *Memory) Dir(path string) fs.Filesystem { + return &Memory{ + base: fs.Join(fs.base, path), + s: fs.s, + } +} + +func (fs *Memory) Base() string { + return fs.base +} + +type file struct { + fs.BaseFile + + content *content + position int64 + flag int +} + +func newFile(base, fullpath string, flag int) *file { + filename, _ := filepath.Rel(base, fullpath) + + return &file{ + BaseFile: fs.BaseFile{BaseFilename: filename}, + content: &content{}, + flag: flag, + } +} + +func (f *file) Read(b []byte) (int, error) { + if f.IsClosed() { + return 0, fs.ErrClosed + } + + if !isReadAndWrite(f.flag) && !isReadOnly(f.flag) { + return 0, errors.New("read not supported") + } + + n, err := f.content.ReadAt(b, f.position) + f.position += int64(n) + + return n, err +} + +func (f *file) Seek(offset int64, whence int) (int64, error) { + if f.IsClosed() { + return 0, fs.ErrClosed + } + + switch whence { + case io.SeekCurrent: + f.position += offset + case io.SeekStart: + f.position = offset + case io.SeekEnd: + f.position = int64(f.content.Len()) - offset + } + + return f.position, nil +} + +func (f *file) Write(p []byte) (int, error) { + if f.IsClosed() { + return 0, fs.ErrClosed + } + + if !isReadAndWrite(f.flag) && !isWriteOnly(f.flag) { + return 0, errors.New("write not supported") + } + + n, err := f.content.WriteAt(p, f.position) + f.position += int64(n) + + return n, err +} + +func (f *file) Close() error { + if f.IsClosed() { + return errors.New("file already closed") + } + + f.Closed = true + return nil +} + +func (f *file) Open() error { + f.Closed = false + return nil +} + +type fileInfo struct { + name string + size int + isDir bool +} + +func newFileInfo(base, fullpath string, size int) *fileInfo { + filename, _ := filepath.Rel(base, fullpath) + + return &fileInfo{ + name: filename, + size: size, + } +} + +func (fi *fileInfo) Name() string { + return fi.name +} + +func (fi *fileInfo) Size() int64 { + return int64(fi.size) +} + +func (fi *fileInfo) Mode() os.FileMode { + return os.FileMode(0) +} + +func (*fileInfo) ModTime() time.Time { + return time.Now() +} + +func (fi *fileInfo) IsDir() bool { + return fi.isDir +} + +func (*fileInfo) Sys() interface{} { + return nil +} + +type storage struct { + files map[string]*file +} + +type content struct { + bytes []byte +} + +func (c *content) WriteAt(p []byte, off int64) (int, error) { + prev := len(c.bytes) + c.bytes = append(c.bytes[:off], p...) + if len(c.bytes) < prev { + c.bytes = c.bytes[:prev] + } + + return len(p), nil +} + +func (c *content) ReadAt(b []byte, off int64) (int, error) { + size := int64(len(c.bytes)) + if off >= size { + return 0, io.EOF + } + + l := int64(len(b)) + if off+l > size { + l = size - off + } + + n := copy(b, c.bytes[off:off+l]) + return n, nil +} + +func (c *content) Truncate() { + c.bytes = make([]byte, 0) +} + +func (c *content) Len() int { + return len(c.bytes) +} + +func isCreate(flag int) bool { + return flag&os.O_CREATE != 0 +} + +func isAppend(flag int) bool { + return flag&os.O_APPEND != 0 +} + +func isTruncate(flag int) bool { + return flag&os.O_TRUNC != 0 +} + +func isReadAndWrite(flag int) bool { + return flag&os.O_RDWR != 0 +} + +func isReadOnly(flag int) bool { + return flag == os.O_RDONLY +} + +func isWriteOnly(flag int) bool { + return flag&os.O_WRONLY != 0 +} diff --git a/utils/fs/memory/memory_test.go b/utils/fs/memory/memory_test.go new file mode 100644 index 0000000..4645fe7 --- /dev/null +++ b/utils/fs/memory/memory_test.go @@ -0,0 +1,33 @@ +package memory + +import ( + "testing" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git.v4/utils/fs/test" +) + +func Test(t *testing.T) { TestingT(t) } + +type MemorySuite struct { + test.FilesystemSuite + path string +} + +var _ = Suite(&MemorySuite{}) + +func (s *MemorySuite) SetUpTest(c *C) { + s.FilesystemSuite.Fs = New() +} + +func (s *MemorySuite) TestTempFileMaxTempFiles(c *C) { + for i := 0; i < maxTempFiles; i++ { + f, err := s.FilesystemSuite.Fs.TempFile("", "") + c.Assert(err, IsNil) + c.Assert(f, NotNil) + } + + f, err := s.FilesystemSuite.Fs.TempFile("", "") + c.Assert(err, NotNil) + c.Assert(f, IsNil) +} diff --git a/utils/fs/os/os.go b/utils/fs/os/os.go index 02395d6..3ff825e 100644 --- a/utils/fs/os/os.go +++ b/utils/fs/os/os.go @@ -14,8 +14,8 @@ type OS struct { base string } -// NewOS returns a new OS filesystem -func NewOS(baseDir string) *OS { +// New returns a new OS filesystem +func New(baseDir string) *OS { return &OS{ base: baseDir, } @@ -139,7 +139,7 @@ func (fs *OS) Join(elem ...string) string { // Dir returns a new Filesystem from the same type of fs using as baseDir the // given path func (fs *OS) Dir(path string) fs.Filesystem { - return NewOS(fs.Join(fs.base, path)) + return New(fs.Join(fs.base, path)) } // Base returns the base path of the filesytem diff --git a/utils/fs/os/os_test.go b/utils/fs/os/os_test.go index 02a0a6c..756e284 100644 --- a/utils/fs/os/os_test.go +++ b/utils/fs/os/os_test.go @@ -21,7 +21,7 @@ var _ = Suite(&OSSuite{}) func (s *OSSuite) SetUpTest(c *C) { s.path, _ = ioutil.TempDir(stdos.TempDir(), "go-git-os-fs-test") - s.FilesystemSuite.Fs = os.NewOS(s.path) + s.FilesystemSuite.Fs = os.New(s.path) } func (s *OSSuite) TearDownTest(c *C) { err := stdos.RemoveAll(s.path) diff --git a/utils/fs/test/fs_suite.go b/utils/fs/test/fs_suite.go index 4c0fd09..74301f5 100644 --- a/utils/fs/test/fs_suite.go +++ b/utils/fs/test/fs_suite.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "io" "io/ioutil" "os" "strings" @@ -184,6 +185,32 @@ func (s *FilesystemSuite) testReadClose(c *C, f File, content string) { c.Assert(f.Close(), IsNil) } +func (s *FilesystemSuite) TestFileReadSeek(c *C) { + f, err := s.Fs.Create("foo") + c.Assert(err, IsNil) + + n, err := f.Write([]byte("0123456789abcdefghijklmnopqrstuvwxyz")) + c.Assert(err, IsNil) + c.Assert(n, Equals, 36) + + p, err := f.Seek(10, io.SeekStart) + c.Assert(err, IsNil) + c.Assert(int(p), Equals, 10) + + all, err := ioutil.ReadAll(f) + c.Assert(err, IsNil) + c.Assert(string(all), Equals, "abcdefghijklmnopqrstuvwxyz") + c.Assert(f.Close(), IsNil) +} + +func (s *FilesystemSuite) TestFileCloseTwice(c *C) { + f, err := s.Fs.Create("foo") + c.Assert(err, IsNil) + + c.Assert(f.Close(), IsNil) + c.Assert(f.Close(), NotNil) +} + func (s *FilesystemSuite) TestReadDirAndDir(c *C) { files := []string{"foo", "bar", "qux/baz", "qux/qux"} for _, name := range files { @@ -268,6 +295,20 @@ func (s *FilesystemSuite) TestTempFileFullWithPath(c *C) { c.Assert(strings.HasPrefix(f.Filename(), s.Fs.Join("foo", "bar")), Equals, true) } +func (s *FilesystemSuite) TestOpenAndWrite(c *C) { + f, err := s.Fs.Create("foo") + c.Assert(err, IsNil) + c.Assert(f.Close(), IsNil) + + foo, err := s.Fs.Open("foo") + c.Assert(foo, NotNil) + c.Assert(err, IsNil) + + n, err := foo.Write([]byte("foo")) + c.Assert(err, NotNil) + c.Assert(n, Equals, 0) +} + func (s *FilesystemSuite) TestOpenAndStat(c *C) { f, err := s.Fs.Create("foo") c.Assert(err, IsNil) @@ -299,6 +340,8 @@ func (s *FilesystemSuite) TestRemoveNonExisting(c *C) { func (s *FilesystemSuite) TestRemoveTempFile(c *C) { f, err := s.Fs.TempFile("test-dir", "test-prefix") + c.Assert(err, IsNil) + fn := f.Filename() c.Assert(err, IsNil) c.Assert(f.Close(), IsNil) -- cgit