From 5e73f01cb2e027a8f02801635b79d3a9bc866914 Mon Sep 17 00:00:00 2001 From: Alberto Cortés Date: Mon, 4 Jul 2016 17:09:22 +0200 Subject: Adds support to open local repositories and to use file-based object storage (#55) * remove some comments * idx writer/reader * Shut up ssh tests, they are annoying * Add file scheme test to clients * Add dummy file client * Add test fot file client * Make tests use fixture endpoint * add parser for packed-refs format * add parser for packed-refs format * WIP adding dir.Refs() tests * Add test for fixture refs * refs parser for the refs directory * Documentation * Add Capabilities to file client * tgz.Exatract now accpets a path instead of a Reader * fix bug in idxfile fanout calculation * remove dead code * packfile documentation * clean packfile parser code * add core.Object.Content() and returns errors for core.ObjectStorage.Iter() * add seekable storage * add dir repos to NewRepository * clean prints * Add dir client documentation to README * Organize the README * README * Clean tgz package * Clean temp dirs after tgz tests * Gometalinter on gitdir * Clean pattern function * metalinter tgz * metalinter gitdir * gitdir coverage and remove seekable packfile filedescriptor leak * gitdir Idxfile tests and remove file descriptor leak * gitdir Idxfile tests when no idx is found * clean storage/seekable/internal/index and some formats/idxfile API issues * clean storage/seekable * clean formats/idx * turn packfile/doc.go into packfile/doc.txt * move formats/packfile/reader to decoder * fix packfile decoder error names * improve documentation * comment packfile decoder errors * comment public API (format/packfile) * remve duplicated code in packfile decoder test * move tracking_reader into an internal package and clean it * use iota for packfile format * rename packfile parse.go to packfile object_at.go * clean packfile deltas * fix delta header size bug * improve delta documentation * clean packfile deltas * clean packfiles deltas * clean repository.go * Remove go 1.5 from Travis CI Because go 1.5 does not suport internal packages. * change local repo scheme to local:// * change "local://" to "file://" as the local scheme * fix broken indentation * shortens names of variables in short scopes * more shortening of variable names * more shortening of variable names * Rename git dir client to "file", as the scheme used for it * Fix file format ctor name, now that the package name has change * Sortcut local repo constructor to not use remotes The object storage is build directly in the repository ctor, instead of creating a remote and waiting for the user to pull it. * update README and fix some errors in it * remove file scheme client * Local respositories has now a new ctor This is, they are no longer identified by the scheme of the URL, but are created different from inception. * remove unused URL field form Repository * move all git dir logic to seekable sotrage ctor * fix documentation * Make formats/file/dir an internal package to storage/seekable * change package storage/seekable to storage/fs * clean storage/fs * overall storage/fs clean * more cleaning * some metalinter fixes * upgrade cshared to last changes * remove dead code * fix test error info * remove file scheme check from clients * fix test error message * fix test error message * fix error messages * style changes * fix comments everywhere * style changes * style changes * scaffolding and tests for local packfiles without ifx files * outsource index building from packfile to the packfile decoder * refactor packfile header reading into a new function * move code to generate index from packfile back to index package * add header parsing * fix documentation errata * add undeltified and OFS delta support for index building from the packfile * add tests for packfile with ref-deltas * support for packfiles with ref-deltas and no idx * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * WIP refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * remove prints from tests * remove prints from tests * refactor packfile.core into packfile.parser * rename packfile reader to something that shows it is a recaller * rename cannot recall error * rename packfile.Reader to packfile.ReadRecaller and document * speed up test by using StreamReader instead of SeekableReader when possible * clean packfile StreamReader * stream_reader tests * refactor packfile.StreamReader into packfile.StreamReadRecaller * refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it * generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations * speed up storage/fs tests * speed up tests in . by loading packfiles in memory * speed up repository tests by using and smaller fixture * restore doc.go files * rename packfile.ReadRecaller implementations to shorter names * update comments to type changes * packfile.Parser test (WIP) * packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller * add test for packfile.ReadRecaller.ForgetAll() * clarify seekable being able to recallByOffset forgetted objects * use better names for internal maps * metalinter packfile package * speed up some tests * documentation fixes * change storage.fs package name to storage.proxy to avoid confusion with new filesystem support * New fs package and os transparent implementation Now NewRepositoryFromFS receives a fs and a path and tests are modified accordingly, but it is still not using for anything. * add fs to gitdir and proxy.store * reduce fs interface for easier implementation * remove garbage dirs from tgz tests * change file name gitdir/dir.go to gitdir/gitdir.go * fs.OS tests * metalinter utils/fs * add NewRepositoryFromFS documentation to README * Readability fixes to README * move tgz to an external dependency * move filesystem impl. example to example dir * rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go * rename proxy package to seekable --- utils/fs/fs.go | 21 ++++++++ utils/fs/os.go | 36 +++++++++++++ utils/fs/os_test.go | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 utils/fs/fs.go create mode 100644 utils/fs/os.go create mode 100644 utils/fs/os_test.go (limited to 'utils') diff --git a/utils/fs/fs.go b/utils/fs/fs.go new file mode 100644 index 0000000..df771dd --- /dev/null +++ b/utils/fs/fs.go @@ -0,0 +1,21 @@ +package fs + +import ( + "io" + "os" +) + +// FS interface represent an abstracted filesystem, so you can +// use NewRepositoryFromFS from any medium. +type FS interface { + Stat(path string) (os.FileInfo, error) + Open(path string) (ReadSeekCloser, error) + ReadDir(path string) ([]os.FileInfo, error) + Join(elem ...string) string +} + +// ReadSeekCloser is a Reader, Seeker and Closer. +type ReadSeekCloser interface { + io.ReadCloser + io.Seeker +} diff --git a/utils/fs/os.go b/utils/fs/os.go new file mode 100644 index 0000000..37ad75a --- /dev/null +++ b/utils/fs/os.go @@ -0,0 +1,36 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// OS is a simple FS implementation for the current host filesystem. +type OS struct{} + +// NewOS returns a new OS. +func NewOS() FS { + return &OS{} +} + +// Stat returns the filesystem info for a path. +func (o *OS) Stat(path string) (os.FileInfo, error) { + return os.Stat(path) +} + +// Open returns a ReadSeekCloser for the specified path. +func (o *OS) Open(path string) (ReadSeekCloser, error) { + return os.Open(path) +} + +// ReadDir returns the filesystem info for all the archives under the +// specified path. +func (o *OS) ReadDir(path string) ([]os.FileInfo, error) { + return ioutil.ReadDir(path) +} + +// Join joins the specified elements using the filesystem separator. +func (o *OS) Join(elem ...string) string { + return filepath.Join(elem...) +} diff --git a/utils/fs/os_test.go b/utils/fs/os_test.go new file mode 100644 index 0000000..84fe895 --- /dev/null +++ b/utils/fs/os_test.go @@ -0,0 +1,151 @@ +package fs + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/alcortesm/tgz" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type FSImplSuite struct { + dir string +} + +var _ = Suite(&FSImplSuite{}) + +func (s *FSImplSuite) SetUpSuite(c *C) { + dir, err := tgz.Extract("../../storage/seekable/internal/gitdir/fixtures/spinnaker-gc.tgz") + c.Assert(err, IsNil) + s.dir = dir +} + +func (s *FSImplSuite) TearDownSuite(c *C) { + err := os.RemoveAll(s.dir) + c.Assert(err, IsNil) +} + +func (s *FSImplSuite) TestJoin(c *C) { + fs := NewOS() + for i, test := range [...]struct { + input []string + expected string + }{ + { + input: []string{}, + expected: "", + }, { + input: []string{"a"}, + expected: "a", + }, { + input: []string{"a", "b"}, + expected: "a/b", + }, { + input: []string{"a", "b", "c"}, + expected: "a/b/c", + }, + } { + obtained := fs.Join(test.input...) + com := Commentf("test %d:\n\tinput = %v", i, test.input) + c.Assert(obtained, Equals, test.expected, com) + } +} + +func (s *FSImplSuite) TestStat(c *C) { + fs := NewOS() + for i, path := range [...]string{ + ".git/index", + ".git/info/refs", + ".git/objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack", + } { + path := fs.Join(s.dir, path) + com := Commentf("test %d", i) + + real, err := os.Open(path) + c.Assert(err, IsNil, com) + + expected, err := real.Stat() + c.Assert(err, IsNil, com) + + obtained, err := fs.Stat(path) + c.Assert(err, IsNil, com) + + c.Assert(obtained, DeepEquals, expected, com) + + err = real.Close() + c.Assert(err, IsNil, com) + } +} + +func (s *FSImplSuite) TestStatErrors(c *C) { + fs := NewOS() + for i, test := range [...]struct { + input string + errRegExp string + }{ + { + input: "bla", + errRegExp: ".*bla: no such file or directory", + }, { + input: "bla/foo", + errRegExp: ".*bla/foo: no such file or directory", + }, + } { + com := Commentf("test %d", i) + _, err := fs.Stat(test.input) + c.Assert(err, ErrorMatches, test.errRegExp, com) + } +} + +func (s *FSImplSuite) TestOpen(c *C) { + fs := NewOS() + for i, test := range [...]string{ + ".git/index", + ".git/info/refs", + ".git/objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack", + } { + com := Commentf("test %d", i) + path := fs.Join(s.dir, test) + + real, err := os.Open(path) + c.Assert(err, IsNil, com) + realData, err := ioutil.ReadAll(real) + c.Assert(err, IsNil, com) + err = real.Close() + c.Assert(err, IsNil, com) + + obtained, err := fs.Open(path) + c.Assert(err, IsNil, com) + obtainedData, err := ioutil.ReadAll(obtained) + c.Assert(err, IsNil, com) + err = obtained.Close() + c.Assert(err, IsNil, com) + + c.Assert(obtainedData, DeepEquals, realData, com) + } +} + +func (s *FSImplSuite) TestReadDir(c *C) { + fs := NewOS() + for i, test := range [...]string{ + ".git/info", + ".", + "", + ".git/objects", + ".git/objects/pack", + } { + com := Commentf("test %d", i) + path := fs.Join(s.dir, test) + + expected, err := ioutil.ReadDir(path) + c.Assert(err, IsNil, com) + + obtained, err := fs.ReadDir(path) + c.Assert(err, IsNil, com) + + c.Assert(obtained, DeepEquals, expected, com) + } +} -- cgit