From 5e73f01cb2e027a8f02801635b79d3a9bc866914 Mon Sep 17 00:00:00 2001 From: Alberto Cortés Date: Mon, 4 Jul 2016 17:09:22 +0200 Subject: Adds support to open local repositories and to use file-based object storage (#55) * remove some comments * idx writer/reader * Shut up ssh tests, they are annoying * Add file scheme test to clients * Add dummy file client * Add test fot file client * Make tests use fixture endpoint * add parser for packed-refs format * add parser for packed-refs format * WIP adding dir.Refs() tests * Add test for fixture refs * refs parser for the refs directory * Documentation * Add Capabilities to file client * tgz.Exatract now accpets a path instead of a Reader * fix bug in idxfile fanout calculation * remove dead code * packfile documentation * clean packfile parser code * add core.Object.Content() and returns errors for core.ObjectStorage.Iter() * add seekable storage * add dir repos to NewRepository * clean prints * Add dir client documentation to README * Organize the README * README * Clean tgz package * Clean temp dirs after tgz tests * Gometalinter on gitdir * Clean pattern function * metalinter tgz * metalinter gitdir * gitdir coverage and remove seekable packfile filedescriptor leak * gitdir Idxfile tests and remove file descriptor leak * gitdir Idxfile tests when no idx is found * clean storage/seekable/internal/index and some formats/idxfile API issues * clean storage/seekable * clean formats/idx * turn packfile/doc.go into packfile/doc.txt * move formats/packfile/reader to decoder * fix packfile decoder error names * improve documentation * comment packfile decoder errors * comment public API (format/packfile) * remve duplicated code in packfile decoder test * move tracking_reader into an internal package and clean it * use iota for packfile format * rename packfile parse.go to packfile object_at.go * clean packfile deltas * fix delta header size bug * improve delta documentation * clean packfile deltas * clean packfiles deltas * clean repository.go * Remove go 1.5 from Travis CI Because go 1.5 does not suport internal packages. * change local repo scheme to local:// * change "local://" to "file://" as the local scheme * fix broken indentation * shortens names of variables in short scopes * more shortening of variable names * more shortening of variable names * Rename git dir client to "file", as the scheme used for it * Fix file format ctor name, now that the package name has change * Sortcut local repo constructor to not use remotes The object storage is build directly in the repository ctor, instead of creating a remote and waiting for the user to pull it. * update README and fix some errors in it * remove file scheme client * Local respositories has now a new ctor This is, they are no longer identified by the scheme of the URL, but are created different from inception. * remove unused URL field form Repository * move all git dir logic to seekable sotrage ctor * fix documentation * Make formats/file/dir an internal package to storage/seekable * change package storage/seekable to storage/fs * clean storage/fs * overall storage/fs clean * more cleaning * some metalinter fixes * upgrade cshared to last changes * remove dead code * fix test error info * remove file scheme check from clients * fix test error message * fix test error message * fix error messages * style changes * fix comments everywhere * style changes * style changes * scaffolding and tests for local packfiles without ifx files * outsource index building from packfile to the packfile decoder * refactor packfile header reading into a new function * move code to generate index from packfile back to index package * add header parsing * fix documentation errata * add undeltified and OFS delta support for index building from the packfile * add tests for packfile with ref-deltas * support for packfiles with ref-deltas and no idx * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * WIP refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * remove prints from tests * remove prints from tests * refactor packfile.core into packfile.parser * rename packfile reader to something that shows it is a recaller * rename cannot recall error * rename packfile.Reader to packfile.ReadRecaller and document * speed up test by using StreamReader instead of SeekableReader when possible * clean packfile StreamReader * stream_reader tests * refactor packfile.StreamReader into packfile.StreamReadRecaller * refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it * generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations * speed up storage/fs tests * speed up tests in . by loading packfiles in memory * speed up repository tests by using and smaller fixture * restore doc.go files * rename packfile.ReadRecaller implementations to shorter names * update comments to type changes * packfile.Parser test (WIP) * packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller * add test for packfile.ReadRecaller.ForgetAll() * clarify seekable being able to recallByOffset forgetted objects * use better names for internal maps * metalinter packfile package * speed up some tests * documentation fixes * change storage.fs package name to storage.proxy to avoid confusion with new filesystem support * New fs package and os transparent implementation Now NewRepositoryFromFS receives a fs and a path and tests are modified accordingly, but it is still not using for anything. * add fs to gitdir and proxy.store * reduce fs interface for easier implementation * remove garbage dirs from tgz tests * change file name gitdir/dir.go to gitdir/gitdir.go * fs.OS tests * metalinter utils/fs * add NewRepositoryFromFS documentation to README * Readability fixes to README * move tgz to an external dependency * move filesystem impl. example to example dir * rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go * rename proxy package to seekable --- storage/seekable/storage_test.go | 326 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 storage/seekable/storage_test.go (limited to 'storage/seekable/storage_test.go') diff --git a/storage/seekable/storage_test.go b/storage/seekable/storage_test.go new file mode 100644 index 0000000..bc0ad3d --- /dev/null +++ b/storage/seekable/storage_test.go @@ -0,0 +1,326 @@ +package seekable_test + +import ( + "fmt" + "os" + "reflect" + "sort" + "testing" + + "gopkg.in/src-d/go-git.v3/core" + "gopkg.in/src-d/go-git.v3/formats/packfile" + "gopkg.in/src-d/go-git.v3/storage/memory" + "gopkg.in/src-d/go-git.v3/storage/seekable" + "gopkg.in/src-d/go-git.v3/storage/seekable/internal/gitdir" + "gopkg.in/src-d/go-git.v3/utils/fs" + + "github.com/alcortesm/tgz" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type FsSuite struct{} + +var _ = Suite(&FsSuite{}) + +var fixtures map[string]string // id to git dir paths (see initFixtures below) + +func fixture(id string, c *C) string { + path, ok := fixtures[id] + c.Assert(ok, Equals, true, Commentf("fixture %q not found", id)) + + return path +} + +var initFixtures = [...]struct { + id string + tgz string +}{ + { + id: "binary-relations", + tgz: "internal/gitdir/fixtures/alcortesm-binary-relations.tgz", + }, { + id: "binary-relations-no-idx", + tgz: "internal/gitdir/fixtures/alcortesm-binary-relations-no-idx.tgz", + }, { + id: "ref-deltas-no-idx", + tgz: "internal/gitdir/fixtures/ref-deltas-no-idx.tgz", + }, +} + +func (s *FsSuite) SetUpSuite(c *C) { + fixtures = make(map[string]string, len(initFixtures)) + for _, init := range initFixtures { + path, err := tgz.Extract(init.tgz) + c.Assert(err, IsNil, Commentf("error extracting %s\n", init.tgz)) + fixtures[init.id] = path + } +} + +func (s *FsSuite) TearDownSuite(c *C) { + for _, v := range fixtures { + err := os.RemoveAll(v) + c.Assert(err, IsNil, Commentf("error removing fixture %q\n", v)) + } +} + +func (s *FsSuite) TestNewErrorNotFound(c *C) { + fs := fs.NewOS() + _, err := seekable.New(fs, "not_found/.git") + c.Assert(err, Equals, gitdir.ErrNotFound) +} + +func (s *FsSuite) TestHashNotFound(c *C) { + path := fixture("binary-relations", c) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + sto, err := seekable.New(fs, gitPath) + c.Assert(err, IsNil) + + _, err = sto.Get(core.ZeroHash) + c.Assert(err, Equals, core.ErrObjectNotFound) +} + +func (s *FsSuite) TestGetCompareWithMemoryStorage(c *C) { + for i, fixId := range [...]string{ + "binary-relations", + "binary-relations-no-idx", + "ref-deltas-no-idx", + } { + path := fixture(fixId, c) + com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", + i, fixId, path) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + memSto, err := memStorageFromGitDir(fs, gitPath) + c.Assert(err, IsNil, com) + + seekableSto, err := seekable.New(fs, gitPath) + c.Assert(err, IsNil, com) + + equal, reason, err := equalsStorages(memSto, seekableSto) + c.Assert(err, IsNil, com) + c.Assert(equal, Equals, true, + Commentf("%s - %s\n", com.CheckCommentString(), reason)) + } +} + +func memStorageFromGitDir(fs fs.FS, path string) (*memory.ObjectStorage, error) { + dir, err := gitdir.New(fs, path) + if err != nil { + return nil, err + } + + fs, packfilePath, err := dir.Packfile() + if err != nil { + return nil, err + } + + f, err := fs.Open(packfilePath) + if err != nil { + return nil, err + } + + sto := memory.NewObjectStorage() + r := packfile.NewStream(f) + d := packfile.NewDecoder(r) + err = d.Decode(sto) + if err != nil { + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + return sto, nil +} + +func equalsStorages(a, b core.ObjectStorage) (bool, string, error) { + for _, typ := range [...]core.ObjectType{ + core.CommitObject, + core.TreeObject, + core.BlobObject, + core.TagObject, + } { + iter, err := a.Iter(typ) + if err != nil { + return false, "", fmt.Errorf("cannot get iterator: %s", err) + } + + for { + ao, err := iter.Next() + if err != nil { + iter.Close() + break + } + + bo, err := b.Get(ao.Hash()) + if err != nil { + return false, "", fmt.Errorf("getting object with hash %s: %s", + ao.Hash(), err) + } + + equal, reason, err := equalsObjects(ao, bo) + if !equal || err != nil { + return equal, reason, fmt.Errorf("comparing objects: %s", err) + } + } + } + + return true, "", nil +} + +func equalsObjects(a, b core.Object) (bool, string, error) { + ah := a.Hash() + bh := b.Hash() + if ah != bh { + return false, fmt.Sprintf("object hashes differ: %s and %s\n", + ah, bh), nil + } + + atyp := a.Type() + btyp := b.Type() + if atyp != btyp { + return false, fmt.Sprintf("object types differ: %d and %d\n", + atyp, btyp), nil + } + + asz := a.Size() + bsz := b.Size() + if asz != bsz { + return false, fmt.Sprintf("object sizes differ: %d and %d\n", + asz, bsz), nil + } + + ac := a.Content() + if ac != nil { + bc := b.Content() + if !reflect.DeepEqual(ac, bc) { + return false, fmt.Sprintf("object contents differ"), nil + } + } + + return true, "", nil +} + +func (s *FsSuite) TestIterCompareWithMemoryStorage(c *C) { + for i, fixId := range [...]string{ + "binary-relations", + "binary-relations-no-idx", + "ref-deltas-no-idx", + } { + + path := fixture(fixId, c) + com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", + i, fixId, path) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + memSto, err := memStorageFromDirPath(fs, gitPath) + c.Assert(err, IsNil, com) + + seekableSto, err := seekable.New(fs, gitPath) + c.Assert(err, IsNil, com) + + for _, typ := range [...]core.ObjectType{ + core.CommitObject, + core.TreeObject, + core.BlobObject, + core.TagObject, + } { + + memObjs, err := iterToSortedSlice(memSto, typ) + c.Assert(err, IsNil, com) + + seekableObjs, err := iterToSortedSlice(seekableSto, typ) + c.Assert(err, IsNil, com) + + for i, o := range memObjs { + c.Assert(seekableObjs[i].Hash(), Equals, o.Hash(), com) + } + } + } +} + +func memStorageFromDirPath(fs fs.FS, path string) (*memory.ObjectStorage, error) { + dir, err := gitdir.New(fs, path) + if err != nil { + return nil, err + } + + fs, packfilePath, err := dir.Packfile() + if err != nil { + return nil, err + } + + sto := memory.NewObjectStorage() + f, err := fs.Open(packfilePath) + if err != nil { + return nil, err + } + + r := packfile.NewStream(f) + d := packfile.NewDecoder(r) + err = d.Decode(sto) + if err != nil { + return nil, err + } + + if err = f.Close(); err != nil { + return nil, err + } + + return sto, nil +} + +func iterToSortedSlice(storage core.ObjectStorage, typ core.ObjectType) ([]core.Object, + error) { + + iter, err := storage.Iter(typ) + if err != nil { + return nil, err + } + + r := make([]core.Object, 0) + for { + obj, err := iter.Next() + if err != nil { + iter.Close() + break + } + r = append(r, obj) + } + + sort.Sort(byHash(r)) + + return r, nil +} + +type byHash []core.Object + +func (a byHash) Len() int { return len(a) } +func (a byHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byHash) Less(i, j int) bool { + return a[i].Hash().String() < a[j].Hash().String() +} + +func (s *FsSuite) TestSet(c *C) { + path := fixture("binary-relations", c) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + sto, err := seekable.New(fs, gitPath) + c.Assert(err, IsNil) + + _, err = sto.Set(&memory.Object{}) + c.Assert(err, ErrorMatches, "not implemented yet") +} -- cgit