From ae999ede139f5fa5601ffb7c55979608b112d274 Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Fri, 12 Aug 2016 19:11:27 +0200 Subject: storage: Storage entity support, and DotGit support for References --- storage/filesystem/internal/dotgit/dotgit.go | 105 +++++++ storage/filesystem/internal/dotgit/dotgit_test.go | 224 +++++++++++++++ storage/filesystem/internal/dotgit/refs.go | 144 ++++++++++ storage/filesystem/internal/gitdir/gitdir.go | 145 ---------- storage/filesystem/internal/gitdir/gitdir_test.go | 263 ------------------ storage/filesystem/internal/gitdir/refs.go | 152 ---------- storage/filesystem/object.go | 142 ++++++++++ storage/filesystem/object_test.go | 315 +++++++++++++++++++++ storage/filesystem/reference.go | 61 ++++ storage/filesystem/storage.go | 195 ++----------- storage/filesystem/storage_test.go | 321 +--------------------- 11 files changed, 1018 insertions(+), 1049 deletions(-) create mode 100644 storage/filesystem/internal/dotgit/dotgit.go create mode 100644 storage/filesystem/internal/dotgit/dotgit_test.go create mode 100644 storage/filesystem/internal/dotgit/refs.go delete mode 100644 storage/filesystem/internal/gitdir/gitdir.go delete mode 100644 storage/filesystem/internal/gitdir/gitdir_test.go delete mode 100644 storage/filesystem/internal/gitdir/refs.go create mode 100644 storage/filesystem/object.go create mode 100644 storage/filesystem/object_test.go create mode 100644 storage/filesystem/reference.go (limited to 'storage/filesystem') diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/internal/dotgit/dotgit.go new file mode 100644 index 0000000..f365f13 --- /dev/null +++ b/storage/filesystem/internal/dotgit/dotgit.go @@ -0,0 +1,105 @@ +package dotgit + +import ( + "errors" + "os" + "strings" + + "gopkg.in/src-d/go-git.v4/core" + "gopkg.in/src-d/go-git.v4/utils/fs" +) + +const ( + suffix = ".git" + packedRefsPath = "packed-refs" +) + +var ( + // ErrNotFound is returned by New when the path is not found. + ErrNotFound = errors.New("path not found") + // ErrIdxNotFound is returned by Idxfile when the idx file is not found on the + // repository. + ErrIdxNotFound = errors.New("idx file not found") + // ErrPackfileNotFound is returned by Packfile when the packfile is not found + // on the repository. + ErrPackfileNotFound = errors.New("packfile not found") +) + +// The DotGit type represents a local git repository on disk. This +// type is not zero-value-safe, use the New function to initialize it. +type DotGit struct { + fs fs.FS + path string +} + +// New returns a DotGit value ready to be used. The path argument must +// be the absolute path of a git repository directory (e.g. +// "/foo/bar/.git"). +func New(fs fs.FS, path string) (*DotGit, error) { + d := &DotGit{fs: fs, path: path} + if _, err := fs.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, err + } + + return d, nil +} + +// Refs scans the git directory collecting references, which it returns. +// Symbolic references are resolved and included in the output. +func (d *DotGit) Refs() ([]*core.Reference, error) { + var refs []*core.Reference + if err := d.addRefsFromPackedRefs(&refs); err != nil { + return nil, err + } + + if err := d.addRefsFromRefDir(&refs); err != nil { + return nil, err + } + + if err := d.addRefFromHEAD(&refs); err != nil { + return nil, err + } + + return refs, nil +} + +// Packfile returns the path of the packfile (really, it returns the +// path of the first file in the "objects/pack/" directory with a +// ".pack" extension. +func (d *DotGit) Packfile() (fs.FS, string, error) { + packDir := d.fs.Join(d.path, "objects", "pack") + files, err := d.fs.ReadDir(packDir) + if err != nil { + return nil, "", err + } + + for _, f := range files { + if strings.HasSuffix(f.Name(), ".pack") { + return d.fs, d.fs.Join(packDir, f.Name()), nil + } + } + + return nil, "", ErrPackfileNotFound +} + +// Idxfile returns the path of the idx file (really, it returns the +// path of the first file in the "objects/pack/" directory with an +// ".idx" extension. +func (d *DotGit) Idxfile() (fs.FS, string, error) { + packDir := d.fs.Join(d.path, "objects", "pack") + files, err := d.fs.ReadDir(packDir) + if err != nil { + return nil, "", err + } + + for _, f := range files { + if strings.HasSuffix(f.Name(), ".idx") { + return d.fs, d.fs.Join(packDir, f.Name()), nil + } + } + + return nil, "", ErrIdxNotFound +} diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/internal/dotgit/dotgit_test.go new file mode 100644 index 0000000..6125114 --- /dev/null +++ b/storage/filesystem/internal/dotgit/dotgit_test.go @@ -0,0 +1,224 @@ +package dotgit + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/src-d/go-git.v4/clients/common" + "gopkg.in/src-d/go-git.v4/core" + "gopkg.in/src-d/go-git.v4/utils/fs" + + "github.com/alcortesm/tgz" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +var initFixtures = [...]struct { + name string + tgz string + capabilities [][2]string + packfile string + idxfile string +}{ + { + name: "spinnaker", + tgz: "fixtures/spinnaker-gc.tgz", + capabilities: [][2]string{ + {"symref", "HEAD:refs/heads/master"}, + }, + packfile: "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack", + idxfile: "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.idx", + }, { + name: "no-packfile-no-idx", + tgz: "fixtures/no-packfile-no-idx.tgz", + }, { + name: "empty", + tgz: "fixtures/empty-gitdir.tgz", + }, +} + +type fixture struct { + installDir string + fs fs.FS + path string // repo names to paths of the extracted tgz + capabilities *common.Capabilities // expected capabilities + packfile string // path of the packfile + idxfile string // path of the idxfile +} + +type SuiteDotGit struct { + fixtures map[string]fixture +} + +var _ = Suite(&SuiteDotGit{}) + +func (s *SuiteDotGit) SetUpSuite(c *C) { + s.fixtures = make(map[string]fixture, len(initFixtures)) + + for _, init := range initFixtures { + com := Commentf("fixture name = %s\n", init.name) + + path, err := tgz.Extract(init.tgz) + c.Assert(err, IsNil, com) + + f := fixture{} + + f.installDir = path + f.fs = fs.NewOS() + f.path = f.fs.Join(path, ".git") + + f.capabilities = common.NewCapabilities() + for _, pair := range init.capabilities { + f.capabilities.Add(pair[0], pair[1]) + } + + f.packfile = init.packfile + f.idxfile = init.idxfile + + s.fixtures[init.name] = f + } +} + +func (s *SuiteDotGit) TearDownSuite(c *C) { + for n, f := range s.fixtures { + err := os.RemoveAll(f.installDir) + c.Assert(err, IsNil, Commentf("cannot delete tmp dir for fixture %s: %s\n", + n, f.installDir)) + } +} + +func (s *SuiteDotGit) TestNewErrors(c *C) { + for i, test := range [...]struct { + input string + err error + }{ + { + input: "./tmp/foo", + err: ErrNotFound, + }, { + input: "./tmp/foo/.git", + err: ErrNotFound, + }, + } { + com := Commentf("subtest %d", i) + + _, err := New(fs.NewOS(), test.input) + c.Assert(err, Equals, test.err, com) + } +} + +func (s *SuiteDotGit) TestRefsFromPackedRefs(c *C) { + _, d := s.newFixtureDir(c, "spinnaker") + refs, err := d.Refs() + c.Assert(err, IsNil) + + ref := findReference(refs, "refs/tags/v0.37.0") + c.Assert(ref, NotNil) + c.Assert(ref.Hash().String(), Equals, "85ec60477681933961c9b64c18ada93220650ac5") + +} +func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) { + _, d := s.newFixtureDir(c, "spinnaker") + refs, err := d.Refs() + c.Assert(err, IsNil) + + ref := findReference(refs, "refs/remotes/origin/HEAD") + c.Assert(ref, NotNil) + c.Assert(ref.Type(), Equals, core.SymbolicReference) + c.Assert(string(ref.Target()), Equals, "refs/remotes/origin/master") + +} + +func (s *SuiteDotGit) TestRefsFromHEADFile(c *C) { + _, d := s.newFixtureDir(c, "spinnaker") + refs, err := d.Refs() + c.Assert(err, IsNil) + + ref := findReference(refs, "HEAD") + c.Assert(ref, NotNil) + c.Assert(ref.Type(), Equals, core.SymbolicReference) + c.Assert(string(ref.Target()), Equals, "refs/heads/master") +} + +func findReference(refs []*core.Reference, name string) *core.Reference { + n := core.ReferenceName(name) + for _, ref := range refs { + if ref.Name() == n { + return ref + } + } + + return nil +} + +func (s *SuiteDotGit) newFixtureDir(c *C, fixName string) (*fixture, *DotGit) { + f, ok := s.fixtures[fixName] + c.Assert(ok, Equals, true) + + d, err := New(fs.NewOS(), f.path) + c.Assert(err, IsNil) + + return &f, d +} + +func (s *SuiteDotGit) TestPackfile(c *C) { + packfile := func(d *DotGit) (fs.FS, string, error) { + return d.Packfile() + } + idxfile := func(d *DotGit) (fs.FS, string, error) { + return d.Idxfile() + } + for _, test := range [...]struct { + fixture string + fn getPathFn + err string // error regexp + }{ + { + fixture: "spinnaker", + fn: packfile, + }, { + fixture: "spinnaker", + fn: idxfile, + }, { + fixture: "empty", + fn: packfile, + err: ".* no such file or directory", + }, { + fixture: "empty", + fn: idxfile, + err: ".* no such file or directory", + }, { + fixture: "no-packfile-no-idx", + fn: packfile, + err: "packfile not found", + }, { + fixture: "no-packfile-no-idx", + fn: idxfile, + err: "idx file not found", + }, + } { + com := Commentf("fixture = %s", test.fixture) + + fix, dir := s.newFixtureDir(c, test.fixture) + + _, path, err := test.fn(dir) + + if test.err != "" { + c.Assert(err, ErrorMatches, test.err, com) + } else { + c.Assert(err, IsNil, com) + c.Assert(strings.HasSuffix(noExt(path), noExt(fix.packfile)), + Equals, true, com) + } + } +} + +type getPathFn func(*DotGit) (fs.FS, string, error) + +func noExt(path string) string { + ext := filepath.Ext(path) + return path[0 : len(path)-len(ext)] +} diff --git a/storage/filesystem/internal/dotgit/refs.go b/storage/filesystem/internal/dotgit/refs.go new file mode 100644 index 0000000..894732f --- /dev/null +++ b/storage/filesystem/internal/dotgit/refs.go @@ -0,0 +1,144 @@ +package dotgit + +import ( + "bufio" + "errors" + "io/ioutil" + "os" + "strings" + + "gopkg.in/src-d/go-git.v4/core" +) + +var ( + // ErrPackedRefsDuplicatedRef is returned when a duplicated + // reference is found in the packed-ref file. This is usually the + // case for corrupted git repositories. + ErrPackedRefsDuplicatedRef = errors.New("duplicated ref found in packed-ref file") + // ErrPackedRefsBadFormat is returned when the packed-ref file + // corrupt. + ErrPackedRefsBadFormat = errors.New("malformed packed-ref") + // ErrSymRefTargetNotFound is returned when a symbolic reference is + // targeting a non-existing object. This usually means the + // repository is corrupt. + ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") +) + +const ( + refsPath = "refs" +) + +func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) { + path := d.fs.Join(d.path, packedRefsPath) + f, err := d.fs.Open(path) + if err != nil { + if err == os.ErrNotExist { + return nil + } + + return err + } + + defer func() { + if errClose := f.Close(); err == nil { + err = errClose + } + }() + + s := bufio.NewScanner(f) + for s.Scan() { + ref, err := d.processLine(s.Text()) + if err != nil { + return err + } + + if ref != nil { + *refs = append(*refs, ref) + } + } + + return s.Err() +} + +// process lines from a packed-refs file +func (d *DotGit) processLine(line string) (*core.Reference, error) { + switch line[0] { + case '#': // comment - ignore + return nil, nil + case '^': // annotated tag commit of the previous line - ignore + return nil, nil + default: + ws := strings.Split(line, " ") // hash then ref + if len(ws) != 2 { + return nil, ErrPackedRefsBadFormat + } + + return core.NewReferenceFromStrings(ws[1], ws[0]), nil + } +} + +func (d *DotGit) addRefsFromRefDir(refs *[]*core.Reference) error { + return d.walkReferencesTree(refs, refsPath) +} + +func (d *DotGit) walkReferencesTree(refs *[]*core.Reference, relPath string) error { + files, err := d.fs.ReadDir(d.fs.Join(d.path, relPath)) + if err != nil { + return err + } + + for _, f := range files { + newRelPath := d.fs.Join(relPath, f.Name()) + if f.IsDir() { + if err = d.walkReferencesTree(refs, newRelPath); err != nil { + return err + } + + continue + } + + ref, err := d.readReferenceFile(d.path, newRelPath) + if err != nil { + return err + } + + if ref != nil { + *refs = append(*refs, ref) + } + } + + return nil +} + +func (d *DotGit) addRefFromHEAD(refs *[]*core.Reference) error { + ref, err := d.readReferenceFile(d.path, "HEAD") + if err != nil { + return err + } + + *refs = append(*refs, ref) + return nil +} + +func (d *DotGit) readReferenceFile(refsPath, refFile string) (ref *core.Reference, err error) { + path := d.fs.Join(refsPath, refFile) + + f, err := d.fs.Open(path) + if err != nil { + return nil, err + } + + defer func() { + if errClose := f.Close(); err == nil { + err = errClose + } + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + line := strings.TrimSpace(string(b)) + return core.NewReferenceFromStrings(refFile, line), nil +} diff --git a/storage/filesystem/internal/gitdir/gitdir.go b/storage/filesystem/internal/gitdir/gitdir.go deleted file mode 100644 index b5497b8..0000000 --- a/storage/filesystem/internal/gitdir/gitdir.go +++ /dev/null @@ -1,145 +0,0 @@ -package gitdir - -import ( - "errors" - "io/ioutil" - "os" - "strings" - - "gopkg.in/src-d/go-git.v4/clients/common" - "gopkg.in/src-d/go-git.v4/core" - "gopkg.in/src-d/go-git.v4/utils/fs" -) - -const ( - suffix = ".git" - packedRefsPath = "packed-refs" -) - -var ( - // ErrNotFound is returned by New when the path is not found. - ErrNotFound = errors.New("path not found") - // ErrIdxNotFound is returned by Idxfile when the idx file is not found on the - // repository. - ErrIdxNotFound = errors.New("idx file not found") - // ErrPackfileNotFound is returned by Packfile when the packfile is not found - // on the repository. - ErrPackfileNotFound = errors.New("packfile not found") -) - -// The GitDir type represents a local git repository on disk. This -// type is not zero-value-safe, use the New function to initialize it. -type GitDir struct { - fs fs.FS - path string - refs map[string]core.Hash - packDir string -} - -// New returns a GitDir value ready to be used. The path argument must -// be the absolute path of a git repository directory (e.g. -// "/foo/bar/.git"). -func New(fs fs.FS, path string) (*GitDir, error) { - d := &GitDir{} - d.fs = fs - d.path = path - d.packDir = d.fs.Join(d.path, "objects", "pack") - - if _, err := fs.Stat(path); err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - return nil, err - } - - return d, nil -} - -// Refs scans the git directory collecting references, which it returns. -// Symbolic references are resolved and included in the output. -func (d *GitDir) Refs() (map[string]core.Hash, error) { - var err error - - d.refs = make(map[string]core.Hash) - - if err = d.addRefsFromPackedRefs(); err != nil { - return nil, err - } - - if err = d.addRefsFromRefDir(); err != nil { - return nil, err - } - - return d.refs, err -} - -// Capabilities scans the git directory collection capabilities, which it returns. -func (d *GitDir) Capabilities() (*common.Capabilities, error) { - c := common.NewCapabilities() - - err := d.addSymRefCapability(c) - - return c, err -} - -func (d *GitDir) addSymRefCapability(cap *common.Capabilities) (err error) { - f, err := d.fs.Open(d.fs.Join(d.path, "HEAD")) - if err != nil { - return err - } - - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - b, err := ioutil.ReadAll(f) - if err != nil { - return err - } - data := strings.TrimSpace(string(b)) - - c := "symref" - ref := strings.TrimPrefix(data, symRefPrefix) - cap.Set(c, "HEAD:"+ref) - - return nil -} - -// Packfile returns the path of the packfile (really, it returns the -// path of the first file in the "objects/pack/" directory with a -// ".pack" extension. -func (d *GitDir) Packfile() (fs.FS, string, error) { - files, err := d.fs.ReadDir(d.packDir) - if err != nil { - return nil, "", err - } - - for _, f := range files { - if strings.HasSuffix(f.Name(), ".pack") { - return d.fs, d.fs.Join(d.packDir, f.Name()), nil - } - } - - return nil, "", ErrPackfileNotFound -} - -// Idxfile returns the path of the idx file (really, it returns the -// path of the first file in the "objects/pack/" directory with an -// ".idx" extension. -func (d *GitDir) Idxfile() (fs.FS, string, error) { - files, err := d.fs.ReadDir(d.packDir) - if err != nil { - return nil, "", err - } - - for _, f := range files { - if strings.HasSuffix(f.Name(), ".idx") { - return d.fs, d.fs.Join(d.packDir, f.Name()), nil - } - } - - return nil, "", ErrIdxNotFound -} diff --git a/storage/filesystem/internal/gitdir/gitdir_test.go b/storage/filesystem/internal/gitdir/gitdir_test.go deleted file mode 100644 index a02e0f4..0000000 --- a/storage/filesystem/internal/gitdir/gitdir_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package gitdir - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "gopkg.in/src-d/go-git.v4/clients/common" - "gopkg.in/src-d/go-git.v4/core" - "gopkg.in/src-d/go-git.v4/utils/fs" - - "github.com/alcortesm/tgz" - . "gopkg.in/check.v1" -) - -func Test(t *testing.T) { TestingT(t) } - -var initFixtures = [...]struct { - name string - tgz string - capabilities [][2]string - packfile string - idxfile string -}{ - { - name: "spinnaker", - tgz: "fixtures/spinnaker-gc.tgz", - capabilities: [][2]string{ - {"symref", "HEAD:refs/heads/master"}, - }, - packfile: "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.pack", - idxfile: "objects/pack/pack-584416f86235cac0d54bfabbdc399fb2b09a5269.idx", - }, { - name: "no-packfile-no-idx", - tgz: "fixtures/no-packfile-no-idx.tgz", - }, { - name: "empty", - tgz: "fixtures/empty-gitdir.tgz", - }, -} - -type fixture struct { - installDir string - fs fs.FS - path string // repo names to paths of the extracted tgz - capabilities *common.Capabilities // expected capabilities - packfile string // path of the packfile - idxfile string // path of the idxfile -} - -type SuiteGitDir struct { - fixtures map[string]fixture -} - -var _ = Suite(&SuiteGitDir{}) - -func (s *SuiteGitDir) SetUpSuite(c *C) { - s.fixtures = make(map[string]fixture, len(initFixtures)) - - for _, init := range initFixtures { - com := Commentf("fixture name = %s\n", init.name) - - path, err := tgz.Extract(init.tgz) - c.Assert(err, IsNil, com) - - f := fixture{} - - f.installDir = path - f.fs = fs.NewOS() - f.path = f.fs.Join(path, ".git") - - f.capabilities = common.NewCapabilities() - for _, pair := range init.capabilities { - f.capabilities.Add(pair[0], pair[1]) - } - - f.packfile = init.packfile - f.idxfile = init.idxfile - - s.fixtures[init.name] = f - } -} - -func (s *SuiteGitDir) TearDownSuite(c *C) { - for n, f := range s.fixtures { - err := os.RemoveAll(f.installDir) - c.Assert(err, IsNil, Commentf("cannot delete tmp dir for fixture %s: %s\n", - n, f.installDir)) - } -} - -func (s *SuiteGitDir) TestNewErrors(c *C) { - for i, test := range [...]struct { - input string - err error - }{ - { - input: "./tmp/foo", - err: ErrNotFound, - }, { - input: "./tmp/foo/.git", - err: ErrNotFound, - }, - } { - com := Commentf("subtest %d", i) - - _, err := New(fs.NewOS(), test.input) - c.Assert(err, Equals, test.err, com) - } -} - -func (s *SuiteGitDir) TestRefs(c *C) { - for i, test := range [...]struct { - fixture string - refs map[string]core.Hash - }{ - { - fixture: "spinnaker", - refs: map[string]core.Hash{ - "refs/heads/master": core.NewHash("409db80e56365049edb704f2ecbd449ddf64dc0d"), - "refs/remotes/origin/HEAD": core.NewHash("409db80e56365049edb704f2ecbd449ddf64dc0d"), - "refs/remotes/origin/explicit-machine-type": core.NewHash("f262e833a215c90b703115691f03f182c1be4b91"), - "refs/remotes/origin/fix-aws-creds-copy": core.NewHash("871cf4d673e0d94c6eb2558bfc7a525c2bc7e538"), - "refs/remotes/origin/kubernetes-no-gcloud": core.NewHash("0b553b5b6fa773f3d7a38b229d9f75627c0762aa"), - "refs/remotes/origin/lwander-patch-igor": core.NewHash("9c987f44908bc9aa05e950347cd03228ba199630"), - "refs/remotes/origin/master": core.NewHash("409db80e56365049edb704f2ecbd449ddf64dc0d"), - "refs/remotes/origin/revert-898-codelab-script-fix": core.NewHash("426cd84d1741d0ff68bad646bc8499b1f163a893"), - "refs/remotes/origin/terraform-aws-prototype": core.NewHash("a34445e7d2e758a8c953fa3a357198ec09fcba88"), - "refs/remotes/origin/typo": core.NewHash("86b48b962e599c096a5870cd8047778bb32a6e1e"), - "refs/tags/v0.10.0": core.NewHash("d081d66c2a76d04ff479a3431dc36e44116fde40"), - "refs/tags/v0.11.0": core.NewHash("3e349f806a0d02bf658c3544c46a0a7a9ee78673"), - "refs/tags/v0.12.0": core.NewHash("82562fa518f0a2e2187ea2604b07b67f2e7049ae"), - "refs/tags/v0.13.0": core.NewHash("48b655898fa9c72d62e8dd73b022ecbddd6e4cc2"), - "refs/tags/v0.14.0": core.NewHash("7ecc2ad58e24a5b52504985467a10c6a3bb85b9b"), - "refs/tags/v0.15.0": core.NewHash("740e3adff4c350899db7772f8f537d1d0d96ec75"), - "refs/tags/v0.16.0": core.NewHash("466ca58a3129f1b2ead117a43535ecb410d621ac"), - "refs/tags/v0.17.0": core.NewHash("48020cb7a45603d47e6041de072fe0665e47676f"), - "refs/tags/v0.18.0": core.NewHash("6fcb9036ab4d921dbdab41baf923320484a11188"), - "refs/tags/v0.19.0": core.NewHash("a2ce1f4c9d0bde4e93dfcb90a445ed069030640c"), - "refs/tags/v0.20.0": core.NewHash("974f476f0ec5a9dcc4bb005384d449f0a5122da4"), - "refs/tags/v0.21.0": core.NewHash("e08e3917f3a0487e33cd6dcef24fe03e570b73f5"), - "refs/tags/v0.22.0": core.NewHash("834612b4f181171d5e1e263b4e7e55d609ab19f5"), - "refs/tags/v0.23.0": core.NewHash("65558da39c07a6f9104651281c226981e880b49c"), - "refs/tags/v0.24.0": core.NewHash("5c97aa1f2f784e92f065055f9e79df83fac7a4aa"), - "refs/tags/v0.25.0": core.NewHash("d6e696f9d5e2dac968638665886e2300ae15709a"), - "refs/tags/v0.26.0": core.NewHash("974861702abd8388e0507cf3f348d6d3c40acef4"), - "refs/tags/v0.27.0": core.NewHash("65771ef145b3e07e130abc84fb07f0b8044fcf59"), - "refs/tags/v0.28.0": core.NewHash("5d86433d6dc4358277a5e9a834948f0822225a6d"), - "refs/tags/v0.29.0": core.NewHash("c1582497c23d81e61963841861c5aebbf10e12ab"), - "refs/tags/v0.3.0": core.NewHash("8b6002b614b454d45bafbd244b127839421f92ff"), - "refs/tags/v0.30.0": core.NewHash("b0f26484aab0afe2f342be84583213c3c64b7eb3"), - "refs/tags/v0.31.0": core.NewHash("8a2da11c9d29e3a879a068c197568c108b9e5f88"), - "refs/tags/v0.32.0": core.NewHash("5c5fc48a1506bb4609ca5588f90cf021a29a4a37"), - "refs/tags/v0.33.0": core.NewHash("d443f1f61e23411d9ac08f0fc6bbeb8e4c46ee39"), - "refs/tags/v0.34.0": core.NewHash("0168d74697d65cde65f931254c09a6bd7ff4f0d5"), - "refs/tags/v0.35.0": core.NewHash("a46303084ad9decf71a8ea9fd1529e22c6fdd2c4"), - "refs/tags/v0.36.0": core.NewHash("4da0d7bb89e85bd5f14ff36d983a0ae773473b2d"), - "refs/tags/v0.37.0": core.NewHash("85ec60477681933961c9b64c18ada93220650ac5"), - "refs/tags/v0.4.0": core.NewHash("95ee6e6c750ded1f4dc5499bad730ce3f58c6c3a"), - "refs/tags/v0.5.0": core.NewHash("0a3fb06ff80156fb153bcdcc58b5e16c2d27625c"), - "refs/tags/v0.6.0": core.NewHash("dc22e2035292ccf020c30d226f3cc2da651773f6"), - "refs/tags/v0.7.0": core.NewHash("3f36d8f1d67538afd1f089ffd0d242fc4fda736f"), - "refs/tags/v0.8.0": core.NewHash("8526c58617f68de076358873b8aa861a354b48a9"), - "refs/tags/v0.9.0": core.NewHash("776914ef8a097f5683957719c49215a5db17c2cb"), - }, - }, - } { - com := Commentf("subtest %d", i) - _, d := s.newFixtureDir(c, test.fixture) - - refs, err := d.Refs() - c.Assert(err, IsNil, com) - c.Assert(refs, DeepEquals, test.refs, com) - } -} - -func (s *SuiteGitDir) newFixtureDir(c *C, fixName string) (*fixture, *GitDir) { - f, ok := s.fixtures[fixName] - c.Assert(ok, Equals, true) - - d, err := New(fs.NewOS(), f.path) - c.Assert(err, IsNil) - - return &f, d -} - -func (s *SuiteGitDir) TestCapabilities(c *C) { - for i, test := range [...]struct { - fixture string - capabilities *common.Capabilities - }{ - { - fixture: "spinnaker", - }, - } { - com := Commentf("subtest %d", i) - f, d := s.newFixtureDir(c, test.fixture) - - caps, err := d.Capabilities() - c.Assert(err, IsNil, com) - c.Assert(caps, DeepEquals, f.capabilities, com) - } -} - -func (s *SuiteGitDir) TestPackfile(c *C) { - packfile := func(d *GitDir) (fs.FS, string, error) { - return d.Packfile() - } - idxfile := func(d *GitDir) (fs.FS, string, error) { - return d.Idxfile() - } - for _, test := range [...]struct { - fixture string - fn getPathFn - err string // error regexp - }{ - { - fixture: "spinnaker", - fn: packfile, - }, { - fixture: "spinnaker", - fn: idxfile, - }, { - fixture: "empty", - fn: packfile, - err: ".* no such file or directory", - }, { - fixture: "empty", - fn: idxfile, - err: ".* no such file or directory", - }, { - fixture: "no-packfile-no-idx", - fn: packfile, - err: "packfile not found", - }, { - fixture: "no-packfile-no-idx", - fn: idxfile, - err: "idx file not found", - }, - } { - com := Commentf("fixture = %s", test.fixture) - - fix, dir := s.newFixtureDir(c, test.fixture) - - _, path, err := test.fn(dir) - - if test.err != "" { - c.Assert(err, ErrorMatches, test.err, com) - } else { - c.Assert(err, IsNil, com) - c.Assert(strings.HasSuffix(noExt(path), noExt(fix.packfile)), - Equals, true, com) - } - } -} - -type getPathFn func(*GitDir) (fs.FS, string, error) - -func noExt(path string) string { - ext := filepath.Ext(path) - return path[0 : len(path)-len(ext)] -} diff --git a/storage/filesystem/internal/gitdir/refs.go b/storage/filesystem/internal/gitdir/refs.go deleted file mode 100644 index cfd42fd..0000000 --- a/storage/filesystem/internal/gitdir/refs.go +++ /dev/null @@ -1,152 +0,0 @@ -package gitdir - -import ( - "bufio" - "errors" - "io/ioutil" - "os" - "strings" - - "gopkg.in/src-d/go-git.v4/core" -) - -var ( - // ErrPackedRefsDuplicatedRef is returned when a duplicated - // reference is found in the packed-ref file. This is usually the - // case for corrupted git repositories. - ErrPackedRefsDuplicatedRef = errors.New("duplicated ref found in packed-ref file") - // ErrPackedRefsBadFormat is returned when the packed-ref file - // corrupt. - ErrPackedRefsBadFormat = errors.New("malformed packed-ref") - // ErrSymRefTargetNotFound is returned when a symbolic reference is - // targeting a non-existing object. This usually means the - // repository is corrupt. - ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") -) - -const ( - symRefPrefix = "ref: " -) - -func (d *GitDir) addRefsFromPackedRefs() (err error) { - path := d.fs.Join(d.path, packedRefsPath) - f, err := d.fs.Open(path) - if err != nil { - if err == os.ErrNotExist { - return nil - } - return err - } - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - s := bufio.NewScanner(f) - for s.Scan() { - line := s.Text() - if err = d.processLine(line); err != nil { - return err - } - } - - return s.Err() -} - -// process lines from a packed-refs file -func (d *GitDir) processLine(line string) error { - switch line[0] { - case '#': // comment - ignore - return nil - case '^': // annotated tag commit of the previous line - ignore - return nil - default: - ws := strings.Split(line, " ") // hash then ref - if len(ws) != 2 { - return ErrPackedRefsBadFormat - } - h, r := ws[0], ws[1] - - if _, ok := d.refs[r]; ok { - return ErrPackedRefsDuplicatedRef - } - d.refs[r] = core.NewHash(h) - } - - return nil -} - -func (d *GitDir) addRefsFromRefDir() error { - return d.walkTree("refs") -} - -func (d *GitDir) walkTree(relPath string) error { - files, err := d.fs.ReadDir(d.fs.Join(d.path, relPath)) - if err != nil { - return err - } - - for _, f := range files { - newRelPath := d.fs.Join(relPath, f.Name()) - - if f.IsDir() { - if err = d.walkTree(newRelPath); err != nil { - return err - } - } else { - filePath := d.fs.Join(d.path, newRelPath) - h, err := d.readHashFile(filePath) - if err != nil { - return err - } - d.refs[newRelPath] = h - } - } - - return nil -} - -// ReadHashFile reads a single hash from a file. If a symbolic -// reference is found instead of a hash, the reference is resolved and -// the proper hash is returned. -func (d *GitDir) readHashFile(path string) (h core.Hash, err error) { - f, err := d.fs.Open(path) - if err != nil { - return core.ZeroHash, err - } - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - b, err := ioutil.ReadAll(f) - if err != nil { - return core.ZeroHash, err - } - line := strings.TrimSpace(string(b)) - - if isSymRef(line) { - return d.resolveSymRef(line) - } - - return core.NewHash(line), nil -} - -func isSymRef(contents string) bool { - return strings.HasPrefix(contents, symRefPrefix) -} - -func (d *GitDir) resolveSymRef(symRef string) (core.Hash, error) { - ref := strings.TrimPrefix(symRef, symRefPrefix) - - hash, ok := d.refs[ref] - if !ok { - return core.ZeroHash, ErrSymRefTargetNotFound - } - - return hash, nil -} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go new file mode 100644 index 0000000..f3a1dda --- /dev/null +++ b/storage/filesystem/object.go @@ -0,0 +1,142 @@ +package filesystem + +import ( + "fmt" + "os" + + "gopkg.in/src-d/go-git.v4/core" + "gopkg.in/src-d/go-git.v4/formats/packfile" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index" + "gopkg.in/src-d/go-git.v4/utils/fs" +) + +// ObjectStorage is an implementation of core.ObjectStorage that stores +// data on disk in the standard git format (this is, the .git directory). +// +// Zero values of this type are not safe to use, see the New function below. +// +// Currently only reads are supported, no writting. +// +// Also values from this type are not yet able to track changes on disk, this is, +// Gitdir values will get outdated as soon as repositories change on disk. +type ObjectStorage struct { + dir *dotgit.DotGit + index index.Index +} + +func (s *ObjectStorage) NewObject() core.Object { + return &core.MemoryObject{} +} + +// Set adds a new object to the storage. As this functionality is not +// yet supported, this method always returns a "not implemented yet" +// error an zero hash. +func (s *ObjectStorage) Set(core.Object) (core.Hash, error) { + return core.ZeroHash, fmt.Errorf("not implemented yet") +} + +// Get returns the object with the given hash, by searching for it in +// the packfile. +func (s *ObjectStorage) Get(h core.Hash) (core.Object, error) { + offset, err := s.index.Get(h) + if err != nil { + return nil, err + } + + fs, path, err := s.dir.Packfile() + if err != nil { + return nil, err + } + + f, err := fs.Open(path) + if err != nil { + return nil, err + } + + defer func() { + errClose := f.Close() + if err == nil { + err = errClose + } + }() + + _, err = f.Seek(offset, os.SEEK_SET) + if err != nil { + return nil, err + } + + r := packfile.NewSeekable(f) + r.HashToOffset = map[core.Hash]int64(s.index) + p := packfile.NewParser(r) + + obj := s.NewObject() + return obj, p.FillObject(obj) +} + +// Iter returns an iterator for all the objects in the packfile with the +// given type. +func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) { + var objects []core.Object + + for hash := range s.index { + object, err := s.Get(hash) + if err != nil { + return nil, err + } + if object.Type() == t { + objects = append(objects, object) + } + } + + return core.NewObjectSliceIter(objects), nil +} + +func buildIndex(dir *dotgit.DotGit) (index.Index, error) { + fs, idxfile, err := dir.Idxfile() + if err != nil { + if err == dotgit.ErrIdxNotFound { + return buildIndexFromPackfile(dir) + } + return nil, err + } + + return buildIndexFromIdxfile(fs, idxfile) +} + +func buildIndexFromPackfile(dir *dotgit.DotGit) (index.Index, error) { + fs, packfile, err := dir.Packfile() + if err != nil { + return nil, err + } + + f, err := fs.Open(packfile) + if err != nil { + return nil, err + } + + defer func() { + errClose := f.Close() + if err == nil { + err = errClose + } + }() + + return index.NewFromPackfile(f) +} + +func buildIndexFromIdxfile(fs fs.FS, path string) (index.Index, error) { + f, err := fs.Open(path) + if err != nil { + return nil, err + } + + defer func() { + errClose := f.Close() + if err == nil { + err = errClose + } + }() + + return index.NewFromIdx(f) +} diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go new file mode 100644 index 0000000..a784525 --- /dev/null +++ b/storage/filesystem/object_test.go @@ -0,0 +1,315 @@ +package filesystem + +import ( + "fmt" + "os" + "reflect" + "sort" + + "gopkg.in/src-d/go-git.v4/core" + "gopkg.in/src-d/go-git.v4/formats/packfile" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/memory" + "gopkg.in/src-d/go-git.v4/utils/fs" + + "github.com/alcortesm/tgz" + . "gopkg.in/check.v1" +) + +type FsSuite struct{} + +var _ = Suite(&FsSuite{}) + +var fixtures map[string]string // id to git dir paths (see initFixtures below) + +func fixture(id string, c *C) string { + path, ok := fixtures[id] + c.Assert(ok, Equals, true, Commentf("fixture %q not found", id)) + + return path +} + +var initFixtures = [...]struct { + id string + tgz string +}{ + { + id: "binary-relations", + tgz: "internal/dotgit/fixtures/alcortesm-binary-relations.tgz", + }, { + id: "binary-relations-no-idx", + tgz: "internal/dotgit/fixtures/alcortesm-binary-relations-no-idx.tgz", + }, { + id: "ref-deltas-no-idx", + tgz: "internal/dotgit/fixtures/ref-deltas-no-idx.tgz", + }, +} + +func (s *FsSuite) SetUpSuite(c *C) { + fixtures = make(map[string]string, len(initFixtures)) + for _, init := range initFixtures { + path, err := tgz.Extract(init.tgz) + c.Assert(err, IsNil, Commentf("error extracting %s\n", init.tgz)) + fixtures[init.id] = path + } +} + +func (s *FsSuite) TearDownSuite(c *C) { + for _, v := range fixtures { + err := os.RemoveAll(v) + c.Assert(err, IsNil, Commentf("error removing fixture %q\n", v)) + } +} + +func (s *FsSuite) TestHashNotFound(c *C) { + sto := s.newObjectStorage(c, "binary-relations") + + _, err := sto.Get(core.ZeroHash) + c.Assert(err, Equals, core.ErrObjectNotFound) +} + +func (s *FsSuite) newObjectStorage(c *C, fixtureName string) core.ObjectStorage { + path := fixture(fixtureName, c) + fs := fs.NewOS() + + store, err := NewStorage(fs, fs.Join(path, ".git/")) + c.Assert(err, IsNil) + + obj, err := store.ObjectStorage() + c.Assert(err, IsNil) + + return obj +} + +func (s *FsSuite) TestGetCompareWithMemoryStorage(c *C) { + for i, fixId := range [...]string{ + "binary-relations", + "binary-relations-no-idx", + "ref-deltas-no-idx", + } { + path := fixture(fixId, c) + com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", + i, fixId, path) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + memSto, err := memStorageFromGitDir(fs, gitPath) + c.Assert(err, IsNil, com) + + filesystemSto := s.newObjectStorage(c, fixId) + + equal, reason, err := equalsStorages(memSto, filesystemSto) + c.Assert(err, IsNil, com) + c.Assert(equal, Equals, true, + Commentf("%s - %s\n", com.CheckCommentString(), reason)) + } +} + +func memStorageFromGitDir(fs fs.FS, path string) (*memory.ObjectStorage, error) { + dir, err := dotgit.New(fs, path) + if err != nil { + return nil, err + } + + fs, packfilePath, err := dir.Packfile() + if err != nil { + return nil, err + } + + f, err := fs.Open(packfilePath) + if err != nil { + return nil, err + } + + sto := memory.NewObjectStorage() + r := packfile.NewStream(f) + d := packfile.NewDecoder(r) + err = d.Decode(sto) + if err != nil { + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + return sto, nil +} + +func equalsStorages(a, b core.ObjectStorage) (bool, string, error) { + for _, typ := range [...]core.ObjectType{ + core.CommitObject, + core.TreeObject, + core.BlobObject, + core.TagObject, + } { + iter, err := a.Iter(typ) + if err != nil { + return false, "", fmt.Errorf("cannot get iterator: %s", err) + } + + for { + ao, err := iter.Next() + if err != nil { + iter.Close() + break + } + + bo, err := b.Get(ao.Hash()) + if err != nil { + return false, "", fmt.Errorf("getting object with hash %s: %s", + ao.Hash(), err) + } + + equal, reason, err := equalsObjects(ao, bo) + if !equal || err != nil { + return equal, reason, fmt.Errorf("comparing objects: %s", err) + } + } + } + + return true, "", nil +} + +func equalsObjects(a, b core.Object) (bool, string, error) { + ah := a.Hash() + bh := b.Hash() + if ah != bh { + return false, fmt.Sprintf("object hashes differ: %s and %s\n", + ah, bh), nil + } + + atyp := a.Type() + btyp := b.Type() + if atyp != btyp { + return false, fmt.Sprintf("object types differ: %d and %d\n", + atyp, btyp), nil + } + + asz := a.Size() + bsz := b.Size() + if asz != bsz { + return false, fmt.Sprintf("object sizes differ: %d and %d\n", + asz, bsz), nil + } + + ac := a.Content() + if ac != nil { + bc := b.Content() + if !reflect.DeepEqual(ac, bc) { + return false, fmt.Sprintf("object contents differ"), nil + } + } + + return true, "", nil +} + +func (s *FsSuite) TestIterCompareWithMemoryStorage(c *C) { + for i, fixId := range [...]string{ + "binary-relations", + "binary-relations-no-idx", + "ref-deltas-no-idx", + } { + + path := fixture(fixId, c) + com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", + i, fixId, path) + + fs := fs.NewOS() + gitPath := fs.Join(path, ".git/") + + memSto, err := memStorageFromDirPath(fs, gitPath) + c.Assert(err, IsNil, com) + + filesystemSto := s.newObjectStorage(c, fixId) + + for _, typ := range [...]core.ObjectType{ + core.CommitObject, + core.TreeObject, + core.BlobObject, + core.TagObject, + } { + + memObjs, err := iterToSortedSlice(memSto, typ) + c.Assert(err, IsNil, com) + + filesystemObjs, err := iterToSortedSlice(filesystemSto, typ) + c.Assert(err, IsNil, com) + + for i, o := range memObjs { + c.Assert(filesystemObjs[i].Hash(), Equals, o.Hash(), com) + } + } + } +} + +func memStorageFromDirPath(fs fs.FS, path string) (*memory.ObjectStorage, error) { + dir, err := dotgit.New(fs, path) + if err != nil { + return nil, err + } + + fs, packfilePath, err := dir.Packfile() + if err != nil { + return nil, err + } + + sto := memory.NewObjectStorage() + f, err := fs.Open(packfilePath) + if err != nil { + return nil, err + } + + r := packfile.NewStream(f) + d := packfile.NewDecoder(r) + err = d.Decode(sto) + if err != nil { + return nil, err + } + + if err = f.Close(); err != nil { + return nil, err + } + + return sto, nil +} + +func iterToSortedSlice(storage core.ObjectStorage, typ core.ObjectType) ([]core.Object, + error) { + + iter, err := storage.Iter(typ) + if err != nil { + return nil, err + } + + r := make([]core.Object, 0) + for { + obj, err := iter.Next() + if err != nil { + iter.Close() + break + } + r = append(r, obj) + } + + sort.Sort(byHash(r)) + + return r, nil +} + +type byHash []core.Object + +func (a byHash) Len() int { return len(a) } +func (a byHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byHash) Less(i, j int) bool { + return a[i].Hash().String() < a[j].Hash().String() +} + +func (s *FsSuite) TestSet(c *C) { + sto := s.newObjectStorage(c, "binary-relations") + + _, err := sto.Set(&core.MemoryObject{}) + c.Assert(err, ErrorMatches, "not implemented yet") +} diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go new file mode 100644 index 0000000..c8e5434 --- /dev/null +++ b/storage/filesystem/reference.go @@ -0,0 +1,61 @@ +package filesystem + +import ( + "fmt" + + "gopkg.in/src-d/go-git.v4/core" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" +) + +type ReferenceStorage struct { + dir *dotgit.DotGit + refs map[core.ReferenceName]*core.Reference +} + +func (r *ReferenceStorage) Set(ref *core.Reference) error { + return fmt.Errorf("not implemented yet") +} + +func (r *ReferenceStorage) Get(n core.ReferenceName) (*core.Reference, error) { + if err := r.load(); err != nil { + return nil, err + } + + ref, ok := r.refs[n] + if !ok { + return nil, core.ErrReferenceNotFound + } + + return ref, nil +} + +func (r *ReferenceStorage) Iter() (core.ReferenceIter, error) { + if err := r.load(); err != nil { + return nil, err + } + + var refs []*core.Reference + for _, ref := range r.refs { + refs = append(refs, ref) + } + + return core.NewReferenceSliceIter(refs), nil +} + +func (r *ReferenceStorage) load() error { + if len(r.refs) != 0 { + return nil + } + + refs, err := r.dir.Refs() + if err != nil { + return err + } + + r.refs = make(map[core.ReferenceName]*core.Reference, 0) + for _, ref := range refs { + r.refs[ref.Name()] = ref + } + + return nil +} diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 427de94..8f9c555 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -1,203 +1,46 @@ package filesystem import ( - "fmt" - "os" - "strings" - "gopkg.in/src-d/go-git.v4/core" - "gopkg.in/src-d/go-git.v4/formats/packfile" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/gitdir" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" "gopkg.in/src-d/go-git.v4/utils/fs" ) -// ObjectStorage is an implementation of core.ObjectStorage that stores -// data on disk in the standard git format (this is, the .git directory). -// -// Zero values of this type are not safe to use, see the New function below. -// -// Currently only reads are supported, no writting. -// -// Also values from this type are not yet able to track changes on disk, this is, -// Gitdir values will get outdated as soon as repositories change on disk. -type ObjectStorage struct { - dir *gitdir.GitDir - index index.Index -} - -// New returns a new ObjectStorage for the git directory at the specified path. -func New(fs fs.FS, path string) (*ObjectStorage, error) { - s := &ObjectStorage{} - - var err error - s.dir, err = gitdir.New(fs, path) - if err != nil { - return nil, err - } - - s.index, err = buildIndex(s.dir) - - return s, err -} - -func buildIndex(dir *gitdir.GitDir) (index.Index, error) { - fs, idxfile, err := dir.Idxfile() - if err != nil { - if err == gitdir.ErrIdxNotFound { - return buildIndexFromPackfile(dir) - } - return nil, err - } - - return buildIndexFromIdxfile(fs, idxfile) -} - -func buildIndexFromPackfile(dir *gitdir.GitDir) (index.Index, error) { - fs, packfile, err := dir.Packfile() - if err != nil { - return nil, err - } +type Storage struct { + dir *dotgit.DotGit - f, err := fs.Open(packfile) - if err != nil { - return nil, err - } - - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - return index.NewFromPackfile(f) + o *ObjectStorage + r *ReferenceStorage } -func buildIndexFromIdxfile(fs fs.FS, path string) (index.Index, error) { - f, err := fs.Open(path) +func NewStorage(fs fs.FS, path string) (*Storage, error) { + dir, err := dotgit.New(fs, path) if err != nil { return nil, err } - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - return index.NewFromIdx(f) -} - -func (s *ObjectStorage) NewObject() core.Object { - return &core.MemoryObject{} -} - -// Set adds a new object to the storage. As this functionality is not -// yet supported, this method always returns a "not implemented yet" -// error an zero hash. -func (s *ObjectStorage) Set(core.Object) (core.Hash, error) { - return core.ZeroHash, fmt.Errorf("not implemented yet") + return &Storage{dir: dir}, nil } -// Get returns the object with the given hash, by searching for it in -// the packfile. -func (s *ObjectStorage) Get(h core.Hash) (core.Object, error) { - offset, err := s.index.Get(h) - if err != nil { - return nil, err - } - - fs, path, err := s.dir.Packfile() - if err != nil { - return nil, err +func (s *Storage) ObjectStorage() (core.ObjectStorage, error) { + if s.o != nil { + return s.o, nil } - f, err := fs.Open(path) + i, err := buildIndex(s.dir) if err != nil { return nil, err } - defer func() { - errClose := f.Close() - if err == nil { - err = errClose - } - }() - - _, err = f.Seek(offset, os.SEEK_SET) - if err != nil { - return nil, err - } - - r := packfile.NewSeekable(f) - r.HashToOffset = map[core.Hash]int64(s.index) - p := packfile.NewParser(r) - - obj := s.NewObject() - return obj, p.FillObject(obj) -} - -// Iter returns an iterator for all the objects in the packfile with the -// given type. -func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) { - var objects []core.Object - - for hash := range s.index { - object, err := s.Get(hash) - if err != nil { - return nil, err - } - if object.Type() == t { - objects = append(objects, object) - } - } - - return core.NewObjectSliceIter(objects), nil + s.o = &ObjectStorage{dir: s.dir, index: i} + return s.o, nil } -const ( - headErrPrefix = "cannot get HEAD reference:" - symrefCapability = "symref" - headRefPrefix = "HEAD:" -) - -// Head returns the hash of the HEAD reference -func (s *ObjectStorage) Head() (core.Hash, error) { - cap, err := s.dir.Capabilities() - if err != nil { - return core.ZeroHash, fmt.Errorf("%s %s", headErrPrefix, err) - } - - ok := cap.Supports(symrefCapability) - if !ok { - return core.ZeroHash, - fmt.Errorf("%s symref capability not supported", headErrPrefix) - } - - symrefs := cap.Get(symrefCapability) - var headRef string - for _, ref := range symrefs.Values { - if strings.HasPrefix(ref, headRefPrefix) { - headRef = strings.TrimPrefix(ref, headRefPrefix) - } - } - if headRef == "" { - return core.ZeroHash, fmt.Errorf("%s HEAD reference not found", - headErrPrefix) - } - - refs, err := s.dir.Refs() - if err != nil { - return core.ZeroHash, fmt.Errorf("%s %s", headErrPrefix, err) - } - - head, ok := refs[headRef] - if !ok { - return core.ZeroHash, fmt.Errorf("%s reference %q not found", - headErrPrefix, headRef) +func (s *Storage) ReferenceStorage() (core.ReferenceStorage, error) { + if s.r != nil { + return s.r, nil } - return head, nil + s.r = &ReferenceStorage{dir: s.dir} + return s.r, nil } diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 035a206..3cb7dd8 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -1,326 +1,21 @@ -package filesystem_test +package filesystem import ( - "fmt" - "os" - "reflect" - "sort" "testing" - "gopkg.in/src-d/go-git.v4/core" - "gopkg.in/src-d/go-git.v4/formats/packfile" - "gopkg.in/src-d/go-git.v4/storage/filesystem" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/gitdir" - "gopkg.in/src-d/go-git.v4/storage/memory" - "gopkg.in/src-d/go-git.v4/utils/fs" - - "github.com/alcortesm/tgz" . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/utils/fs" ) func Test(t *testing.T) { TestingT(t) } -type FsSuite struct{} - -var _ = Suite(&FsSuite{}) - -var fixtures map[string]string // id to git dir paths (see initFixtures below) - -func fixture(id string, c *C) string { - path, ok := fixtures[id] - c.Assert(ok, Equals, true, Commentf("fixture %q not found", id)) - - return path -} - -var initFixtures = [...]struct { - id string - tgz string -}{ - { - id: "binary-relations", - tgz: "internal/gitdir/fixtures/alcortesm-binary-relations.tgz", - }, { - id: "binary-relations-no-idx", - tgz: "internal/gitdir/fixtures/alcortesm-binary-relations-no-idx.tgz", - }, { - id: "ref-deltas-no-idx", - tgz: "internal/gitdir/fixtures/ref-deltas-no-idx.tgz", - }, -} - -func (s *FsSuite) SetUpSuite(c *C) { - fixtures = make(map[string]string, len(initFixtures)) - for _, init := range initFixtures { - path, err := tgz.Extract(init.tgz) - c.Assert(err, IsNil, Commentf("error extracting %s\n", init.tgz)) - fixtures[init.id] = path - } -} - -func (s *FsSuite) TearDownSuite(c *C) { - for _, v := range fixtures { - err := os.RemoveAll(v) - c.Assert(err, IsNil, Commentf("error removing fixture %q\n", v)) - } -} - -func (s *FsSuite) TestNewErrorNotFound(c *C) { - fs := fs.NewOS() - _, err := filesystem.New(fs, "not_found/.git") - c.Assert(err, Equals, gitdir.ErrNotFound) -} - -func (s *FsSuite) TestHashNotFound(c *C) { - path := fixture("binary-relations", c) - - fs := fs.NewOS() - gitPath := fs.Join(path, ".git/") - - sto, err := filesystem.New(fs, gitPath) - c.Assert(err, IsNil) - - _, err = sto.Get(core.ZeroHash) - c.Assert(err, Equals, core.ErrObjectNotFound) -} - -func (s *FsSuite) TestGetCompareWithMemoryStorage(c *C) { - for i, fixId := range [...]string{ - "binary-relations", - "binary-relations-no-idx", - "ref-deltas-no-idx", - } { - path := fixture(fixId, c) - com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", - i, fixId, path) - - fs := fs.NewOS() - gitPath := fs.Join(path, ".git/") - - memSto, err := memStorageFromGitDir(fs, gitPath) - c.Assert(err, IsNil, com) - - filesystemSto, err := filesystem.New(fs, gitPath) - c.Assert(err, IsNil, com) - - equal, reason, err := equalsStorages(memSto, filesystemSto) - c.Assert(err, IsNil, com) - c.Assert(equal, Equals, true, - Commentf("%s - %s\n", com.CheckCommentString(), reason)) - } -} - -func memStorageFromGitDir(fs fs.FS, path string) (*memory.ObjectStorage, error) { - dir, err := gitdir.New(fs, path) - if err != nil { - return nil, err - } - - fs, packfilePath, err := dir.Packfile() - if err != nil { - return nil, err - } - - f, err := fs.Open(packfilePath) - if err != nil { - return nil, err - } - - sto := memory.NewObjectStorage() - r := packfile.NewStream(f) - d := packfile.NewDecoder(r) - err = d.Decode(sto) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - - return sto, nil -} - -func equalsStorages(a, b core.ObjectStorage) (bool, string, error) { - for _, typ := range [...]core.ObjectType{ - core.CommitObject, - core.TreeObject, - core.BlobObject, - core.TagObject, - } { - iter, err := a.Iter(typ) - if err != nil { - return false, "", fmt.Errorf("cannot get iterator: %s", err) - } - - for { - ao, err := iter.Next() - if err != nil { - iter.Close() - break - } +type StorageSuite struct{} - bo, err := b.Get(ao.Hash()) - if err != nil { - return false, "", fmt.Errorf("getting object with hash %s: %s", - ao.Hash(), err) - } - - equal, reason, err := equalsObjects(ao, bo) - if !equal || err != nil { - return equal, reason, fmt.Errorf("comparing objects: %s", err) - } - } - } - - return true, "", nil -} - -func equalsObjects(a, b core.Object) (bool, string, error) { - ah := a.Hash() - bh := b.Hash() - if ah != bh { - return false, fmt.Sprintf("object hashes differ: %s and %s\n", - ah, bh), nil - } - - atyp := a.Type() - btyp := b.Type() - if atyp != btyp { - return false, fmt.Sprintf("object types differ: %d and %d\n", - atyp, btyp), nil - } - - asz := a.Size() - bsz := b.Size() - if asz != bsz { - return false, fmt.Sprintf("object sizes differ: %d and %d\n", - asz, bsz), nil - } - - ac := a.Content() - if ac != nil { - bc := b.Content() - if !reflect.DeepEqual(ac, bc) { - return false, fmt.Sprintf("object contents differ"), nil - } - } - - return true, "", nil -} - -func (s *FsSuite) TestIterCompareWithMemoryStorage(c *C) { - for i, fixId := range [...]string{ - "binary-relations", - "binary-relations-no-idx", - "ref-deltas-no-idx", - } { - - path := fixture(fixId, c) - com := Commentf("at subtest %d, (fixture id = %q, extracted to %q)", - i, fixId, path) - - fs := fs.NewOS() - gitPath := fs.Join(path, ".git/") - - memSto, err := memStorageFromDirPath(fs, gitPath) - c.Assert(err, IsNil, com) - - filesystemSto, err := filesystem.New(fs, gitPath) - c.Assert(err, IsNil, com) - - for _, typ := range [...]core.ObjectType{ - core.CommitObject, - core.TreeObject, - core.BlobObject, - core.TagObject, - } { - - memObjs, err := iterToSortedSlice(memSto, typ) - c.Assert(err, IsNil, com) - - filesystemObjs, err := iterToSortedSlice(filesystemSto, typ) - c.Assert(err, IsNil, com) - - for i, o := range memObjs { - c.Assert(filesystemObjs[i].Hash(), Equals, o.Hash(), com) - } - } - } -} - -func memStorageFromDirPath(fs fs.FS, path string) (*memory.ObjectStorage, error) { - dir, err := gitdir.New(fs, path) - if err != nil { - return nil, err - } - - fs, packfilePath, err := dir.Packfile() - if err != nil { - return nil, err - } - - sto := memory.NewObjectStorage() - f, err := fs.Open(packfilePath) - if err != nil { - return nil, err - } - - r := packfile.NewStream(f) - d := packfile.NewDecoder(r) - err = d.Decode(sto) - if err != nil { - return nil, err - } - - if err = f.Close(); err != nil { - return nil, err - } - - return sto, nil -} - -func iterToSortedSlice(storage core.ObjectStorage, typ core.ObjectType) ([]core.Object, - error) { - - iter, err := storage.Iter(typ) - if err != nil { - return nil, err - } - - r := make([]core.Object, 0) - for { - obj, err := iter.Next() - if err != nil { - iter.Close() - break - } - r = append(r, obj) - } - - sort.Sort(byHash(r)) - - return r, nil -} - -type byHash []core.Object - -func (a byHash) Len() int { return len(a) } -func (a byHash) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byHash) Less(i, j int) bool { - return a[i].Hash().String() < a[j].Hash().String() -} - -func (s *FsSuite) TestSet(c *C) { - path := fixture("binary-relations", c) +var _ = Suite(&StorageSuite{}) +func (s *StorageSuite) TestNewErrorNotFound(c *C) { fs := fs.NewOS() - gitPath := fs.Join(path, ".git/") - - sto, err := filesystem.New(fs, gitPath) - c.Assert(err, IsNil) - - _, err = sto.Set(&core.MemoryObject{}) - c.Assert(err, ErrorMatches, "not implemented yet") + _, err := NewStorage(fs, "not_found/.git") + c.Assert(err, Equals, dotgit.ErrNotFound) } -- cgit