From c64eb817d5e5cbaec10dea1342e1ec95721e886b Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Tue, 25 Jul 2017 10:08:36 +0200 Subject: packfile: create packfile.Index and reuse it There was an internal type (i.e. storage/filesystem.idx) to use as in-memory index for packfiles. This was not convenient to reuse in the packfile. This commit creates a new representation (format/packfile.Index) that can be converted to and from idxfile.Idxfile. A packfile.Index now contains the functionality that was scattered on storage/filesystem.idx and packfile.Decoder's internals. storage/filesystem now reuses packfile.Index instances and this also results in higher cache hit ratios when resolving deltas. --- storage/filesystem/internal/dotgit/writers.go | 20 +++++------ storage/filesystem/object.go | 51 ++++++++++----------------- 2 files changed, 26 insertions(+), 45 deletions(-) (limited to 'storage/filesystem') diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/internal/dotgit/writers.go index 531259b..a7525d4 100644 --- a/storage/filesystem/internal/dotgit/writers.go +++ b/storage/filesystem/internal/dotgit/writers.go @@ -20,13 +20,13 @@ import ( // is renamed/moved (depends on the Filesystem implementation) to the final // location, if the PackWriter is not used, nothing is written type PackWriter struct { - Notify func(h plumbing.Hash, i idxfile.Idxfile) + Notify func(plumbing.Hash, *packfile.Index) fs billy.Filesystem fr, fw billy.File synced *syncedReader checksum plumbing.Hash - index idxfile.Idxfile + index *packfile.Index result chan error } @@ -68,14 +68,7 @@ func (w *PackWriter) buildIndex() { } w.checksum = checksum - w.index.PackfileChecksum = checksum - w.index.Version = idxfile.VersionSupported - - offsets := d.Offsets() - for h, crc := range d.CRCs() { - w.index.Add(h, uint64(offsets[h]), crc) - } - + w.index = d.Index() w.result <- err } @@ -122,7 +115,7 @@ func (w *PackWriter) Close() error { return err } - if len(w.index.Entries) == 0 { + if w.index == nil || w.index.Size() == 0 { return w.clean() } @@ -152,8 +145,11 @@ func (w *PackWriter) save() error { } func (w *PackWriter) encodeIdx(writer io.Writer) error { + idx := w.index.ToIdxFile() + idx.PackfileChecksum = w.checksum + idx.Version = idxfile.VersionSupported e := idxfile.NewEncoder(writer) - _, err := e.Encode(&w.index) + _, err := e.Encode(idx) return err } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 8bf84f2..e235b33 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -18,7 +18,7 @@ import ( type ObjectStorage struct { dir *dotgit.DotGit - index map[plumbing.Hash]idx + index map[plumbing.Hash]*packfile.Index } func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { @@ -34,7 +34,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]idx, 0) + s.index = make(map[plumbing.Hash]*packfile.Index, 0) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -50,14 +50,19 @@ func (s *ObjectStorage) requireIndex() error { } func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) error { - idxfile, err := s.dir.ObjectPackIdx(h) + f, err := s.dir.ObjectPackIdx(h) if err != nil { return err } - defer ioutil.CheckClose(idxfile, &err) - s.index[h] = make(idx) - err = s.index[h].Decode(idxfile) + defer ioutil.CheckClose(f, &err) + idxf := idxfile.NewIdxfile() + d := idxfile.NewDecoder(f) + if err = d.Decode(idxf); err != nil { + return err + } + + s.index[h] = packfile.NewIndexFromIdxFile(idxf) return err } @@ -75,11 +80,8 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { return nil, err } - w.Notify = func(h plumbing.Hash, idxfile idxfile.Idxfile) { - s.index[h] = make(idx) - for _, e := range idxfile.Entries { - s.index[h][e.Hash] = int64(e.Offset) - } + w.Notify = func(h plumbing.Hash, idx *packfile.Index) { + s.index[h] = idx } return w, nil @@ -196,15 +198,15 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash) (plumbing.EncodedObject return nil, err } - d.SetOffsets(s.index[pack]) + d.SetIndex(s.index[pack]) obj, err := d.DecodeObjectAt(offset) return obj, err } func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, int64) { for packfile, index := range s.index { - if offset, ok := index[h]; ok { - return packfile, offset + if e, ok := index.LookupHash(h); ok { + return packfile, int64(e.Offset) } } @@ -263,23 +265,6 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb return iters, nil } -type idx map[plumbing.Hash]int64 - -func (i idx) Decode(r io.Reader) error { - idx := &idxfile.Idxfile{} - - d := idxfile.NewDecoder(r) - if err := d.Decode(idx); err != nil { - return err - } - - for _, e := range idx.Entries { - i[e.Hash] = int64(e.Offset) - } - - return nil -} - type packfileIter struct { f billy.File d *packfile.Decoder @@ -295,7 +280,7 @@ func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectI } func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]bool, - index idx) (storer.EncodedObjectIter, error) { + index *packfile.Index) (storer.EncodedObjectIter, error) { s := packfile.NewScanner(f) _, total, err := s.Header() if err != nil { @@ -307,7 +292,7 @@ func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash return nil, err } - d.SetOffsets(index) + d.SetIndex(index) return &packfileIter{ f: f, -- cgit