path: root/storage/filesystem/object.go

package filesystem

import (
	"io"
	"os"

	"gopkg.in/src-d/go-git.v4/core"
	"gopkg.in/src-d/go-git.v4/formats/idxfile"
	"gopkg.in/src-d/go-git.v4/formats/objfile"
	"gopkg.in/src-d/go-git.v4/formats/packfile"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
	"gopkg.in/src-d/go-git.v4/storage/memory"
	"gopkg.in/src-d/go-git.v4/utils/fs"
)

type ObjectStorage struct {
	dir   *dotgit.DotGit
	index map[core.Hash]index
}

func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) {
	s := ObjectStorage{
		dir:   dir,
		index: make(map[core.Hash]index, 0),
	}

	return s, s.loadIdxFiles()
}

func (s *ObjectStorage) loadIdxFiles() error {
	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return err
	}

	for _, h := range packs {
		if err := s.loadIdxFile(h); err != nil {
			return err
		}
	}

	return nil
}

func (s *ObjectStorage) loadIdxFile(h core.Hash) error {
	idx, err := s.dir.ObjectPackIdx(h)
	if err != nil {
		return err
	}

	s.index[h] = make(index)
	return s.index[h].Decode(idx)
}

func (s *ObjectStorage) NewObject() core.Object {
	return &core.MemoryObject{}
}

func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) {
	w, err := s.dir.NewObjectPack()
	if err != nil {
		return nil, err
	}

	w.Notify = func(h core.Hash, idx idxfile.Idxfile) {
		s.index[h] = make(index)
		for _, e := range idx.Entries {
			s.index[h][e.Hash] = int64(e.Offset)
		}
	}

	return w, nil
}

// Set adds a new object to the storage.
func (s *ObjectStorage) SetObject(o core.Object) (core.Hash, error) {
	if o.Type() == core.OFSDeltaObject || o.Type() == core.REFDeltaObject {
		return core.ZeroHash, core.ErrInvalidType
	}

	ow, err := s.dir.NewObject()
	if err != nil {
		return core.ZeroHash, err
	}

	defer ow.Close()

	or, err := o.Reader()
	if err != nil {
		return core.ZeroHash, err
	}

	defer or.Close()

	if err := ow.WriteHeader(o.Type(), o.Size()); err != nil {
		return core.ZeroHash, err
	}

	if _, err := io.Copy(ow, or); err != nil {
		return core.ZeroHash, err
	}

	return o.Hash(), nil
}

// Get returns the object with the given hash, by searching for it in
// the packfile and the git object directories.
func (s *ObjectStorage) Object(t core.ObjectType, h core.Hash) (core.Object, error) {
	obj, err := s.getFromUnpacked(h)
	if err == core.ErrObjectNotFound {
		obj, err = s.getFromPackfile(h)
	}

	if err != nil {
		return nil, err
	}

	if core.AnyObject != t && obj.Type() != t {
		return nil, core.ErrObjectNotFound
	}

	return obj, nil
}

func (s *ObjectStorage) getFromUnpacked(h core.Hash) (obj core.Object, err error) {
	f, err := s.dir.Object(h)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, core.ErrObjectNotFound
		}

		return nil, err
	}

	defer f.Close()

	obj = s.NewObject()
	r, err := objfile.NewReader(f)
	if err != nil {
		return nil, err
	}

	defer r.Close()

	t, size, err := r.Header()
	if err != nil {
		return nil, err
	}

	obj.SetType(t)
	obj.SetSize(size)
	w, err := obj.Writer()
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(w, r)
	return obj, err
}

// Get returns the object with the given hash, by searching for it in
// the packfile.
func (s *ObjectStorage) getFromPackfile(h core.Hash) (core.Object, error) {
	pack, offset := s.findObjectInPackfile(h)
	if offset == -1 {
		return nil, core.ErrObjectNotFound
	}

	f, err := s.dir.ObjectPack(pack)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	p := packfile.NewScanner(f)
	d, err := packfile.NewDecoder(p, memory.NewStorage())
	if err != nil {
		return nil, err
	}

	d.SetOffsets(s.index[pack])
	return d.ReadObjectAt(offset)
}

func (s *ObjectStorage) findObjectInPackfile(h core.Hash) (core.Hash, int64) {
	for packfile, index := range s.index {
		if offset, ok := index[h]; ok {
			return packfile, offset
		}
	}

	return core.ZeroHash, -1
}

// Iter returns an iterator for all the objects in the packfile with the
// given type.
func (s *ObjectStorage) IterObjects(t core.ObjectType) (core.ObjectIter, error) {
	objects, err := s.dir.Objects()
	if err != nil {
		return nil, err
	}

	seen := make(map[core.Hash]bool, 0)
	var iters []core.ObjectIter
	if len(objects) != 0 {
		iters = append(iters, &objectsIter{s: s, t: t, h: objects})
		seen = hashListAsMap(objects)
	}

	packi, err := s.buildPackfileIters(t, seen)
	if err != nil {
		return nil, err
	}

	iters = append(iters, packi...)
	return core.NewMultiObjectIter(iters), nil
}

func (s *ObjectStorage) buildPackfileIters(
	t core.ObjectType, seen map[core.Hash]bool) ([]core.ObjectIter, error) {
	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return nil, err
	}

	var iters []core.ObjectIter
	for _, h := range packs {
		pack, err := s.dir.ObjectPack(h)
		if err != nil {
			return nil, err
		}

		iter, err := newPackfileIter(pack, t, seen)
		if err != nil {
			return nil, err
		}

		iters = append(iters, iter)
	}

	return iters, nil
}

type index map[core.Hash]int64

func (i index) Decode(r io.Reader) error {
	idx := &idxfile.Idxfile{}

	d := idxfile.NewDecoder(r)
	if err := d.Decode(idx); err != nil {
		return err
	}

	for _, e := range idx.Entries {
		i[e.Hash] = int64(e.Offset)
	}

	return nil
}

type packfileIter struct {
	f fs.File
	d *packfile.Decoder
	t core.ObjectType

	seen     map[core.Hash]bool
	position uint32
	total    uint32
}

func newPackfileIter(f fs.File, t core.ObjectType, seen map[core.Hash]bool) (core.ObjectIter, error) {
	s := packfile.NewScanner(f)
	_, total, err := s.Header()
	if err != nil {
		return nil, err
	}

	d, err := packfile.NewDecoder(s, memory.NewStorage())
	if err != nil {
		return nil, err
	}

	return &packfileIter{
		f: f,
		d: d,
		t: t,

		total: total,
		seen:  seen,
	}, nil
}

func (iter *packfileIter) Next() (core.Object, error) {
	if iter.position >= iter.total {
		return nil, io.EOF
	}

	obj, err := iter.d.ReadObject()
	if err != nil {
		return nil, err
	}

	iter.position++
	if iter.seen[obj.Hash()] {
		return iter.Next()
	}

	if iter.t != core.AnyObject && iter.t != obj.Type() {
		return iter.Next()
	}

	return obj, nil
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *packfileIter) ForEach(cb func(core.Object) error) error {
	return nil
}

func (iter *packfileIter) Close() {
	iter.f.Close()
	iter.d.Close()
}

type objectsIter struct {
	s *ObjectStorage
	t core.ObjectType
	h []core.Hash
}

func (iter *objectsIter) Next() (core.Object, error) {
	if len(iter.h) == 0 {
		return nil, io.EOF
	}

	obj, err := iter.s.getFromUnpacked(iter.h[0])
	iter.h = iter.h[1:]

	if err != nil {
		return nil, err
	}

	if iter.t != core.AnyObject && iter.t != obj.Type() {
		return iter.Next()
	}

	return obj, err
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *objectsIter) ForEach(cb func(core.Object) error) error {
	return nil
}

func (iter *objectsIter) Close() {
	iter.h = []core.Hash{}
}

func hashListAsMap(l []core.Hash) map[core.Hash]bool {
	m := make(map[core.Hash]bool, len(l))
	for _, h := range l {
		m[h] = true
	}

	return m
}