package packfile
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"io/ioutil"
"gopkg.in/src-d/go-git.v3/core"
"github.com/klauspost/compress/zlib"
)
type Format int
var (
EmptyRepositoryErr = newError("empty repository")
UnsupportedVersionErr = newError("unsupported packfile version")
MaxObjectsLimitReachedErr = newError("max. objects limit reached")
MalformedPackfileErr = newError("malformed pack file, does not start with 'PACK'")
InvalidObjectErr = newError("invalid git object")
PatchingErr = newError("patching error")
PackEntryNotFoundErr = newError("can't find a pack entry")
ObjectNotFoundErr = newError("can't find a object")
ZLibErr = newError("zlib reading error")
)
const (
DefaultMaxObjectsLimit = 1 << 20
VersionSupported = 2
UnknownFormat Format = 0
OFSDeltaFormat Format = 1
REFDeltaFormat Format = 2
)
// Reader reads a packfile from a binary string splitting it on objects
type Reader struct {
// MaxObjectsLimit is the limit of objects to be load in the packfile, if
// a packfile excess this number an error is throw, the default value
// is defined by DefaultMaxObjectsLimit, usually the default limit is more
// than enough to work with any repository, working extremly big repositories
// where the number of object is bigger the memory can be exhausted.
MaxObjectsLimit uint32
// Format specifies if we are using ref-delta's or ofs-delta's, choosing the
// correct format the memory usage is optimized
// https://github.com/git/git/blob/8d530c4d64ffcc853889f7b385f554d53db375ed/Documentation/technical/protocol-capabilities.txt#L154
Format Format
r *trackingReader
s core.ObjectStorage
offsets map[int64]core.Hash
}
// NewReader returns a new Reader that reads from a io.Reader
func NewReader(r io.Reader) *Reader {
return &Reader{
MaxObjectsLimit: DefaultMaxObjectsLimit,
r: NewTrackingReader(r),
offsets: make(map[int64]core.Hash, 0),
}
}
// Read reads the objects and stores it at the ObjectStorage
func (r *Reader) Read(s core.ObjectStorage) (int64, error) {
r.s = s
if err := r.validateHeader(); err != nil {
if err == io.EOF {
return -1, EmptyRepositoryErr
}
return -1, err
}
version, err := r.readInt32()
if err != nil {
return -1, err
}
if version > VersionSupported {
return -1, UnsupportedVersionErr
}
count, err := r.readInt32()
if err != nil {
return -1, err
}
if count > r.MaxObjectsLimit {
return -1, MaxObjectsLimitReachedErr
}
return r.r.position, r.readObjects(count)
}
func (r *Reader) validateHeader() error {
var header = make([]byte, 4)
if _, err := io.ReadFull(r.r, header); err != nil {
return err
}
if !bytes.Equal(header, []byte{'P', 'A', 'C', 'K'}) {
return MalformedPackfileErr
}
return nil
}
func (r *Reader) readInt32() (uint32, error) {
var value uint32
if err := binary.Read(r.r, binary.BigEndian, &value); err != nil {
return 0, err
}
return value, nil
}
func (r *Reader) readObjects(count uint32) error {
// This code has 50-80 µs of overhead per object not counting zlib inflation.
// Together with zlib inflation, it's 400-410 µs for small objects.
// That's 1 sec for ~2450 objects, ~4.20 MB, or ~250 ms per MB,
// of which 12-20 % is _not_ zlib inflation (ie. is our code).
for i := 0; i < int(count); i++ {
start := r.r.position
obj, err := r.newObject()
if err != nil && err != io.EOF {
return err
}
if r.Format == UnknownFormat || r.Format == OFSDeltaFormat {
r.offsets[start] = obj.Hash()
}
r.s.Set(obj)
if err == io.EOF {
break
}
}
return nil
}
func (r *Reader) newObject() (core.Object, error) {
raw, err := r.s.New()
if err != nil {
return nil, err
}
var steps int64
var buf [1]byte
if _, err := r.r.Read(buf[:]); err != nil {
return nil, err
}
typ := core.ObjectType((buf[0] >> 4) & 7)
size := int64(buf[0] & 15)
steps++ // byte we just read to get `o.typ` and `o.size`
var shift uint = 4
for buf[0]&0x80 == 0x80 {
if _, err := r.r.Read(buf[:]); err != nil {
return nil, err
}
size += int64(buf[0]&0x7f) << shift
steps++ // byte we just read to update `o.size`
shift += 7
}
raw.SetType(typ)
raw.SetSize(size)
switch raw.Type() {
case core.REFDeltaObject:
err = r.readREFDelta(raw)
case core.OFSDeltaObject:
err = r.readOFSDelta(raw, steps)
case core.CommitObject, core.TreeObject, core.BlobObject, core.TagObject:
err = r.readObject(raw)
default:
err = InvalidObjectErr.n("tag %q", raw.Type)
}
return raw, err
}
func (r *Reader) readREFDelta(raw core.Object) (err error) {
var ref core.Hash
if _, err := io.ReadFull(r.r, ref[:]); err != nil {
return err
}
buf := bytes.NewBuffer(nil)
if err := r.inflate(buf); err != nil {
return err
}
referenced, err := r.s.Get(ref)
if err != nil {
if err == core.ObjectNotFoundErr {
return ObjectNotFoundErr.n("%s", ref)
}
return err
}
reader, err := referenced.Reader()
if err != nil {
return err
}
defer close(reader, &err)
d, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
patched := patchDelta(d, buf.Bytes())
if patched == nil {
return PatchingErr.n("hash %q", ref)
}
raw.SetType(referenced.Type())
raw.SetSize(int64(len(patched)))
writer, err := raw.Writer()
if err != nil {
return err
}
defer close(writer, &err)
writer.Write(patched)
return nil
}
func (r *Reader) readOFSDelta(raw core.Object, steps int64) (err error) {
start := r.r.position
offset, err := decodeOffset(r.r, steps)
if err != nil {
return err
}
buf := bytes.NewBuffer(nil)
if err = r.inflate(buf); err != nil {
return err
}
ref, ok := r.offsets[start+offset]
if !ok {
return PackEntryNotFoundErr.n("offset %d", start+offset)
}
referenced, err := r.s.Get(ref)
if err != nil {
return err
}
reader, err := referenced.Reader()
if err != nil {
return err
}
defer close(reader, &err)
d, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
patched := patchDelta(d, buf.Bytes())
if patched == nil {
return PatchingErr.n("hash %q", ref)
}
raw.SetType(referenced.Type())
raw.SetSize(int64(len(patched)))
writer, err := raw.Writer()
if err != nil {
return err
}
defer close(writer, &err)
writer.Write(patched)
return nil
}
func (r *Reader) readObject(raw core.Object) (err error) {
writer, err := raw.Writer()
if err != nil {
return err
}
defer close(writer, &err)
return r.inflate(writer)
}
func (r *Reader) inflate(w io.Writer) error {
zr, err := zlib.NewReader(r.r)
if err != nil {
if err == zlib.ErrHeader {
return zlib.ErrHeader
}
return ZLibErr.n("%s", err)
}
defer zr.Close()
_, err = io.Copy(w, zr)
return err
}
type ReaderError struct {
reason, additional string
}
func newError(reason string) *ReaderError {
return &ReaderError{reason: reason}
}
func (e *ReaderError) Error() string {
if e.additional == "" {
return e.reason
}
return fmt.Sprintf("%s: %s", e.reason, e.additional)
}
func (e *ReaderError) n(format string, args ...interface{}) *ReaderError {
return &ReaderError{
reason: e.reason,
additional: fmt.Sprintf(format, args...),
}
}
