new plumbing package (#118)

* plumbing: now core was renamed to core, and formats and clients moved inside

6 files changed, 0 insertions, 590 deletions

diff --git a/formats/idxfile/decoder.go b/formats/idxfile/decoder.go
deleted file mode 100644
index 884d32b..0000000
--- a/formats/idxfile/decoder.go
+++ /dev/null
@@ -1,148 +0,0 @@
-package idxfile
-
-import (
-	"bytes"
-	"errors"
-	"io"
-
-	"gopkg.in/src-d/go-git.v4/core"
-	"gopkg.in/src-d/go-git.v4/utils/binary"
-)
-
-var (
-	// ErrUnsupportedVersion is returned by Decode when the idx file version
-	// is not supported.
-	ErrUnsupportedVersion = errors.New("Unsuported version")
-	// ErrMalformedIdxFile is returned by Decode when the idx file is corrupted.
-	ErrMalformedIdxFile = errors.New("Malformed IDX file")
-)
-
-// A Decoder reads and decodes idx files from an input stream.
-type Decoder struct {
-	io.Reader
-}
-
-// NewDecoder returns a new decoder that reads from r.
-func NewDecoder(r io.Reader) *Decoder {
-	return &Decoder{r}
-}
-
-// Decode reads the whole idx object from its input and stores it in the
-// value pointed to by idx.
-func (d *Decoder) Decode(idx *Idxfile) error {
-	if err := validateHeader(d); err != nil {
-		return err
-	}
-
-	flow := []func(*Idxfile, io.Reader) error{
-		readVersion,
-		readFanout,
-		readObjectNames,
-		readCRC32,
-		readOffsets,
-		readChecksums,
-	}
-
-	for _, f := range flow {
-		if err := f(idx, d); err != nil {
-			return err
-		}
-	}
-
-	if !idx.isValid() {
-		return ErrMalformedIdxFile
-	}
-
-	return nil
-}
-
-func validateHeader(r io.Reader) error {
-	var h = make([]byte, 4)
-	if _, err := r.Read(h); err != nil {
-		return err
-	}
-
-	if !bytes.Equal(h, idxHeader) {
-		return ErrMalformedIdxFile
-	}
-
-	return nil
-}
-
-func readVersion(idx *Idxfile, r io.Reader) error {
-	v, err := binary.ReadUint32(r)
-	if err != nil {
-		return err
-	}
-
-	if v > VersionSupported {
-		return ErrUnsupportedVersion
-	}
-
-	idx.Version = v
-	return nil
-}
-
-func readFanout(idx *Idxfile, r io.Reader) error {
-	var err error
-	for i := 0; i < 255; i++ {
-		idx.Fanout[i], err = binary.ReadUint32(r)
-		if err != nil {
-			return err
-		}
-	}
-
-	idx.ObjectCount, err = binary.ReadUint32(r)
-	return err
-}
-
-func readObjectNames(idx *Idxfile, r io.Reader) error {
-	c := int(idx.ObjectCount)
-	for i := 0; i < c; i++ {
-		var ref core.Hash
-		if _, err := r.Read(ref[:]); err != nil {
-			return err
-		}
-
-		idx.Entries = append(idx.Entries, Entry{Hash: ref})
-	}
-
-	return nil
-}
-
-func readCRC32(idx *Idxfile, r io.Reader) error {
-	c := int(idx.ObjectCount)
-	for i := 0; i < c; i++ {
-		if err := binary.Read(r, &idx.Entries[i].CRC32); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func readOffsets(idx *Idxfile, r io.Reader) error {
-	c := int(idx.ObjectCount)
-	for i := 0; i < c; i++ {
-		o, err := binary.ReadUint32(r)
-		if err != nil {
-			return err
-		}
-
-		idx.Entries[i].Offset = uint64(o)
-	}
-
-	return nil
-}
-
-func readChecksums(idx *Idxfile, r io.Reader) error {
-	if _, err := r.Read(idx.PackfileChecksum[:]); err != nil {
-		return err
-	}
-
-	if _, err := r.Read(idx.IdxChecksum[:]); err != nil {
-		return err
-	}
-
-	return nil
-}
diff --git a/formats/idxfile/decoder_test.go b/formats/idxfile/decoder_test.go
deleted file mode 100644
index 18546d2..0000000
--- a/formats/idxfile/decoder_test.go
+++ /dev/null
@@ -1,69 +0,0 @@
-package idxfile
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	. "gopkg.in/check.v1"
-	"gopkg.in/src-d/go-git.v4/fixtures"
-	"gopkg.in/src-d/go-git.v4/formats/packfile"
-	"gopkg.in/src-d/go-git.v4/storage/memory"
-)
-
-func Test(t *testing.T) { TestingT(t) }
-
-type IdxfileSuite struct {
-	fixtures.Suite
-}
-
-var _ = Suite(&IdxfileSuite{})
-
-func (s *IdxfileSuite) TestDecode(c *C) {
-	f := fixtures.Basic().One()
-
-	d := NewDecoder(f.Idx())
-	idx := &Idxfile{}
-	err := d.Decode(idx)
-	c.Assert(err, IsNil)
-
-	c.Assert(idx.Entries, HasLen, 31)
-	c.Assert(idx.Entries[0].Hash.String(), Equals, "1669dce138d9b841a518c64b10914d88f5e488ea")
-	c.Assert(idx.Entries[0].Offset, Equals, uint64(615))
-	c.Assert(idx.Entries[0].CRC32, Equals, uint32(3645019190))
-
-	c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9")
-	c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String())
-}
-
-func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
-	f := fixtures.Basic().ByTag("ofs-delta").One()
-
-	scanner := packfile.NewScanner(f.Packfile())
-	storage := memory.NewStorage()
-
-	pd, err := packfile.NewDecoder(scanner, storage)
-	c.Assert(err, IsNil)
-	_, err = pd.Decode()
-	c.Assert(err, IsNil)
-
-	i := &Idxfile{Version: VersionSupported}
-
-	offsets := pd.Offsets()
-	for h, crc := range pd.CRCs() {
-		i.Add(h, uint64(offsets[h]), crc)
-	}
-
-	buf := bytes.NewBuffer(nil)
-	e := NewEncoder(buf)
-	_, err = e.Encode(i)
-	c.Assert(err, IsNil)
-
-	idx := &Idxfile{}
-
-	d := NewDecoder(buf)
-	err = d.Decode(idx)
-	c.Assert(err, IsNil)
-
-	c.Assert(idx.Entries, DeepEquals, i.Entries)
-}
diff --git a/formats/idxfile/doc.go b/formats/idxfile/doc.go
deleted file mode 100644
index 8a76853..0000000
--- a/formats/idxfile/doc.go
+++ /dev/null
@@ -1,132 +0,0 @@
-// Package idxfile implements a encoder/decoder of idx files
-package idxfile
-
-/*
-== Original (version 1) pack-*.idx files have the following format:
-
-  - The header consists of 256 4-byte network byte order
-    integers.  N-th entry of this table records the number of
-    objects in the corresponding pack, the first byte of whose
-    object name is less than or equal to N.  This is called the
-    'first-level fan-out' table.
-
-  - The header is followed by sorted 24-byte entries, one entry
-    per object in the pack.  Each entry is:
-
-    4-byte network byte order integer, recording where the
-    object is stored in the packfile as the offset from the
-    beginning.
-
-    20-byte object name.
-
-  - The file is concluded with a trailer:
-
-    A copy of the 20-byte SHA1 checksum at the end of
-    corresponding packfile.
-
-    20-byte SHA1-checksum of all of the above.
-
-Pack Idx file:
-
-    --  +--------------------------------+
-fanout  | fanout[0] = 2 (for example)    |-.
-table   +--------------------------------+ |
-        | fanout[1]                      | |
-        +--------------------------------+ |
-        | fanout[2]                      | |
-        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
-        | fanout[255] = total objects    |---.
-    --  +--------------------------------+ | |
-main        | offset                         | | |
-index       | object name 00XXXXXXXXXXXXXXXX | | |
-table       +--------------------------------+ | |
-        | offset                         | | |
-        | object name 00XXXXXXXXXXXXXXXX | | |
-        +--------------------------------+<+ |
-      .-| offset                         |   |
-      | | object name 01XXXXXXXXXXXXXXXX |   |
-      | +--------------------------------+   |
-      | | offset                         |   |
-      | | object name 01XXXXXXXXXXXXXXXX |   |
-      | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~   |
-      | | offset                         |   |
-      | | object name FFXXXXXXXXXXXXXXXX |   |
-    --| +--------------------------------+<--+
-trailer   | | packfile checksum              |
-      | +--------------------------------+
-      | | idxfile checksum               |
-      | +--------------------------------+
-          .-------.
-                  |
-Pack file entry: <+
-
-     packed object header:
-    1-byte size extension bit (MSB)
-           type (next 3 bit)
-           size0 (lower 4-bit)
-        n-byte sizeN (as long as MSB is set, each 7-bit)
-        size0..sizeN form 4+7+7+..+7 bit integer, size0
-        is the least significant part, and sizeN is the
-        most significant part.
-     packed object data:
-        If it is not DELTA, then deflated bytes (the size above
-        is the size before compression).
-    If it is REF_DELTA, then
-      20-byte base object name SHA1 (the size above is the
-        size of the delta data that follows).
-          delta data, deflated.
-    If it is OFS_DELTA, then
-      n-byte offset (see below) interpreted as a negative
-        offset from the type-byte of the header of the
-        ofs-delta entry (the size above is the size of
-        the delta data that follows).
-      delta data, deflated.
-
-     offset encoding:
-      n bytes with MSB set in all but the last one.
-      The offset is then the number constructed by
-      concatenating the lower 7 bit of each byte, and
-      for n >= 2 adding 2^7 + 2^14 + ... + 2^(7*(n-1))
-      to the result.
-
-
-
-== Version 2 pack-*.idx files support packs larger than 4 GiB, and
-   have some other reorganizations.  They have the format:
-
-  - A 4-byte magic number '\377tOc' which is an unreasonable
-    fanout[0] value.
-
-  - A 4-byte version number (= 2)
-
-  - A 256-entry fan-out table just like v1.
-
-  - A table of sorted 20-byte SHA1 object names.  These are
-    packed together without offset values to reduce the cache
-    footprint of the binary search for a specific object name.
-
-  - A table of 4-byte CRC32 values of the packed object data.
-    This is new in v2 so compressed data can be copied directly
-    from pack to pack during repacking without undetected
-    data corruption.
-
-  - A table of 4-byte offset values (in network byte order).
-    These are usually 31-bit pack file offsets, but large
-    offsets are encoded as an index into the next table with
-    the msbit set.
-
-  - A table of 8-byte offset entries (empty for pack files less
-    than 2 GiB).  Pack files are organized with heavily used
-    objects toward the front, so most object references should
-    not need to refer to this table.
-
-  - The same trailer as a v1 pack file:
-
-    A copy of the 20-byte SHA1 checksum at the end of
-    corresponding packfile.
-
-    20-byte SHA1-checksum of all of the above.
-
-From:
-https://www.kernel.org/pub/software/scm/git/docs/v1.7.5/technical/pack-protocol.txt
-*/
diff --git a/formats/idxfile/encoder.go b/formats/idxfile/encoder.go
deleted file mode 100644
index 164414a..0000000
--- a/formats/idxfile/encoder.go
+++ /dev/null
@@ -1,131 +0,0 @@
-package idxfile
-
-import (
-	"crypto/sha1"
-	"hash"
-	"io"
-	"sort"
-
-	"gopkg.in/src-d/go-git.v4/utils/binary"
-)
-
-// An Encoder writes idx files to an output stream.
-type Encoder struct {
-	io.Writer
-	hash hash.Hash
-}
-
-// NewEncoder returns a new encoder that writes to w.
-func NewEncoder(w io.Writer) *Encoder {
-	h := sha1.New()
-	mw := io.MultiWriter(w, h)
-	return &Encoder{mw, h}
-}
-
-// Encode writes the idx in an idx file format to the stream of the encoder.
-func (e *Encoder) Encode(idx *Idxfile) (int, error) {
-	idx.Entries.Sort()
-
-	flow := []func(*Idxfile) (int, error){
-		e.encodeHeader,
-		e.encodeFanout,
-		e.encodeHashes,
-		e.encodeCRC32,
-		e.encodeOffsets,
-		e.encodeChecksums,
-	}
-
-	sz := 0
-	for _, f := range flow {
-		i, err := f(idx)
-		sz += i
-
-		if err != nil {
-			return sz, err
-		}
-	}
-
-	return sz, nil
-}
-
-func (e *Encoder) encodeHeader(idx *Idxfile) (int, error) {
-	c, err := e.Write(idxHeader)
-	if err != nil {
-		return c, err
-	}
-
-	return c + 4, binary.WriteUint32(e, idx.Version)
-}
-
-func (e *Encoder) encodeFanout(idx *Idxfile) (int, error) {
-	fanout := idx.calculateFanout()
-	for _, c := range fanout {
-		if err := binary.WriteUint32(e, c); err != nil {
-			return 0, err
-		}
-	}
-
-	return 1024, nil
-}
-
-func (e *Encoder) encodeHashes(idx *Idxfile) (int, error) {
-	sz := 0
-	for _, ent := range idx.Entries {
-		i, err := e.Write(ent.Hash[:])
-		sz += i
-
-		if err != nil {
-			return sz, err
-		}
-	}
-
-	return sz, nil
-}
-
-func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
-	sz := 0
-	for _, ent := range idx.Entries {
-		err := binary.Write(e, ent.CRC32)
-		sz += 4
-
-		if err != nil {
-			return sz, err
-		}
-	}
-
-	return sz, nil
-}
-
-func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) {
-	sz := 0
-	for _, ent := range idx.Entries {
-		if err := binary.WriteUint32(e, uint32(ent.Offset)); err != nil {
-			return sz, err
-		}
-
-		sz += 4
-
-	}
-
-	return sz, nil
-}
-
-func (e *Encoder) encodeChecksums(idx *Idxfile) (int, error) {
-	if _, err := e.Write(idx.PackfileChecksum[:]); err != nil {
-		return 0, err
-	}
-
-	copy(idx.IdxChecksum[:], e.hash.Sum(nil)[:20])
-	if _, err := e.Write(idx.IdxChecksum[:]); err != nil {
-		return 0, err
-	}
-
-	return 40, nil
-}
-
-type EntryList []Entry
-
-func (p EntryList) Len() int           { return len(p) }
-func (p EntryList) Less(i, j int) bool { return p[i].Hash.String() < p[j].Hash.String() }
-func (p EntryList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p EntryList) Sort()              { sort.Sort(p) }
diff --git a/formats/idxfile/encoder_test.go b/formats/idxfile/encoder_test.go
deleted file mode 100644
index d9d83eb..0000000
--- a/formats/idxfile/encoder_test.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package idxfile
-
-import (
-	"bytes"
-	"io/ioutil"
-
-	. "gopkg.in/check.v1"
-	"gopkg.in/src-d/go-git.v4/core"
-	"gopkg.in/src-d/go-git.v4/fixtures"
-)
-
-func (s *IdxfileSuite) TestEncode(c *C) {
-	expected := &Idxfile{}
-	expected.Add(core.NewHash("4bfc730165c370df4a012afbb45ba3f9c332c0d4"), 82, 82)
-	expected.Add(core.NewHash("8fa2238efdae08d83c12ee176fae65ff7c99af46"), 42, 42)
-
-	buf := bytes.NewBuffer(nil)
-	e := NewEncoder(buf)
-	_, err := e.Encode(expected)
-	c.Assert(err, IsNil)
-
-	idx := &Idxfile{}
-	d := NewDecoder(buf)
-	err = d.Decode(idx)
-	c.Assert(err, IsNil)
-
-	c.Assert(idx.Entries, DeepEquals, expected.Entries)
-}
-
-func (s *IdxfileSuite) TestDecodeEncode(c *C) {
-	fixtures.ByTag("packfile").Test(c, func(f *fixtures.Fixture) {
-		expected, err := ioutil.ReadAll(f.Idx())
-		c.Assert(err, IsNil)
-
-		idx := &Idxfile{}
-		d := NewDecoder(bytes.NewBuffer(expected))
-		err = d.Decode(idx)
-		c.Assert(err, IsNil)
-
-		result := bytes.NewBuffer(nil)
-		e := NewEncoder(result)
-		size, err := e.Encode(idx)
-		c.Assert(err, IsNil)
-
-		c.Assert(size, Equals, len(expected))
-		c.Assert(result.Bytes(), DeepEquals, expected)
-	})
-}
diff --git a/formats/idxfile/idxfile.go b/formats/idxfile/idxfile.go
deleted file mode 100644
index 8549d3f..0000000
--- a/formats/idxfile/idxfile.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package idxfile
-
-import "gopkg.in/src-d/go-git.v4/core"
-
-const (
-	// VersionSupported is the only idx version supported.
-	VersionSupported = 2
-)
-
-var (
-	idxHeader = []byte{255, 't', 'O', 'c'}
-)
-
-// An Idxfile represents an idx file in memory.
-type Idxfile struct {
-	Version          uint32
-	Fanout           [255]uint32
-	ObjectCount      uint32
-	Entries          EntryList
-	PackfileChecksum [20]byte
-	IdxChecksum      [20]byte
-}
-
-// An Entry represents data about an object in the packfile: its hash,
-// offset and CRC32 checksum.
-type Entry struct {
-	Hash   core.Hash
-	CRC32  uint32
-	Offset uint64
-}
-
-func (idx *Idxfile) Add(h core.Hash, offset uint64, crc32 uint32) {
-	idx.Entries = append(idx.Entries, Entry{
-		Hash:   h,
-		Offset: offset,
-		CRC32:  crc32,
-	})
-}
-
-func (idx *Idxfile) isValid() bool {
-	fanout := idx.calculateFanout()
-	for k, c := range idx.Fanout {
-		if fanout[k] != c {
-			return false
-		}
-	}
-
-	return true
-}
-
-func (idx *Idxfile) calculateFanout() [256]uint32 {
-	fanout := [256]uint32{}
-	for _, e := range idx.Entries {
-		fanout[e.Hash[0]]++
-	}
-
-	for i := 1; i < 256; i++ {
-		fanout[i] += fanout[i-1]
-	}
-
-	return fanout
-}