move: format/packp -> protocol/packp (#141)

* move: format/packp -> protocol/packp

* format/packp -> protocol/packp
* format/packp/pktline -> format/pktline.

* move: protocol/packp/ulreq/* -> protocol/packp/*

* protocol/packp: rename UlReq types to make them unique.

* * protocol/packp: namespace UlReq encoder.

* protocol/packp: namespace UlReq decoder.

* protocol/packp: fix example names

* move: protocol/packp/advrefs/* -> protocol/packp/*

* further ulreq namespacing

* protocol/packp: namespace AdvRefs types.

4 files changed, 730 insertions, 0 deletions

diff --git a/plumbing/format/pktline/encoder.go b/plumbing/format/pktline/encoder.go
new file mode 100644
index 0000000..0a88a9b
--- /dev/null
+++ b/plumbing/format/pktline/encoder.go
@@ -0,0 +1,123 @@
+// Package pktline implements reading payloads form pkt-lines and encoding pkt-lines from payloads.
+package pktline
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+)
+
+// An Encoder writes pkt-lines to an output stream.
+type Encoder struct {
+	w io.Writer
+}
+
+const (
+	// MaxPayloadSize is the maximum payload size of a pkt-line in bytes.
+	MaxPayloadSize = 65516
+)
+
+var (
+	// FlushPkt are the contents of a flush-pkt pkt-line.
+	FlushPkt = []byte{'0', '0', '0', '0'}
+	// Flush is the payload to use with the Encode method to encode a flush-pkt.
+	Flush = []byte{}
+	// FlushString is the payload to use with the EncodeString method to encode a flush-pkt.
+	FlushString = ""
+	// ErrPayloadTooLong is returned by the Encode methods when any of the
+	// provided payloads is bigger than MaxPayloadSize.
+	ErrPayloadTooLong = errors.New("payload is too long")
+)
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		w: w,
+	}
+}
+
+// Flush encodes a flush-pkt to the output stream.
+func (e *Encoder) Flush() error {
+	_, err := e.w.Write(FlushPkt)
+	return err
+}
+
+// Encode encodes a pkt-line with the payload specified and write it to
+// the output stream.  If several payloads are specified, each of them
+// will get streamed in their own pkt-lines.
+func (e *Encoder) Encode(payloads ...[]byte) error {
+	for _, p := range payloads {
+		if err := e.encodeLine(p); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (e *Encoder) encodeLine(p []byte) error {
+	if len(p) > MaxPayloadSize {
+		return ErrPayloadTooLong
+	}
+
+	if bytes.Equal(p, Flush) {
+		if err := e.Flush(); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	n := len(p) + 4
+	if _, err := e.w.Write(asciiHex16(n)); err != nil {
+		return err
+	}
+	if _, err := e.w.Write(p); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Returns the hexadecimal ascii representation of the 16 less
+// significant bits of n.  The length of the returned slice will always
+// be 4.  Example: if n is 1234 (0x4d2), the return value will be
+// []byte{'0', '4', 'd', '2'}.
+func asciiHex16(n int) []byte {
+	var ret [4]byte
+	ret[0] = byteToASCIIHex(byte(n & 0xf000 >> 12))
+	ret[1] = byteToASCIIHex(byte(n & 0x0f00 >> 8))
+	ret[2] = byteToASCIIHex(byte(n & 0x00f0 >> 4))
+	ret[3] = byteToASCIIHex(byte(n & 0x000f))
+
+	return ret[:]
+}
+
+// turns a byte into its hexadecimal ascii representation.  Example:
+// from 11 (0xb) to 'b'.
+func byteToASCIIHex(n byte) byte {
+	if n < 10 {
+		return '0' + n
+	}
+
+	return 'a' - 10 + n
+}
+
+// EncodeString works similarly as Encode but payloads are specified as strings.
+func (e *Encoder) EncodeString(payloads ...string) error {
+	for _, p := range payloads {
+		if err := e.Encode([]byte(p)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Encodef encodes a single pkt-line with the payload formatted as
+// the format specifier and the rest of the arguments suggest.
+func (e *Encoder) Encodef(format string, a ...interface{}) error {
+	return e.EncodeString(
+		fmt.Sprintf(format, a...),
+	)
+}
diff --git a/plumbing/format/pktline/encoder_test.go b/plumbing/format/pktline/encoder_test.go
new file mode 100644
index 0000000..d1258af
--- /dev/null
+++ b/plumbing/format/pktline/encoder_test.go
@@ -0,0 +1,249 @@
+package pktline_test
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"gopkg.in/src-d/go-git.v4/plumbing/format/pktline"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type SuiteEncoder struct{}
+
+var _ = Suite(&SuiteEncoder{})
+
+func (s *SuiteEncoder) TestFlush(c *C) {
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+
+	err := e.Flush()
+	c.Assert(err, IsNil)
+
+	obtained := buf.Bytes()
+	c.Assert(obtained, DeepEquals, pktline.FlushPkt)
+}
+
+func (s *SuiteEncoder) TestEncode(c *C) {
+	for i, test := range [...]struct {
+		input    [][]byte
+		expected []byte
+	}{
+		{
+			input: [][]byte{
+				[]byte("hello\n"),
+			},
+			expected: []byte("000ahello\n"),
+		}, {
+			input: [][]byte{
+				[]byte("hello\n"),
+				pktline.Flush,
+			},
+			expected: []byte("000ahello\n0000"),
+		}, {
+			input: [][]byte{
+				[]byte("hello\n"),
+				[]byte("world!\n"),
+				[]byte("foo"),
+			},
+			expected: []byte("000ahello\n000bworld!\n0007foo"),
+		}, {
+			input: [][]byte{
+				[]byte("hello\n"),
+				pktline.Flush,
+				[]byte("world!\n"),
+				[]byte("foo"),
+				pktline.Flush,
+			},
+			expected: []byte("000ahello\n0000000bworld!\n0007foo0000"),
+		}, {
+			input: [][]byte{
+				[]byte(strings.Repeat("a", pktline.MaxPayloadSize)),
+			},
+			expected: []byte(
+				"fff0" + strings.Repeat("a", pktline.MaxPayloadSize)),
+		}, {
+			input: [][]byte{
+				[]byte(strings.Repeat("a", pktline.MaxPayloadSize)),
+				[]byte(strings.Repeat("b", pktline.MaxPayloadSize)),
+			},
+			expected: []byte(
+				"fff0" + strings.Repeat("a", pktline.MaxPayloadSize) +
+					"fff0" + strings.Repeat("b", pktline.MaxPayloadSize)),
+		},
+	} {
+		comment := Commentf("input %d = %v\n", i, test.input)
+
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+
+		err := e.Encode(test.input...)
+		c.Assert(err, IsNil, comment)
+
+		c.Assert(buf.Bytes(), DeepEquals, test.expected, comment)
+	}
+}
+
+func (s *SuiteEncoder) TestEncodeErrPayloadTooLong(c *C) {
+	for i, input := range [...][][]byte{
+		{
+			[]byte(strings.Repeat("a", pktline.MaxPayloadSize+1)),
+		},
+		{
+			[]byte("hello world!"),
+			[]byte(strings.Repeat("a", pktline.MaxPayloadSize+1)),
+		},
+		{
+			[]byte("hello world!"),
+			[]byte(strings.Repeat("a", pktline.MaxPayloadSize+1)),
+			[]byte("foo"),
+		},
+	} {
+		comment := Commentf("input %d = %v\n", i, input)
+
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+
+		err := e.Encode(input...)
+		c.Assert(err, Equals, pktline.ErrPayloadTooLong, comment)
+	}
+}
+
+func (s *SuiteEncoder) TestEncodeStrings(c *C) {
+	for i, test := range [...]struct {
+		input    []string
+		expected []byte
+	}{
+		{
+			input: []string{
+				"hello\n",
+			},
+			expected: []byte("000ahello\n"),
+		}, {
+			input: []string{
+				"hello\n",
+				pktline.FlushString,
+			},
+			expected: []byte("000ahello\n0000"),
+		}, {
+			input: []string{
+				"hello\n",
+				"world!\n",
+				"foo",
+			},
+			expected: []byte("000ahello\n000bworld!\n0007foo"),
+		}, {
+			input: []string{
+				"hello\n",
+				pktline.FlushString,
+				"world!\n",
+				"foo",
+				pktline.FlushString,
+			},
+			expected: []byte("000ahello\n0000000bworld!\n0007foo0000"),
+		}, {
+			input: []string{
+				strings.Repeat("a", pktline.MaxPayloadSize),
+			},
+			expected: []byte(
+				"fff0" + strings.Repeat("a", pktline.MaxPayloadSize)),
+		}, {
+			input: []string{
+				strings.Repeat("a", pktline.MaxPayloadSize),
+				strings.Repeat("b", pktline.MaxPayloadSize),
+			},
+			expected: []byte(
+				"fff0" + strings.Repeat("a", pktline.MaxPayloadSize) +
+					"fff0" + strings.Repeat("b", pktline.MaxPayloadSize)),
+		},
+	} {
+		comment := Commentf("input %d = %v\n", i, test.input)
+
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+
+		err := e.EncodeString(test.input...)
+		c.Assert(err, IsNil, comment)
+		c.Assert(buf.Bytes(), DeepEquals, test.expected, comment)
+	}
+}
+
+func (s *SuiteEncoder) TestEncodeStringErrPayloadTooLong(c *C) {
+	for i, input := range [...][]string{
+		{
+			strings.Repeat("a", pktline.MaxPayloadSize+1),
+		},
+		{
+			"hello world!",
+			strings.Repeat("a", pktline.MaxPayloadSize+1),
+		},
+		{
+			"hello world!",
+			strings.Repeat("a", pktline.MaxPayloadSize+1),
+			"foo",
+		},
+	} {
+		comment := Commentf("input %d = %v\n", i, input)
+
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+
+		err := e.EncodeString(input...)
+		c.Assert(err, Equals, pktline.ErrPayloadTooLong, comment)
+	}
+}
+
+func (s *SuiteEncoder) TestEncodef(c *C) {
+	format := " %s %d\n"
+	str := "foo"
+	d := 42
+
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+
+	err := e.Encodef(format, str, d)
+	c.Assert(err, IsNil)
+
+	expected := []byte("000c foo 42\n")
+	c.Assert(buf.Bytes(), DeepEquals, expected)
+}
+
+func ExampleEncoder() {
+	// Create an encoder that writes pktlines to stdout.
+	e := pktline.NewEncoder(os.Stdout)
+
+	// Encode some data as a new pkt-line.
+	_ = e.Encode([]byte("data\n")) // error checks removed for brevity
+
+	// Encode a flush-pkt.
+	_ = e.Flush()
+
+	// Encode a couple of byte slices and a flush in one go. Each of
+	// them will end up as payloads of their own pktlines.
+	_ = e.Encode(
+		[]byte("hello\n"),
+		[]byte("world!\n"),
+		pktline.Flush,
+	)
+
+	// You can also encode strings:
+	_ = e.EncodeString(
+		"foo\n",
+		"bar\n",
+		pktline.FlushString,
+	)
+
+	// You can also format and encode a payload:
+	_ = e.Encodef(" %s %d\n", "foo", 42)
+	// Output:
+	// 0009data
+	// 0000000ahello
+	// 000bworld!
+	// 00000008foo
+	// 0008bar
+	// 0000000c foo 42
+}
diff --git a/plumbing/format/pktline/scanner.go b/plumbing/format/pktline/scanner.go
new file mode 100644
index 0000000..3ce2adf
--- /dev/null
+++ b/plumbing/format/pktline/scanner.go
@@ -0,0 +1,133 @@
+package pktline
+
+import (
+	"errors"
+	"io"
+)
+
+const (
+	lenSize = 4
+)
+
+// ErrInvalidPktLen is returned by Err() when an invalid pkt-len is found.
+var ErrInvalidPktLen = errors.New("invalid pkt-len found")
+
+// Scanner provides a convenient interface for reading the payloads of a
+// series of pkt-lines.  It takes an io.Reader providing the source,
+// which then can be tokenized through repeated calls to the Scan
+// method.
+//
+// After each Scan call, the Bytes method will return the payload of the
+// corresponding pkt-line on a shared buffer, which will be 65516 bytes
+// or smaller.  Flush pkt-lines are represented by empty byte slices.
+//
+// Scanning stops at EOF or the first I/O error.
+type Scanner struct {
+	r       io.Reader     // The reader provided by the client
+	err     error         // Sticky error
+	payload []byte        // Last pkt-payload
+	len     [lenSize]byte // Last pkt-len
+}
+
+// NewScanner returns a new Scanner to read from r.
+func NewScanner(r io.Reader) *Scanner {
+	return &Scanner{
+		r: r,
+	}
+}
+
+// Err returns the first error encountered by the Scanner.
+func (s *Scanner) Err() error {
+	return s.err
+}
+
+// Scan advances the Scanner to the next pkt-line, whose payload will
+// then be available through the Bytes method.  Scanning stops at EOF
+// or the first I/O error.  After Scan returns false, the Err method
+// will return any error that occurred during scanning, except that if
+// it was io.EOF, Err will return nil.
+func (s *Scanner) Scan() bool {
+	var l int
+	l, s.err = s.readPayloadLen()
+	if s.err == io.EOF {
+		s.err = nil
+		return false
+	}
+	if s.err != nil {
+		return false
+	}
+
+	if cap(s.payload) < l {
+		s.payload = make([]byte, 0, l)
+	}
+
+	if _, s.err = io.ReadFull(s.r, s.payload[:l]); s.err != nil {
+		return false
+	}
+	s.payload = s.payload[:l]
+
+	return true
+}
+
+// Bytes returns the most recent payload generated by a call to Scan.
+// The underlying array may point to data that will be overwritten by a
+// subsequent call to Scan. It does no allocation.
+func (s *Scanner) Bytes() []byte {
+	return s.payload
+}
+
+// Method readPayloadLen returns the payload length by reading the
+// pkt-len and substracting the pkt-len size.
+func (s *Scanner) readPayloadLen() (int, error) {
+	if _, err := io.ReadFull(s.r, s.len[:]); err != nil {
+		if err == io.EOF {
+			return 0, err
+		}
+		return 0, ErrInvalidPktLen
+	}
+
+	n, err := hexDecode(s.len)
+	if err != nil {
+		return 0, err
+	}
+
+	switch {
+	case n == 0:
+		return 0, nil
+	case n <= lenSize:
+		return 0, ErrInvalidPktLen
+	case n > MaxPayloadSize+lenSize:
+		return 0, ErrInvalidPktLen
+	default:
+		return n - lenSize, nil
+	}
+}
+
+// Turns the hexadecimal representation of a number in a byte slice into
+// a number. This function substitute strconv.ParseUint(string(buf), 16,
+// 16) and/or hex.Decode, to avoid generating new strings, thus helping the
+// GC.
+func hexDecode(buf [lenSize]byte) (int, error) {
+	var ret int
+	for i := 0; i < lenSize; i++ {
+		n, err := asciiHexToByte(buf[i])
+		if err != nil {
+			return 0, ErrInvalidPktLen
+		}
+		ret = 16*ret + int(n)
+	}
+	return ret, nil
+}
+
+// turns the hexadecimal ascii representation of a byte into its
+// numerical value.  Example: from 'b' to 11 (0xb).
+func asciiHexToByte(b byte) (byte, error) {
+	switch {
+	case b >= '0' && b <= '9':
+		return b - '0', nil
+	case b >= 'a' && b <= 'f':
+		return b - 'a' + 10, nil
+	default:
+		return 0, ErrInvalidPktLen
+	}
+}
diff --git a/plumbing/format/pktline/scanner_test.go b/plumbing/format/pktline/scanner_test.go
new file mode 100644
index 0000000..9f440a4
--- /dev/null
+++ b/plumbing/format/pktline/scanner_test.go
@@ -0,0 +1,225 @@
+package pktline_test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+
+	"gopkg.in/src-d/go-git.v4/plumbing/format/pktline"
+
+	. "gopkg.in/check.v1"
+)
+
+type SuiteScanner struct{}
+
+var _ = Suite(&SuiteScanner{})
+
+func (s *SuiteScanner) TestInvalid(c *C) {
+	for _, test := range [...]string{
+		"0001", "0002", "0003", "0004",
+		"0001asdfsadf", "0004foo",
+		"fff1", "fff2",
+		"gorka",
+		"0", "003",
+		"   5a", "5   a", "5   \n",
+		"-001", "-000",
+	} {
+		r := strings.NewReader(test)
+		sc := pktline.NewScanner(r)
+		_ = sc.Scan()
+		c.Assert(sc.Err(), ErrorMatches, pktline.ErrInvalidPktLen.Error(),
+			Commentf("data = %q", test))
+	}
+}
+
+func (s *SuiteScanner) TestEmptyReader(c *C) {
+	r := strings.NewReader("")
+	sc := pktline.NewScanner(r)
+	hasPayload := sc.Scan()
+	c.Assert(hasPayload, Equals, false)
+	c.Assert(sc.Err(), Equals, nil)
+}
+
+func (s *SuiteScanner) TestFlush(c *C) {
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+	err := e.Flush()
+	c.Assert(err, IsNil)
+
+	sc := pktline.NewScanner(&buf)
+	c.Assert(sc.Scan(), Equals, true)
+
+	payload := sc.Bytes()
+	c.Assert(len(payload), Equals, 0)
+}
+
+func (s *SuiteScanner) TestPktLineTooShort(c *C) {
+	r := strings.NewReader("010cfoobar")
+
+	sc := pktline.NewScanner(r)
+
+	c.Assert(sc.Scan(), Equals, false)
+	c.Assert(sc.Err(), ErrorMatches, "unexpected EOF")
+}
+
+func (s *SuiteScanner) TestScanAndPayload(c *C) {
+	for _, test := range [...]string{
+		"a",
+		"a\n",
+		strings.Repeat("a", 100),
+		strings.Repeat("a", 100) + "\n",
+		strings.Repeat("\x00", 100),
+		strings.Repeat("\x00", 100) + "\n",
+		strings.Repeat("a", pktline.MaxPayloadSize),
+		strings.Repeat("a", pktline.MaxPayloadSize-1) + "\n",
+	} {
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+		err := e.EncodeString(test)
+		c.Assert(err, IsNil,
+			Commentf("input len=%x, contents=%.10q\n", len(test), test))
+
+		sc := pktline.NewScanner(&buf)
+		c.Assert(sc.Scan(), Equals, true,
+			Commentf("test = %.20q...", test))
+
+		obtained := sc.Bytes()
+		c.Assert(obtained, DeepEquals, []byte(test),
+			Commentf("in = %.20q out = %.20q", test, string(obtained)))
+	}
+}
+
+func (s *SuiteScanner) TestSkip(c *C) {
+	for _, test := range [...]struct {
+		input    []string
+		n        int
+		expected []byte
+	}{
+		{
+			input: []string{
+				"first",
+				"second",
+				"third"},
+			n:        1,
+			expected: []byte("second"),
+		},
+		{
+			input: []string{
+				"first",
+				"second",
+				"third"},
+			n:        2,
+			expected: []byte("third"),
+		},
+	} {
+		var buf bytes.Buffer
+		e := pktline.NewEncoder(&buf)
+		err := e.EncodeString(test.input...)
+		c.Assert(err, IsNil)
+
+		sc := pktline.NewScanner(&buf)
+		for i := 0; i < test.n; i++ {
+			c.Assert(sc.Scan(), Equals, true,
+				Commentf("scan error = %s", sc.Err()))
+		}
+		c.Assert(sc.Scan(), Equals, true,
+			Commentf("scan error = %s", sc.Err()))
+
+		obtained := sc.Bytes()
+		c.Assert(obtained, DeepEquals, test.expected,
+			Commentf("\nin = %.20q\nout = %.20q\nexp = %.20q",
+				test.input, obtained, test.expected))
+	}
+}
+
+func (s *SuiteScanner) TestEOF(c *C) {
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+	err := e.EncodeString("first", "second")
+	c.Assert(err, IsNil)
+
+	sc := pktline.NewScanner(&buf)
+	for sc.Scan() {
+	}
+	c.Assert(sc.Err(), IsNil)
+}
+
+// A section are several non flush-pkt lines followed by a flush-pkt, which
+// how the git protocol sends long messages.
+func (s *SuiteScanner) TestReadSomeSections(c *C) {
+	nSections := 2
+	nLines := 4
+	data := sectionsExample(c, nSections, nLines)
+	sc := pktline.NewScanner(data)
+
+	sectionCounter := 0
+	lineCounter := 0
+	for sc.Scan() {
+		if len(sc.Bytes()) == 0 {
+			sectionCounter++
+		}
+		lineCounter++
+	}
+	c.Assert(sc.Err(), IsNil)
+	c.Assert(sectionCounter, Equals, nSections)
+	c.Assert(lineCounter, Equals, (1+nLines)*nSections)
+}
+
+// returns nSection sections, each of them with nLines pkt-lines (not
+// counting the flush-pkt:
+//
+// 0009 0.0\n
+// 0009 0.1\n
+// ...
+// 0000
+// and so on
+func sectionsExample(c *C, nSections, nLines int) io.Reader {
+	var buf bytes.Buffer
+	e := pktline.NewEncoder(&buf)
+
+	for section := 0; section < nSections; section++ {
+		ss := []string{}
+		for line := 0; line < nLines; line++ {
+			line := fmt.Sprintf(" %d.%d\n", section, line)
+			ss = append(ss, line)
+		}
+		err := e.EncodeString(ss...)
+		c.Assert(err, IsNil)
+		err = e.Flush()
+		c.Assert(err, IsNil)
+	}
+
+	return &buf
+}
+
+func ExampleScanner() {
+	// A reader is needed as input.
+	input := strings.NewReader("000ahello\n" +
+		"000bworld!\n" +
+		"0000",
+	)
+
+	// Create the scanner...
+	s := pktline.NewScanner(input)
+
+	// and scan every pkt-line found in the input.
+	for s.Scan() {
+		payload := s.Bytes()
+		if len(payload) == 0 { // zero sized payloads correspond to flush-pkts.
+			fmt.Println("FLUSH-PKT DETECTED\n")
+		} else { // otherwise, you will be able to access the full payload.
+			fmt.Printf("PAYLOAD = %q\n", string(payload))
+		}
+	}
+
+	// this will catch any error when reading from the input, if any.
+	if s.Err() != nil {
+		fmt.Println(s.Err())
+	}
+
+	// Output:
+	// PAYLOAD = "hello\n"
+	// PAYLOAD = "world!\n"
+	// FLUSH-PKT DETECTED
+}