some refactor in folders and crawler

5 files changed, 320 insertions, 0 deletions

diff --git a/pktline/decoder.go b/pktline/decoder.go
new file mode 100644
index 0000000..194a3e6
--- /dev/null
+++ b/pktline/decoder.go
@@ -0,0 +1,104 @@
+package pktline
+
+import (
+	"errors"
+	"io"
+	"strconv"
+)
+
+var (
+	ErrUnderflow     = errors.New("unexepected string length")
+	ErrInvalidHeader = errors.New("invalid header")
+	ErrInvalidLen    = errors.New("invalid length")
+)
+
+type Decoder struct {
+	r io.Reader
+}
+
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{r}
+}
+
+func (d *Decoder) readLine() (string, error) {
+	raw := make([]byte, HEADER_LENGTH)
+	if _, err := d.r.Read(raw); err != nil {
+		return "", err
+	}
+
+	header, err := strconv.ParseInt(string(raw), 16, 16)
+	if err != nil {
+		return "", ErrInvalidHeader
+	}
+
+	if header == 0 {
+		return "", nil
+	}
+
+	exp := int(header - HEADER_LENGTH)
+	if exp < 0 {
+		return "", ErrInvalidLen
+	}
+
+	line := make([]byte, exp)
+	if read, err := d.r.Read(line); err != nil {
+		return "", err
+	} else if read != exp {
+		return "", ErrUnderflow
+	}
+
+	return string(line), nil
+}
+
+func (d *Decoder) ReadLine() (string, error) {
+	return d.readLine()
+}
+
+func (d *Decoder) ReadBlock() ([]string, error) {
+	o := make([]string, 0)
+
+	for {
+		line, err := d.readLine()
+		if err == io.EOF {
+			return o, nil
+		}
+
+		if err != nil {
+			return o, err
+		}
+
+		if err == nil && line == "" {
+			return o, nil
+		}
+
+		o = append(o, line)
+	}
+
+	return o, nil
+}
+
+func (d *Decoder) ReadAll() ([]string, error) {
+	result, err := d.ReadBlock()
+	if err != nil {
+		return result, err
+	}
+
+	for {
+		lines, err := d.ReadBlock()
+		if err == io.EOF {
+			return result, nil
+		}
+
+		if err != nil {
+			return result, err
+		}
+
+		if err == nil && len(lines) == 0 {
+			return result, nil
+		}
+
+		result = append(result, lines...)
+	}
+
+	return result, nil
+}
diff --git a/pktline/decoder_test.go b/pktline/decoder_test.go
new file mode 100644
index 0000000..a0f85ce
--- /dev/null
+++ b/pktline/decoder_test.go
@@ -0,0 +1,70 @@
+package pktline
+
+import (
+	"strings"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type DecoderSuite struct{}
+
+var _ = Suite(&DecoderSuite{})
+
+func (s *DecoderSuite) TestReadLine(c *C) {
+	j := &Decoder{strings.NewReader("0006a\n")}
+
+	line, err := j.ReadLine()
+	c.Assert(err, IsNil)
+	c.Assert(line, Equals, "a\n")
+}
+
+func (s *DecoderSuite) TestReadLineBufferUnderflow(c *C) {
+	j := &Decoder{strings.NewReader("00e7a\n")}
+
+	line, err := j.ReadLine()
+	c.Assert(err, ErrorMatches, "unexepected string length")
+	c.Assert(line, Equals, "")
+}
+
+func (s *DecoderSuite) TestReadLineBufferInvalidLen(c *C) {
+	j := &Decoder{strings.NewReader("0001foo\n")}
+
+	line, err := j.ReadLine()
+	c.Assert(err, ErrorMatches, "invalid length")
+	c.Assert(line, Equals, "")
+}
+
+func (s *DecoderSuite) TestReadBlock(c *C) {
+	j := &Decoder{strings.NewReader("0006a\n")}
+
+	lines, err := j.ReadBlock()
+	c.Assert(err, IsNil)
+	c.Assert(lines, HasLen, 1)
+	c.Assert(lines[0], Equals, "a\n")
+}
+
+func (s *DecoderSuite) TestReadBlockWithFlush(c *C) {
+	j := &Decoder{strings.NewReader("0006a\n0006b\n00000006c\n")}
+
+	lines, err := j.ReadBlock()
+	c.Assert(err, IsNil)
+	c.Assert(lines, HasLen, 2)
+	c.Assert(lines[0], Equals, "a\n")
+	c.Assert(lines[1], Equals, "b\n")
+}
+
+func (s *DecoderSuite) TestReadAll(c *C) {
+	j := &Decoder{strings.NewReader("0006a\n0006b\n00000006c\n0006d\n0006e\n")}
+
+	lines, err := j.ReadAll()
+	c.Assert(err, IsNil)
+	c.Assert(lines, HasLen, 5)
+	c.Assert(lines[0], Equals, "a\n")
+	c.Assert(lines[1], Equals, "b\n")
+	c.Assert(lines[2], Equals, "c\n")
+	c.Assert(lines[3], Equals, "d\n")
+	c.Assert(lines[4], Equals, "e\n")
+}
diff --git a/pktline/doc.go b/pktline/doc.go
new file mode 100644
index 0000000..a976b54
--- /dev/null
+++ b/pktline/doc.go
@@ -0,0 +1,56 @@
+package pktline
+
+// pkt-line Format
+// ---------------
+//
+// Much (but not all) of the payload is described around pkt-lines.
+//
+// A pkt-line is a variable length binary string.  The first four bytes
+// of the line, the pkt-len, indicates the total length of the line,
+// in hexadecimal.  The pkt-len includes the 4 bytes used to contain
+// the length's hexadecimal representation.
+//
+// A pkt-line MAY contain binary data, so implementors MUST ensure
+// pkt-line parsing/formatting routines are 8-bit clean.
+//
+// A non-binary line SHOULD BE terminated by an LF, which if present
+// MUST be included in the total length.
+//
+// The maximum length of a pkt-line's data component is 65520 bytes.
+// Implementations MUST NOT send pkt-line whose length exceeds 65524
+// (65520 bytes of payload + 4 bytes of length data).
+//
+// Implementations SHOULD NOT send an empty pkt-line ("0004").
+//
+// A pkt-line with a length field of 0 ("0000"), called a flush-pkt,
+// is a special case and MUST be handled differently than an empty
+// pkt-line ("0004").
+//
+// ----
+//   pkt-line     =  data-pkt / flush-pkt
+//
+//   data-pkt     =  pkt-len pkt-payload
+//   pkt-len      =  4*(HEXDIG)
+//   pkt-payload  =  (pkt-len - 4)*(OCTET)
+//
+//   flush-pkt    = "0000"
+// ----
+//
+// Examples (as C-style strings):
+//
+// ----
+//   pkt-line          actual value
+//   ---------------------------------
+//   "0006a\n"         "a\n"
+//   "0005a"           "a"
+//   "000bfoobar\n"    "foobar\n"
+//   "0004"            ""
+// ----
+//
+// Extracted from:
+// https://github.com/git/git/blob/master/Documentation/technical/protocol-common.txt
+
+const (
+	HEADER_LENGTH = 4
+	MAX_LENGTH    = 65524
+)
diff --git a/pktline/encoder.go b/pktline/encoder.go
new file mode 100644
index 0000000..b439c54
--- /dev/null
+++ b/pktline/encoder.go
@@ -0,0 +1,56 @@
+package pktline
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+var (
+	ErrOverflow = errors.New("unexepected string length")
+)
+
+type Encoder struct {
+	lines []string
+}
+
+func NewEncoder() *Encoder {
+	return &Encoder{make([]string, 0)}
+}
+
+func (e *Encoder) AddLine(line string) error {
+	le, err := EncodeFromString(line + "\n")
+	if err != nil {
+		return err
+	}
+
+	e.lines = append(e.lines, le)
+	return nil
+}
+
+func (e *Encoder) AddFlush() {
+	e.lines = append(e.lines, "0000")
+}
+
+func (e *Encoder) GetReader() *strings.Reader {
+	data := strings.Join(e.lines, "")
+
+	return strings.NewReader(data)
+}
+
+func EncodeFromString(line string) (string, error) {
+	return Encode([]byte(line))
+}
+
+func Encode(line []byte) (string, error) {
+	if line == nil {
+		return "0000", nil
+	}
+
+	l := len(line) + HEADER_LENGTH
+	if l > MAX_LENGTH {
+		return "", ErrOverflow
+	}
+
+	return fmt.Sprintf("%04x%s", l, line), nil
+}
diff --git a/pktline/encoder_test.go b/pktline/encoder_test.go
new file mode 100644
index 0000000..091ad1c
--- /dev/null
+++ b/pktline/encoder_test.go
@@ -0,0 +1,34 @@
+package pktline
+
+import (
+	"io/ioutil"
+
+	. "gopkg.in/check.v1"
+)
+
+type EncoderSuite struct{}
+
+var _ = Suite(&EncoderSuite{})
+
+func (s *EncoderSuite) TestEncode(c *C) {
+	line, err := Encode([]byte("a\n"))
+	c.Assert(err, IsNil)
+	c.Assert(string(line), Equals, "0006a\n")
+}
+
+func (s *EncoderSuite) TestEncodeFromString(c *C) {
+	line, err := EncodeFromString("a\n")
+	c.Assert(err, IsNil)
+	c.Assert(string(line), Equals, "0006a\n")
+}
+
+func (s *EncoderSuite) TestEncoder(c *C) {
+	e := NewEncoder()
+	e.AddLine("a")
+	e.AddFlush()
+	e.AddLine("b")
+
+	r := e.GetReader()
+	a, _ := ioutil.ReadAll(r)
+	c.Assert(string(a), Equals, "0006a\n00000006b\n")
+}