From ac095bb12c4d29722b60ba9f20590fa7cfa6bc7d Mon Sep 17 00:00:00 2001 From: Máximo Cuadros Date: Tue, 8 Nov 2016 23:46:38 +0100 Subject: new plumbing package (#118) * plumbing: now core was renamed to core, and formats and clients moved inside --- formats/objfile/common_test.go | 69 ------------------------ formats/objfile/reader.go | 118 ----------------------------------------- formats/objfile/reader_test.go | 67 ----------------------- formats/objfile/writer.go | 109 ------------------------------------- formats/objfile/writer_test.go | 80 ---------------------------- 5 files changed, 443 deletions(-) delete mode 100644 formats/objfile/common_test.go delete mode 100644 formats/objfile/reader.go delete mode 100644 formats/objfile/reader_test.go delete mode 100644 formats/objfile/writer.go delete mode 100644 formats/objfile/writer_test.go (limited to 'formats/objfile') diff --git a/formats/objfile/common_test.go b/formats/objfile/common_test.go deleted file mode 100644 index 682dfbb..0000000 --- a/formats/objfile/common_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package objfile - -import ( - "encoding/base64" - "testing" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-git.v4/core" -) - -type objfileFixture struct { - hash string // hash of data - t core.ObjectType // object type - content string // base64-encoded content - data string // base64-encoded objfile data -} - -var objfileFixtures = []objfileFixture{ - { - "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("")), - "eAFLyslPUjBgAAAJsAHw", - }, - { - "a8a940627d132695a9769df883f85992f0ff4a43", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("this is a test")), - "eAFLyslPUjA0YSjJyCxWAKJEhZLU4hIAUDYHOg==", - }, - { - "4dc2174801ac4a3d36886210fd086fbe134cf7b2", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("this\nis\n\n\na\nmultiline\n\ntest.\n")), - "eAFLyslPUjCyZCjJyCzmAiIurkSu3NKcksyczLxULq6S1OISPS4A1I8LMQ==", - }, - { - "13e6f47dd57798bfdc728d91f5c6d7f40c5bb5fc", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("this tests\r\nCRLF\r\nencoded files.\r\n")), - "eAFLyslPUjA2YSjJyCxWKEktLinm5XIO8nHj5UrNS85PSU1RSMvMSS3W4+UCABp3DNE=", - }, - { - "72a7bc4667ab068e954172437b993d9fbaa137cb", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("test@example.com")), - "eAFLyslPUjA0YyhJLS5xSK1IzC3ISdVLzs8FAGVtCIA=", - }, - { - "bb2b40e85ec0455d1de72daff71583f0dd72a33f", - core.BlobObject, - base64.StdEncoding.EncodeToString([]byte("package main\r\n\r\nimport (\r\n\t\"fmt\"\r\n\t\"io\"\r\n\t\"os\"\r\n\r\n\t\"gopkg.in/src-d/go-git.v3\"\r\n)\r\n\r\nfunc main() {\r\n\tfmt.Printf(\"Retrieving %q ...\\n\", os.Args[2])\r\n\tr, err := git.NewRepository(os.Args[2], nil)\r\n\tif err != nil {\r\n\t\tpanic(err)\r\n\t}\r\n\r\n\tif err := r.Pull(\"origin\", \"refs/heads/master\"); err != nil {\r\n\t\tpanic(err)\r\n\t}\r\n\r\n\tdumpCommits(r)\r\n}\r\n\r\nfunc dumpCommits(r *git.Repository) {\r\n\titer := r.Commits()\r\n\tdefer iter.Close()\r\n\r\n\tfor {\r\n\t\tcommit, err := iter.Next()\r\n\t\tif err != nil {\r\n\t\t\tif err == io.EOF {\r\n\t\t\t\tbreak\r\n\t\t\t}\r\n\r\n\t\t\tpanic(err)\r\n\t\t}\r\n\r\n\t\tfmt.Println(commit)\r\n\t}\r\n}\r\n")), - "eAGNUU1LAzEU9JpC/0NcEFJps2ARQdmDFD3W0qt6SHez8dHdZH1JqyL+d/Oy/aDgQVh47LzJTGayatyKX99MzzpVrpXRvFVgh4PhANrOYeBiOGBZ3YaMJrg0nI+D/o3r1kaCzT2Wkyo3bmIgyO00rkfEqDe2TIJixL/jgagjFwg21CJb6oCgt2ANv3jnUsoXm4258/IejX++eo0CDMdcI/LbgpPuXH8sdec8BIdf4sgccwsN0aFO9POCgGTIOmWhFFGE9j/p1jtWFEW52DSNyByCAXLPUNc+f9Oq8nmrfNCYje7+o1lt2m7m2haCF2SVnFL6kw2/pBzHEH0rEH0oI8q9BF220nWEaSdnjfNaRDDCtcM+WZnsDgUl4lx/BuKxv6rYY0XBwcmHp8deh7EVarWmQ7uC2Glre/TweI0VvTk5xaTx+wWX66Gs", - }, - { - "e94db0f9ffca44dc7bade6a3591f544183395a7c", - core.TreeObject, - "MTAwNjQ0IFRlc3QgMS50eHQAqKlAYn0TJpWpdp34g/hZkvD/SkMxMDA2NDQgVGVzdCAyLnR4dABNwhdIAaxKPTaIYhD9CG++E0z3sjEwMDY0NCBUZXN0IDMudHh0ABPm9H3Vd5i/3HKNkfXG1/QMW7X8MTAwNjQ0IFRlc3QgNC50eHQAcqe8RmerBo6VQXJDe5k9n7qhN8sxMDA2NDQgVGVzdCA1LnR4dAC7K0DoXsBFXR3nLa/3FYPw3XKjPw==", - "eAErKUpNVTC0NGAwNDAwMzFRCEktLlEw1CupKGFYsdIhqVZYberKsrk/mn9ETvrw38sZWZURWJXvIXEPxjVetmYdSQJ/OfL3Cft834SsyhisSvjZl9qr5TP23ynqnfj12PUvPNFb/yCrMgGrKlq+xy19NVvfVMci5+qZtvN3LTQ/jazKFKxqt7bDi7gDrrGyz3XXfxdt/nC3aLE9AA2STmk=", - }, - { - "9d7f8a56eaf92469dee8a856e716a03387ddb076", - core.CommitObject, - "dHJlZSBlOTRkYjBmOWZmY2E0NGRjN2JhZGU2YTM1OTFmNTQ0MTgzMzk1YTdjCmF1dGhvciBKb3NodWEgU2pvZGluZyA8am9zaHVhLnNqb2RpbmdAc2NqYWxsaWFuY2UuY29tPiAxNDU2NTMxNTgzIC0wODAwCmNvbW1pdHRlciBKb3NodWEgU2pvZGluZyA8am9zaHVhLnNqb2RpbmdAc2NqYWxsaWFuY2UuY29tPiAxNDU2NTMxNTgzIC0wODAwCgpUZXN0IENvbW1pdAo=", - "eAGtjksOgjAUAF33FO8CktZ+aBNjTNy51Qs8Xl8FAjSh5f4SvILLmcVkKM/zUOEi3amuzMDBxE6mkBKhMZHaDiM71DaoZI1RXutgsSWBW+3zCs9c+g3hNeY4LB+4jgc35cf3QiNO04ALcUN5voEy1lmtrNdwll5Ksdt9oPIfUuLNpcLjCIov3ApFmQ==", - }, -} - -func Test(t *testing.T) { TestingT(t) } diff --git a/formats/objfile/reader.go b/formats/objfile/reader.go deleted file mode 100644 index 5c319f6..0000000 --- a/formats/objfile/reader.go +++ /dev/null @@ -1,118 +0,0 @@ -package objfile - -import ( - "compress/zlib" - "errors" - "io" - "strconv" - - "gopkg.in/src-d/go-git.v4/core" - "gopkg.in/src-d/go-git.v4/formats/packfile" -) - -var ( - ErrClosed = errors.New("objfile: already closed") - ErrHeader = errors.New("objfile: invalid header") - ErrNegativeSize = errors.New("objfile: negative object size") -) - -// Reader reads and decodes compressed objfile data from a provided io.Reader. -// Reader implements io.ReadCloser. Close should be called when finished with -// the Reader. Close will not close the underlying io.Reader. -type Reader struct { - multi io.Reader - zlib io.ReadCloser - hasher core.Hasher -} - -// NewReader returns a new Reader reading from r. -func NewReader(r io.Reader) (*Reader, error) { - zlib, err := zlib.NewReader(r) - if err != nil { - return nil, packfile.ErrZLib.AddDetails(err.Error()) - } - - return &Reader{ - zlib: zlib, - }, nil -} - -// Header reads the type and the size of object, and prepares the reader for read -func (r *Reader) Header() (t core.ObjectType, size int64, err error) { - var raw []byte - raw, err = r.readUntil(' ') - if err != nil { - return - } - - t, err = core.ParseObjectType(string(raw)) - if err != nil { - return - } - - raw, err = r.readUntil(0) - if err != nil { - return - } - - size, err = strconv.ParseInt(string(raw), 10, 64) - if err != nil { - err = ErrHeader - return - } - - defer r.prepareForRead(t, size) - return -} - -// readSlice reads one byte at a time from r until it encounters delim or an -// error. -func (r *Reader) readUntil(delim byte) ([]byte, error) { - var buf [1]byte - value := make([]byte, 0, 16) - for { - if n, err := r.zlib.Read(buf[:]); err != nil && (err != io.EOF || n == 0) { - if err == io.EOF { - return nil, ErrHeader - } - return nil, err - } - - if buf[0] == delim { - return value, nil - } - - value = append(value, buf[0]) - } -} - -func (r *Reader) prepareForRead(t core.ObjectType, size int64) { - r.hasher = core.NewHasher(t, size) - r.multi = io.TeeReader(r.zlib, r.hasher) -} - -// Read reads len(p) bytes into p from the object data stream. It returns -// the number of bytes read (0 <= n <= len(p)) and any error encountered. Even -// if Read returns n < len(p), it may use all of p as scratch space during the -// call. -// -// If Read encounters the end of the data stream it will return err == io.EOF, -// either in the current call if n > 0 or in a subsequent call. -func (r *Reader) Read(p []byte) (n int, err error) { - return r.multi.Read(p) -} - -// Hash returns the hash of the object data stream that has been read so far. -func (r *Reader) Hash() core.Hash { - return r.hasher.Sum() -} - -// Close releases any resources consumed by the Reader. Calling Close does not -// close the wrapped io.Reader originally passed to NewReader. -func (r *Reader) Close() error { - if err := r.zlib.Close(); err != nil { - return err - } - - return nil -} diff --git a/formats/objfile/reader_test.go b/formats/objfile/reader_test.go deleted file mode 100644 index a383fd2..0000000 --- a/formats/objfile/reader_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package objfile - -import ( - "bytes" - "encoding/base64" - "fmt" - "io" - "io/ioutil" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-git.v4/core" -) - -type SuiteReader struct{} - -var _ = Suite(&SuiteReader{}) - -func (s *SuiteReader) TestReadObjfile(c *C) { - for k, fixture := range objfileFixtures { - com := fmt.Sprintf("test %d: ", k) - hash := core.NewHash(fixture.hash) - content, _ := base64.StdEncoding.DecodeString(fixture.content) - data, _ := base64.StdEncoding.DecodeString(fixture.data) - - testReader(c, bytes.NewReader(data), hash, fixture.t, content, com) - } -} - -func testReader(c *C, source io.Reader, hash core.Hash, t core.ObjectType, content []byte, com string) { - r, err := NewReader(source) - c.Assert(err, IsNil) - - typ, size, err := r.Header() - c.Assert(err, IsNil) - c.Assert(typ, Equals, t) - c.Assert(content, HasLen, int(size)) - - rc, err := ioutil.ReadAll(r) - c.Assert(err, IsNil) - c.Assert(rc, DeepEquals, content, Commentf("%scontent=%s, expected=%s", base64.StdEncoding.EncodeToString(rc), base64.StdEncoding.EncodeToString(content))) - - c.Assert(r.Hash(), Equals, hash) // Test Hash() before close - c.Assert(r.Close(), IsNil) - -} - -func (s *SuiteReader) TestReadEmptyObjfile(c *C) { - source := bytes.NewReader([]byte{}) - _, err := NewReader(source) - c.Assert(err, NotNil) -} - -func (s *SuiteReader) TestReadGarbage(c *C) { - source := bytes.NewReader([]byte("!@#$RO!@NROSADfinq@o#irn@oirfn")) - _, err := NewReader(source) - c.Assert(err, NotNil) -} - -func (s *SuiteReader) TestReadCorruptZLib(c *C) { - data, _ := base64.StdEncoding.DecodeString("eAFLysaalPUjBgAAAJsAHw") - source := bytes.NewReader(data) - r, err := NewReader(source) - c.Assert(err, IsNil) - - _, _, err = r.Header() - c.Assert(err, NotNil) -} diff --git a/formats/objfile/writer.go b/formats/objfile/writer.go deleted file mode 100644 index d2f2314..0000000 --- a/formats/objfile/writer.go +++ /dev/null @@ -1,109 +0,0 @@ -package objfile - -import ( - "compress/zlib" - "errors" - "io" - "strconv" - - "gopkg.in/src-d/go-git.v4/core" -) - -var ( - ErrOverflow = errors.New("objfile: declared data length exceeded (overflow)") -) - -// Writer writes and encodes data in compressed objfile format to a provided -// io.Writerº. Close should be called when finished with the Writer. Close will -// not close the underlying io.Writer. -type Writer struct { - raw io.Writer - zlib io.WriteCloser - hasher core.Hasher - multi io.Writer - - closed bool - pending int64 // number of unwritten bytes -} - -// NewWriter returns a new Writer writing to w. -// -// The returned Writer implements io.WriteCloser. Close should be called when -// finished with the Writer. Close will not close the underlying io.Writer. -func NewWriter(w io.Writer) *Writer { - return &Writer{ - raw: w, - zlib: zlib.NewWriter(w), - } -} - -// WriteHeader writes the type and the size and prepares to accept the object's -// contents. If an invalid t is provided, core.ErrInvalidType is returned. If a -// negative size is provided, ErrNegativeSize is returned. -func (w *Writer) WriteHeader(t core.ObjectType, size int64) error { - if !t.Valid() { - return core.ErrInvalidType - } - if size < 0 { - return ErrNegativeSize - } - - b := t.Bytes() - b = append(b, ' ') - b = append(b, []byte(strconv.FormatInt(size, 10))...) - b = append(b, 0) - - defer w.prepareForWrite(t, size) - _, err := w.zlib.Write(b) - - return err -} - -func (w *Writer) prepareForWrite(t core.ObjectType, size int64) { - w.pending = size - - w.hasher = core.NewHasher(t, size) - w.multi = io.MultiWriter(w.zlib, w.hasher) -} - -// Write writes the object's contents. Write returns the error ErrOverflow if -// more than size bytes are written after WriteHeader. -func (w *Writer) Write(p []byte) (n int, err error) { - if w.closed { - return 0, ErrClosed - } - - overwrite := false - if int64(len(p)) > w.pending { - p = p[0:w.pending] - overwrite = true - } - - n, err = w.multi.Write(p) - w.pending -= int64(n) - if err == nil && overwrite { - err = ErrOverflow - return - } - - return -} - -// Hash returns the hash of the object data stream that has been written so far. -// It can be called before or after Close. -func (w *Writer) Hash() core.Hash { - return w.hasher.Sum() // Not yet closed, return hash of data written so far -} - -// Close releases any resources consumed by the Writer. -// -// Calling Close does not close the wrapped io.Writer originally passed to -// NewWriter. -func (w *Writer) Close() error { - if err := w.zlib.Close(); err != nil { - return err - } - - w.closed = true - return nil -} diff --git a/formats/objfile/writer_test.go b/formats/objfile/writer_test.go deleted file mode 100644 index 18bba79..0000000 --- a/formats/objfile/writer_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package objfile - -import ( - "bytes" - "encoding/base64" - "fmt" - "io" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-git.v4/core" -) - -type SuiteWriter struct{} - -var _ = Suite(&SuiteWriter{}) - -func (s *SuiteWriter) TestWriteObjfile(c *C) { - for k, fixture := range objfileFixtures { - buffer := bytes.NewBuffer(nil) - - com := fmt.Sprintf("test %d: ", k) - hash := core.NewHash(fixture.hash) - content, _ := base64.StdEncoding.DecodeString(fixture.content) - - // Write the data out to the buffer - testWriter(c, buffer, hash, fixture.t, content) - - // Read the data back in from the buffer to be sure it matches - testReader(c, buffer, hash, fixture.t, content, com) - } -} - -func testWriter(c *C, dest io.Writer, hash core.Hash, t core.ObjectType, content []byte) { - size := int64(len(content)) - w := NewWriter(dest) - - err := w.WriteHeader(t, size) - c.Assert(err, IsNil) - - written, err := io.Copy(w, bytes.NewReader(content)) - c.Assert(err, IsNil) - c.Assert(written, Equals, size) - - c.Assert(w.Hash(), Equals, hash) - c.Assert(w.Close(), IsNil) -} - -func (s *SuiteWriter) TestWriteOverflow(c *C) { - buf := bytes.NewBuffer(nil) - w := NewWriter(buf) - - err := w.WriteHeader(core.BlobObject, 8) - c.Assert(err, IsNil) - - n, err := w.Write([]byte("1234")) - c.Assert(err, IsNil) - c.Assert(n, Equals, 4) - - n, err = w.Write([]byte("56789")) - c.Assert(err, Equals, ErrOverflow) - c.Assert(n, Equals, 4) -} - -func (s *SuiteWriter) TestNewWriterInvalidType(c *C) { - buf := bytes.NewBuffer(nil) - w := NewWriter(buf) - - err := w.WriteHeader(core.InvalidObject, 8) - c.Assert(err, Equals, core.ErrInvalidType) -} - -func (s *SuiteWriter) TestNewWriterInvalidSize(c *C) { - buf := bytes.NewBuffer(nil) - w := NewWriter(buf) - - err := w.WriteHeader(core.BlobObject, -1) - c.Assert(err, Equals, ErrNegativeSize) - err = w.WriteHeader(core.BlobObject, -1651860) - c.Assert(err, Equals, ErrNegativeSize) -} -- cgit