aboutsummaryrefslogtreecommitdiffstats
path: root/formats/idxfile/decoder.go
diff options
context:
space:
mode:
authorAlberto Cortés <alcortesm@gmail.com>2016-07-04 17:09:22 +0200
committerMáximo Cuadros <mcuadros@gmail.com>2016-07-04 17:09:22 +0200
commit5e73f01cb2e027a8f02801635b79d3a9bc866914 (patch)
treec0e7eb355c9b8633d99bab9295cb72b6c3a9c0e1 /formats/idxfile/decoder.go
parent808076af869550a200a3a544c9ee2fa22a8b6a85 (diff)
downloadgo-git-5e73f01cb2e027a8f02801635b79d3a9bc866914.tar.gz
Adds support to open local repositories and to use file-based object storage (#55)v3.1.0
* remove some comments * idx writer/reader * Shut up ssh tests, they are annoying * Add file scheme test to clients * Add dummy file client * Add test fot file client * Make tests use fixture endpoint * add parser for packed-refs format * add parser for packed-refs format * WIP adding dir.Refs() tests * Add test for fixture refs * refs parser for the refs directory * Documentation * Add Capabilities to file client * tgz.Exatract now accpets a path instead of a Reader * fix bug in idxfile fanout calculation * remove dead code * packfile documentation * clean packfile parser code * add core.Object.Content() and returns errors for core.ObjectStorage.Iter() * add seekable storage * add dir repos to NewRepository * clean prints * Add dir client documentation to README * Organize the README * README * Clean tgz package * Clean temp dirs after tgz tests * Gometalinter on gitdir * Clean pattern function * metalinter tgz * metalinter gitdir * gitdir coverage and remove seekable packfile filedescriptor leak * gitdir Idxfile tests and remove file descriptor leak * gitdir Idxfile tests when no idx is found * clean storage/seekable/internal/index and some formats/idxfile API issues * clean storage/seekable * clean formats/idx * turn packfile/doc.go into packfile/doc.txt * move formats/packfile/reader to decoder * fix packfile decoder error names * improve documentation * comment packfile decoder errors * comment public API (format/packfile) * remve duplicated code in packfile decoder test * move tracking_reader into an internal package and clean it * use iota for packfile format * rename packfile parse.go to packfile object_at.go * clean packfile deltas * fix delta header size bug * improve delta documentation * clean packfile deltas * clean packfiles deltas * clean repository.go * Remove go 1.5 from Travis CI Because go 1.5 does not suport internal packages. * change local repo scheme to local:// * change "local://" to "file://" as the local scheme * fix broken indentation * shortens names of variables in short scopes * more shortening of variable names * more shortening of variable names * Rename git dir client to "file", as the scheme used for it * Fix file format ctor name, now that the package name has change * Sortcut local repo constructor to not use remotes The object storage is build directly in the repository ctor, instead of creating a remote and waiting for the user to pull it. * update README and fix some errors in it * remove file scheme client * Local respositories has now a new ctor This is, they are no longer identified by the scheme of the URL, but are created different from inception. * remove unused URL field form Repository * move all git dir logic to seekable sotrage ctor * fix documentation * Make formats/file/dir an internal package to storage/seekable * change package storage/seekable to storage/fs * clean storage/fs * overall storage/fs clean * more cleaning * some metalinter fixes * upgrade cshared to last changes * remove dead code * fix test error info * remove file scheme check from clients * fix test error message * fix test error message * fix error messages * style changes * fix comments everywhere * style changes * style changes * scaffolding and tests for local packfiles without ifx files * outsource index building from packfile to the packfile decoder * refactor packfile header reading into a new function * move code to generate index from packfile back to index package * add header parsing * fix documentation errata * add undeltified and OFS delta support for index building from the packfile * add tests for packfile with ref-deltas * support for packfiles with ref-deltas and no idx * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * WIP refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * remove prints from tests * remove prints from tests * refactor packfile.core into packfile.parser * rename packfile reader to something that shows it is a recaller * rename cannot recall error * rename packfile.Reader to packfile.ReadRecaller and document * speed up test by using StreamReader instead of SeekableReader when possible * clean packfile StreamReader * stream_reader tests * refactor packfile.StreamReader into packfile.StreamReadRecaller * refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it * generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations * speed up storage/fs tests * speed up tests in . by loading packfiles in memory * speed up repository tests by using and smaller fixture * restore doc.go files * rename packfile.ReadRecaller implementations to shorter names * update comments to type changes * packfile.Parser test (WIP) * packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller * add test for packfile.ReadRecaller.ForgetAll() * clarify seekable being able to recallByOffset forgetted objects * use better names for internal maps * metalinter packfile package * speed up some tests * documentation fixes * change storage.fs package name to storage.proxy to avoid confusion with new filesystem support * New fs package and os transparent implementation Now NewRepositoryFromFS receives a fs and a path and tests are modified accordingly, but it is still not using for anything. * add fs to gitdir and proxy.store * reduce fs interface for easier implementation * remove garbage dirs from tgz tests * change file name gitdir/dir.go to gitdir/gitdir.go * fs.OS tests * metalinter utils/fs * add NewRepositoryFromFS documentation to README * Readability fixes to README * move tgz to an external dependency * move filesystem impl. example to example dir * rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go * rename proxy package to seekable
Diffstat (limited to 'formats/idxfile/decoder.go')
-rw-r--r--formats/idxfile/decoder.go160
1 files changed, 160 insertions, 0 deletions
diff --git a/formats/idxfile/decoder.go b/formats/idxfile/decoder.go
new file mode 100644
index 0000000..72a9338
--- /dev/null
+++ b/formats/idxfile/decoder.go
@@ -0,0 +1,160 @@
+package idxfile
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "io"
+
+ "gopkg.in/src-d/go-git.v3/core"
+)
+
+var (
+ // ErrUnsupportedVersion is returned by Decode when the idx file version
+ // is not supported.
+ ErrUnsupportedVersion = errors.New("Unsuported version")
+ // ErrMalformedIdxFile is returned by Decode when the idx file is corrupted.
+ ErrMalformedIdxFile = errors.New("Malformed IDX file")
+)
+
+// A Decoder reads and decodes idx files from an input stream.
+type Decoder struct {
+ io.Reader
+}
+
+// NewDecoder returns a new decoder that reads from r.
+func NewDecoder(r io.Reader) *Decoder {
+ return &Decoder{r}
+}
+
+// Decode reads the whole idx object from its input and stores it in the
+// value pointed to by idx.
+func (d *Decoder) Decode(idx *Idxfile) error {
+ if err := validateHeader(d); err != nil {
+ return err
+ }
+
+ flow := []func(*Idxfile, io.Reader) error{
+ readVersion,
+ readFanout,
+ readObjectNames,
+ readCRC32,
+ readOffsets,
+ readChecksums,
+ }
+
+ for _, f := range flow {
+ if err := f(idx, d); err != nil {
+ return err
+ }
+ }
+
+ if !idx.isValid() {
+ return ErrMalformedIdxFile
+ }
+
+ return nil
+}
+
+func validateHeader(r io.Reader) error {
+ var h = make([]byte, 4)
+ if _, err := r.Read(h); err != nil {
+ return err
+ }
+
+ if !bytes.Equal(h, idxHeader) {
+ return ErrMalformedIdxFile
+ }
+
+ return nil
+}
+
+func readVersion(idx *Idxfile, r io.Reader) error {
+ v, err := readInt32(r)
+ if err != nil {
+ return err
+ }
+
+ if v > VersionSupported {
+ return ErrUnsupportedVersion
+ }
+
+ idx.Version = v
+
+ return nil
+}
+
+func readFanout(idx *Idxfile, r io.Reader) error {
+ var err error
+
+ for i := 0; i < 255; i++ {
+ idx.Fanout[i], err = readInt32(r)
+ if err != nil {
+ return err
+ }
+ }
+
+ idx.ObjectCount, err = readInt32(r)
+
+ return err
+}
+
+func readObjectNames(idx *Idxfile, r io.Reader) error {
+ c := int(idx.ObjectCount)
+ for i := 0; i < c; i++ {
+ var ref core.Hash
+ if _, err := r.Read(ref[:]); err != nil {
+ return err
+ }
+
+ idx.Entries = append(idx.Entries, Entry{Hash: ref})
+ }
+
+ return nil
+}
+
+func readCRC32(idx *Idxfile, r io.Reader) error {
+ c := int(idx.ObjectCount)
+ for i := 0; i < c; i++ {
+ if _, err := r.Read(idx.Entries[i].CRC32[:]); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func readOffsets(idx *Idxfile, r io.Reader) error {
+ c := int(idx.ObjectCount)
+ for i := 0; i < c; i++ {
+ o, err := readInt32(r)
+ if err != nil {
+ return err
+ }
+
+ idx.Entries[i].Offset = uint64(o)
+ }
+
+ return nil
+}
+
+func readChecksums(idx *Idxfile, r io.Reader) error {
+ if _, err := r.Read(idx.PackfileChecksum[:]); err != nil {
+ return err
+ }
+
+ if _, err := r.Read(idx.IdxChecksum[:]); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func readInt32(r io.Reader) (uint32, error) {
+ var v uint32
+ if err := binary.Read(r, binary.BigEndian, &v); err != nil {
+ return 0, err
+ }
+
+ return v, nil
+}