From 355cfc3df3a64d1bd438e0e17e1c4ba21350badf Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 19 Jul 2018 23:27:16 +0200 Subject: plumbing: idxfile, add idxfile.Writer with Observer interface It's still not complete: * 64 bit offsets * IdxChecksum Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 132 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 plumbing/format/idxfile/writer.go (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go new file mode 100644 index 0000000..aac68b5 --- /dev/null +++ b/plumbing/format/idxfile/writer.go @@ -0,0 +1,132 @@ +package idxfile + +import ( + "bytes" + "math" + "sort" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/utils/binary" +) + +type object struct { + hash plumbing.Hash + offset int64 + crc uint32 +} + +type objects []object + +// Writer implements a packfile Observer interface and is used to generate +// indexes. +type Writer struct { + count uint32 + checksum plumbing.Hash + objects objects +} + +// Create index returns a filled MemoryIndex with the information filled by +// the observer callbacks. +func (w *Writer) CreateIndex() (*MemoryIndex, error) { + idx := new(MemoryIndex) + sort.Sort(w.objects) + + // unmap all fans by default + for i := range idx.FanoutMapping { + idx.FanoutMapping[i] = noMapping + } + + buf := new(bytes.Buffer) + + last := -1 + bucket := -1 + for i, o := range w.objects { + fan := o.hash[0] + + // fill the gaps between fans + for j := last + 1; j < int(fan); j++ { + idx.Fanout[j] = uint32(i) + } + + // update the number of objects for this position + idx.Fanout[fan] = uint32(i + 1) + + // we move from one bucket to another, update counters and allocate + // memory + if last != int(fan) { + bucket++ + idx.FanoutMapping[fan] = bucket + last = int(fan) + + idx.Names = append(idx.Names, make([]byte, 0)) + idx.Offset32 = append(idx.Offset32, make([]byte, 0)) + idx.Crc32 = append(idx.Crc32, make([]byte, 0)) + } + + idx.Names[bucket] = append(idx.Names[bucket], o.hash[:]...) + + // TODO: implement 64 bit offsets + if o.offset > math.MaxInt32 { + panic("64 bit offsets not implemented") + } + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(o.offset)) + idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(o.crc)) + idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) + } + + for j := last + 1; j < 256; j++ { + idx.Fanout[j] = uint32(len(w.objects)) + } + + idx.PackfileChecksum = w.checksum + // TODO: fill IdxChecksum + + return idx, nil +} + +// Add appends new object data. +func (w *Writer) Add(h plumbing.Hash, pos int64, crc uint32) { + w.objects = append(w.objects, object{h, pos, crc}) +} + +// OnHeader implements packfile.Observer interface. +func (w *Writer) OnHeader(count uint32) error { + w.count = count + w.objects = make(objects, 0, count) + return nil +} + +// OnInflatedObjectHeader implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { + return nil +} + +// OnInflatedObjectContent implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { + w.Add(h, pos, crc) + return nil +} + +// OnFooter implements packfile.Observer interface. +func (w *Writer) OnFooter(h plumbing.Hash) error { + w.checksum = h + return nil +} + +func (o objects) Len() int { + return len(o) +} + +func (o objects) Less(i int, j int) bool { + cmp := bytes.Compare(o[i].hash[:], o[j].hash[:]) + return cmp < 0 +} + +func (o objects) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] +} -- cgit From 4e3765aef344eae2fbcd977fefd66b6571638d59 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 20 Jul 2018 12:22:55 +0200 Subject: plumbing/idxfile: use Entry to hold object data Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index aac68b5..3c5a00e 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -9,13 +9,8 @@ import ( "gopkg.in/src-d/go-git.v4/utils/binary" ) -type object struct { - hash plumbing.Hash - offset int64 - crc uint32 -} - -type objects []object +// objects implements sort.Interface and uses hash as sorting key. +type objects []Entry // Writer implements a packfile Observer interface and is used to generate // indexes. @@ -41,7 +36,7 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { last := -1 bucket := -1 for i, o := range w.objects { - fan := o.hash[0] + fan := o.Hash[0] // fill the gaps between fans for j := last + 1; j < int(fan); j++ { @@ -63,19 +58,19 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Crc32 = append(idx.Crc32, make([]byte, 0)) } - idx.Names[bucket] = append(idx.Names[bucket], o.hash[:]...) + idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) // TODO: implement 64 bit offsets - if o.offset > math.MaxInt32 { + if o.Offset > math.MaxInt32 { panic("64 bit offsets not implemented") } buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.offset)) + binary.WriteUint32(buf, uint32(o.Offset)) idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.crc)) + binary.WriteUint32(buf, uint32(o.CRC32)) idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) } @@ -90,8 +85,8 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { } // Add appends new object data. -func (w *Writer) Add(h plumbing.Hash, pos int64, crc uint32) { - w.objects = append(w.objects, object{h, pos, crc}) +func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { + w.objects = append(w.objects, Entry{h, crc, pos}) } // OnHeader implements packfile.Observer interface. @@ -108,7 +103,7 @@ func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, po // OnInflatedObjectContent implements packfile.Observer interface. func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { - w.Add(h, pos, crc) + w.Add(h, uint64(pos), crc) return nil } @@ -123,7 +118,7 @@ func (o objects) Len() int { } func (o objects) Less(i int, j int) bool { - cmp := bytes.Compare(o[i].hash[:], o[j].hash[:]) + cmp := bytes.Compare(o[i].Hash[:], o[j].Hash[:]) return cmp < 0 } -- cgit From 65e8359db00ae79838d19e19f69594f6a262c3d4 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 20 Jul 2018 13:01:27 +0200 Subject: plumbing/idxfile: support offset64 generating indexes Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index 3c5a00e..ea54081 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -18,12 +18,16 @@ type Writer struct { count uint32 checksum plumbing.Hash objects objects + offset64 uint32 + idx *MemoryIndex } // Create index returns a filled MemoryIndex with the information filled by // the observer callbacks. func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx := new(MemoryIndex) + w.idx = idx + sort.Sort(w.objects) // unmap all fans by default @@ -60,13 +64,13 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) - // TODO: implement 64 bit offsets - if o.Offset > math.MaxInt32 { - panic("64 bit offsets not implemented") + offset := o.Offset + if offset > math.MaxInt32 { + offset = w.addOffset64(offset) } buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.Offset)) + binary.WriteUint32(buf, uint32(offset)) idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) @@ -78,12 +82,23 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Fanout[j] = uint32(len(w.objects)) } + idx.Version = VersionSupported idx.PackfileChecksum = w.checksum - // TODO: fill IdxChecksum return idx, nil } +func (w *Writer) addOffset64(pos uint64) uint64 { + buf := new(bytes.Buffer) + binary.WriteUint64(buf, pos) + w.idx.Offset64 = append(w.idx.Offset64, buf.Bytes()...) + + index := uint64(w.offset64 | (1 << 31)) + w.offset64++ + + return index +} + // Add appends new object data. func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { w.objects = append(w.objects, Entry{h, crc, pos}) -- cgit From 74f56f388bbe8072bfcd976add2373f9a7e20341 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 13:14:02 +0200 Subject: plumbing/idxfile: index is created only once and retrieved with Index Index is also automatically generated when OnFooter is called. Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 103 +++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 34 deletions(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index ea54081..efcdcc6 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -2,8 +2,10 @@ package idxfile import ( "bytes" + "fmt" "math" "sort" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" @@ -15,18 +17,80 @@ type objects []Entry // Writer implements a packfile Observer interface and is used to generate // indexes. type Writer struct { + m sync.Mutex + count uint32 checksum plumbing.Hash objects objects offset64 uint32 - idx *MemoryIndex + finished bool + index *MemoryIndex +} + +// Index returns a previously created MemoryIndex or creates a new one if +// needed. +func (w *Writer) Index() (*MemoryIndex, error) { + w.m.Lock() + defer w.m.Unlock() + + if w.index == nil { + return w.createIndex() + } + + return w.index, nil +} + +// Add appends new object data. +func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { + w.m.Lock() + defer w.m.Unlock() + + w.objects = append(w.objects, Entry{h, crc, pos}) +} + +func (w *Writer) Finished() bool { + return w.finished +} + +// OnHeader implements packfile.Observer interface. +func (w *Writer) OnHeader(count uint32) error { + w.count = count + w.objects = make(objects, 0, count) + return nil +} + +// OnInflatedObjectHeader implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { + return nil +} + +// OnInflatedObjectContent implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { + w.Add(h, uint64(pos), crc) + return nil } -// Create index returns a filled MemoryIndex with the information filled by +// OnFooter implements packfile.Observer interface. +func (w *Writer) OnFooter(h plumbing.Hash) error { + w.checksum = h + w.finished = true + _, err := w.createIndex() + if err != nil { + return err + } + + return nil +} + +// creatIndex returns a filled MemoryIndex with the information filled by // the observer callbacks. -func (w *Writer) CreateIndex() (*MemoryIndex, error) { +func (w *Writer) createIndex() (*MemoryIndex, error) { + if !w.finished { + return nil, fmt.Errorf("the index still hasn't finished building") + } + idx := new(MemoryIndex) - w.idx = idx + w.index = idx sort.Sort(w.objects) @@ -91,7 +155,7 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { func (w *Writer) addOffset64(pos uint64) uint64 { buf := new(bytes.Buffer) binary.WriteUint64(buf, pos) - w.idx.Offset64 = append(w.idx.Offset64, buf.Bytes()...) + w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) index := uint64(w.offset64 | (1 << 31)) w.offset64++ @@ -99,35 +163,6 @@ func (w *Writer) addOffset64(pos uint64) uint64 { return index } -// Add appends new object data. -func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { - w.objects = append(w.objects, Entry{h, crc, pos}) -} - -// OnHeader implements packfile.Observer interface. -func (w *Writer) OnHeader(count uint32) error { - w.count = count - w.objects = make(objects, 0, count) - return nil -} - -// OnInflatedObjectHeader implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { - return nil -} - -// OnInflatedObjectContent implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { - w.Add(h, uint64(pos), crc) - return nil -} - -// OnFooter implements packfile.Observer interface. -func (w *Writer) OnFooter(h plumbing.Hash) error { - w.checksum = h - return nil -} - func (o objects) Len() int { return len(o) } -- cgit From 6f7fc05543861ee074aa17f75e1d1b5c1b948d48 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Mon, 30 Jul 2018 17:11:01 +0200 Subject: plumbing: packfile, fix package tests Signed-off-by: Miguel Molina --- plumbing/format/idxfile/writer.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index efcdcc6..a22cf16 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -25,6 +25,7 @@ type Writer struct { offset64 uint32 finished bool index *MemoryIndex + added map[plumbing.Hash]struct{} } // Index returns a previously created MemoryIndex or creates a new one if @@ -45,7 +46,15 @@ func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { w.m.Lock() defer w.m.Unlock() - w.objects = append(w.objects, Entry{h, crc, pos}) + if w.added == nil { + w.added = make(map[plumbing.Hash]struct{}) + } + + if _, ok := w.added[h]; !ok { + w.added[h] = struct{}{} + w.objects = append(w.objects, Entry{h, crc, pos}) + } + } func (w *Writer) Finished() bool { -- cgit From 6a24b4c1f0cb9e5daf30fa7979f2643a967af1ad Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Tue, 7 Aug 2018 18:41:19 +0200 Subject: *: use parser to populate non writable storages and bug fixes Signed-off-by: Miguel Molina --- plumbing/format/idxfile/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index a22cf16..89b79cd 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -74,7 +74,7 @@ func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, po } // OnInflatedObjectContent implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, _ []byte) error { w.Add(h, uint64(pos), crc) return nil } -- cgit From 8d75d239e93474e4287870e4e5143da14e2c360d Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 10 Aug 2018 12:33:56 +0200 Subject: plumbing: idxfile, Crc32 to CRC32 and return ok from findHashIndex Signed-off-by: Miguel Molina --- plumbing/format/idxfile/writer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plumbing/format/idxfile/writer.go') diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index 89b79cd..aa919e7 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -132,7 +132,7 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { idx.Names = append(idx.Names, make([]byte, 0)) idx.Offset32 = append(idx.Offset32, make([]byte, 0)) - idx.Crc32 = append(idx.Crc32, make([]byte, 0)) + idx.CRC32 = append(idx.CRC32, make([]byte, 0)) } idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) @@ -148,7 +148,7 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { buf.Truncate(0) binary.WriteUint32(buf, uint32(o.CRC32)) - idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) + idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...) } for j := last + 1; j < 256; j++ { -- cgit