From 026d7c48163a9d246820c84693673a13f42f9145 Mon Sep 17 00:00:00 2001 From: Jeremy Stribling Date: Tue, 31 Oct 2017 15:15:58 -0700 Subject: filesystem: implement PackRefs() Currently this implementation is only valid for kbfsgit, since it assumes some things about the filesystem not being updated during the packing, and about conflict resolution rules. In the future, it would be nice to replace this with a more general one, and move this kbfsgit-optimized implementation into kbfsgit. Issue: KBFS-2517 --- plumbing/storer/reference.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'plumbing/storer') diff --git a/plumbing/storer/reference.go b/plumbing/storer/reference.go index ae80a39..5e85a3b 100644 --- a/plumbing/storer/reference.go +++ b/plumbing/storer/reference.go @@ -24,6 +24,8 @@ type ReferenceStorer interface { Reference(plumbing.ReferenceName) (*plumbing.Reference, error) IterReferences() (ReferenceIter, error) RemoveReference(plumbing.ReferenceName) error + CountLooseRefs() (int, error) + PackRefs() error } // ReferenceIter is a generic closable interface for iterating over references. -- cgit From ac1914eac3c20efa63de8809229994364ad9639b Mon Sep 17 00:00:00 2001 From: Taru Karttunen Date: Wed, 1 Nov 2017 21:06:37 +0200 Subject: First pass of prune design --- plumbing/storer/object.go | 14 ++++++++++++++ plumbing/storer/object_test.go | 13 +++++++++++++ 2 files changed, 27 insertions(+) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index e793211..5d043cb 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -3,6 +3,7 @@ package storer import ( "errors" "io" + "time" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -36,6 +37,19 @@ type EncodedObjectStorer interface { // // Valid plumbing.ObjectType values are CommitObject, BlobObject, TagObject, IterEncodedObjects(plumbing.ObjectType) (EncodedObjectIter, error) + // HasEncodedObject returns ErrObjNotFound if the object doesn't + // exist. If the object does exist, it returns nil. + HasEncodedObject(plumbing.Hash) error + // ForEachObjectHash iterates over all the (loose) object hashes + // in the repository without necessarily having to read those objects. + // Objects only inside pack files may be omitted. + ForEachObjectHash(func(plumbing.Hash) error) error + // LooseObjectTime looks up the (m)time associated with the + // loose object (that is not in a pack file). Implementations + // may + LooseObjectTime(plumbing.Hash) (time.Time, error) + // DeleteLooseObject deletes a loose object if it exists. + DeleteLooseObject(plumbing.Hash) error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index 6bdd25c..f8b1f73 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -3,6 +3,7 @@ package storer import ( "fmt" "testing" + "time" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git.v4/plumbing" @@ -148,3 +149,15 @@ func (o *MockObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (EncodedOb func (o *MockObjectStorage) Begin() Transaction { return nil } + +func (o *MockObjectStorage) ForEachObjectHash(fun func(plumbing.Hash) error) error { + return nil +} + +func (o *MockObjectStorage) LooseObjectTime(plumbing.Hash) (time.Time, error) { + return time.Time{}, plumbing.ErrObjectNotFound +} + +func (o *MockObjectStorage) DeleteLooseObject(plumbing.Hash) error { + return plumbing.ErrObjectNotFound +} -- cgit From 3f0b1ff37b64108cfed1b57ea4ae1f1566592905 Mon Sep 17 00:00:00 2001 From: Taru Karttunen Date: Mon, 6 Nov 2017 13:30:42 +0200 Subject: Address CI and move code around --- plumbing/storer/object.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index 5d043cb..bd34be8 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -43,10 +43,12 @@ type EncodedObjectStorer interface { // ForEachObjectHash iterates over all the (loose) object hashes // in the repository without necessarily having to read those objects. // Objects only inside pack files may be omitted. + // If ErrStop is sent the iteration is stop but no error is returned. ForEachObjectHash(func(plumbing.Hash) error) error // LooseObjectTime looks up the (m)time associated with the - // loose object (that is not in a pack file). Implementations - // may + // loose object (that is not in a pack file). Some + // implementations (e.g. without loose objects) + // always return an error. LooseObjectTime(plumbing.Hash) (time.Time, error) // DeleteLooseObject deletes a loose object if it exists. DeleteLooseObject(plumbing.Hash) error -- cgit From fae438980c3e17cb04f84ce92b99cd3c835e3e18 Mon Sep 17 00:00:00 2001 From: Taru Karttunen Date: Wed, 15 Nov 2017 18:33:41 +0200 Subject: Support for repacking objects --- plumbing/storer/object.go | 5 +++++ plumbing/storer/object_test.go | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index bd34be8..29e0090 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -52,6 +52,11 @@ type EncodedObjectStorer interface { LooseObjectTime(plumbing.Hash) (time.Time, error) // DeleteLooseObject deletes a loose object if it exists. DeleteLooseObject(plumbing.Hash) error + // ObjectPacks returns hashes of object packs if the underlying + // implementation has pack files. + ObjectPacks() ([]plumbing.Hash, error) + // DeleteObjectPackAndIndex deletes an object pack and the corresponding index file if they exist. + DeleteObjectPackAndIndex(plumbing.Hash) error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index f8b1f73..68e8e10 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -161,3 +161,7 @@ func (o *MockObjectStorage) LooseObjectTime(plumbing.Hash) (time.Time, error) { func (o *MockObjectStorage) DeleteLooseObject(plumbing.Hash) error { return plumbing.ErrObjectNotFound } + +func (o *MockObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { + return nil, nil +} -- cgit From d96582a6fb7df092c2856f56decd33034fe0ade3 Mon Sep 17 00:00:00 2001 From: Taru Karttunen Date: Thu, 16 Nov 2017 21:00:51 +0200 Subject: Make object repacking more configurable --- plumbing/storer/object.go | 5 +++-- plumbing/storer/object_test.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index 29e0090..e5f98d7 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -55,8 +55,9 @@ type EncodedObjectStorer interface { // ObjectPacks returns hashes of object packs if the underlying // implementation has pack files. ObjectPacks() ([]plumbing.Hash, error) - // DeleteObjectPackAndIndex deletes an object pack and the corresponding index file if they exist. - DeleteObjectPackAndIndex(plumbing.Hash) error + // DeleteOldObjectPackAndIndex deletes an object pack and the corresponding index file if they exist. + // Deletion is only performed if the pack is older than the supplied time (or the time is zero). + DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index 68e8e10..da0db81 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -165,3 +165,7 @@ func (o *MockObjectStorage) DeleteLooseObject(plumbing.Hash) error { func (o *MockObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { return nil, nil } + +func (o *MockObjectStorage) DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error { + return plumbing.ErrObjectNotFound +} -- cgit From aa092f5474da3e9c3ff4f40b88849726b645f39f Mon Sep 17 00:00:00 2001 From: Jeremy Stribling Date: Fri, 22 Sep 2017 21:14:17 -0700 Subject: plumbing: add `HasEncodedObject` method to Storer This allows the user to check whether an object exists, without reading all the object data from storage. Issue: KBFS-2445 --- plumbing/storer/object_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index da0db81..9a6959d 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -133,6 +133,15 @@ func (o *MockObjectStorage) SetEncodedObject(obj plumbing.EncodedObject) (plumbi return plumbing.ZeroHash, nil } +func (o *MockObjectStorage) HasEncodedObject(h plumbing.Hash) error { + for _, o := range o.db { + if o.Hash() == h { + return nil + } + } + return plumbing.ErrObjectNotFound +} + func (o *MockObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { for _, o := range o.db { if o.Hash() == h { -- cgit From 4c1569511db5e1d26e42e9cd8dadb9e65ccafb20 Mon Sep 17 00:00:00 2001 From: Jeremy Stribling Date: Wed, 29 Nov 2017 14:15:32 -0800 Subject: storer: separate loose and packed object mgmt into optional ifaces Suggested by mcuadros. --- plumbing/storer/object.go | 40 +++++++++++++++++++++++++--------------- plumbing/storer/object_test.go | 21 --------------------- 2 files changed, 25 insertions(+), 36 deletions(-) (limited to 'plumbing/storer') diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index e5f98d7..f1d19ef 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -40,6 +40,26 @@ type EncodedObjectStorer interface { // HasEncodedObject returns ErrObjNotFound if the object doesn't // exist. If the object does exist, it returns nil. HasEncodedObject(plumbing.Hash) error +} + +// DeltaObjectStorer is an EncodedObjectStorer that can return delta +// objects. +type DeltaObjectStorer interface { + // DeltaObject is the same as EncodedObject but without resolving deltas. + // Deltas will be returned as plumbing.DeltaObject instances. + DeltaObject(plumbing.ObjectType, plumbing.Hash) (plumbing.EncodedObject, error) +} + +// Transactioner is a optional method for ObjectStorer, it enable transaction +// base write and read operations in the storage +type Transactioner interface { + // Begin starts a transaction. + Begin() Transaction +} + +// LooseObjectStorer is an optional interface for managing "loose" +// objects, i.e. those not in packfiles. +type LooseObjectStorer interface { // ForEachObjectHash iterates over all the (loose) object hashes // in the repository without necessarily having to read those objects. // Objects only inside pack files may be omitted. @@ -52,6 +72,11 @@ type EncodedObjectStorer interface { LooseObjectTime(plumbing.Hash) (time.Time, error) // DeleteLooseObject deletes a loose object if it exists. DeleteLooseObject(plumbing.Hash) error +} + +// PackedObjectStorer is an optional interface for managing objects in +// packfiles. +type PackedObjectStorer interface { // ObjectPacks returns hashes of object packs if the underlying // implementation has pack files. ObjectPacks() ([]plumbing.Hash, error) @@ -60,21 +85,6 @@ type EncodedObjectStorer interface { DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error } -// DeltaObjectStorer is an EncodedObjectStorer that can return delta -// objects. -type DeltaObjectStorer interface { - // DeltaObject is the same as EncodedObject but without resolving deltas. - // Deltas will be returned as plumbing.DeltaObject instances. - DeltaObject(plumbing.ObjectType, plumbing.Hash) (plumbing.EncodedObject, error) -} - -// Transactioner is a optional method for ObjectStorer, it enable transaction -// base write and read operations in the storage -type Transactioner interface { - // Begin starts a transaction. - Begin() Transaction -} - // PackfileWriter is a optional method for ObjectStorer, it enable direct write // of packfile to the storage type PackfileWriter interface { diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index 9a6959d..6b4fe0f 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -3,7 +3,6 @@ package storer import ( "fmt" "testing" - "time" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git.v4/plumbing" @@ -158,23 +157,3 @@ func (o *MockObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (EncodedOb func (o *MockObjectStorage) Begin() Transaction { return nil } - -func (o *MockObjectStorage) ForEachObjectHash(fun func(plumbing.Hash) error) error { - return nil -} - -func (o *MockObjectStorage) LooseObjectTime(plumbing.Hash) (time.Time, error) { - return time.Time{}, plumbing.ErrObjectNotFound -} - -func (o *MockObjectStorage) DeleteLooseObject(plumbing.Hash) error { - return plumbing.ErrObjectNotFound -} - -func (o *MockObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { - return nil, nil -} - -func (o *MockObjectStorage) DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error { - return plumbing.ErrObjectNotFound -} -- cgit