aboutsummaryrefslogtreecommitdiffstats
path: root/repository
diff options
context:
space:
mode:
Diffstat (limited to 'repository')
-rw-r--r--repository/gogit.go156
-rw-r--r--repository/gogit_test.go11
-rw-r--r--repository/hash.go2
-rw-r--r--repository/index_bleve.go154
-rw-r--r--repository/mock_repo.go142
-rw-r--r--repository/repo.go54
-rw-r--r--repository/repo_testing.go84
7 files changed, 384 insertions, 219 deletions
diff --git a/repository/gogit.go b/repository/gogit.go
index 35934c91..01c47d41 100644
--- a/repository/gogit.go
+++ b/repository/gogit.go
@@ -13,7 +13,6 @@ import (
"time"
"github.com/ProtonMail/go-crypto/openpgp"
- "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
@@ -46,7 +45,7 @@ type GoGitRepo struct {
clocks map[string]lamport.Clock
indexesMutex sync.Mutex
- indexes map[string]bleve.Index
+ indexes map[string]Index
keyring Keyring
localStorage billy.Filesystem
@@ -76,7 +75,7 @@ func OpenGoGitRepo(path, namespace string, clockLoaders []ClockLoader) (*GoGitRe
r: r,
path: path,
clocks: make(map[string]lamport.Clock),
- indexes: make(map[string]bleve.Index),
+ indexes: make(map[string]Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, namespace)),
}
@@ -130,7 +129,7 @@ func InitGoGitRepo(path, namespace string) (*GoGitRepo, error) {
r: r,
path: filepath.Join(path, ".git"),
clocks: make(map[string]lamport.Clock),
- indexes: make(map[string]bleve.Index),
+ indexes: make(map[string]Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, ".git", namespace)),
}, nil
@@ -155,7 +154,7 @@ func InitBareGoGitRepo(path, namespace string) (*GoGitRepo, error) {
r: r,
path: path,
clocks: make(map[string]lamport.Clock),
- indexes: make(map[string]bleve.Index),
+ indexes: make(map[string]Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, namespace)),
}, nil
@@ -219,11 +218,12 @@ func isGitDir(path string) (bool, error) {
func (repo *GoGitRepo) Close() error {
var firstErr error
- for _, index := range repo.indexes {
+ for name, index := range repo.indexes {
err := index.Close()
if err != nil && firstErr == nil {
firstErr = err
}
+ delete(repo.indexes, name)
}
return firstErr
}
@@ -324,8 +324,7 @@ func (repo *GoGitRepo) LocalStorage() billy.Filesystem {
return repo.localStorage
}
-// GetBleveIndex return a bleve.Index that can be used to index documents
-func (repo *GoGitRepo) GetBleveIndex(name string) (bleve.Index, error) {
+func (repo *GoGitRepo) GetIndex(name string) (Index, error) {
repo.indexesMutex.Lock()
defer repo.indexesMutex.Unlock()
@@ -335,63 +334,28 @@ func (repo *GoGitRepo) GetBleveIndex(name string) (bleve.Index, error) {
path := filepath.Join(repo.localStorage.Root(), indexPath, name)
- index, err := bleve.Open(path)
+ index, err := openBleveIndex(path)
if err == nil {
repo.indexes[name] = index
- return index, nil
- }
-
- err = os.MkdirAll(path, os.ModePerm)
- if err != nil {
- return nil, err
- }
-
- mapping := bleve.NewIndexMapping()
- mapping.DefaultAnalyzer = "en"
-
- index, err = bleve.New(path, mapping)
- if err != nil {
- return nil, err
}
-
- repo.indexes[name] = index
-
- return index, nil
-}
-
-// ClearBleveIndex will wipe the given index
-func (repo *GoGitRepo) ClearBleveIndex(name string) error {
- repo.indexesMutex.Lock()
- defer repo.indexesMutex.Unlock()
-
- if index, ok := repo.indexes[name]; ok {
- err := index.Close()
- if err != nil {
- return err
- }
- delete(repo.indexes, name)
- }
-
- path := filepath.Join(repo.localStorage.Root(), indexPath, name)
- err := os.RemoveAll(path)
- if err != nil {
- return err
- }
-
- return nil
+ return index, err
}
// FetchRefs fetch git refs matching a directory prefix to a remote
// Ex: prefix="foo" will fetch any remote refs matching "refs/foo/*" locally.
// The equivalent git refspec would be "refs/foo/*:refs/remotes/<remote>/foo/*"
-func (repo *GoGitRepo) FetchRefs(remote string, prefix string) (string, error) {
- refspec := fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix)
+func (repo *GoGitRepo) FetchRefs(remote string, prefixes ...string) (string, error) {
+ refSpecs := make([]config.RefSpec, len(prefixes))
+
+ for i, prefix := range prefixes {
+ refSpecs[i] = config.RefSpec(fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix))
+ }
buf := bytes.NewBuffer(nil)
err := repo.r.Fetch(&gogit.FetchOptions{
RemoteName: remote,
- RefSpecs: []config.RefSpec{config.RefSpec(refspec)},
+ RefSpecs: refSpecs,
Progress: buf,
})
if err == gogit.NoErrAlreadyUpToDate {
@@ -410,35 +374,41 @@ func (repo *GoGitRepo) FetchRefs(remote string, prefix string) (string, error) {
//
// Additionally, PushRefs will update the local references in refs/remotes/<remote>/foo to match
// the remote state.
-func (repo *GoGitRepo) PushRefs(remote string, prefix string) (string, error) {
- refspec := fmt.Sprintf("refs/%s/*:refs/%s/*", prefix, prefix)
-
+func (repo *GoGitRepo) PushRefs(remote string, prefixes ...string) (string, error) {
remo, err := repo.r.Remote(remote)
if err != nil {
return "", err
}
- // to make sure that the push also create the corresponding refs/remotes/<remote>/... references,
- // we need to have a default fetch refspec configured on the remote, to make our refs "track" the remote ones.
- // This does not change the config on disk, only on memory.
- hasCustomFetch := false
- fetchRefspec := fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix)
- for _, r := range remo.Config().Fetch {
- if string(r) == fetchRefspec {
- hasCustomFetch = true
- break
+ refSpecs := make([]config.RefSpec, len(prefixes))
+
+ for i, prefix := range prefixes {
+ refspec := fmt.Sprintf("refs/%s/*:refs/%s/*", prefix, prefix)
+
+ // to make sure that the push also create the corresponding refs/remotes/<remote>/... references,
+ // we need to have a default fetch refspec configured on the remote, to make our refs "track" the remote ones.
+ // This does not change the config on disk, only on memory.
+ hasCustomFetch := false
+ fetchRefspec := fmt.Sprintf("refs/%s/*:refs/remotes/%s/%s/*", prefix, remote, prefix)
+ for _, r := range remo.Config().Fetch {
+ if string(r) == fetchRefspec {
+ hasCustomFetch = true
+ break
+ }
}
- }
- if !hasCustomFetch {
- remo.Config().Fetch = append(remo.Config().Fetch, config.RefSpec(fetchRefspec))
+ if !hasCustomFetch {
+ remo.Config().Fetch = append(remo.Config().Fetch, config.RefSpec(fetchRefspec))
+ }
+
+ refSpecs[i] = config.RefSpec(refspec)
}
buf := bytes.NewBuffer(nil)
err = remo.Push(&gogit.PushOptions{
RemoteName: remote,
- RefSpecs: []config.RefSpec{config.RefSpec(refspec)},
+ RefSpecs: refSpecs,
Progress: buf,
})
if err == gogit.NoErrAlreadyUpToDate {
@@ -480,6 +450,9 @@ func (repo *GoGitRepo) ReadData(hash Hash) ([]byte, error) {
defer repo.rMutex.Unlock()
obj, err := repo.r.BlobObject(plumbing.NewHash(hash.String()))
+ if err == plumbing.ErrObjectNotFound {
+ return nil, ErrNotFound
+ }
if err != nil {
return nil, err
}
@@ -549,6 +522,9 @@ func (repo *GoGitRepo) ReadTree(hash Hash) ([]TreeEntry, error) {
// the given hash could be a tree or a commit
obj, err := repo.r.Storer.EncodedObject(plumbing.AnyObject, h)
+ if err == plumbing.ErrObjectNotFound {
+ return nil, ErrNotFound
+ }
if err != nil {
return nil, err
}
@@ -655,43 +631,11 @@ func (repo *GoGitRepo) StoreSignedCommit(treeHash Hash, signKey *openpgp.Entity,
return Hash(hash.String()), nil
}
-// GetTreeHash return the git tree hash referenced in a commit
-func (repo *GoGitRepo) GetTreeHash(commit Hash) (Hash, error) {
- repo.rMutex.Lock()
- defer repo.rMutex.Unlock()
-
- obj, err := repo.r.CommitObject(plumbing.NewHash(commit.String()))
- if err != nil {
- return "", err
- }
-
- return Hash(obj.TreeHash.String()), nil
-}
-
-// FindCommonAncestor will return the last common ancestor of two chain of commit
-func (repo *GoGitRepo) FindCommonAncestor(commit1 Hash, commit2 Hash) (Hash, error) {
- repo.rMutex.Lock()
- defer repo.rMutex.Unlock()
-
- obj1, err := repo.r.CommitObject(plumbing.NewHash(commit1.String()))
- if err != nil {
- return "", err
- }
- obj2, err := repo.r.CommitObject(plumbing.NewHash(commit2.String()))
- if err != nil {
- return "", err
- }
-
- commits, err := obj1.MergeBase(obj2)
- if err != nil {
- return "", err
- }
-
- return Hash(commits[0].Hash.String()), nil
-}
-
func (repo *GoGitRepo) ResolveRef(ref string) (Hash, error) {
r, err := repo.r.Reference(plumbing.ReferenceName(ref), false)
+ if err == plumbing.ErrReferenceNotFound {
+ return "", ErrNotFound
+ }
if err != nil {
return "", err
}
@@ -744,6 +688,9 @@ func (repo *GoGitRepo) RefExist(ref string) (bool, error) {
// CopyRef will create a new reference with the same value as another one
func (repo *GoGitRepo) CopyRef(source string, dest string) error {
r, err := repo.r.Reference(plumbing.ReferenceName(source), false)
+ if err == plumbing.ErrReferenceNotFound {
+ return ErrNotFound
+ }
if err != nil {
return err
}
@@ -760,6 +707,9 @@ func (repo *GoGitRepo) ReadCommit(hash Hash) (Commit, error) {
defer repo.rMutex.Unlock()
commit, err := repo.r.CommitObject(plumbing.NewHash(hash.String()))
+ if err == plumbing.ErrObjectNotFound {
+ return Commit{}, ErrNotFound
+ }
if err != nil {
return Commit{}, err
}
diff --git a/repository/gogit_test.go b/repository/gogit_test.go
index a3de0a03..02bd42fd 100644
--- a/repository/gogit_test.go
+++ b/repository/gogit_test.go
@@ -65,24 +65,19 @@ func TestGoGitRepo_Indexes(t *testing.T) {
plainRoot := goGitRepoDir(t, repo)
// Can create indices
- indexA, err := repo.GetBleveIndex("a")
+ indexA, err := repo.GetIndex("a")
require.NoError(t, err)
require.NotZero(t, indexA)
require.FileExists(t, filepath.Join(plainRoot, ".git", namespace, "indexes", "a", "index_meta.json"))
require.FileExists(t, filepath.Join(plainRoot, ".git", namespace, "indexes", "a", "store"))
- indexB, err := repo.GetBleveIndex("b")
+ indexB, err := repo.GetIndex("b")
require.NoError(t, err)
require.NotZero(t, indexB)
require.DirExists(t, filepath.Join(plainRoot, ".git", namespace, "indexes", "b"))
// Can get an existing index
- indexA, err = repo.GetBleveIndex("a")
+ indexA, err = repo.GetIndex("a")
require.NoError(t, err)
require.NotZero(t, indexA)
-
- // Can delete an index
- err = repo.ClearBleveIndex("a")
- require.NoError(t, err)
- require.NoDirExists(t, filepath.Join(plainRoot, ".git", namespace, "indexes", "a"))
}
diff --git a/repository/hash.go b/repository/hash.go
index 6a11558f..ad0206c3 100644
--- a/repository/hash.go
+++ b/repository/hash.go
@@ -43,7 +43,7 @@ func (h *Hash) IsValid() bool {
return false
}
for _, r := range *h {
- if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
+ if (r < 'a' || r > 'f') && (r < '0' || r > '9') {
return false
}
}
diff --git a/repository/index_bleve.go b/repository/index_bleve.go
new file mode 100644
index 00000000..aae41d5f
--- /dev/null
+++ b/repository/index_bleve.go
@@ -0,0 +1,154 @@
+package repository
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "sync"
+ "unicode/utf8"
+
+ "github.com/blevesearch/bleve"
+)
+
+var _ Index = &bleveIndex{}
+
+type bleveIndex struct {
+ path string
+
+ mu sync.RWMutex
+ index bleve.Index
+}
+
+func openBleveIndex(path string) (*bleveIndex, error) {
+ index, err := bleve.Open(path)
+ if err == nil {
+ return &bleveIndex{path: path, index: index}, nil
+ }
+
+ b := &bleveIndex{path: path}
+ err = b.makeIndex()
+ if err != nil {
+ return nil, err
+ }
+
+ return b, nil
+}
+
+func (b *bleveIndex) makeIndex() error {
+ err := os.MkdirAll(b.path, os.ModePerm)
+ if err != nil {
+ return err
+ }
+
+ // TODO: follow https://github.com/blevesearch/bleve/issues/1576 recommendations
+
+ mapping := bleve.NewIndexMapping()
+ mapping.DefaultAnalyzer = "en"
+
+ index, err := bleve.New(b.path, mapping)
+ if err != nil {
+ return err
+ }
+ b.index = index
+ return nil
+}
+
+func (b *bleveIndex) IndexOne(id string, texts []string) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b._index(b.index.Index, id, texts)
+}
+
+func (b *bleveIndex) IndexBatch() (indexer func(id string, texts []string) error, closer func() error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ batch := b.index.NewBatch()
+
+ indexer = func(id string, texts []string) error {
+ return b._index(batch.Index, id, texts)
+ }
+
+ closer = func() error {
+ return b.index.Batch(batch)
+ }
+
+ return indexer, closer
+}
+
+func (b *bleveIndex) _index(indexer func(string, interface{}) error, id string, texts []string) error {
+ searchable := struct{ Text []string }{Text: texts}
+
+ // See https://github.com/blevesearch/bleve/issues/1576
+ var sb strings.Builder
+ normalize := func(text string) string {
+ sb.Reset()
+ for _, field := range strings.Fields(text) {
+ if utf8.RuneCountInString(field) < 100 {
+ sb.WriteString(field)
+ sb.WriteRune(' ')
+ }
+ }
+ return sb.String()
+ }
+
+ for i, s := range searchable.Text {
+ searchable.Text[i] = normalize(s)
+ }
+
+ return indexer(id, searchable)
+}
+
+func (b *bleveIndex) Search(terms []string) ([]string, error) {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+
+ for i, term := range terms {
+ if strings.Contains(term, " ") {
+ terms[i] = fmt.Sprintf("\"%s\"", term)
+ }
+ }
+
+ query := bleve.NewQueryStringQuery(strings.Join(terms, " "))
+ search := bleve.NewSearchRequest(query)
+
+ res, err := b.index.Search(search)
+ if err != nil {
+ return nil, err
+ }
+
+ ids := make([]string, len(res.Hits))
+ for i, hit := range res.Hits {
+ ids[i] = hit.ID
+ }
+
+ return ids, nil
+}
+
+func (b *bleveIndex) DocCount() (uint64, error) {
+ return b.index.DocCount()
+}
+
+func (b *bleveIndex) Clear() error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ err := b.index.Close()
+ if err != nil {
+ return err
+ }
+
+ err = os.RemoveAll(b.path)
+ if err != nil {
+ return err
+ }
+
+ return b.makeIndex()
+}
+
+func (b *bleveIndex) Close() error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ return b.index.Close()
+}
diff --git a/repository/mock_repo.go b/repository/mock_repo.go
index 3d7f0e73..c2cef8ef 100644
--- a/repository/mock_repo.go
+++ b/repository/mock_repo.go
@@ -9,7 +9,6 @@ import (
"github.com/99designs/keyring"
"github.com/ProtonMail/go-crypto/openpgp"
- "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
@@ -25,7 +24,7 @@ type mockRepo struct {
*mockRepoKeyring
*mockRepoCommon
*mockRepoStorage
- *mockRepoBleve
+ *mockRepoIndex
*mockRepoData
*mockRepoClock
*mockRepoTest
@@ -39,7 +38,7 @@ func NewMockRepo() *mockRepo {
mockRepoKeyring: NewMockRepoKeyring(),
mockRepoCommon: NewMockRepoCommon(),
mockRepoStorage: NewMockRepoStorage(),
- mockRepoBleve: newMockRepoBleve(),
+ mockRepoIndex: newMockRepoIndex(),
mockRepoData: NewMockRepoData(),
mockRepoClock: NewMockRepoClock(),
mockRepoTest: NewMockRepoTest(),
@@ -135,20 +134,20 @@ func (m *mockRepoStorage) LocalStorage() billy.Filesystem {
return m.localFs
}
-var _ RepoBleve = &mockRepoBleve{}
+var _ RepoIndex = &mockRepoIndex{}
-type mockRepoBleve struct {
+type mockRepoIndex struct {
indexesMutex sync.Mutex
- indexes map[string]bleve.Index
+ indexes map[string]Index
}
-func newMockRepoBleve() *mockRepoBleve {
- return &mockRepoBleve{
- indexes: make(map[string]bleve.Index),
+func newMockRepoIndex() *mockRepoIndex {
+ return &mockRepoIndex{
+ indexes: make(map[string]Index),
}
}
-func (m *mockRepoBleve) GetBleveIndex(name string) (bleve.Index, error) {
+func (m *mockRepoIndex) GetIndex(name string) (Index, error) {
m.indexesMutex.Lock()
defer m.indexesMutex.Unlock()
@@ -156,24 +155,63 @@ func (m *mockRepoBleve) GetBleveIndex(name string) (bleve.Index, error) {
return index, nil
}
- mapping := bleve.NewIndexMapping()
- mapping.DefaultAnalyzer = "en"
+ index := newIndex()
+ m.indexes[name] = index
+ return index, nil
+}
- index, err := bleve.NewMemOnly(mapping)
- if err != nil {
- return nil, err
- }
+var _ Index = &mockIndex{}
- m.indexes[name] = index
+type mockIndex map[string][]string
- return index, nil
+func newIndex() *mockIndex {
+ m := make(map[string][]string)
+ return (*mockIndex)(&m)
}
-func (m *mockRepoBleve) ClearBleveIndex(name string) error {
- m.indexesMutex.Lock()
- defer m.indexesMutex.Unlock()
+func (m *mockIndex) IndexOne(id string, texts []string) error {
+ (*m)[id] = texts
+ return nil
+}
+
+func (m *mockIndex) IndexBatch() (indexer func(id string, texts []string) error, closer func() error) {
+ indexer = func(id string, texts []string) error {
+ (*m)[id] = texts
+ return nil
+ }
+ closer = func() error { return nil }
+ return indexer, closer
+}
+
+func (m *mockIndex) Search(terms []string) (ids []string, err error) {
+loop:
+ for id, texts := range *m {
+ for _, text := range texts {
+ for _, s := range strings.Fields(text) {
+ for _, term := range terms {
+ if s == term {
+ ids = append(ids, id)
+ continue loop
+ }
+ }
+ }
+ }
+ }
+ return ids, nil
+}
+
+func (m *mockIndex) DocCount() (uint64, error) {
+ return uint64(len(*m)), nil
+}
+
+func (m *mockIndex) Clear() error {
+ for k, _ := range *m {
+ delete(*m, k)
+ }
+ return nil
+}
- delete(m.indexes, name)
+func (m *mockIndex) Close() error {
return nil
}
@@ -201,12 +239,12 @@ func NewMockRepoData() *mockRepoData {
}
}
-func (r *mockRepoData) FetchRefs(remote string, prefix string) (string, error) {
+func (r *mockRepoData) FetchRefs(remote string, prefixes ...string) (string, error) {
panic("implement me")
}
// PushRefs push git refs to a remote
-func (r *mockRepoData) PushRefs(remote string, prefix string) (string, error) {
+func (r *mockRepoData) PushRefs(remote string, prefixes ...string) (string, error) {
panic("implement me")
}
@@ -220,7 +258,7 @@ func (r *mockRepoData) StoreData(data []byte) (Hash, error) {
func (r *mockRepoData) ReadData(hash Hash) ([]byte, error) {
data, ok := r.blobs[hash]
if !ok {
- return nil, fmt.Errorf("unknown hash")
+ return nil, ErrNotFound
}
return data, nil
@@ -245,13 +283,13 @@ func (r *mockRepoData) ReadTree(hash Hash) ([]TreeEntry, error) {
commit, ok := r.commits[hash]
if !ok {
- return nil, fmt.Errorf("unknown hash")
+ return nil, ErrNotFound
}
data, ok = r.trees[commit.treeHash]
if !ok {
- return nil, fmt.Errorf("unknown hash")
+ return nil, ErrNotFound
}
}
@@ -289,7 +327,7 @@ func (r *mockRepoData) StoreSignedCommit(treeHash Hash, signKey *openpgp.Entity,
func (r *mockRepoData) ReadCommit(hash Hash) (Commit, error) {
c, ok := r.commits[hash]
if !ok {
- return Commit{}, fmt.Errorf("unknown commit")
+ return Commit{}, ErrNotFound
}
result := Commit{
@@ -308,19 +346,10 @@ func (r *mockRepoData) ReadCommit(hash Hash) (Commit, error) {
return result, nil
}
-func (r *mockRepoData) GetTreeHash(commit Hash) (Hash, error) {
- c, ok := r.commits[commit]
- if !ok {
- return "", fmt.Errorf("unknown commit")
- }
-
- return c.treeHash, nil
-}
-
func (r *mockRepoData) ResolveRef(ref string) (Hash, error) {
h, ok := r.refs[ref]
if !ok {
- return "", fmt.Errorf("unknown ref")
+ return "", ErrNotFound
}
return h, nil
}
@@ -356,48 +385,13 @@ func (r *mockRepoData) CopyRef(source string, dest string) error {
hash, exist := r.refs[source]
if !exist {
- return fmt.Errorf("Unknown ref")
+ return ErrNotFound
}
r.refs[dest] = hash
return nil
}
-func (r *mockRepoData) FindCommonAncestor(hash1 Hash, hash2 Hash) (Hash, error) {
- ancestor1 := []Hash{hash1}
-
- for hash1 != "" {
- c, ok := r.commits[hash1]
- if !ok {
- return "", fmt.Errorf("unknown commit %v", hash1)
- }
- if len(c.parents) == 0 {
- break
- }
- ancestor1 = append(ancestor1, c.parents[0])
- hash1 = c.parents[0]
- }
-
- for {
- for _, ancestor := range ancestor1 {
- if ancestor == hash2 {
- return ancestor, nil
- }
- }
-
- c, ok := r.commits[hash2]
- if !ok {
- return "", fmt.Errorf("unknown commit %v", hash1)
- }
-
- if c.parents[0] == "" {
- return "", fmt.Errorf("no ancestor found")
- }
-
- hash2 = c.parents[0]
- }
-}
-
func (r *mockRepoData) ListCommits(ref string) ([]Hash, error) {
return nonNativeListCommits(r, ref)
}
diff --git a/repository/repo.go b/repository/repo.go
index 2f90b437..66baec65 100644
--- a/repository/repo.go
+++ b/repository/repo.go
@@ -6,7 +6,6 @@ import (
"io"
"github.com/ProtonMail/go-crypto/openpgp"
- "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/MichaelMure/git-bug/util/lamport"
@@ -17,6 +16,8 @@ var (
ErrNotARepo = errors.New("not a git repository")
// ErrClockNotExist is the error returned when a clock can't be found
ErrClockNotExist = errors.New("clock doesn't exist")
+ // ErrNotFound is the error returned when a git object can't be found
+ ErrNotFound = errors.New("ref not found")
)
// Repo represents a source code repository.
@@ -25,7 +26,7 @@ type Repo interface {
RepoKeyring
RepoCommon
RepoStorage
- RepoBleve
+ RepoIndex
RepoData
Close() error
@@ -81,13 +82,33 @@ type RepoStorage interface {
LocalStorage() billy.Filesystem
}
-// RepoBleve give access to Bleve to implement full-text search indexes.
-type RepoBleve interface {
- // GetBleveIndex return a bleve.Index that can be used to index documents
- GetBleveIndex(name string) (bleve.Index, error)
+// RepoIndex gives access to full-text search indexes
+type RepoIndex interface {
+ GetIndex(name string) (Index, error)
+}
+
+// Index is a full-text search index
+type Index interface {
+ // IndexOne indexes one document, for the given ID. If the document already exist,
+ // it replaces it.
+ IndexOne(id string, texts []string) error
+
+ // IndexBatch start a batch indexing. The returned indexer function is used the same
+ // way as IndexOne, and the closer function complete the batch insertion.
+ IndexBatch() (indexer func(id string, texts []string) error, closer func() error)
+
+ // Search returns the list of IDs matching the given terms.
+ Search(terms []string) (ids []string, err error)
+
+ // DocCount returns the number of document in the index.
+ DocCount() (uint64, error)
- // ClearBleveIndex will wipe the given index
- ClearBleveIndex(name string) error
+ // Clear empty the index.
+ Clear() error
+
+ // Close closes the index and make sure everything is safely written. After this call
+ // the index can't be used anymore.
+ Close() error
}
type Commit struct {
@@ -103,7 +124,7 @@ type RepoData interface {
// FetchRefs fetch git refs matching a directory prefix to a remote
// Ex: prefix="foo" will fetch any remote refs matching "refs/foo/*" locally.
// The equivalent git refspec would be "refs/foo/*:refs/remotes/<remote>/foo/*"
- FetchRefs(remote string, prefix string) (string, error)
+ FetchRefs(remote string, prefixes ...string) (string, error)
// PushRefs push git refs matching a directory prefix to a remote
// Ex: prefix="foo" will push any local refs matching "refs/foo/*" to the remote.
@@ -111,12 +132,13 @@ type RepoData interface {
//
// Additionally, PushRefs will update the local references in refs/remotes/<remote>/foo to match
// the remote state.
- PushRefs(remote string, prefix string) (string, error)
+ PushRefs(remote string, prefixes ...string) (string, error)
// StoreData will store arbitrary data and return the corresponding hash
StoreData(data []byte) (Hash, error)
// ReadData will attempt to read arbitrary data from the given hash
+ // Returns ErrNotFound if not found.
ReadData(hash Hash) ([]byte, error)
// StoreTree will store a mapping key-->Hash as a Git tree
@@ -124,6 +146,7 @@ type RepoData interface {
// ReadTree will return the list of entries in a Git tree
// The given hash could be from either a commit or a tree
+ // Returns ErrNotFound if not found.
ReadTree(hash Hash) ([]TreeEntry, error)
// StoreCommit will store a Git commit with the given Git tree
@@ -134,13 +157,11 @@ type RepoData interface {
StoreSignedCommit(treeHash Hash, signKey *openpgp.Entity, parents ...Hash) (Hash, error)
// ReadCommit read a Git commit and returns some of its characteristic
+ // Returns ErrNotFound if not found.
ReadCommit(hash Hash) (Commit, error)
- // GetTreeHash return the git tree hash referenced in a commit
- // Deprecated
- GetTreeHash(commit Hash) (Hash, error)
-
// ResolveRef returns the hash of the target commit of the given ref
+ // Returns ErrNotFound if not found.
ResolveRef(ref string) (Hash, error)
// UpdateRef will create or update a Git reference
@@ -157,12 +178,9 @@ type RepoData interface {
RefExist(ref string) (bool, error)
// CopyRef will create a new reference with the same value as another one
+ // Returns ErrNotFound if not found.
CopyRef(source string, dest string) error
- // FindCommonAncestor will return the last common ancestor of two chain of commit
- // Deprecated
- FindCommonAncestor(commit1 Hash, commit2 Hash) (Hash, error)
-
// ListCommits will return the list of tree hashes of a ref, in chronological order
ListCommits(ref string) ([]Hash, error)
}
diff --git a/repository/repo_testing.go b/repository/repo_testing.go
index 5d51d23f..821eb762 100644
--- a/repository/repo_testing.go
+++ b/repository/repo_testing.go
@@ -10,7 +10,6 @@ import (
"github.com/MichaelMure/git-bug/util/lamport"
)
-// TODO: add tests for RepoBleve
// TODO: add tests for RepoStorage
type RepoCreator func(t testing.TB, bare bool) TestedRepo
@@ -33,6 +32,10 @@ func RepoTest(t *testing.T, creator RepoCreator) {
RepoConfigTest(t, repo)
})
+ t.Run("Index", func(t *testing.T) {
+ RepoIndexTest(t, repo)
+ })
+
t.Run("Clocks", func(t *testing.T) {
RepoClockTest(t, repo)
})
@@ -45,6 +48,15 @@ func RepoConfigTest(t *testing.T, repo RepoConfig) {
testConfig(t, repo.LocalConfig())
}
+func randomHash() Hash {
+ var letterRunes = "abcdef0123456789"
+ b := make([]byte, idLengthSHA256)
+ for i := range b {
+ b[i] = letterRunes[rand.Intn(len(letterRunes))]
+ }
+ return Hash(b)
+}
+
// helper to test a RepoData
func RepoDataTest(t *testing.T, repo RepoData) {
// Blob
@@ -59,6 +71,9 @@ func RepoDataTest(t *testing.T, repo RepoData) {
require.NoError(t, err)
require.Equal(t, data, blob1Read)
+ _, err = repo.ReadData(randomHash())
+ require.ErrorIs(t, err, ErrNotFound)
+
// Tree
blobHash2, err := repo.StoreData(randomData())
@@ -108,25 +123,20 @@ func RepoDataTest(t *testing.T, repo RepoData) {
require.NoError(t, err)
require.ElementsMatch(t, tree2, tree2Read)
+ _, err = repo.ReadTree(randomHash())
+ require.ErrorIs(t, err, ErrNotFound)
+
// Commit
commit1, err := repo.StoreCommit(treeHash1)
require.NoError(t, err)
require.True(t, commit1.IsValid())
- treeHash1Read, err := repo.GetTreeHash(commit1)
- require.NoError(t, err)
- require.Equal(t, treeHash1, treeHash1Read)
-
// commit with a parent
commit2, err := repo.StoreCommit(treeHash2, commit1)
require.NoError(t, err)
require.True(t, commit2.IsValid())
- treeHash2Read, err := repo.GetTreeHash(commit2)
- require.NoError(t, err)
- require.Equal(t, treeHash2, treeHash2Read)
-
// ReadTree should accept tree and commit hashes
tree1read, err := repo.ReadTree(commit1)
require.NoError(t, err)
@@ -137,6 +147,9 @@ func RepoDataTest(t *testing.T, repo RepoData) {
c2expected := Commit{Hash: commit2, Parents: []Hash{commit1}, TreeHash: treeHash2}
require.Equal(t, c2expected, c2)
+ _, err = repo.ReadCommit(randomHash())
+ require.ErrorIs(t, err, ErrNotFound)
+
// Ref
exist1, err := repo.RefExist("refs/bugs/ref1")
@@ -169,14 +182,13 @@ func RepoDataTest(t *testing.T, repo RepoData) {
require.NoError(t, err)
require.Equal(t, []Hash{commit1, commit2}, commits)
- // Graph
+ _, err = repo.ResolveRef("/refs/bugs/refnotexist")
+ require.ErrorIs(t, err, ErrNotFound)
- commit3, err := repo.StoreCommit(treeHash1, commit1)
- require.NoError(t, err)
+ err = repo.CopyRef("/refs/bugs/refnotexist", "refs/foo")
+ require.ErrorIs(t, err, ErrNotFound)
- ancestorHash, err := repo.FindCommonAncestor(commit2, commit3)
- require.NoError(t, err)
- require.Equal(t, commit1, ancestorHash)
+ // Cleanup
err = repo.RemoveRef("refs/bugs/ref1")
require.NoError(t, err)
@@ -234,6 +246,48 @@ func RepoDataSignatureTest(t *testing.T, repo RepoData) {
require.Error(t, err)
}
+func RepoIndexTest(t *testing.T, repo RepoIndex) {
+ idx, err := repo.GetIndex("a")
+ require.NoError(t, err)
+
+ // simple indexing
+ err = idx.IndexOne("id1", []string{"foo", "bar", "foobar barfoo"})
+ require.NoError(t, err)
+
+ // batched indexing
+ indexer, closer := idx.IndexBatch()
+ err = indexer("id2", []string{"hello", "foo bar"})
+ require.NoError(t, err)
+ err = indexer("id3", []string{"Hola", "Esta bien"})
+ require.NoError(t, err)
+ err = closer()
+ require.NoError(t, err)
+
+ // search
+ res, err := idx.Search([]string{"foobar"})
+ require.NoError(t, err)
+ require.ElementsMatch(t, []string{"id1"}, res)
+
+ res, err = idx.Search([]string{"foo"})
+ require.NoError(t, err)
+ require.ElementsMatch(t, []string{"id1", "id2"}, res)
+
+ // re-indexing an item replace previous versions
+ err = idx.IndexOne("id2", []string{"hello"})
+ require.NoError(t, err)
+
+ res, err = idx.Search([]string{"foo"})
+ require.NoError(t, err)
+ require.ElementsMatch(t, []string{"id1"}, res)
+
+ err = idx.Clear()
+ require.NoError(t, err)
+
+ res, err = idx.Search([]string{"foo"})
+ require.NoError(t, err)
+ require.Empty(t, res)
+}
+
// helper to test a RepoClock
func RepoClockTest(t *testing.T, repo RepoClock) {
allClocks, err := repo.AllClocks()
generated by cgit (git 2.34.1) at 2024-09-25 12:35:05 +0000