repo: move bleve there

11 files changed, 262 insertions, 59 deletions

diff --git a/api/graphql/resolvers/repo.go b/api/graphql/resolvers/repo.go
index 5d96428e..c2163cbe 100644
--- a/api/graphql/resolvers/repo.go
+++ b/api/graphql/resolvers/repo.go
@@ -41,7 +41,10 @@ func (repoResolver) AllBugs(_ context.Context, obj *models.Repository, after *st
 	}
 
 	// Simply pass a []string with the ids to the pagination algorithm
-	source := obj.Repo.QueryBugs(q)
+	source, err := obj.Repo.QueryBugs(q)
+	if err != nil {
+		return nil, err
+	}
 
 	// The edger create a custom edge holding just the id
 	edger := func(id entity.Id, offset int) connections.Edge {
diff --git a/cache/repo_cache.go b/cache/repo_cache.go
index a08744cc..b5b9ee54 100644
--- a/cache/repo_cache.go
+++ b/cache/repo_cache.go
@@ -13,7 +13,6 @@ import (
 	"github.com/MichaelMure/git-bug/identity"
 	"github.com/MichaelMure/git-bug/repository"
 	"github.com/MichaelMure/git-bug/util/process"
-	"github.com/blevesearch/bleve"
 )
 
 // 1: original format
@@ -55,8 +54,6 @@ type RepoCache struct {
 	muBug sync.RWMutex
 	// excerpt of bugs data for all bugs
 	bugExcerpts map[entity.Id]*BugExcerpt
-	// searchable cache of all bugs
-	searchCache bleve.Index
 	// bug loaded in memory
 	bugs map[entity.Id]*BugCache
 	// loadedBugs is an LRU cache that records which bugs the cache has loaded in
@@ -161,9 +158,9 @@ func (c *RepoCache) Close() error {
 	c.bugs = make(map[entity.Id]*BugCache)
 	c.bugExcerpts = nil
 
-	if c.searchCache != nil {
-		c.searchCache.Close()
-		c.searchCache = nil
+	err := c.repo.Close()
+	if err != nil {
+		return err
 	}
 
 	return c.repo.LocalStorage().Remove(lockfile)
@@ -199,9 +196,10 @@ func (c *RepoCache) buildCache() error {
 
 	allBugs := bug.ReadAllLocal(c.repo)
 
-	err := c.createBleveIndex()
+	// wipe the index just to be sure
+	err := c.repo.ClearBleveIndex("bug")
 	if err != nil {
-		return fmt.Errorf("Unable to create search cache. Error: %v", err)
+		return err
 	}
 
 	for b := range allBugs {
diff --git a/cache/repo_cache_bug.go b/cache/repo_cache_bug.go
index f540e51b..1701f66d 100644
--- a/cache/repo_cache_bug.go
+++ b/cache/repo_cache_bug.go
@@ -23,10 +23,6 @@ const (
 
 var errBugNotInCache = errors.New("bug missing from cache")
 
-func searchCacheDirPath(repo repository.Repo) string {
-	return path.Join(repo.GetPath(), "git-bug", searchCacheDir)
-}
-
 // bugUpdated is a callback to trigger when the excerpt of a bug changed,
 // that is each time a bug is updated
 func (c *RepoCache) bugUpdated(id entity.Id) error {
@@ -82,14 +78,13 @@ func (c *RepoCache) loadBugCache() error {
 
 	c.bugExcerpts = aux.Excerpts
 
-	blevePath := searchCacheDirPath(c.repo)
-	searchCache, err := bleve.Open(blevePath)
+	index, err := c.repo.GetBleveIndex("bug")
 	if err != nil {
-		return fmt.Errorf("Unable to open search cache. Error: %v", err)
+		return err
 	}
-	c.searchCache = searchCache
 
-	count, err := c.searchCache.DocCount()
+	// simple heuristic to detect a mismatch between the index and the bugs
+	count, err := index.DocCount()
 	if err != nil {
 		return err
 	}
@@ -100,26 +95,6 @@ func (c *RepoCache) loadBugCache() error {
 	return nil
 }
 
-func (c *RepoCache) createBleveIndex() error {
-	blevePath := searchCacheDirPath(c.repo)
-
-	_ = os.RemoveAll(blevePath)
-
-	mapping := bleve.NewIndexMapping()
-	mapping.DefaultAnalyzer = "en"
-
-	dir := searchCacheDirPath(c.repo)
-
-	bleveIndex, err := bleve.New(dir, mapping)
-	if err != nil {
-		return err
-	}
-
-	c.searchCache = bleveIndex
-
-	return nil
-}
-
 // write will serialize on disk the bug cache file
 func (c *RepoCache) writeBugCache() error {
 	c.muBug.RLock()
@@ -287,12 +262,12 @@ func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, erro
 }
 
 // QueryBugs return the id of all Bug matching the given Query
-func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
+func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) {
 	c.muBug.RLock()
 	defer c.muBug.RUnlock()
 
 	if q == nil {
-		return c.AllBugsIds()
+		return c.AllBugsIds(), nil
 	}
 
 	matcher := compileMatcher(q.Filters)
@@ -313,9 +288,15 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
 
 		bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " "))
 		bleveSearch := bleve.NewSearchRequest(bleveQuery)
-		searchResults, err := c.searchCache.Search(bleveSearch)
+
+		index, err := c.repo.GetBleveIndex("bug")
+		if err != nil {
+			return nil, err
+		}
+
+		searchResults, err := index.Search(bleveSearch)
 		if err != nil {
-			panic("bleve search failed")
+			return nil, err
 		}
 
 		for _, hit := range searchResults.Hits {
@@ -341,7 +322,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
 	case query.OrderByEdit:
 		sorter = BugsByEditTime(filtered)
 	default:
-		panic("missing sort type")
+		return nil, errors.New("missing sort type")
 	}
 
 	switch q.OrderDirection {
@@ -350,7 +331,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
 	case query.OrderDescending:
 		sorter = sort.Reverse(sorter)
 	default:
-		panic("missing sort direction")
+		return nil, errors.New("missing sort direction")
 	}
 
 	sort.Sort(sorter)
@@ -361,7 +342,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
 		result[i] = val.Id
 	}
 
-	return result
+	return result, nil
 }
 
 // AllBugsIds return all known bug ids
@@ -504,7 +485,12 @@ func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error {
 
 	searchableBug.Text = append(searchableBug.Text, snap.Title)
 
-	err := c.searchCache.Index(snap.Id().String(), searchableBug)
+	index, err := c.repo.GetBleveIndex("bug")
+	if err != nil {
+		return err
+	}
+
+	err = index.Index(snap.Id().String(), searchableBug)
 	if err != nil {
 		return err
 	}
diff --git a/cache/repo_cache_test.go b/cache/repo_cache_test.go
index 1c5c41d2..bd06e84d 100644
--- a/cache/repo_cache_test.go
+++ b/cache/repo_cache_test.go
@@ -73,7 +73,9 @@ func TestCache(t *testing.T) {
 	// Querying
 	q, err := query.Parse("status:open author:descartes sort:edit-asc")
 	require.NoError(t, err)
-	require.Len(t, cache.QueryBugs(q), 2)
+	res, err := cache.QueryBugs(q)
+	require.NoError(t, err)
+	require.Len(t, res, 2)
 
 	// Close
 	require.NoError(t, cache.Close())
diff --git a/commands/ls.go b/commands/ls.go
index f6d654b1..327fd37f 100644
--- a/commands/ls.go
+++ b/commands/ls.go
@@ -110,7 +110,10 @@ func runLs(env *Env, opts lsOptions, args []string) error {
 		return err
 	}
 
-	allIds := env.backend.QueryBugs(q)
+	allIds, err := env.backend.QueryBugs(q)
+	if err != nil {
+		return err
+	}
 
 	bugExcerpt := make([]*cache.BugExcerpt, len(allIds))
 	for i, id := range allIds {
diff --git a/identity/identity_test.go b/identity/identity_test.go
index baf933c8..82e58b01 100644
--- a/identity/identity_test.go
+++ b/identity/identity_test.go
@@ -273,10 +273,10 @@ func TestIdentityRemove(t *testing.T) {
 	remoteB := repository.CreateGoGitTestRepo(true)
 	defer repository.CleanupTestRepos(repo, remoteA, remoteB)
 
-	err := repo.AddRemote("remoteA", "file://"+remoteA.GetPath())
+	err := repo.AddRemote("remoteA", remoteA.GetLocalRemote())
 	require.NoError(t, err)
 
-	err = repo.AddRemote("remoteB", "file://"+remoteB.GetPath())
+	err = repo.AddRemote("remoteB", remoteB.GetLocalRemote())
 	require.NoError(t, err)
 
 	// generate an identity for testing
diff --git a/repository/git.go b/repository/git.go
index 2348f5d5..993e6cc6 100644
--- a/repository/git.go
+++ b/repository/git.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/blevesearch/bleve"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/osfs"
 
@@ -30,12 +31,15 @@ type GitRepo struct {
 	clocksMutex sync.Mutex
 	clocks      map[string]lamport.Clock
 
+	indexesMutex sync.Mutex
+	indexes      map[string]bleve.Index
+
 	keyring Keyring
 }
 
-// NewGitRepo determines if the given working directory is inside of a git repository,
+// OpenGitRepo determines if the given working directory is inside of a git repository,
 // and returns the corresponding GitRepo instance if it is.
-func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
+func OpenGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
 	k, err := defaultKeyring()
 	if err != nil {
 		return nil, err
@@ -45,6 +49,7 @@ func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
 		gitCli:  gitCli{path: path},
 		path:    path,
 		clocks:  make(map[string]lamport.Clock),
+		indexes: make(map[string]bleve.Index),
 		keyring: k,
 	}
 
@@ -84,9 +89,10 @@ func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
 // InitGitRepo create a new empty git repo at the given path
 func InitGitRepo(path string) (*GitRepo, error) {
 	repo := &GitRepo{
-		gitCli: gitCli{path: path},
-		path:   path + "/.git",
-		clocks: make(map[string]lamport.Clock),
+		gitCli:  gitCli{path: path},
+		path:    path + "/.git",
+		clocks:  make(map[string]lamport.Clock),
+		indexes: make(map[string]bleve.Index),
 	}
 
 	_, err := repo.runGitCommand("init", path)
@@ -100,9 +106,10 @@ func InitGitRepo(path string) (*GitRepo, error) {
 // InitBareGitRepo create a new --bare empty git repo at the given path
 func InitBareGitRepo(path string) (*GitRepo, error) {
 	repo := &GitRepo{
-		gitCli: gitCli{path: path},
-		path:   path,
-		clocks: make(map[string]lamport.Clock),
+		gitCli:  gitCli{path: path},
+		path:    path,
+		clocks:  make(map[string]lamport.Clock),
+		indexes: make(map[string]bleve.Index),
 	}
 
 	_, err := repo.runGitCommand("init", "--bare", path)
@@ -113,6 +120,17 @@ func InitBareGitRepo(path string) (*GitRepo, error) {
 	return repo, nil
 }
 
+func (repo *GitRepo) Close() error {
+	var firstErr error
+	for _, index := range repo.indexes {
+		err := index.Close()
+		if err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
 // LocalConfig give access to the repository scoped configuration
 func (repo *GitRepo) LocalConfig() Config {
 	return newGitConfig(repo.gitCli, false)
@@ -183,6 +201,58 @@ func (repo *GitRepo) LocalStorage() billy.Filesystem {
 	return osfs.New(repo.path)
 }
 
+// GetBleveIndex return a bleve.Index that can be used to index documents
+func (repo *GitRepo) GetBleveIndex(name string) (bleve.Index, error) {
+	repo.indexesMutex.Lock()
+	defer repo.indexesMutex.Unlock()
+
+	if index, ok := repo.indexes[name]; ok {
+		return index, nil
+	}
+
+	path := filepath.Join(repo.path, "indexes", name)
+
+	index, err := bleve.Open(path)
+	if err == nil {
+		repo.indexes[name] = index
+		return index, nil
+	}
+
+	err = os.MkdirAll(path, os.ModeDir)
+	if err != nil {
+		return nil, err
+	}
+
+	mapping := bleve.NewIndexMapping()
+	mapping.DefaultAnalyzer = "en"
+
+	index, err = bleve.New(path, mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	repo.indexes[name] = index
+
+	return index, nil
+}
+
+// ClearBleveIndex will wipe the given index
+func (repo *GitRepo) ClearBleveIndex(name string) error {
+	repo.indexesMutex.Lock()
+	defer repo.indexesMutex.Unlock()
+
+	path := filepath.Join(repo.path, "indexes", name)
+
+	err := os.RemoveAll(path)
+	if err != nil {
+		return err
+	}
+
+	delete(repo.indexes, name)
+
+	return nil
+}
+
 // FetchRefs fetch git refs from a remote
 func (repo *GitRepo) FetchRefs(remote, refSpec string) (string, error) {
 	stdout, err := repo.runGitCommand("fetch", remote, refSpec)
diff --git a/repository/gogit.go b/repository/gogit.go
index 741982aa..74fe3fc5 100644
--- a/repository/gogit.go
+++ b/repository/gogit.go
@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/blevesearch/bleve"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/osfs"
 	gogit "github.com/go-git/go-git/v5"
@@ -33,6 +34,9 @@ type GoGitRepo struct {
 	clocksMutex sync.Mutex
 	clocks      map[string]lamport.Clock
 
+	indexesMutex sync.Mutex
+	indexes      map[string]bleve.Index
+
 	keyring      Keyring
 	localStorage billy.Filesystem
 }
@@ -58,6 +62,7 @@ func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error)
 		r:            r,
 		path:         path,
 		clocks:       make(map[string]lamport.Clock),
+		indexes:      make(map[string]bleve.Index),
 		keyring:      k,
 		localStorage: osfs.New(filepath.Join(path, "git-bug")),
 	}
@@ -97,6 +102,7 @@ func InitGoGitRepo(path string) (*GoGitRepo, error) {
 		r:            r,
 		path:         filepath.Join(path, ".git"),
 		clocks:       make(map[string]lamport.Clock),
+		indexes:      make(map[string]bleve.Index),
 		keyring:      k,
 		localStorage: osfs.New(filepath.Join(path, ".git", "git-bug")),
 	}, nil
@@ -118,6 +124,7 @@ func InitBareGoGitRepo(path string) (*GoGitRepo, error) {
 		r:            r,
 		path:         path,
 		clocks:       make(map[string]lamport.Clock),
+		indexes:      make(map[string]bleve.Index),
 		keyring:      k,
 		localStorage: osfs.New(filepath.Join(path, "git-bug")),
 	}, nil
@@ -179,6 +186,17 @@ func isGitDir(path string) (bool, error) {
 	return true, nil
 }
 
+func (repo *GoGitRepo) Close() error {
+	var firstErr error
+	for _, index := range repo.indexes {
+		err := index.Close()
+		if err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
 // LocalConfig give access to the repository scoped configuration
 func (repo *GoGitRepo) LocalConfig() Config {
 	return newGoGitLocalConfig(repo.r)
@@ -274,6 +292,64 @@ func (repo *GoGitRepo) LocalStorage() billy.Filesystem {
 	return repo.localStorage
 }
 
+// GetBleveIndex return a bleve.Index that can be used to index documents
+func (repo *GoGitRepo) GetBleveIndex(name string) (bleve.Index, error) {
+	repo.indexesMutex.Lock()
+	defer repo.indexesMutex.Unlock()
+
+	if index, ok := repo.indexes[name]; ok {
+		return index, nil
+	}
+
+	path := filepath.Join(repo.path, "git-bug", "indexes", name)
+
+	index, err := bleve.Open(path)
+	if err == nil {
+		repo.indexes[name] = index
+		return index, nil
+	}
+
+	err = os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+
+	mapping := bleve.NewIndexMapping()
+	mapping.DefaultAnalyzer = "en"
+
+	index, err = bleve.New(path, mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	repo.indexes[name] = index
+
+	return index, nil
+}
+
+// ClearBleveIndex will wipe the given index
+func (repo *GoGitRepo) ClearBleveIndex(name string) error {
+	repo.indexesMutex.Lock()
+	defer repo.indexesMutex.Unlock()
+
+	path := filepath.Join(repo.path, "indexes", name)
+
+	err := os.RemoveAll(path)
+	if err != nil {
+		return err
+	}
+
+	if index, ok := repo.indexes[name]; ok {
+		err = index.Close()
+		if err != nil {
+			return err
+		}
+		delete(repo.indexes, name)
+	}
+
+	return nil
+}
+
 // FetchRefs fetch git refs from a remote
 func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) {
 	buf := bytes.NewBuffer(nil)
diff --git a/repository/mock_repo.go b/repository/mock_repo.go
index 02e5010f..8a1724ef 100644
--- a/repository/mock_repo.go
+++ b/repository/mock_repo.go
@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/99designs/keyring"
+	"github.com/blevesearch/bleve"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
 
@@ -22,16 +23,20 @@ type mockRepoForTest struct {
 	*mockRepoKeyring
 	*mockRepoCommon
 	*mockRepoStorage
+	*mockRepoBleve
 	*mockRepoData
 	*mockRepoClock
 }
 
+func (m *mockRepoForTest) Close() error { return nil }
+
 func NewMockRepoForTest() *mockRepoForTest {
 	return &mockRepoForTest{
 		mockRepoConfig:  NewMockRepoConfig(),
 		mockRepoKeyring: NewMockRepoKeyring(),
 		mockRepoCommon:  NewMockRepoCommon(),
 		mockRepoStorage: NewMockRepoStorage(),
+		mockRepoBleve:   newMockRepoBleve(),
 		mockRepoData:    NewMockRepoData(),
 		mockRepoClock:   NewMockRepoClock(),
 	}
@@ -126,6 +131,48 @@ func (m *mockRepoStorage) LocalStorage() billy.Filesystem {
 	return m.localFs
 }
 
+var _ RepoBleve = &mockRepoBleve{}
+
+type mockRepoBleve struct {
+	indexesMutex sync.Mutex
+	indexes      map[string]bleve.Index
+}
+
+func newMockRepoBleve() *mockRepoBleve {
+	return &mockRepoBleve{
+		indexes: make(map[string]bleve.Index),
+	}
+}
+
+func (m *mockRepoBleve) GetBleveIndex(name string) (bleve.Index, error) {
+	m.indexesMutex.Lock()
+	defer m.indexesMutex.Unlock()
+
+	if index, ok := m.indexes[name]; ok {
+		return index, nil
+	}
+
+	mapping := bleve.NewIndexMapping()
+	mapping.DefaultAnalyzer = "en"
+
+	index, err := bleve.NewMemOnly(mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	m.indexes[name] = index
+
+	return index, nil
+}
+
+func (m *mockRepoBleve) ClearBleveIndex(name string) error {
+	m.indexesMutex.Lock()
+	defer m.indexesMutex.Unlock()
+
+	delete(m.indexes, name)
+	return nil
+}
+
 var _ RepoData = &mockRepoData{}
 
 type commit struct {
diff --git a/repository/repo.go b/repository/repo.go
index d8fe44e6..eb9296d4 100644
--- a/repository/repo.go
+++ b/repository/repo.go
@@ -4,6 +4,7 @@ package repository
 import (
 	"errors"
 
+	"github.com/blevesearch/bleve"
 	"github.com/go-git/go-billy/v5"
 
 	"github.com/MichaelMure/git-bug/util/lamport"
@@ -23,6 +24,9 @@ type Repo interface {
 	RepoCommon
 	RepoData
 	RepoStorage
+	RepoBleve
+
+	Close() error
 }
 
 type RepoCommonStorage interface {
@@ -69,11 +73,21 @@ type RepoCommon interface {
 	GetRemotes() (map[string]string, error)
 }
 
+// RepoStorage give access to the filesystem
 type RepoStorage interface {
 	// LocalStorage return a billy.Filesystem giving access to $RepoPath/.git/git-bug
 	LocalStorage() billy.Filesystem
 }
 
+// RepoBleve give access to Bleve to implement full-text search indexes.
+type RepoBleve interface {
+	// GetBleveIndex return a bleve.Index that can be used to index documents
+	GetBleveIndex(name string) (bleve.Index, error)
+
+	// ClearBleveIndex will wipe the given index
+	ClearBleveIndex(name string) error
+}
+
 // RepoData give access to the git data storage
 type RepoData interface {
 	// FetchRefs fetch git refs from a remote
diff --git a/termui/bug_table.go b/termui/bug_table.go
index d3a187db..94185a29 100644
--- a/termui/bug_table.go
+++ b/termui/bug_table.go
@@ -237,7 +237,11 @@ func (bt *bugTable) disable(g *gocui.Gui) error {
 }
 
 func (bt *bugTable) paginate(max int) error {
-	bt.allIds = bt.repo.QueryBugs(bt.query)
+	var err error
+	bt.allIds, err = bt.repo.QueryBugs(bt.query)
+	if err != nil {
+		return err
+	}
 
 	return bt.doPaginate(max)
 }