From 4ef2c1104079032336da9f2a7879f2432c2609ce Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sat, 5 Dec 2020 03:08:54 +0100 Subject: repo: finish RepoStorage move --- cache/repo_cache.go | 31 ++++++++----------------------- cache/repo_cache_bug.go | 10 ++-------- cache/repo_cache_common.go | 24 +++++++++++++----------- cache/repo_cache_identity.go | 11 ++--------- cache/repo_cache_test.go | 4 ++-- 5 files changed, 27 insertions(+), 53 deletions(-) (limited to 'cache') diff --git a/cache/repo_cache.go b/cache/repo_cache.go index 8bce3d8c..a08744cc 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -5,8 +5,6 @@ import ( "io" "io/ioutil" "os" - "path" - "path/filepath" "strconv" "sync" @@ -133,25 +131,18 @@ func (c *RepoCache) write() error { } func (c *RepoCache) lock() error { - lockPath := repoLockFilePath(c.repo) - err := repoIsAvailable(c.repo) if err != nil { return err } - err = os.MkdirAll(filepath.Dir(lockPath), 0777) - if err != nil { - return err - } - - f, err := os.Create(lockPath) + f, err := c.repo.LocalStorage().Create(lockfile) if err != nil { return err } pid := fmt.Sprintf("%d", os.Getpid()) - _, err = f.WriteString(pid) + _, err = f.Write([]byte(pid)) if err != nil { return err } @@ -175,11 +166,12 @@ func (c *RepoCache) Close() error { c.searchCache = nil } - lockPath := repoLockFilePath(c.repo) - return os.Remove(lockPath) + return c.repo.LocalStorage().Remove(lockfile) } func (c *RepoCache) buildCache() error { + // TODO: make that parallel + c.muBug.Lock() defer c.muBug.Unlock() c.muIdentity.Lock() @@ -230,17 +222,11 @@ func (c *RepoCache) buildCache() error { return nil } -func repoLockFilePath(repo repository.Repo) string { - return path.Join(repo.GetPath(), "git-bug", lockfile) -} - // repoIsAvailable check is the given repository is locked by a Cache. // Note: this is a smart function that will cleanup the lock file if the // corresponding process is not there anymore. // If no error is returned, the repo is free to edit. -func repoIsAvailable(repo repository.Repo) error { - lockPath := repoLockFilePath(repo) - +func repoIsAvailable(repo repository.RepoStorage) error { // Todo: this leave way for a racey access to the repo between the test // if the file exist and the actual write. It's probably not a problem in // practice because using a repository will be done from user interaction @@ -252,8 +238,7 @@ func repoIsAvailable(repo repository.Repo) error { // computer. Should add a configuration that prevent the cleaning of the // lock file - f, err := os.Open(lockPath) - + f, err := repo.LocalStorage().Open(lockfile) if err != nil && !os.IsNotExist(err) { return err } @@ -285,7 +270,7 @@ func repoIsAvailable(repo repository.Repo) error { return err } - err = os.Remove(lockPath) + err = repo.LocalStorage().Remove(lockfile) if err != nil { return err } diff --git a/cache/repo_cache_bug.go b/cache/repo_cache_bug.go index d57e1bce..f540e51b 100644 --- a/cache/repo_cache_bug.go +++ b/cache/repo_cache_bug.go @@ -5,8 +5,6 @@ import ( "encoding/gob" "errors" "fmt" - "os" - "path" "sort" "strings" "time" @@ -25,10 +23,6 @@ const ( var errBugNotInCache = errors.New("bug missing from cache") -func bugCacheFilePath(repo repository.Repo) string { - return path.Join(repo.GetPath(), "git-bug", bugCacheFile) -} - func searchCacheDirPath(repo repository.Repo) string { return path.Join(repo.GetPath(), "git-bug", searchCacheDir) } @@ -65,7 +59,7 @@ func (c *RepoCache) loadBugCache() error { c.muBug.Lock() defer c.muBug.Unlock() - f, err := os.Open(bugCacheFilePath(c.repo)) + f, err := c.repo.LocalStorage().Open(bugCacheFile) if err != nil { return err } @@ -148,7 +142,7 @@ func (c *RepoCache) writeBugCache() error { return err } - f, err := os.Create(bugCacheFilePath(c.repo)) + f, err := c.repo.LocalStorage().Create(bugCacheFile) if err != nil { return err } diff --git a/cache/repo_cache_common.go b/cache/repo_cache_common.go index 95e2f7bb..5dc19d22 100644 --- a/cache/repo_cache_common.go +++ b/cache/repo_cache_common.go @@ -3,6 +3,7 @@ package cache import ( "fmt" + "github.com/go-git/go-billy/v5" "github.com/pkg/errors" "github.com/MichaelMure/git-bug/bug" @@ -30,13 +31,19 @@ func (c *RepoCache) AnyConfig() repository.ConfigRead { return c.repo.AnyConfig() } +// Keyring give access to a user-wide storage for secrets func (c *RepoCache) Keyring() repository.Keyring { return c.repo.Keyring() } -// GetPath returns the path to the repo. -func (c *RepoCache) GetPath() string { - return c.repo.GetPath() +// GetUserName returns the name the the user has used to configure git +func (c *RepoCache) GetUserName() (string, error) { + return c.repo.GetUserName() +} + +// GetUserEmail returns the email address that the user has used to configure git. +func (c *RepoCache) GetUserEmail() (string, error) { + return c.repo.GetUserEmail() } // GetCoreEditor returns the name of the editor that the user has used to configure git. @@ -49,14 +56,9 @@ func (c *RepoCache) GetRemotes() (map[string]string, error) { return c.repo.GetRemotes() } -// GetUserName returns the name the the user has used to configure git -func (c *RepoCache) GetUserName() (string, error) { - return c.repo.GetUserName() -} - -// GetUserEmail returns the email address that the user has used to configure git. -func (c *RepoCache) GetUserEmail() (string, error) { - return c.repo.GetUserEmail() +// LocalStorage return a billy.Filesystem giving access to $RepoPath/.git/git-bug +func (c *RepoCache) LocalStorage() billy.Filesystem { + return c.repo.LocalStorage() } // ReadData will attempt to read arbitrary data from the given hash diff --git a/cache/repo_cache_identity.go b/cache/repo_cache_identity.go index 8957d4aa..8df5b810 100644 --- a/cache/repo_cache_identity.go +++ b/cache/repo_cache_identity.go @@ -4,20 +4,13 @@ import ( "bytes" "encoding/gob" "fmt" - "os" - "path" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/identity" - "github.com/MichaelMure/git-bug/repository" ) const identityCacheFile = "identity-cache" -func identityCacheFilePath(repo repository.Repo) string { - return path.Join(repo.GetPath(), "git-bug", identityCacheFile) -} - // identityUpdated is a callback to trigger when the excerpt of an identity // changed, that is each time an identity is updated func (c *RepoCache) identityUpdated(id entity.Id) error { @@ -41,7 +34,7 @@ func (c *RepoCache) loadIdentityCache() error { c.muIdentity.Lock() defer c.muIdentity.Unlock() - f, err := os.Open(identityCacheFilePath(c.repo)) + f, err := c.repo.LocalStorage().Open(identityCacheFile) if err != nil { return err } @@ -88,7 +81,7 @@ func (c *RepoCache) writeIdentityCache() error { return err } - f, err := os.Create(identityCacheFilePath(c.repo)) + f, err := c.repo.LocalStorage().Create(identityCacheFile) if err != nil { return err } diff --git a/cache/repo_cache_test.go b/cache/repo_cache_test.go index 0037c7bb..1c5c41d2 100644 --- a/cache/repo_cache_test.go +++ b/cache/repo_cache_test.go @@ -167,10 +167,10 @@ func TestRemove(t *testing.T) { remoteB := repository.CreateGoGitTestRepo(true) defer repository.CleanupTestRepos(repo, remoteA, remoteB) - err := repo.AddRemote("remoteA", "file://"+remoteA.GetPath()) + err := repo.AddRemote("remoteA", remoteA.GetLocalRemote()) require.NoError(t, err) - err = repo.AddRemote("remoteB", "file://"+remoteB.GetPath()) + err = repo.AddRemote("remoteB", remoteB.GetLocalRemote()) require.NoError(t, err) repoCache, err := NewRepoCache(repo) -- cgit From c884d557bf5e0ebdbe6e3b20535af24c2e97d29f Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 8 Dec 2020 14:42:13 +0100 Subject: repo: move bleve there --- cache/repo_cache.go | 14 +++++------ cache/repo_cache_bug.go | 60 +++++++++++++++++++----------------------------- cache/repo_cache_test.go | 4 +++- 3 files changed, 32 insertions(+), 46 deletions(-) (limited to 'cache') diff --git a/cache/repo_cache.go b/cache/repo_cache.go index a08744cc..b5b9ee54 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -13,7 +13,6 @@ import ( "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/repository" "github.com/MichaelMure/git-bug/util/process" - "github.com/blevesearch/bleve" ) // 1: original format @@ -55,8 +54,6 @@ type RepoCache struct { muBug sync.RWMutex // excerpt of bugs data for all bugs bugExcerpts map[entity.Id]*BugExcerpt - // searchable cache of all bugs - searchCache bleve.Index // bug loaded in memory bugs map[entity.Id]*BugCache // loadedBugs is an LRU cache that records which bugs the cache has loaded in @@ -161,9 +158,9 @@ func (c *RepoCache) Close() error { c.bugs = make(map[entity.Id]*BugCache) c.bugExcerpts = nil - if c.searchCache != nil { - c.searchCache.Close() - c.searchCache = nil + err := c.repo.Close() + if err != nil { + return err } return c.repo.LocalStorage().Remove(lockfile) @@ -199,9 +196,10 @@ func (c *RepoCache) buildCache() error { allBugs := bug.ReadAllLocal(c.repo) - err := c.createBleveIndex() + // wipe the index just to be sure + err := c.repo.ClearBleveIndex("bug") if err != nil { - return fmt.Errorf("Unable to create search cache. Error: %v", err) + return err } for b := range allBugs { diff --git a/cache/repo_cache_bug.go b/cache/repo_cache_bug.go index f540e51b..1701f66d 100644 --- a/cache/repo_cache_bug.go +++ b/cache/repo_cache_bug.go @@ -23,10 +23,6 @@ const ( var errBugNotInCache = errors.New("bug missing from cache") -func searchCacheDirPath(repo repository.Repo) string { - return path.Join(repo.GetPath(), "git-bug", searchCacheDir) -} - // bugUpdated is a callback to trigger when the excerpt of a bug changed, // that is each time a bug is updated func (c *RepoCache) bugUpdated(id entity.Id) error { @@ -82,14 +78,13 @@ func (c *RepoCache) loadBugCache() error { c.bugExcerpts = aux.Excerpts - blevePath := searchCacheDirPath(c.repo) - searchCache, err := bleve.Open(blevePath) + index, err := c.repo.GetBleveIndex("bug") if err != nil { - return fmt.Errorf("Unable to open search cache. Error: %v", err) + return err } - c.searchCache = searchCache - count, err := c.searchCache.DocCount() + // simple heuristic to detect a mismatch between the index and the bugs + count, err := index.DocCount() if err != nil { return err } @@ -100,26 +95,6 @@ func (c *RepoCache) loadBugCache() error { return nil } -func (c *RepoCache) createBleveIndex() error { - blevePath := searchCacheDirPath(c.repo) - - _ = os.RemoveAll(blevePath) - - mapping := bleve.NewIndexMapping() - mapping.DefaultAnalyzer = "en" - - dir := searchCacheDirPath(c.repo) - - bleveIndex, err := bleve.New(dir, mapping) - if err != nil { - return err - } - - c.searchCache = bleveIndex - - return nil -} - // write will serialize on disk the bug cache file func (c *RepoCache) writeBugCache() error { c.muBug.RLock() @@ -287,12 +262,12 @@ func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, erro } // QueryBugs return the id of all Bug matching the given Query -func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id { +func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) { c.muBug.RLock() defer c.muBug.RUnlock() if q == nil { - return c.AllBugsIds() + return c.AllBugsIds(), nil } matcher := compileMatcher(q.Filters) @@ -313,9 +288,15 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id { bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " ")) bleveSearch := bleve.NewSearchRequest(bleveQuery) - searchResults, err := c.searchCache.Search(bleveSearch) + + index, err := c.repo.GetBleveIndex("bug") + if err != nil { + return nil, err + } + + searchResults, err := index.Search(bleveSearch) if err != nil { - panic("bleve search failed") + return nil, err } for _, hit := range searchResults.Hits { @@ -341,7 +322,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id { case query.OrderByEdit: sorter = BugsByEditTime(filtered) default: - panic("missing sort type") + return nil, errors.New("missing sort type") } switch q.OrderDirection { @@ -350,7 +331,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id { case query.OrderDescending: sorter = sort.Reverse(sorter) default: - panic("missing sort direction") + return nil, errors.New("missing sort direction") } sort.Sort(sorter) @@ -361,7 +342,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id { result[i] = val.Id } - return result + return result, nil } // AllBugsIds return all known bug ids @@ -504,7 +485,12 @@ func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error { searchableBug.Text = append(searchableBug.Text, snap.Title) - err := c.searchCache.Index(snap.Id().String(), searchableBug) + index, err := c.repo.GetBleveIndex("bug") + if err != nil { + return err + } + + err = index.Index(snap.Id().String(), searchableBug) if err != nil { return err } diff --git a/cache/repo_cache_test.go b/cache/repo_cache_test.go index 1c5c41d2..bd06e84d 100644 --- a/cache/repo_cache_test.go +++ b/cache/repo_cache_test.go @@ -73,7 +73,9 @@ func TestCache(t *testing.T) { // Querying q, err := query.Parse("status:open author:descartes sort:edit-asc") require.NoError(t, err) - require.Len(t, cache.QueryBugs(q), 2) + res, err := cache.QueryBugs(q) + require.NoError(t, err) + require.Len(t, res, 2) // Close require.NoError(t, cache.Close()) -- cgit