diff options
author | Michael Muré <batolettre@gmail.com> | 2020-02-09 02:56:37 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-02-09 02:56:37 +0100 |
commit | 3caffeef4d2ed25d4eb5d4bfd262f4fc3b92561f (patch) | |
tree | 40730a7ff92a420af65d9d2560a07d47a6a72174 /cache | |
parent | 9e1a987b4d94dc5c2115423ede5954d4faf1d342 (diff) | |
parent | 2af456f5e05f242d9e13049e038b9bd2d6dfc1cc (diff) | |
download | git-bug-3caffeef4d2ed25d4eb5d4bfd262f4fc3b92561f.tar.gz |
Merge pull request #316 from MichaelMure/lazy-graphql
graphql: use the cache in priority for fast browsing in the WebUI
Diffstat (limited to 'cache')
-rw-r--r-- | cache/filter.go | 65 | ||||
-rw-r--r-- | cache/filter_test.go | 2 | ||||
-rw-r--r-- | cache/repo_cache.go | 195 |
3 files changed, 209 insertions, 53 deletions
diff --git a/cache/filter.go b/cache/filter.go index 9b1de1d5..f840067b 100644 --- a/cache/filter.go +++ b/cache/filter.go @@ -4,10 +4,17 @@ import ( "strings" "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/entity" ) +// resolver has the resolving functions needed by filters. +// This exist mainly to go through the functions of the cache with proper locking. +type resolver interface { + ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) +} + // Filter is a predicate that match a subset of bugs -type Filter func(repoCache *RepoCache, excerpt *BugExcerpt) bool +type Filter func(excerpt *BugExcerpt, resolver resolver) bool // StatusFilter return a Filter that match a bug status func StatusFilter(query string) (Filter, error) { @@ -16,21 +23,21 @@ func StatusFilter(query string) (Filter, error) { return nil, err } - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { return excerpt.Status == status }, nil } // AuthorFilter return a Filter that match a bug author func AuthorFilter(query string) Filter { - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { query = strings.ToLower(query) // Normal identity if excerpt.AuthorId != "" { - author, ok := repoCache.identitiesExcerpts[excerpt.AuthorId] - if !ok { - panic("missing identity in the cache") + author, err := resolver.ResolveIdentityExcerpt(excerpt.AuthorId) + if err != nil { + panic(err) } return author.Match(query) @@ -43,7 +50,7 @@ func AuthorFilter(query string) Filter { // LabelFilter return a Filter that match a label func LabelFilter(label string) Filter { - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { for _, l := range excerpt.Labels { if string(l) == label { return true @@ -55,13 +62,13 @@ func LabelFilter(label string) Filter { // ActorFilter return a Filter that match a bug actor func ActorFilter(query string) Filter { - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { query = strings.ToLower(query) for _, id := range excerpt.Actors { - identityExcerpt, ok := repoCache.identitiesExcerpts[id] - if !ok { - panic("missing identity in the cache") + identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + if err != nil { + panic(err) } if identityExcerpt.Match(query) { @@ -74,13 +81,13 @@ func ActorFilter(query string) Filter { // ParticipantFilter return a Filter that match a bug participant func ParticipantFilter(query string) Filter { - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { query = strings.ToLower(query) for _, id := range excerpt.Participants { - identityExcerpt, ok := repoCache.identitiesExcerpts[id] - if !ok { - panic("missing identity in the cache") + identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + if err != nil { + panic(err) } if identityExcerpt.Match(query) { @@ -93,7 +100,7 @@ func ParticipantFilter(query string) Filter { // TitleFilter return a Filter that match if the title contains the given query func TitleFilter(query string) Filter { - return func(repo *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { return strings.Contains( strings.ToLower(excerpt.Title), strings.ToLower(query), @@ -103,7 +110,7 @@ func TitleFilter(query string) Filter { // NoLabelFilter return a Filter that match the absence of labels func NoLabelFilter() Filter { - return func(repoCache *RepoCache, excerpt *BugExcerpt) bool { + return func(excerpt *BugExcerpt, resolver resolver) bool { return len(excerpt.Labels) == 0 } } @@ -120,32 +127,32 @@ type Filters struct { } // Match check if a bug match the set of filters -func (f *Filters) Match(repoCache *RepoCache, excerpt *BugExcerpt) bool { - if match := f.orMatch(f.Status, repoCache, excerpt); !match { +func (f *Filters) Match(excerpt *BugExcerpt, resolver resolver) bool { + if match := f.orMatch(f.Status, excerpt, resolver); !match { return false } - if match := f.orMatch(f.Author, repoCache, excerpt); !match { + if match := f.orMatch(f.Author, excerpt, resolver); !match { return false } - if match := f.orMatch(f.Participant, repoCache, excerpt); !match { + if match := f.orMatch(f.Participant, excerpt, resolver); !match { return false } - if match := f.orMatch(f.Actor, repoCache, excerpt); !match { + if match := f.orMatch(f.Actor, excerpt, resolver); !match { return false } - if match := f.andMatch(f.Label, repoCache, excerpt); !match { + if match := f.andMatch(f.Label, excerpt, resolver); !match { return false } - if match := f.andMatch(f.NoFilters, repoCache, excerpt); !match { + if match := f.andMatch(f.NoFilters, excerpt, resolver); !match { return false } - if match := f.andMatch(f.Title, repoCache, excerpt); !match { + if match := f.andMatch(f.Title, excerpt, resolver); !match { return false } @@ -153,28 +160,28 @@ func (f *Filters) Match(repoCache *RepoCache, excerpt *BugExcerpt) bool { } // Check if any of the filters provided match the bug -func (*Filters) orMatch(filters []Filter, repoCache *RepoCache, excerpt *BugExcerpt) bool { +func (*Filters) orMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { if len(filters) == 0 { return true } match := false for _, f := range filters { - match = match || f(repoCache, excerpt) + match = match || f(excerpt, resolver) } return match } // Check if all of the filters provided match the bug -func (*Filters) andMatch(filters []Filter, repoCache *RepoCache, excerpt *BugExcerpt) bool { +func (*Filters) andMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { if len(filters) == 0 { return true } match := true for _, f := range filters { - match = match && f(repoCache, excerpt) + match = match && f(excerpt, resolver) } return match diff --git a/cache/filter_test.go b/cache/filter_test.go index a47d2ad7..2c7fabc0 100644 --- a/cache/filter_test.go +++ b/cache/filter_test.go @@ -28,7 +28,7 @@ func TestTitleFilter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { filter := TitleFilter(tt.query) excerpt := &BugExcerpt{Title: tt.title} - assert.Equal(t, tt.match, filter(nil, excerpt)) + assert.Equal(t, tt.match, filter(excerpt, nil)) }) } } diff --git a/cache/repo_cache.go b/cache/repo_cache.go index 18be9b5a..c8d6bd69 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -10,6 +10,7 @@ import ( "path" "sort" "strconv" + "sync" "time" "github.com/pkg/errors" @@ -57,11 +58,13 @@ type RepoCache struct { // the underlying repo repo repository.ClockedRepo + muBug sync.RWMutex // excerpt of bugs data for all bugs bugExcerpts map[entity.Id]*BugExcerpt // bug loaded in memory bugs map[entity.Id]*BugCache + muIdentity sync.RWMutex // excerpt of identities data for all identities identitiesExcerpts map[entity.Id]*IdentityExcerpt // identities loaded in memory @@ -157,6 +160,11 @@ func (c *RepoCache) lock() error { } func (c *RepoCache) Close() error { + c.muBug.Lock() + defer c.muBug.Unlock() + c.muIdentity.Lock() + defer c.muIdentity.Unlock() + c.identities = make(map[entity.Id]*IdentityCache) c.identitiesExcerpts = nil c.bugs = make(map[entity.Id]*BugCache) @@ -169,12 +177,16 @@ func (c *RepoCache) Close() error { // bugUpdated is a callback to trigger when the excerpt of a bug changed, // that is each time a bug is updated func (c *RepoCache) bugUpdated(id entity.Id) error { + c.muBug.Lock() + b, ok := c.bugs[id] if !ok { + c.muBug.Unlock() panic("missing bug in the cache") } c.bugExcerpts[id] = NewBugExcerpt(b.bug, b.Snapshot()) + c.muBug.Unlock() // we only need to write the bug cache return c.writeBugCache() @@ -183,12 +195,16 @@ func (c *RepoCache) bugUpdated(id entity.Id) error { // identityUpdated is a callback to trigger when the excerpt of an identity // changed, that is each time an identity is updated func (c *RepoCache) identityUpdated(id entity.Id) error { + c.muIdentity.Lock() + i, ok := c.identities[id] if !ok { + c.muIdentity.Unlock() panic("missing identity in the cache") } c.identitiesExcerpts[id] = NewIdentityExcerpt(i.Identity) + c.muIdentity.Unlock() // we only need to write the identity cache return c.writeIdentityCache() @@ -205,6 +221,9 @@ func (c *RepoCache) load() error { // load will try to read from the disk the bug cache file func (c *RepoCache) loadBugCache() error { + c.muBug.Lock() + defer c.muBug.Unlock() + f, err := os.Open(bugCacheFilePath(c.repo)) if err != nil { return err @@ -234,6 +253,9 @@ func (c *RepoCache) loadBugCache() error { // load will try to read from the disk the identity cache file func (c *RepoCache) loadIdentityCache() error { + c.muIdentity.Lock() + defer c.muIdentity.Unlock() + f, err := os.Open(identityCacheFilePath(c.repo)) if err != nil { return err @@ -272,6 +294,9 @@ func (c *RepoCache) write() error { // write will serialize on disk the bug cache file func (c *RepoCache) writeBugCache() error { + c.muBug.RLock() + defer c.muBug.RUnlock() + var data bytes.Buffer aux := struct { @@ -304,6 +329,9 @@ func (c *RepoCache) writeBugCache() error { // write will serialize on disk the identity cache file func (c *RepoCache) writeIdentityCache() error { + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + var data bytes.Buffer aux := struct { @@ -343,6 +371,11 @@ func identityCacheFilePath(repo repository.Repo) string { } func (c *RepoCache) buildCache() error { + c.muBug.Lock() + defer c.muBug.Unlock() + c.muIdentity.Lock() + defer c.muIdentity.Unlock() + _, _ = fmt.Fprintf(os.Stderr, "Building identity cache... ") c.identitiesExcerpts = make(map[entity.Id]*IdentityExcerpt) @@ -378,9 +411,24 @@ func (c *RepoCache) buildCache() error { return nil } +// ResolveBugExcerpt retrieve a BugExcerpt matching the exact given id +func (c *RepoCache) ResolveBugExcerpt(id entity.Id) (*BugExcerpt, error) { + c.muBug.RLock() + defer c.muBug.RUnlock() + + e, ok := c.bugExcerpts[id] + if !ok { + return nil, bug.ErrBugNotExist + } + + return e, nil +} + // ResolveBug retrieve a bug matching the exact given id func (c *RepoCache) ResolveBug(id entity.Id) (*BugCache, error) { + c.muBug.RLock() cached, ok := c.bugs[id] + c.muBug.RUnlock() if ok { return cached, nil } @@ -391,19 +439,20 @@ func (c *RepoCache) ResolveBug(id entity.Id) (*BugCache, error) { } cached = NewBugCache(c, b) + + c.muBug.Lock() c.bugs[id] = cached + c.muBug.Unlock() return cached, nil } -// ResolveBugExcerpt retrieve a BugExcerpt matching the exact given id -func (c *RepoCache) ResolveBugExcerpt(id entity.Id) (*BugExcerpt, error) { - e, ok := c.bugExcerpts[id] - if !ok { - return nil, bug.ErrBugNotExist - } - - return e, nil +// ResolveBugExcerptPrefix retrieve a BugExcerpt matching an id prefix. It fails if multiple +// bugs match. +func (c *RepoCache) ResolveBugExcerptPrefix(prefix string) (*BugExcerpt, error) { + return c.ResolveBugExcerptMatcher(func(excerpt *BugExcerpt) bool { + return excerpt.Id.HasPrefix(prefix) + }) } // ResolveBugPrefix retrieve a bug matching an id prefix. It fails if multiple @@ -423,7 +472,26 @@ func (c *RepoCache) ResolveBugCreateMetadata(key string, value string) (*BugCach }) } +func (c *RepoCache) ResolveBugExcerptMatcher(f func(*BugExcerpt) bool) (*BugExcerpt, error) { + id, err := c.resolveBugMatcher(f) + if err != nil { + return nil, err + } + return c.ResolveBugExcerpt(id) +} + func (c *RepoCache) ResolveBugMatcher(f func(*BugExcerpt) bool) (*BugCache, error) { + id, err := c.resolveBugMatcher(f) + if err != nil { + return nil, err + } + return c.ResolveBug(id) +} + +func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, error) { + c.muBug.RLock() + defer c.muBug.RUnlock() + // preallocate but empty matching := make([]entity.Id, 0, 5) @@ -434,18 +502,21 @@ func (c *RepoCache) ResolveBugMatcher(f func(*BugExcerpt) bool) (*BugCache, erro } if len(matching) > 1 { - return nil, bug.NewErrMultipleMatchBug(matching) + return entity.UnsetId, bug.NewErrMultipleMatchBug(matching) } if len(matching) == 0 { - return nil, bug.ErrBugNotExist + return entity.UnsetId, bug.ErrBugNotExist } - return c.ResolveBug(matching[0]) + return matching[0], nil } // QueryBugs return the id of all Bug matching the given Query func (c *RepoCache) QueryBugs(query *Query) []entity.Id { + c.muBug.RLock() + defer c.muBug.RUnlock() + if query == nil { return c.AllBugsIds() } @@ -453,7 +524,7 @@ func (c *RepoCache) QueryBugs(query *Query) []entity.Id { var filtered []*BugExcerpt for _, excerpt := range c.bugExcerpts { - if query.Match(c, excerpt) { + if query.Match(excerpt, c) { filtered = append(filtered, excerpt) } } @@ -488,6 +559,9 @@ func (c *RepoCache) QueryBugs(query *Query) []entity.Id { // AllBugsIds return all known bug ids func (c *RepoCache) AllBugsIds() []entity.Id { + c.muBug.RLock() + defer c.muBug.RUnlock() + result := make([]entity.Id, len(c.bugExcerpts)) i := 0 @@ -505,6 +579,9 @@ func (c *RepoCache) AllBugsIds() []entity.Id { // labels are defined in a configuration file. Until that, the default behavior // is to return the list of labels already used. func (c *RepoCache) ValidLabels() []bug.Label { + c.muBug.RLock() + defer c.muBug.RUnlock() + set := map[bug.Label]interface{}{} for _, excerpt := range c.bugExcerpts { @@ -564,12 +641,15 @@ func (c *RepoCache) NewBugRaw(author *IdentityCache, unixTime int64, title strin return nil, nil, err } + c.muBug.Lock() if _, has := c.bugs[b.Id()]; has { + c.muBug.Unlock() return nil, nil, fmt.Errorf("bug %s already exist in the cache", b.Id()) } cached := NewBugCache(c, b) c.bugs[b.Id()] = cached + c.muBug.Unlock() // force the write of the excerpt err = c.bugUpdated(b.Id()) @@ -615,7 +695,9 @@ func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { switch result.Status { case entity.MergeStatusNew, entity.MergeStatusUpdated: i := result.Entity.(*identity.Identity) + c.muIdentity.Lock() c.identitiesExcerpts[result.Id] = NewIdentityExcerpt(i) + c.muIdentity.Unlock() } } @@ -631,7 +713,9 @@ func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { case entity.MergeStatusNew, entity.MergeStatusUpdated: b := result.Entity.(*bug.Bug) snap := b.Compile() + c.muBug.Lock() c.bugExcerpts[result.Id] = NewBugExcerpt(b, &snap) + c.muBug.Unlock() } } @@ -745,9 +829,24 @@ func repoIsAvailable(repo repository.Repo) error { return nil } +// ResolveIdentityExcerpt retrieve a IdentityExcerpt matching the exact given id +func (c *RepoCache) ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) { + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + + e, ok := c.identitiesExcerpts[id] + if !ok { + return nil, identity.ErrIdentityNotExist + } + + return e, nil +} + // ResolveIdentity retrieve an identity matching the exact given id func (c *RepoCache) ResolveIdentity(id entity.Id) (*IdentityCache, error) { + c.muIdentity.RLock() cached, ok := c.identities[id] + c.muIdentity.RUnlock() if ok { return cached, nil } @@ -758,19 +857,20 @@ func (c *RepoCache) ResolveIdentity(id entity.Id) (*IdentityCache, error) { } cached = NewIdentityCache(c, i) + + c.muIdentity.Lock() c.identities[id] = cached + c.muIdentity.Unlock() return cached, nil } -// ResolveIdentityExcerpt retrieve a IdentityExcerpt matching the exact given id -func (c *RepoCache) ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) { - e, ok := c.identitiesExcerpts[id] - if !ok { - return nil, identity.ErrIdentityNotExist - } - - return e, nil +// ResolveIdentityExcerptPrefix retrieve a IdentityExcerpt matching an id prefix. +// It fails if multiple identities match. +func (c *RepoCache) ResolveIdentityExcerptPrefix(prefix string) (*IdentityExcerpt, error) { + return c.ResolveIdentityExcerptMatcher(func(excerpt *IdentityExcerpt) bool { + return excerpt.Id.HasPrefix(prefix) + }) } // ResolveIdentityPrefix retrieve an Identity matching an id prefix. @@ -789,7 +889,26 @@ func (c *RepoCache) ResolveIdentityImmutableMetadata(key string, value string) ( }) } +func (c *RepoCache) ResolveIdentityExcerptMatcher(f func(*IdentityExcerpt) bool) (*IdentityExcerpt, error) { + id, err := c.resolveIdentityMatcher(f) + if err != nil { + return nil, err + } + return c.ResolveIdentityExcerpt(id) +} + func (c *RepoCache) ResolveIdentityMatcher(f func(*IdentityExcerpt) bool) (*IdentityCache, error) { + id, err := c.resolveIdentityMatcher(f) + if err != nil { + return nil, err + } + return c.ResolveIdentity(id) +} + +func (c *RepoCache) resolveIdentityMatcher(f func(*IdentityExcerpt) bool) (entity.Id, error) { + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + // preallocate but empty matching := make([]entity.Id, 0, 5) @@ -800,18 +919,21 @@ func (c *RepoCache) ResolveIdentityMatcher(f func(*IdentityExcerpt) bool) (*Iden } if len(matching) > 1 { - return nil, identity.NewErrMultipleMatch(matching) + return entity.UnsetId, identity.NewErrMultipleMatch(matching) } if len(matching) == 0 { - return nil, identity.ErrIdentityNotExist + return entity.UnsetId, identity.ErrIdentityNotExist } - return c.ResolveIdentity(matching[0]) + return matching[0], nil } // AllIdentityIds return all known identity ids func (c *RepoCache) AllIdentityIds() []entity.Id { + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + result := make([]entity.Id, len(c.identitiesExcerpts)) i := 0 @@ -829,6 +951,9 @@ func (c *RepoCache) SetUserIdentity(i *IdentityCache) error { return err } + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + // Make sure that everything is fine if _, ok := c.identities[i.Id()]; !ok { panic("SetUserIdentity while the identity is not from the cache, something is wrong") @@ -847,6 +972,9 @@ func (c *RepoCache) GetUserIdentity() (*IdentityCache, error) { } } + c.muIdentity.Lock() + defer c.muIdentity.Unlock() + i, err := identity.GetUserIdentity(c.repo) if err != nil { return nil, err @@ -859,6 +987,25 @@ func (c *RepoCache) GetUserIdentity() (*IdentityCache, error) { return cached, nil } +func (c *RepoCache) GetUserIdentityExcerpt() (*IdentityExcerpt, error) { + if c.userIdentityId == "" { + id, err := identity.GetUserIdentityId(c.repo) + if err != nil { + return nil, err + } + c.userIdentityId = id + } + + c.muIdentity.RLock() + defer c.muIdentity.RUnlock() + + excerpt, ok := c.identitiesExcerpts[c.userIdentityId] + if !ok { + return nil, fmt.Errorf("cache: missing identity excerpt %v", c.userIdentityId) + } + return excerpt, nil +} + func (c *RepoCache) IsUserIdentitySet() (bool, error) { return identity.IsUserIdentitySet(c.repo) } @@ -902,12 +1049,14 @@ func (c *RepoCache) finishIdentity(i *identity.Identity, metadata map[string]str return nil, err } + c.muIdentity.Lock() if _, has := c.identities[i.Id()]; has { return nil, fmt.Errorf("identity %s already exist in the cache", i.Id()) } cached := NewIdentityCache(c, i) c.identities[i.Id()] = cached + c.muIdentity.Unlock() // force the write of the excerpt err = c.identityUpdated(i.Id()) |