diff options
author | Michael Muré <batolettre@gmail.com> | 2022-12-21 21:54:36 +0100 |
---|---|---|
committer | Michael Muré <batolettre@gmail.com> | 2022-12-21 21:54:36 +0100 |
commit | 9b98fc06489353053564b431ac0c0d73a5c55a56 (patch) | |
tree | 67cab95c0c6167c0982516a2dda610bd5ece9eab /cache | |
parent | 905c9a90f134842b97e2cf729d8b11ff59bfd766 (diff) | |
download | git-bug-9b98fc06489353053564b431ac0c0d73a5c55a56.tar.gz |
cache: tie up the refactor up to compiling
Diffstat (limited to 'cache')
-rw-r--r-- | cache/bug_excerpt.go | 9 | ||||
-rw-r--r-- | cache/bug_subcache.go | 21 | ||||
-rw-r--r-- | cache/cached.go | 16 | ||||
-rw-r--r-- | cache/filter.go | 56 | ||||
-rw-r--r-- | cache/identity_excerpt.go | 9 | ||||
-rw-r--r-- | cache/identity_subcache.go | 58 | ||||
-rw-r--r-- | cache/multi_repo_cache.go | 16 | ||||
-rw-r--r-- | cache/repo_cache.go | 49 | ||||
-rw-r--r-- | cache/repo_cache_common.go | 97 | ||||
-rw-r--r-- | cache/repo_cache_test.go | 163 | ||||
-rw-r--r-- | cache/subcache.go | 160 |
11 files changed, 372 insertions, 282 deletions
diff --git a/cache/bug_excerpt.go b/cache/bug_excerpt.go index b4748fd2..26b7ec74 100644 --- a/cache/bug_excerpt.go +++ b/cache/bug_excerpt.go @@ -15,6 +15,8 @@ func init() { gob.Register(BugExcerpt{}) } +var _ Excerpt = &BugExcerpt{} + // BugExcerpt hold a subset of the bug values to be able to sort and filter bugs // efficiently without having to read and compile each raw bugs. type BugExcerpt struct { @@ -36,7 +38,8 @@ type BugExcerpt struct { CreateMetadata map[string]string } -func NewBugExcerpt(b bug.Interface, snap *bug.Snapshot) *BugExcerpt { +func NewBugExcerpt(b *BugCache) *BugExcerpt { + snap := b.Snapshot() participantsIds := make([]entity.Id, 0, len(snap.Participants)) for _, participant := range snap.Participants { participantsIds = append(participantsIds, participant.Id()) @@ -66,6 +69,10 @@ func NewBugExcerpt(b bug.Interface, snap *bug.Snapshot) *BugExcerpt { return e } +func (b *BugExcerpt) setId(id entity.Id) { + b.id = id +} + func (b *BugExcerpt) Id() entity.Id { return b.id } diff --git a/cache/bug_subcache.go b/cache/bug_subcache.go index f0a8889b..14b56cdc 100644 --- a/cache/bug_subcache.go +++ b/cache/bug_subcache.go @@ -24,11 +24,7 @@ func NewRepoCacheBug(repo repository.ClockedRepo, return NewBugCache(b, repo, getUserIdentity, entityUpdated) } - makeExcerpt := func(b *bug.Bug) *BugExcerpt { - return NewBugExcerpt(b, b.Compile()) - } - - makeIndex := func(b *BugCache) []string { + makeIndexData := func(b *BugCache) []string { snap := b.Snapshot() var res []string for _, comment := range snap.Comments { @@ -38,9 +34,16 @@ func NewRepoCacheBug(repo repository.ClockedRepo, return res } + actions := Actions[*bug.Bug]{ + ReadWithResolver: bug.ReadWithResolver, + ReadAllWithResolver: bug.ReadAllWithResolver, + Remove: bug.Remove, + MergeAll: bug.MergeAll, + } + sc := NewSubCache[*bug.Bug, *BugExcerpt, *BugCache]( repo, resolvers, getUserIdentity, - makeCached, makeExcerpt, makeIndex, + makeCached, NewBugExcerpt, makeIndexData, actions, "bug", "bugs", formatVersion, defaultMaxLoadedBugs, ) @@ -104,8 +107,8 @@ func (c *RepoCacheBug) ResolveComment(prefix string) (*BugCache, entity.Combined return matchingBug, matchingCommentId, nil } -// QueryBugs return the id of all Bug matching the given Query -func (c *RepoCacheBug) QueryBugs(q *query.Query) ([]entity.Id, error) { +// Query return the id of all Bug matching the given Query +func (c *RepoCacheBug) Query(q *query.Query) ([]entity.Id, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -140,7 +143,7 @@ func (c *RepoCacheBug) QueryBugs(q *query.Query) ([]entity.Id, error) { } for _, excerpt := range foundBySearch { - if matcher.Match(excerpt, c) { + if matcher.Match(excerpt, c.resolvers()) { filtered = append(filtered, excerpt) } } diff --git a/cache/cached.go b/cache/cached.go index 74757356..5a31ee59 100644 --- a/cache/cached.go +++ b/cache/cached.go @@ -3,10 +3,10 @@ package cache import ( "sync" - "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/entity/dag" "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/lamport" ) // type withSnapshot[SnapT dag.Snapshot, OpT dag.OperationWithApply[SnapT]] struct { @@ -97,7 +97,7 @@ func (e *CachedEntityBase[SnapT, OpT]) ResolveOperationWithMetadata(key string, } if len(matching) > 1 { - return "", bug.NewErrMultipleMatchOp(matching) + return "", entity.NewErrMultipleMatch("operation", matching) } return matching[0], nil @@ -136,3 +136,15 @@ func (e *CachedEntityBase[SnapT, OpT]) NeedCommit() bool { defer e.mu.RUnlock() return e.entity.NeedCommit() } + +func (e *CachedEntityBase[SnapT, OpT]) CreateLamportTime() lamport.Time { + return e.entity.CreateLamportTime() +} + +func (e *CachedEntityBase[SnapT, OpT]) EditLamportTime() lamport.Time { + return e.entity.EditLamportTime() +} + +func (e *CachedEntityBase[SnapT, OpT]) FirstOp() OpT { + return e.entity.FirstOp() +} diff --git a/cache/filter.go b/cache/filter.go index 01f635c5..5a15e402 100644 --- a/cache/filter.go +++ b/cache/filter.go @@ -8,28 +8,22 @@ import ( "github.com/MichaelMure/git-bug/query" ) -// resolver has the resolving functions needed by filters. -// This exists mainly to go through the functions of the cache with proper locking. -type resolver interface { - ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) -} - // Filter is a predicate that match a subset of bugs -type Filter func(excerpt *BugExcerpt, resolver resolver) bool +type Filter func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool // StatusFilter return a Filter that match a bug status func StatusFilter(status common.Status) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return excerpt.Status == status } } // AuthorFilter return a Filter that match a bug author func AuthorFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) - author, err := resolver.ResolveIdentityExcerpt(excerpt.AuthorId) + author, err := entity.Resolve[*IdentityExcerpt](resolvers, excerpt.AuthorId) if err != nil { panic(err) } @@ -40,7 +34,7 @@ func AuthorFilter(query string) Filter { // MetadataFilter return a Filter that match a bug metadata at creation time func MetadataFilter(pair query.StringPair) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if value, ok := excerpt.CreateMetadata[pair.Key]; ok { return value == pair.Value } @@ -50,7 +44,7 @@ func MetadataFilter(pair query.StringPair) Filter { // LabelFilter return a Filter that match a label func LabelFilter(label string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { for _, l := range excerpt.Labels { if string(l) == label { return true @@ -62,11 +56,11 @@ func LabelFilter(label string) Filter { // ActorFilter return a Filter that match a bug actor func ActorFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) for _, id := range excerpt.Actors { - identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + identityExcerpt, err := entity.Resolve[*IdentityExcerpt](resolvers, id) if err != nil { panic(err) } @@ -81,11 +75,11 @@ func ActorFilter(query string) Filter { // ParticipantFilter return a Filter that match a bug participant func ParticipantFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) for _, id := range excerpt.Participants { - identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + identityExcerpt, err := entity.Resolve[*IdentityExcerpt](resolvers, id) if err != nil { panic(err) } @@ -100,7 +94,7 @@ func ParticipantFilter(query string) Filter { // TitleFilter return a Filter that match if the title contains the given query func TitleFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return strings.Contains( strings.ToLower(excerpt.Title), strings.ToLower(query), @@ -110,7 +104,7 @@ func TitleFilter(query string) Filter { // NoLabelFilter return a Filter that match the absence of labels func NoLabelFilter() Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return len(excerpt.Labels) == 0 } } @@ -161,36 +155,36 @@ func compileMatcher(filters query.Filters) *Matcher { } // Match check if a bug match the set of filters -func (f *Matcher) Match(excerpt *BugExcerpt, resolver resolver) bool { - if match := f.orMatch(f.Status, excerpt, resolver); !match { +func (f *Matcher) Match(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { + if match := f.orMatch(f.Status, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Author, excerpt, resolver); !match { + if match := f.orMatch(f.Author, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Metadata, excerpt, resolver); !match { + if match := f.orMatch(f.Metadata, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Participant, excerpt, resolver); !match { + if match := f.orMatch(f.Participant, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Actor, excerpt, resolver); !match { + if match := f.orMatch(f.Actor, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.Label, excerpt, resolver); !match { + if match := f.andMatch(f.Label, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.NoFilters, excerpt, resolver); !match { + if match := f.andMatch(f.NoFilters, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.Title, excerpt, resolver); !match { + if match := f.andMatch(f.Title, excerpt, resolvers); !match { return false } @@ -198,28 +192,28 @@ func (f *Matcher) Match(excerpt *BugExcerpt, resolver resolver) bool { } // Check if any of the filters provided match the bug -func (*Matcher) orMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { +func (*Matcher) orMatch(filters []Filter, excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if len(filters) == 0 { return true } match := false for _, f := range filters { - match = match || f(excerpt, resolver) + match = match || f(excerpt, resolvers) } return match } // Check if all the filters provided match the bug -func (*Matcher) andMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { +func (*Matcher) andMatch(filters []Filter, excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if len(filters) == 0 { return true } match := true for _, f := range filters { - match = match && f(excerpt, resolver) + match = match && f(excerpt, resolvers) } return match diff --git a/cache/identity_excerpt.go b/cache/identity_excerpt.go index 126c712b..79d88537 100644 --- a/cache/identity_excerpt.go +++ b/cache/identity_excerpt.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" ) @@ -14,6 +13,8 @@ func init() { gob.Register(IdentityExcerpt{}) } +var _ Excerpt = &IdentityExcerpt{} + // IdentityExcerpt hold a subset of the identity values to be able to sort and // filter identities efficiently without having to read and compile each raw // identity. @@ -25,7 +26,7 @@ type IdentityExcerpt struct { ImmutableMetadata map[string]string } -func NewIdentityExcerpt(i *identity.Identity) *IdentityExcerpt { +func NewIdentityExcerpt(i *IdentityCache) *IdentityExcerpt { return &IdentityExcerpt{ id: i.Id(), Name: i.Name(), @@ -34,6 +35,10 @@ func NewIdentityExcerpt(i *identity.Identity) *IdentityExcerpt { } } +func (i *IdentityExcerpt) setId(id entity.Id) { + i.id = id +} + func (i *IdentityExcerpt) Id() entity.Id { return i.id } diff --git a/cache/identity_subcache.go b/cache/identity_subcache.go index b175d731..a623d0e1 100644 --- a/cache/identity_subcache.go +++ b/cache/identity_subcache.go @@ -20,18 +20,34 @@ func NewRepoCacheIdentity(repo repository.ClockedRepo, return NewIdentityCache(i, repo, entityUpdated) } - makeExcerpt := func(i *identity.Identity) *IdentityExcerpt { - return NewIdentityExcerpt(i) - } - makeIndex := func(i *IdentityCache) []string { // no indexing return nil } + // TODO: this is terribly ugly, but we are currently stuck with the fact that identities are NOT using the fancy dag framework. + // This lead to various complication here and there to handle entities generically, and avoid large code duplication. + // TL;DR: something has to give, and this is the less ugly solution I found. This "normalize" identities as just another "dag framework" + // entity. Ideally identities would be converted to the dag framework, but right now that could lead to potential attack: if an old + // private key is leaked, it would be possible to craft a legal identity update that take over the most recent version. While this is + // meaningless in the case of a normal entity, it's really an issues for identities. + + actions := Actions[*identity.Identity]{ + ReadWithResolver: func(repo repository.ClockedRepo, resolvers entity.Resolvers, id entity.Id) (*identity.Identity, error) { + return identity.ReadLocal(repo, id) + }, + ReadAllWithResolver: func(repo repository.ClockedRepo, resolvers entity.Resolvers) <-chan entity.StreamedEntity[*identity.Identity] { + return identity.ReadAllLocal(repo) + }, + Remove: identity.RemoveIdentity, + MergeAll: func(repo repository.ClockedRepo, resolvers entity.Resolvers, remote string, mergeAuthor identity.Interface) <-chan entity.MergeResult { + return identity.MergeAll(repo, remote) + }, + } + sc := NewSubCache[*identity.Identity, *IdentityExcerpt, *IdentityCache]( repo, resolvers, getUserIdentity, - makeCached, makeExcerpt, makeIndex, + makeCached, NewIdentityExcerpt, makeIndex, actions, "identity", "identities", formatVersion, defaultMaxLoadedBugs, ) @@ -47,32 +63,32 @@ func (c *RepoCacheIdentity) ResolveIdentityImmutableMetadata(key string, value s }) } -func (c *RepoCacheIdentity) NewIdentityFromGitUser() (*IdentityCache, error) { - return c.NewIdentityFromGitUserRaw(nil) +// New create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) New(name string, email string) (*IdentityCache, error) { + return c.NewRaw(name, email, "", "", nil, nil) } -func (c *RepoCacheIdentity) NewIdentityFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewFromGitUser(c.repo) +// NewFull create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) NewFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { + return c.NewRaw(name, email, login, avatarUrl, keys, nil) +} + +func (c *RepoCacheIdentity) NewRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) if err != nil { return nil, err } return c.finishIdentity(i, metadata) } -// NewIdentity create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCacheIdentity) NewIdentity(name string, email string) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, "", "", nil, nil) -} - -// NewIdentityFull create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCacheIdentity) NewIdentityFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, login, avatarUrl, keys, nil) +func (c *RepoCacheIdentity) NewFromGitUser() (*IdentityCache, error) { + return c.NewFromGitUserRaw(nil) } -func (c *RepoCacheIdentity) NewIdentityRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) +func (c *RepoCacheIdentity) NewFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewFromGitUser(c.repo) if err != nil { return nil, err } diff --git a/cache/multi_repo_cache.go b/cache/multi_repo_cache.go index 98f868e9..12d26e26 100644 --- a/cache/multi_repo_cache.go +++ b/cache/multi_repo_cache.go @@ -21,25 +21,25 @@ func NewMultiRepoCache() *MultiRepoCache { } // RegisterRepository register a named repository. Use this for multi-repo setup -func (c *MultiRepoCache) RegisterRepository(ref string, repo repository.ClockedRepo) (*RepoCache, error) { - r, err := NewRepoCache(repo) +func (c *MultiRepoCache) RegisterRepository(ref string, repo repository.ClockedRepo) (*RepoCache, chan BuildEvent, error) { + r, events, err := NewRepoCache(repo) if err != nil { - return nil, err + return nil, nil, err } c.repos[ref] = r - return r, nil + return r, events, nil } // RegisterDefaultRepository register an unnamed repository. Use this for mono-repo setup -func (c *MultiRepoCache) RegisterDefaultRepository(repo repository.ClockedRepo) (*RepoCache, error) { - r, err := NewRepoCache(repo) +func (c *MultiRepoCache) RegisterDefaultRepository(repo repository.ClockedRepo) (*RepoCache, chan BuildEvent, error) { + r, events, err := NewRepoCache(repo) if err != nil { - return nil, err + return nil, nil, err } c.repos[defaultRepoName] = r - return r, nil + return r, events, nil } // DefaultRepo retrieve the default repository diff --git a/cache/repo_cache.go b/cache/repo_cache.go index b2facac3..2cac711b 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -30,8 +30,10 @@ var _ repository.RepoKeyring = &RepoCache{} type cacheMgmt interface { Typename() string Load() error - Write() error Build() error + SetCacheSize(size int) + MergeAll(remote string) <-chan entity.MergeResult + GetNamespace() string Close() error } @@ -86,18 +88,24 @@ func NewNamedRepoCache(r repository.ClockedRepo, name string) (*RepoCache, chan c.subcaches = append(c.subcaches, c.bugs) c.resolvers = entity.Resolvers{ - &IdentityCache{}: entity.ResolverFunc[*IdentityCache](c.identities.Resolve), - &BugCache{}: entity.ResolverFunc[*BugCache](c.bugs.Resolve), + &IdentityCache{}: entity.ResolverFunc[*IdentityCache](c.identities.Resolve), + &IdentityExcerpt{}: entity.ResolverFunc[*IdentityExcerpt](c.identities.ResolveExcerpt), + &BugCache{}: entity.ResolverFunc[*BugCache](c.bugs.Resolve), + &BugExcerpt{}: entity.ResolverFunc[*BugExcerpt](c.bugs.ResolveExcerpt), } err := c.lock() if err != nil { - return &RepoCache{}, nil, err + closed := make(chan BuildEvent) + close(closed) + return &RepoCache{}, closed, err } err = c.load() if err == nil { - return c, nil, nil + closed := make(chan BuildEvent) + close(closed) + return c, closed, nil } // Cache is either missing, broken or outdated. Rebuilding. @@ -122,8 +130,9 @@ func (c *RepoCache) getResolvers() entity.Resolvers { // setCacheSize change the maximum number of loaded bugs func (c *RepoCache) setCacheSize(size int) { - c.maxLoadedBugs = size - c.evictIfNeeded() + for _, subcache := range c.subcaches { + subcache.SetCacheSize(size) + } } // load will try to read from the disk all the cache files @@ -135,15 +144,6 @@ func (c *RepoCache) load() error { return errWait.Wait() } -// write will serialize on disk all the cache files -func (c *RepoCache) write() error { - var errWait multierr.ErrWaitGroup - for _, mgmt := range c.subcaches { - errWait.Go(mgmt.Write) - } - return errWait.Wait() -} - func (c *RepoCache) lock() error { err := repoIsAvailable(c.repo) if err != nil { @@ -190,10 +190,14 @@ const ( BuildEventFinished ) +// BuildEvent carry an event happening during the cache build process. type BuildEvent struct { + // Err carry an error if the build process failed. If set, no other field matter. + Err error + // Typename is the name of the entity of which the event relate to. Typename string - Event BuildEventType - Err error + // Event is the type of the event. + Event BuildEventType } func (c *RepoCache) buildCache() chan BuildEvent { @@ -221,15 +225,6 @@ func (c *RepoCache) buildCache() chan BuildEvent { return } - err = subcache.Write() - if err != nil { - out <- BuildEvent{ - Typename: subcache.Typename(), - Err: err, - } - return - } - out <- BuildEvent{ Typename: subcache.Typename(), Event: BuildEventFinished, diff --git a/cache/repo_cache_common.go b/cache/repo_cache_common.go index 1aed04b2..f768b8e2 100644 --- a/cache/repo_cache_common.go +++ b/cache/repo_cache_common.go @@ -1,10 +1,11 @@ package cache import ( + "sync" + "github.com/go-git/go-billy/v5" "github.com/pkg/errors" - "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" @@ -72,76 +73,40 @@ func (c *RepoCache) StoreData(data []byte) (repository.Hash, error) { // Fetch retrieve updates from a remote // This does not change the local bugs or identities state func (c *RepoCache) Fetch(remote string) (string, error) { - stdout1, err := identity.Fetch(c.repo, remote) - if err != nil { - return stdout1, err - } - - stdout2, err := bug.Fetch(c.repo, remote) - if err != nil { - return stdout2, err + prefixes := make([]string, len(c.subcaches)) + for i, subcache := range c.subcaches { + prefixes[i] = subcache.GetNamespace() } - return stdout1 + stdout2, nil + // fetch everything at once, to have a single auth step if required. + return c.repo.FetchRefs(remote, prefixes...) } // MergeAll will merge all the available remote bug and identities func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { out := make(chan entity.MergeResult) - // Intercept merge results to update the cache properly + dependency := [][]cacheMgmt{ + {c.identities}, + {c.bugs}, + } + + // run MergeAll according to entities dependencies and merge the results go func() { defer close(out) - author, err := c.GetUserIdentity() - if err != nil { - out <- entity.NewMergeError(err, "") - return - } - - results := identity.MergeAll(c.repo, remote) - for result := range results { - out <- result - - if result.Err != nil { - continue - } - - switch result.Status { - case entity.MergeStatusNew, entity.MergeStatusUpdated: - i := result.Entity.(*identity.Identity) - c.muIdentity.Lock() - c.identitiesExcerpts[result.Id] = NewIdentityExcerpt(i) - c.muIdentity.Unlock() + for _, subcaches := range dependency { + var wg sync.WaitGroup + for _, subcache := range subcaches { + wg.Add(1) + go func(subcache cacheMgmt) { + for res := range subcache.MergeAll(remote) { + out <- res + } + wg.Done() + }(subcache) } - } - - results = bug.MergeAll(c.repo, c.resolvers, remote, author) - for result := range results { - out <- result - - if result.Err != nil { - continue - } - - // TODO: have subcache do the merging? - switch result.Status { - case entity.MergeStatusNew: - b := result.Entity.(*bug.Bug) - _, err := c.bugs.add(b) - case entity.MergeStatusUpdated: - _, err := c.bugs.entityUpdated(b) - snap := b.Compile() - c.muBug.Lock() - c.bugExcerpts[result.Id] = NewBugExcerpt(b, snap) - c.muBug.Unlock() - } - } - - err = c.write() - if err != nil { - out <- entity.NewMergeError(err, "") - return + wg.Wait() } }() @@ -150,17 +115,13 @@ func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { // Push update a remote with the local changes func (c *RepoCache) Push(remote string) (string, error) { - stdout1, err := identity.Push(c.repo, remote) - if err != nil { - return stdout1, err - } - - stdout2, err := bug.Push(c.repo, remote) - if err != nil { - return stdout2, err + prefixes := make([]string, len(c.subcaches)) + for i, subcache := range c.subcaches { + prefixes[i] = subcache.GetNamespace() } - return stdout1 + stdout2, nil + // push everything at once, to have a single auth step if required + return c.repo.PushRefs(remote, prefixes...) } // Pull will do a Fetch + MergeAll diff --git a/cache/repo_cache_test.go b/cache/repo_cache_test.go index 58ade144..395872f7 100644 --- a/cache/repo_cache_test.go +++ b/cache/repo_cache_test.go @@ -8,20 +8,27 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/query" "github.com/MichaelMure/git-bug/repository" ) +func noBuildEventErrors(t *testing.T, c chan BuildEvent) { + t.Helper() + for event := range c { + require.NoError(t, event.Err) + } +} + func TestCache(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - cache, err := NewRepoCache(repo) + cache, events, err := NewRepoCache(repo) + noBuildEventErrors(t, events) require.NoError(t, err) // Create, set and get user identity - iden1, err := cache.NewIdentity("René Descartes", "rene@descartes.fr") + iden1, err := cache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = cache.SetUserIdentity(iden1) require.NoError(t, err) @@ -30,102 +37,105 @@ func TestCache(t *testing.T) { require.Equal(t, iden1.Id(), userIden.Id()) // it's possible to create two identical identities - iden2, err := cache.NewIdentity("René Descartes", "rene@descartes.fr") + iden2, err := cache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) // Two identical identities yield a different id require.NotEqual(t, iden1.Id(), iden2.Id()) // There is now two identities in the cache - require.Len(t, cache.AllIdentityIds(), 2) - require.Len(t, cache.identitiesExcerpts, 2) - require.Len(t, cache.identities, 2) + require.Len(t, cache.Identities().AllIds(), 2) + require.Len(t, cache.identities.excerpts, 2) + require.Len(t, cache.identities.cached, 2) // Create a bug - bug1, _, err := cache.NewBug("title", "message") + bug1, _, err := cache.Bugs().New("title", "message") require.NoError(t, err) // It's possible to create two identical bugs - bug2, _, err := cache.NewBug("title", "message") + bug2, _, err := cache.Bugs().New("title", "message") require.NoError(t, err) // two identical bugs yield a different id require.NotEqual(t, bug1.Id(), bug2.Id()) // There is now two bugs in the cache - require.Len(t, cache.AllBugsIds(), 2) - require.Len(t, cache.bugExcerpts, 2) - require.Len(t, cache.bugs, 2) + require.Len(t, cache.Bugs().AllIds(), 2) + require.Len(t, cache.bugs.excerpts, 2) + require.Len(t, cache.bugs.cached, 2) // Resolving - _, err = cache.ResolveIdentity(iden1.Id()) + _, err = cache.Identities().Resolve(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityExcerpt(iden1.Id()) + _, err = cache.Identities().ResolveExcerpt(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityPrefix(iden1.Id().String()[:10]) + _, err = cache.Identities().ResolvePrefix(iden1.Id().String()[:10]) require.NoError(t, err) - _, err = cache.ResolveBug(bug1.Id()) + _, err = cache.Bugs().Resolve(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugExcerpt(bug1.Id()) + _, err = cache.Bugs().ResolveExcerpt(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugPrefix(bug1.Id().String()[:10]) + _, err = cache.Bugs().ResolvePrefix(bug1.Id().String()[:10]) require.NoError(t, err) // Querying q, err := query.Parse("status:open author:descartes sort:edit-asc") require.NoError(t, err) - res, err := cache.QueryBugs(q) + res, err := cache.Bugs().Query(q) require.NoError(t, err) require.Len(t, res, 2) // Close require.NoError(t, cache.Close()) - require.Empty(t, cache.bugs) - require.Empty(t, cache.bugExcerpts) - require.Empty(t, cache.identities) - require.Empty(t, cache.identitiesExcerpts) + require.Empty(t, cache.bugs.cached) + require.Empty(t, cache.bugs.excerpts) + require.Empty(t, cache.identities.cached) + require.Empty(t, cache.identities.excerpts) // Reload, only excerpt are loaded, but as we need to load the identities used in the bugs // to check the signatures, we also load the identity used above - cache, err = NewRepoCache(repo) + cache, events, err = NewRepoCache(repo) + noBuildEventErrors(t, events) require.NoError(t, err) - require.Empty(t, cache.bugs) - require.Len(t, cache.identities, 1) - require.Len(t, cache.bugExcerpts, 2) - require.Len(t, cache.identitiesExcerpts, 2) + require.Empty(t, cache.bugs.cached) + require.Len(t, cache.bugs.excerpts, 2) + require.Len(t, cache.identities.cached, 0) + require.Len(t, cache.identities.excerpts, 2) // Resolving load from the disk - _, err = cache.ResolveIdentity(iden1.Id()) + _, err = cache.Identities().Resolve(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityExcerpt(iden1.Id()) + _, err = cache.Identities().ResolveExcerpt(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityPrefix(iden1.Id().String()[:10]) + _, err = cache.Identities().ResolvePrefix(iden1.Id().String()[:10]) require.NoError(t, err) - _, err = cache.ResolveBug(bug1.Id()) + _, err = cache.Bugs().Resolve(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugExcerpt(bug1.Id()) + _, err = cache.Bugs().ResolveExcerpt(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugPrefix(bug1.Id().String()[:10]) + _, err = cache.Bugs().ResolvePrefix(bug1.Id().String()[:10]) require.NoError(t, err) } func TestCachePushPull(t *testing.T) { repoA, repoB, _ := repository.SetupGoGitReposAndRemote(t) - cacheA, err := NewRepoCache(repoA) + cacheA, events, err := NewRepoCache(repoA) + noBuildEventErrors(t, events) require.NoError(t, err) - cacheB, err := NewRepoCache(repoB) + cacheB, events, err := NewRepoCache(repoB) + noBuildEventErrors(t, events) require.NoError(t, err) // Create, set and get user identity - reneA, err := cacheA.NewIdentity("René Descartes", "rene@descartes.fr") + reneA, err := cacheA.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = cacheA.SetUserIdentity(reneA) require.NoError(t, err) - isaacB, err := cacheB.NewIdentity("Isaac Newton", "isaac@newton.uk") + isaacB, err := cacheB.Identities().New("Isaac Newton", "isaac@newton.uk") require.NoError(t, err) err = cacheB.SetUserIdentity(isaacB) require.NoError(t, err) @@ -137,7 +147,7 @@ func TestCachePushPull(t *testing.T) { require.NoError(t, err) // Create a bug in A - _, _, err = cacheA.NewBug("bug1", "message") + _, _, err = cacheA.Bugs().New("bug1", "message") require.NoError(t, err) // A --> remote --> B @@ -147,17 +157,17 @@ func TestCachePushPull(t *testing.T) { err = cacheB.Pull("origin") require.NoError(t, err) - require.Len(t, cacheB.AllBugsIds(), 1) + require.Len(t, cacheB.Bugs().AllIds(), 1) // retrieve and set identity - reneB, err := cacheB.ResolveIdentity(reneA.Id()) + reneB, err := cacheB.Identities().Resolve(reneA.Id()) require.NoError(t, err) err = cacheB.SetUserIdentity(reneB) require.NoError(t, err) // B --> remote --> A - _, _, err = cacheB.NewBug("bug2", "message") + _, _, err = cacheB.Bugs().New("bug2", "message") require.NoError(t, err) _, err = cacheB.Push("origin") @@ -166,7 +176,7 @@ func TestCachePushPull(t *testing.T) { err = cacheA.Pull("origin") require.NoError(t, err) - require.Len(t, cacheA.AllBugsIds(), 2) + require.Len(t, cacheA.Bugs().AllIds(), 2) } func TestRemove(t *testing.T) { @@ -180,20 +190,21 @@ func TestRemove(t *testing.T) { err = repo.AddRemote("remoteB", remoteB.GetLocalRemote()) require.NoError(t, err) - repoCache, err := NewRepoCache(repo) + repoCache, events, err := NewRepoCache(repo) + noBuildEventErrors(t, events) require.NoError(t, err) - rene, err := repoCache.NewIdentity("René Descartes", "rene@descartes.fr") + rene, err := repoCache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = repoCache.SetUserIdentity(rene) require.NoError(t, err) - _, _, err = repoCache.NewBug("title", "message") + _, _, err = repoCache.Bugs().New("title", "message") require.NoError(t, err) // and one more for testing - b1, _, err := repoCache.NewBug("title", "message") + b1, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) _, err = repoCache.Push("remoteA") @@ -208,72 +219,73 @@ func TestRemove(t *testing.T) { _, err = repoCache.Fetch("remoteB") require.NoError(t, err) - err = repoCache.RemoveBug(b1.Id().String()) + err = repoCache.Bugs().Remove(b1.Id().String()) require.NoError(t, err) - assert.Equal(t, 1, len(repoCache.bugs)) - assert.Equal(t, 1, len(repoCache.bugExcerpts)) + assert.Len(t, repoCache.bugs.cached, 1) + assert.Len(t, repoCache.bugs.excerpts, 1) - _, err = repoCache.ResolveBug(b1.Id()) - assert.ErrorIs(t, entity.ErrNotFound{}, err) + _, err = repoCache.Bugs().Resolve(b1.Id()) + assert.ErrorAs(t, entity.ErrNotFound{}, err) } func TestCacheEviction(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - repoCache, err := NewRepoCache(repo) + repoCache, events, err := NewRepoCache(repo) + noBuildEventErrors(t, events) require.NoError(t, err) repoCache.setCacheSize(2) - require.Equal(t, 2, repoCache.maxLoadedBugs) - require.Equal(t, 0, repoCache.loadedBugs.Len()) - require.Equal(t, 0, len(repoCache.bugs)) + require.Equal(t, 2, repoCache.bugs.maxLoaded) + require.Len(t, repoCache.bugs.cached, 0) + require.Equal(t, repoCache.bugs.lru.Len(), 0) // Generating some bugs - rene, err := repoCache.NewIdentity("René Descartes", "rene@descartes.fr") + rene, err := repoCache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = repoCache.SetUserIdentity(rene) require.NoError(t, err) - bug1, _, err := repoCache.NewBug("title", "message") + bug1, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) checkBugPresence(t, repoCache, bug1, true) - require.Equal(t, 1, repoCache.loadedBugs.Len()) - require.Equal(t, 1, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 1) + require.Equal(t, 1, repoCache.bugs.lru.Len()) - bug2, _, err := repoCache.NewBug("title", "message") + bug2, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) checkBugPresence(t, repoCache, bug1, true) checkBugPresence(t, repoCache, bug2, true) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) // Number of bugs should not exceed max size of lruCache, oldest one should be evicted - bug3, _, err := repoCache.NewBug("title", "message") + bug3, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) checkBugPresence(t, repoCache, bug1, false) checkBugPresence(t, repoCache, bug2, true) checkBugPresence(t, repoCache, bug3, true) // Accessing bug should update position in lruCache and therefore it should not be evicted - repoCache.loadedBugs.Get(bug2.Id()) - oldestId, _ := repoCache.loadedBugs.GetOldest() + repoCache.bugs.lru.Get(bug2.Id()) + oldestId, _ := repoCache.bugs.lru.GetOldest() require.Equal(t, bug3.Id(), oldestId) checkBugPresence(t, repoCache, bug1, false) checkBugPresence(t, repoCache, bug2, true) checkBugPresence(t, repoCache, bug3, true) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) } func checkBugPresence(t *testing.T, cache *RepoCache, bug *BugCache, presence bool) { id := bug.Id() - require.Equal(t, presence, cache.loadedBugs.Contains(id)) - b, ok := cache.bugs[id] + require.Equal(t, presence, cache.bugs.lru.Contains(id)) + b, ok := cache.bugs.cached[id] require.Equal(t, presence, ok) if ok { require.Equal(t, bug, b) @@ -287,12 +299,13 @@ func TestLongDescription(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - backend, err := NewRepoCache(repo) + backend, events, err := NewRepoCache(repo) + noBuildEventErrors(t, events) require.NoError(t, err) - i, err := backend.NewIdentity("René Descartes", "rene@descartes.fr") + i, err := backend.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) - _, _, err = backend.NewBugRaw(i, time.Now().Unix(), text, text, nil, nil) + _, _, err = backend.Bugs().NewRaw(i, time.Now().Unix(), text, text, nil, nil) require.NoError(t, err) } diff --git a/cache/subcache.go b/cache/subcache.go index 1737da43..0678988a 100644 --- a/cache/subcache.go +++ b/cache/subcache.go @@ -4,18 +4,18 @@ import ( "bytes" "encoding/gob" "fmt" - "os" "sync" "github.com/pkg/errors" - "github.com/MichaelMure/git-bug/entities/bug" + "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" ) type Excerpt interface { Id() entity.Id + setId(id entity.Id) } type CacheEntity interface { @@ -25,15 +25,27 @@ type CacheEntity interface { type getUserIdentityFunc func() (*IdentityCache, error) +// Actions expose a number of action functions on Entities, to give upper layers (cache) a way to normalize interactions. +// Note: ideally this wouldn't exist, the cache layer would assume that everything is an entity/dag, and directly use the +// functions from this package, but right now identities are not using that framework. +type Actions[EntityT entity.Interface] struct { + ReadWithResolver func(repo repository.ClockedRepo, resolvers entity.Resolvers, id entity.Id) (EntityT, error) + ReadAllWithResolver func(repo repository.ClockedRepo, resolvers entity.Resolvers) <-chan entity.StreamedEntity[EntityT] + Remove func(repo repository.ClockedRepo, id entity.Id) error + MergeAll func(repo repository.ClockedRepo, resolvers entity.Resolvers, remote string, mergeAuthor identity.Interface) <-chan entity.MergeResult +} + +var _ cacheMgmt = &SubCache[entity.Interface, Excerpt, CacheEntity]{} + type SubCache[EntityT entity.Interface, ExcerptT Excerpt, CacheT CacheEntity] struct { repo repository.ClockedRepo resolvers func() entity.Resolvers - getUserIdentity getUserIdentityFunc - readWithResolver func(repository.ClockedRepo, entity.Resolvers, entity.Id) (EntityT, error) - makeCached func(entity EntityT, entityUpdated func(id entity.Id) error) CacheT - makeExcerpt func(EntityT) ExcerptT - makeIndex func(CacheT) []string + getUserIdentity getUserIdentityFunc + makeCached func(entity EntityT, entityUpdated func(id entity.Id) error) CacheT + makeExcerpt func(CacheT) ExcerptT + makeIndexData func(CacheT) []string + actions Actions[EntityT] typename string namespace string @@ -50,14 +62,19 @@ func NewSubCache[EntityT entity.Interface, ExcerptT Excerpt, CacheT CacheEntity] repo repository.ClockedRepo, resolvers func() entity.Resolvers, getUserIdentity getUserIdentityFunc, makeCached func(entity EntityT, entityUpdated func(id entity.Id) error) CacheT, - makeExcerpt func(EntityT) ExcerptT, - makeIndex func(CacheT) []string, + makeExcerpt func(CacheT) ExcerptT, + makeIndexData func(CacheT) []string, + actions Actions[EntityT], typename, namespace string, version uint, maxLoaded int) *SubCache[EntityT, ExcerptT, CacheT] { return &SubCache[EntityT, ExcerptT, CacheT]{ repo: repo, resolvers: resolvers, getUserIdentity: getUserIdentity, + makeCached: makeCached, + makeExcerpt: makeExcerpt, + makeIndexData: makeIndexData, + actions: actions, typename: typename, namespace: namespace, version: version, @@ -98,6 +115,12 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Load() error { return fmt.Errorf("unknown %s cache format version %v", sc.namespace, aux.Version) } + // the id is not serialized in the excerpt itself (non-exported field in go, long story ...), + // so we fix it here, which doubles as enforcing coherency. + for id, excerpt := range aux.Excerpts { + excerpt.setId(id) + } + sc.excerpts = aux.Excerpts index, err := sc.repo.GetIndex(sc.typename) @@ -118,7 +141,7 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Load() error { } // Write will serialize on disk the entity cache file -func (sc *SubCache[EntityT, ExcerptT, CacheT]) Write() error { +func (sc *SubCache[EntityT, ExcerptT, CacheT]) write() error { sc.mu.RLock() defer sc.mu.RUnlock() @@ -155,9 +178,7 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Write() error { func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error { sc.excerpts = make(map[entity.Id]ExcerptT) - sc.readWithResolver - - allBugs := bug.ReadAllWithResolver(c.repo, c.resolvers) + allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers()) index, err := sc.repo.GetIndex(sc.typename) if err != nil { @@ -172,15 +193,17 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error { indexer, indexEnd := index.IndexBatch() - for b := range allBugs { - if b.Err != nil { - return b.Err + for e := range allEntities { + if e.Err != nil { + return e.Err } - snap := b.Bug.Compile() - c.bugExcerpts[b.Bug.Id()] = NewBugExcerpt(b.Bug, snap) + // TODO: doesn't actually record in cache, should we? + cached := sc.makeCached(e.Entity, sc.entityUpdated) + sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached) - if err := indexer(snap); err != nil { + indexData := sc.makeIndexData(cached) + if err := indexer(e.Entity.Id().String(), indexData); err != nil { return err } } @@ -190,10 +213,19 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error { return err } - _, _ = fmt.Fprintln(os.Stderr, "Done.") + err = sc.write() + if err != nil { + return err + } + return nil } +func (sc *SubCache[EntityT, ExcerptT, CacheT]) SetCacheSize(size int) { + sc.maxLoaded = size + sc.evictIfNeeded() +} + func (sc *SubCache[EntityT, ExcerptT, CacheT]) Close() error { sc.mu.Lock() defer sc.mu.Unlock() @@ -229,7 +261,7 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Resolve(id entity.Id) (CacheT, er } sc.mu.RUnlock() - e, err := sc.readWithResolver(sc.repo, sc.resolvers(), id) + e, err := sc.actions.ReadWithResolver(sc.repo, sc.resolvers(), id) if err != nil { return *new(CacheT), err } @@ -315,8 +347,6 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) resolveMatcher(f func(ExcerptT) b return matching[0], nil } -var errNotInCache = errors.New("entity missing from cache") - func (sc *SubCache[EntityT, ExcerptT, CacheT]) add(e EntityT) (CacheT, error) { sc.mu.Lock() if _, has := sc.cached[e.Id()]; has { @@ -348,26 +378,74 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) Remove(prefix string) error { sc.mu.Lock() - err = bug.Remove(c.repo, b.Id()) + err = sc.actions.Remove(sc.repo, e.Id()) if err != nil { - c.muBug.Unlock() - + sc.mu.Unlock() return err } - delete(c.bugs, b.Id()) - delete(c.bugExcerpts, b.Id()) - c.loadedBugs.Remove(b.Id()) + delete(sc.cached, e.Id()) + delete(sc.excerpts, e.Id()) + sc.lru.Remove(e.Id()) + + sc.mu.Unlock() + + return sc.write() +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) MergeAll(remote string) <-chan entity.MergeResult { + out := make(chan entity.MergeResult) - c.muBug.Unlock() + // Intercept merge results to update the cache properly + go func() { + defer close(out) + + author, err := sc.getUserIdentity() + if err != nil { + out <- entity.NewMergeError(err, "") + return + } + + results := sc.actions.MergeAll(sc.repo, sc.resolvers(), remote, author) + for result := range results { + out <- result + + if result.Err != nil { + continue + } + + switch result.Status { + case entity.MergeStatusNew, entity.MergeStatusUpdated: + e := result.Entity.(EntityT) + + // TODO: doesn't actually record in cache, should we? + cached := sc.makeCached(e, sc.entityUpdated) + + sc.mu.Lock() + sc.excerpts[result.Id] = sc.makeExcerpt(cached) + sc.mu.Unlock() + } + } + + err = sc.write() + if err != nil { + out <- entity.NewMergeError(err, "") + return + } + }() + + return out - return c.writeBugCache() +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) GetNamespace() string { + return sc.namespace } // entityUpdated is a callback to trigger when the excerpt of an entity changed func (sc *SubCache[EntityT, ExcerptT, CacheT]) entityUpdated(id entity.Id) error { sc.mu.Lock() - b, ok := sc.cached[id] + e, ok := sc.cached[id] if !ok { sc.mu.Unlock() @@ -376,19 +454,24 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) entityUpdated(id entity.Id) error // memory and thus concurrent write. // Failing immediately here is the simple and safe solution to avoid // complicated data loss. - return errNotInCache + return errors.New("entity missing from cache") } sc.lru.Get(id) // sc.excerpts[id] = bug2.NewBugExcerpt(b.bug, b.Snapshot()) - sc.excerpts[id] = bug2.NewBugExcerpt(b.bug, b.Snapshot()) + sc.excerpts[id] = sc.makeExcerpt(e) sc.mu.Unlock() - if err := sc.addBugToSearchIndex(b.Snapshot()); err != nil { + index, err := sc.repo.GetIndex(sc.typename) + if err != nil { + return err + } + + err = index.IndexOne(e.Id().String(), sc.makeIndexData(e)) + if err != nil { return err } - // we only need to write the bug cache - return sc.Write() + return sc.write() } // evictIfNeeded will evict an entity from the cache if needed @@ -405,7 +488,8 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) evictIfNeeded() { continue } - b.Lock() + // TODO + // b.Lock() sc.lru.Remove(id) delete(sc.cached, id) |