From 4a341b5e1714a6a36ec7f5839a6a1b73571d4851 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 29 Nov 2022 13:01:53 +0100 Subject: WIP --- cache/bug_subcache.go | 25 -------------- cache/cached.go | 6 ++++ cache/filter.go | 4 +-- cache/identity_subcache.go | 79 ++++++++++++++++++++++++++++++++++++++++++++ cache/repo_cache.go | 75 +++++++++++++++++++++++------------------ cache/repo_cache_identity.go | 79 -------------------------------------------- cache/resolvers.go | 42 ----------------------- cache/subcache.go | 28 ++++++++++++---- 8 files changed, 151 insertions(+), 187 deletions(-) create mode 100644 cache/identity_subcache.go delete mode 100644 cache/repo_cache_identity.go delete mode 100644 cache/resolvers.go (limited to 'cache') diff --git a/cache/bug_subcache.go b/cache/bug_subcache.go index a0c8d84c..e61bbf2b 100644 --- a/cache/bug_subcache.go +++ b/cache/bug_subcache.go @@ -233,31 +233,6 @@ func (c *RepoCacheBug) NewRaw(author identity.Interface, unixTime int64, title s return cached, op, nil } -// Remove removes a bug from the cache and repo given a bug id prefix -func (c *RepoCacheBug) Remove(prefix string) error { - b, err := c.ResolveBugPrefix(prefix) - if err != nil { - return err - } - - c.muBug.Lock() - - err = bug.Remove(c.repo, b.Id()) - if err != nil { - c.muBug.Unlock() - - return err - } - - delete(c.bugs, b.Id()) - delete(c.bugExcerpts, b.Id()) - c.loadedBugs.Remove(b.Id()) - - c.muBug.Unlock() - - return c.writeBugCache() -} - func (c *RepoCacheBug) addBugToSearchIndex(snap *bug.Snapshot) error { searchableBug := struct { Text []string diff --git a/cache/cached.go b/cache/cached.go index 5e24e732..75ca58e5 100644 --- a/cache/cached.go +++ b/cache/cached.go @@ -104,6 +104,12 @@ func (e *CachedEntityBase[SnapT, OpT]) ResolveOperationWithMetadata(key string, return matching[0], nil } +func (e *CachedEntityBase[SnapT, OpT]) Validate() error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.entity.Validate() +} + func (e *CachedEntityBase[SnapT, OpT]) Commit() error { e.mu.Lock() err := e.entity.Commit(e.repo) diff --git a/cache/filter.go b/cache/filter.go index 299e7c83..01f635c5 100644 --- a/cache/filter.go +++ b/cache/filter.go @@ -9,7 +9,7 @@ import ( ) // resolver has the resolving functions needed by filters. -// This exist mainly to go through the functions of the cache with proper locking. +// This exists mainly to go through the functions of the cache with proper locking. type resolver interface { ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) } @@ -211,7 +211,7 @@ func (*Matcher) orMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver return match } -// Check if all of the filters provided match the bug +// Check if all the filters provided match the bug func (*Matcher) andMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { if len(filters) == 0 { return true diff --git a/cache/identity_subcache.go b/cache/identity_subcache.go new file mode 100644 index 00000000..a3b5a0f8 --- /dev/null +++ b/cache/identity_subcache.go @@ -0,0 +1,79 @@ +package cache + +import ( + "fmt" + + "github.com/MichaelMure/git-bug/entities/identity" +) + +type RepoCacheIdentity struct { + SubCache[*IdentityExcerpt, *IdentityCache, identity.Interface] +} + +// ResolveIdentityImmutableMetadata retrieve an Identity that has the exact given metadata on +// one of its version. If multiple version have the same key, the first defined take precedence. +func (c *RepoCacheIdentity) ResolveIdentityImmutableMetadata(key string, value string) (*IdentityCache, error) { + return c.ResolveMatcher(func(excerpt *IdentityExcerpt) bool { + return excerpt.ImmutableMetadata[key] == value + }) +} + +func (c *RepoCacheIdentity) NewIdentityFromGitUser() (*IdentityCache, error) { + return c.NewIdentityFromGitUserRaw(nil) +} + +func (c *RepoCacheIdentity) NewIdentityFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewFromGitUser(c.repo) + if err != nil { + return nil, err + } + return c.finishIdentity(i, metadata) +} + +// NewIdentity create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) NewIdentity(name string, email string) (*IdentityCache, error) { + return c.NewIdentityRaw(name, email, "", "", nil, nil) +} + +// NewIdentityFull create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) NewIdentityFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { + return c.NewIdentityRaw(name, email, login, avatarUrl, keys, nil) +} + +func (c *RepoCacheIdentity) NewIdentityRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) + if err != nil { + return nil, err + } + return c.finishIdentity(i, metadata) +} + +func (c *RepoCacheIdentity) finishIdentity(i *identity.Identity, metadata map[string]string) (*IdentityCache, error) { + for key, value := range metadata { + i.SetMetadata(key, value) + } + + err := i.Commit(c.repo) + if err != nil { + return nil, err + } + + c.mu.Lock() + if _, has := c.cached[i.Id()]; has { + return nil, fmt.Errorf("identity %s already exist in the cache", i.Id()) + } + + cached := NewIdentityCache(c, i) + c.cached[i.Id()] = cached + c.mu.Unlock() + + // force the write of the excerpt + err = c.entityUpdated(i.Id()) + if err != nil { + return nil, err + } + + return cached, nil +} diff --git a/cache/repo_cache.go b/cache/repo_cache.go index c1646d3b..9250bb40 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -6,7 +6,8 @@ import ( "io/ioutil" "os" "strconv" - "sync" + + "golang.org/x/sync/errgroup" "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entities/identity" @@ -52,13 +53,10 @@ type RepoCache struct { // resolvers for all known entities resolvers entity.Resolvers - bugs *RepoCacheBug + bugs *RepoCacheBug + identities *RepoCacheIdentity - muIdentity sync.RWMutex - // excerpt of identities data for all identities - identitiesExcerpts map[entity.Id]*IdentityExcerpt - // identities loaded in memory - identities map[entity.Id]*IdentityCache + subcaches []cacheMgmt // the user identity's id, if known userIdentityId entity.Id @@ -72,14 +70,20 @@ func NewNamedRepoCache(r repository.ClockedRepo, name string) (*RepoCache, error c := &RepoCache{ repo: r, name: name, - bugs: NewCache(r), - // maxLoadedBugs: defaultMaxLoadedBugs, - // bugs: make(map[entity.Id]*BugCache), - // loadedBugs: newLRUIdCache(), - // identities: make(map[entity.Id]*IdentityCache), } - c.resolvers = makeResolvers(c) + bugs := NewSubCache[*BugExcerpt, *BugCache, bug.Interface](r, + c.getResolvers, c.GetUserIdentity, + "bug", "bugs", + formatVersion, defaultMaxLoadedBugs) + + c.subcaches = append(c.subcaches, bugs) + c.bugs = &RepoCacheBug{SubCache: *bugs} + + c.resolvers = entity.Resolvers{ + &IdentityCache{}: entity.ResolverFunc((func(id entity.Id) (entity.Interface, error)(c.identities.Resolve)), + &BugCache{}: c.bugs, + } err := c.lock() if err != nil { @@ -105,6 +109,15 @@ func (c *RepoCache) Bugs() *RepoCacheBug { return c.bugs } +// Identities gives access to the Identity entities +func (c *RepoCache) Identities() *RepoCacheIdentity { + return c.identities +} + +func (c *RepoCache) getResolvers() entity.Resolvers { + return c.resolvers +} + // setCacheSize change the maximum number of loaded bugs func (c *RepoCache) setCacheSize(size int) { c.maxLoadedBugs = size @@ -113,21 +126,20 @@ func (c *RepoCache) setCacheSize(size int) { // load will try to read from the disk all the cache files func (c *RepoCache) load() error { - err := c.loadBugCache() - if err != nil { - return err + var errG errgroup.Group + for _, mgmt := range c.subcaches { + errG.Go(mgmt.Load) } - - return c.loadIdentityCache() + return errG.Wait() } // write will serialize on disk all the cache files func (c *RepoCache) write() error { - err := c.writeBugCache() - if err != nil { - return err + var errG errgroup.Group + for _, mgmt := range c.subcaches { + errG.Go(mgmt.Write) } - return c.writeIdentityCache() + return errG.Wait() } func (c *RepoCache) lock() error { @@ -151,17 +163,16 @@ func (c *RepoCache) lock() error { } func (c *RepoCache) Close() error { - c.muBug.Lock() - defer c.muBug.Unlock() - c.muIdentity.Lock() - defer c.muIdentity.Unlock() - - c.identities = make(map[entity.Id]*IdentityCache) - c.identitiesExcerpts = nil - c.bugs = make(map[entity.Id]*BugCache) - c.bugExcerpts = nil + var errG errgroup.Group + for _, mgmt := range c.subcaches { + errG.Go(mgmt.Close) + } + err := errG.Wait() + if err != nil { + return err + } - err := c.repo.Close() + err = c.repo.Close() if err != nil { return err } diff --git a/cache/repo_cache_identity.go b/cache/repo_cache_identity.go deleted file mode 100644 index a99c7687..00000000 --- a/cache/repo_cache_identity.go +++ /dev/null @@ -1,79 +0,0 @@ -package cache - -import ( - "fmt" - - "github.com/MichaelMure/git-bug/entities/identity" -) - -type RepoCacheIdentity struct { - SubCache[*IdentityExcerpt, *IdentityCache] -} - -// ResolveIdentityImmutableMetadata retrieve an Identity that has the exact given metadata on -// one of its version. If multiple version have the same key, the first defined take precedence. -func (c *RepoCacheIdentity) ResolveIdentityImmutableMetadata(key string, value string) (*IdentityCache, error) { - return c.ResolveMatcher(func(excerpt *IdentityExcerpt) bool { - return excerpt.ImmutableMetadata[key] == value - }) -} - -func (c *RepoCacheIdentity) NewIdentityFromGitUser() (*IdentityCache, error) { - return c.NewIdentityFromGitUserRaw(nil) -} - -func (c *RepoCacheIdentity) NewIdentityFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewFromGitUser(c.repo) - if err != nil { - return nil, err - } - return c.finishIdentity(i, metadata) -} - -// NewIdentity create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCacheIdentity) NewIdentity(name string, email string) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, "", "", nil, nil) -} - -// NewIdentityFull create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCacheIdentity) NewIdentityFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, login, avatarUrl, keys, nil) -} - -func (c *RepoCacheIdentity) NewIdentityRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) - if err != nil { - return nil, err - } - return c.finishIdentity(i, metadata) -} - -func (c *RepoCacheIdentity) finishIdentity(i *identity.Identity, metadata map[string]string) (*IdentityCache, error) { - for key, value := range metadata { - i.SetMetadata(key, value) - } - - err := i.Commit(c.repo) - if err != nil { - return nil, err - } - - c.mu.Lock() - if _, has := c.cached[i.Id()]; has { - return nil, fmt.Errorf("identity %s already exist in the cache", i.Id()) - } - - cached := NewIdentityCache(c, i) - c.cached[i.Id()] = cached - c.mu.Unlock() - - // force the write of the excerpt - err = c.entityUpdated(i.Id()) - if err != nil { - return nil, err - } - - return cached, nil -} diff --git a/cache/resolvers.go b/cache/resolvers.go deleted file mode 100644 index 9ed2fa4c..00000000 --- a/cache/resolvers.go +++ /dev/null @@ -1,42 +0,0 @@ -package cache - -import ( - "github.com/MichaelMure/git-bug/entity" -) - -func makeResolvers(cache *RepoCache) entity.Resolvers { - return entity.Resolvers{ - &IdentityCache{}: newIdentityCacheResolver(cache), - &BugCache{}: newBugCacheResolver(cache), - } -} - -var _ entity.Resolver = &identityCacheResolver{} - -// identityCacheResolver is an identity Resolver that retrieve identities from -// the cache -type identityCacheResolver struct { - cache *RepoCache -} - -func newIdentityCacheResolver(cache *RepoCache) *identityCacheResolver { - return &identityCacheResolver{cache: cache} -} - -func (i *identityCacheResolver) Resolve(id entity.Id) (entity.Interface, error) { - return i.cache.ResolveIdentity(id) -} - -var _ entity.Resolver = &bugCacheResolver{} - -type bugCacheResolver struct { - cache *RepoCache -} - -func newBugCacheResolver(cache *RepoCache) *bugCacheResolver { - return &bugCacheResolver{cache: cache} -} - -func (b *bugCacheResolver) Resolve(id entity.Id) (entity.Interface, error) { - return b.cache.ResolveBug(id) -} diff --git a/cache/subcache.go b/cache/subcache.go index 658781d9..66f72767 100644 --- a/cache/subcache.go +++ b/cache/subcache.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" "github.com/MichaelMure/git-bug/entities/bug" - "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" ) @@ -22,11 +21,18 @@ type CacheEntity interface { NeedCommit() bool } -type getUserIdentityFunc func() (identity.Interface, error) +type cacheMgmt interface { + Load() error + Write() error + Build() error + Close() error +} + +type getUserIdentityFunc func() (*IdentityCache, error) type SubCache[ExcerptT Excerpt, CacheT CacheEntity, EntityT entity.Interface] struct { repo repository.ClockedRepo - resolvers entity.Resolvers + resolvers func() entity.Resolvers getUserIdentity getUserIdentityFunc readWithResolver func(repository.ClockedRepo, entity.Resolvers, entity.Id) (EntityT, error) @@ -46,8 +52,8 @@ type SubCache[ExcerptT Excerpt, CacheT CacheEntity, EntityT entity.Interface] st func NewSubCache[ExcerptT Excerpt, CacheT CacheEntity, EntityT entity.Interface]( repo repository.ClockedRepo, - resolvers entity.Resolvers, - getUserIdentity func() (identity.Interface, error), + resolvers func() entity.Resolvers, + getUserIdentity getUserIdentityFunc, typename, namespace string, version uint, maxLoaded int) *SubCache[ExcerptT, CacheT, EntityT] { return &SubCache[ExcerptT, CacheT, EntityT]{ @@ -144,8 +150,16 @@ func (sc *SubCache[ExcerptT, CacheT, EntityT]) Write() error { return f.Close() } -func (sc *SubCache[ExcerptT, CacheT, EntityT]) Build() { +func (sc *SubCache[ExcerptT, CacheT, EntityT]) Build() error { + +} +func (sc *SubCache[ExcerptT, CacheT, EntityT]) Close() error { + sc.mu.Lock() + defer sc.mu.Unlock() + sc.excerpts = nil + sc.cached = make(map[entity.Id]CacheT) + return nil } // AllIds return all known bug ids @@ -175,7 +189,7 @@ func (sc *SubCache[ExcerptT, CacheT, EntityT]) Resolve(id entity.Id) (CacheT, er } sc.mu.RUnlock() - b, err := sc.readWithResolver(sc.repo, sc.resolvers, id) + b, err := sc.readWithResolver(sc.repo, sc.resolvers(), id) if err != nil { return nil, err } -- cgit