diff options
Diffstat (limited to 'cache')
| -rw-r--r-- | cache/bug_cache.go | 137 | ||||
| -rw-r--r-- | cache/bug_excerpt.go | 48 | ||||
| -rw-r--r-- | cache/bug_subcache.go | 254 | ||||
| -rw-r--r-- | cache/cached.go | 111 | ||||
| -rw-r--r-- | cache/filter.go | 58 | ||||
| -rw-r--r-- | cache/identity_cache.go | 32 | ||||
| -rw-r--r-- | cache/identity_excerpt.go | 21 | ||||
| -rw-r--r-- | cache/identity_subcache.go | 124 | ||||
| -rw-r--r-- | cache/lru_id_cache.go | 36 | ||||
| -rw-r--r-- | cache/multi_repo_cache.go | 26 | ||||
| -rw-r--r-- | cache/repo_cache.go | 232 | ||||
| -rw-r--r-- | cache/repo_cache_bug.go | 556 | ||||
| -rw-r--r-- | cache/repo_cache_common.go | 155 | ||||
| -rw-r--r-- | cache/repo_cache_identity.go | 271 | ||||
| -rw-r--r-- | cache/repo_cache_test.go | 170 | ||||
| -rw-r--r-- | cache/resolvers.go | 42 | ||||
| -rw-r--r-- | cache/subcache.go | 505 | ||||
| -rw-r--r-- | cache/with_snapshot.go | 56 |
18 files changed, 1489 insertions, 1345 deletions
diff --git a/cache/bug_cache.go b/cache/bug_cache.go index 65e2068f..3466f186 100644 --- a/cache/bug_cache.go +++ b/cache/bug_cache.go @@ -2,10 +2,10 @@ package cache import ( "fmt" - "sync" "time" "github.com/MichaelMure/git-bug/entities/bug" + "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/entity/dag" "github.com/MichaelMure/git-bug/repository" @@ -19,63 +19,26 @@ var ErrNoMatchingOp = fmt.Errorf("no matching operation found") // 2. Maintain an up-to-date Snapshot available. // 3. Deal with concurrency. type BugCache struct { - repoCache *RepoCache - mu sync.RWMutex - bug *bug.WithSnapshot + CachedEntityBase[*bug.Snapshot, bug.Operation] } -func NewBugCache(repoCache *RepoCache, b *bug.Bug) *BugCache { +func NewBugCache(b *bug.Bug, repo repository.ClockedRepo, getUserIdentity getUserIdentityFunc, entityUpdated func(id entity.Id) error) *BugCache { return &BugCache{ - repoCache: repoCache, - bug: &bug.WithSnapshot{Bug: b}, + CachedEntityBase: CachedEntityBase[*bug.Snapshot, bug.Operation]{ + repo: repo, + entityUpdated: entityUpdated, + getUserIdentity: getUserIdentity, + entity: &withSnapshot[*bug.Snapshot, bug.Operation]{Interface: b}, + }, } } -func (c *BugCache) Snapshot() *bug.Snapshot { - c.mu.RLock() - defer c.mu.RUnlock() - return c.bug.Compile() -} - -func (c *BugCache) Id() entity.Id { - return c.bug.Id() -} - -func (c *BugCache) notifyUpdated() error { - return c.repoCache.bugUpdated(c.bug.Id()) -} - -// ResolveOperationWithMetadata will find an operation that has the matching metadata -func (c *BugCache) ResolveOperationWithMetadata(key string, value string) (entity.Id, error) { - c.mu.RLock() - defer c.mu.RUnlock() - // preallocate but empty - matching := make([]entity.Id, 0, 5) - - for _, op := range c.bug.Operations() { - opValue, ok := op.GetMetadata(key) - if ok && value == opValue { - matching = append(matching, op.Id()) - } - } - - if len(matching) == 0 { - return "", ErrNoMatchingOp - } - - if len(matching) > 1 { - return "", bug.NewErrMultipleMatchOp(matching) - } - - return matching[0], nil -} - func (c *BugCache) AddComment(message string) (entity.CombinedId, *bug.AddCommentOperation, error) { return c.AddCommentWithFiles(message, nil) } func (c *BugCache) AddCommentWithFiles(message string, files []repository.Hash) (entity.CombinedId, *bug.AddCommentOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return entity.UnsetCombinedId, nil, err } @@ -83,9 +46,9 @@ func (c *BugCache) AddCommentWithFiles(message string, files []repository.Hash) return c.AddCommentRaw(author, time.Now().Unix(), message, files, nil) } -func (c *BugCache) AddCommentRaw(author *IdentityCache, unixTime int64, message string, files []repository.Hash, metadata map[string]string) (entity.CombinedId, *bug.AddCommentOperation, error) { +func (c *BugCache) AddCommentRaw(author identity.Interface, unixTime int64, message string, files []repository.Hash, metadata map[string]string) (entity.CombinedId, *bug.AddCommentOperation, error) { c.mu.Lock() - commentId, op, err := bug.AddComment(c.bug, author, unixTime, message, files, metadata) + commentId, op, err := bug.AddComment(c.entity, author, unixTime, message, files, metadata) c.mu.Unlock() if err != nil { return entity.UnsetCombinedId, nil, err @@ -94,7 +57,7 @@ func (c *BugCache) AddCommentRaw(author *IdentityCache, unixTime int64, message } func (c *BugCache) ChangeLabels(added []string, removed []string) ([]bug.LabelChangeResult, *bug.LabelChangeOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, nil, err } @@ -102,9 +65,9 @@ func (c *BugCache) ChangeLabels(added []string, removed []string) ([]bug.LabelCh return c.ChangeLabelsRaw(author, time.Now().Unix(), added, removed, nil) } -func (c *BugCache) ChangeLabelsRaw(author *IdentityCache, unixTime int64, added []string, removed []string, metadata map[string]string) ([]bug.LabelChangeResult, *bug.LabelChangeOperation, error) { +func (c *BugCache) ChangeLabelsRaw(author identity.Interface, unixTime int64, added []string, removed []string, metadata map[string]string) ([]bug.LabelChangeResult, *bug.LabelChangeOperation, error) { c.mu.Lock() - changes, op, err := bug.ChangeLabels(c.bug, author.Identity, unixTime, added, removed, metadata) + changes, op, err := bug.ChangeLabels(c.entity, author, unixTime, added, removed, metadata) c.mu.Unlock() if err != nil { return changes, nil, err @@ -113,7 +76,7 @@ func (c *BugCache) ChangeLabelsRaw(author *IdentityCache, unixTime int64, added } func (c *BugCache) ForceChangeLabels(added []string, removed []string) (*bug.LabelChangeOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -121,9 +84,9 @@ func (c *BugCache) ForceChangeLabels(added []string, removed []string) (*bug.Lab return c.ForceChangeLabelsRaw(author, time.Now().Unix(), added, removed, nil) } -func (c *BugCache) ForceChangeLabelsRaw(author *IdentityCache, unixTime int64, added []string, removed []string, metadata map[string]string) (*bug.LabelChangeOperation, error) { +func (c *BugCache) ForceChangeLabelsRaw(author identity.Interface, unixTime int64, added []string, removed []string, metadata map[string]string) (*bug.LabelChangeOperation, error) { c.mu.Lock() - op, err := bug.ForceChangeLabels(c.bug, author.Identity, unixTime, added, removed, metadata) + op, err := bug.ForceChangeLabels(c.entity, author, unixTime, added, removed, metadata) c.mu.Unlock() if err != nil { return nil, err @@ -132,7 +95,7 @@ func (c *BugCache) ForceChangeLabelsRaw(author *IdentityCache, unixTime int64, a } func (c *BugCache) Open() (*bug.SetStatusOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -140,9 +103,9 @@ func (c *BugCache) Open() (*bug.SetStatusOperation, error) { return c.OpenRaw(author, time.Now().Unix(), nil) } -func (c *BugCache) OpenRaw(author *IdentityCache, unixTime int64, metadata map[string]string) (*bug.SetStatusOperation, error) { +func (c *BugCache) OpenRaw(author identity.Interface, unixTime int64, metadata map[string]string) (*bug.SetStatusOperation, error) { c.mu.Lock() - op, err := bug.Open(c.bug, author.Identity, unixTime, metadata) + op, err := bug.Open(c.entity, author, unixTime, metadata) c.mu.Unlock() if err != nil { return nil, err @@ -151,7 +114,7 @@ func (c *BugCache) OpenRaw(author *IdentityCache, unixTime int64, metadata map[s } func (c *BugCache) Close() (*bug.SetStatusOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -159,9 +122,9 @@ func (c *BugCache) Close() (*bug.SetStatusOperation, error) { return c.CloseRaw(author, time.Now().Unix(), nil) } -func (c *BugCache) CloseRaw(author *IdentityCache, unixTime int64, metadata map[string]string) (*bug.SetStatusOperation, error) { +func (c *BugCache) CloseRaw(author identity.Interface, unixTime int64, metadata map[string]string) (*bug.SetStatusOperation, error) { c.mu.Lock() - op, err := bug.Close(c.bug, author.Identity, unixTime, metadata) + op, err := bug.Close(c.entity, author, unixTime, metadata) c.mu.Unlock() if err != nil { return nil, err @@ -170,7 +133,7 @@ func (c *BugCache) CloseRaw(author *IdentityCache, unixTime int64, metadata map[ } func (c *BugCache) SetTitle(title string) (*bug.SetTitleOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -178,9 +141,9 @@ func (c *BugCache) SetTitle(title string) (*bug.SetTitleOperation, error) { return c.SetTitleRaw(author, time.Now().Unix(), title, nil) } -func (c *BugCache) SetTitleRaw(author *IdentityCache, unixTime int64, title string, metadata map[string]string) (*bug.SetTitleOperation, error) { +func (c *BugCache) SetTitleRaw(author identity.Interface, unixTime int64, title string, metadata map[string]string) (*bug.SetTitleOperation, error) { c.mu.Lock() - op, err := bug.SetTitle(c.bug, author.Identity, unixTime, title, metadata) + op, err := bug.SetTitle(c.entity, author, unixTime, title, metadata) c.mu.Unlock() if err != nil { return nil, err @@ -190,7 +153,7 @@ func (c *BugCache) SetTitleRaw(author *IdentityCache, unixTime int64, title stri // EditCreateComment is a convenience function to edit the body of a bug (the first comment) func (c *BugCache) EditCreateComment(body string) (entity.CombinedId, *bug.EditCommentOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return entity.UnsetCombinedId, nil, err } @@ -199,9 +162,9 @@ func (c *BugCache) EditCreateComment(body string) (entity.CombinedId, *bug.EditC } // EditCreateCommentRaw is a convenience function to edit the body of a bug (the first comment) -func (c *BugCache) EditCreateCommentRaw(author *IdentityCache, unixTime int64, body string, metadata map[string]string) (entity.CombinedId, *bug.EditCommentOperation, error) { +func (c *BugCache) EditCreateCommentRaw(author identity.Interface, unixTime int64, body string, metadata map[string]string) (entity.CombinedId, *bug.EditCommentOperation, error) { c.mu.Lock() - commentId, op, err := bug.EditCreateComment(c.bug, author.Identity, unixTime, body, nil, metadata) + commentId, op, err := bug.EditCreateComment(c.entity, author, unixTime, body, nil, metadata) c.mu.Unlock() if err != nil { return entity.UnsetCombinedId, nil, err @@ -210,7 +173,7 @@ func (c *BugCache) EditCreateCommentRaw(author *IdentityCache, unixTime int64, b } func (c *BugCache) EditComment(target entity.CombinedId, message string) (*bug.EditCommentOperation, error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -218,14 +181,14 @@ func (c *BugCache) EditComment(target entity.CombinedId, message string) (*bug.E return c.EditCommentRaw(author, time.Now().Unix(), target, message, nil) } -func (c *BugCache) EditCommentRaw(author *IdentityCache, unixTime int64, target entity.CombinedId, message string, metadata map[string]string) (*bug.EditCommentOperation, error) { +func (c *BugCache) EditCommentRaw(author identity.Interface, unixTime int64, target entity.CombinedId, message string, metadata map[string]string) (*bug.EditCommentOperation, error) { comment, err := c.Snapshot().SearchComment(target) if err != nil { return nil, err } c.mu.Lock() - commentId, op, err := bug.EditComment(c.bug, author.Identity, unixTime, comment.TargetId(), message, nil, metadata) + commentId, op, err := bug.EditComment(c.entity, author, unixTime, comment.TargetId(), message, nil, metadata) c.mu.Unlock() if err != nil { return nil, err @@ -237,7 +200,7 @@ func (c *BugCache) EditCommentRaw(author *IdentityCache, unixTime int64, target } func (c *BugCache) SetMetadata(target entity.Id, newMetadata map[string]string) (*dag.SetMetadataOperation[*bug.Snapshot], error) { - author, err := c.repoCache.GetUserIdentity() + author, err := c.getUserIdentity() if err != nil { return nil, err } @@ -245,40 +208,12 @@ func (c *BugCache) SetMetadata(target entity.Id, newMetadata map[string]string) return c.SetMetadataRaw(author, time.Now().Unix(), target, newMetadata) } -func (c *BugCache) SetMetadataRaw(author *IdentityCache, unixTime int64, target entity.Id, newMetadata map[string]string) (*dag.SetMetadataOperation[*bug.Snapshot], error) { +func (c *BugCache) SetMetadataRaw(author identity.Interface, unixTime int64, target entity.Id, newMetadata map[string]string) (*dag.SetMetadataOperation[*bug.Snapshot], error) { c.mu.Lock() - op, err := bug.SetMetadata(c.bug, author.Identity, unixTime, target, newMetadata) + op, err := bug.SetMetadata(c.entity, author, unixTime, target, newMetadata) c.mu.Unlock() if err != nil { return nil, err } return op, c.notifyUpdated() } - -func (c *BugCache) Commit() error { - c.mu.Lock() - err := c.bug.Commit(c.repoCache.repo) - if err != nil { - c.mu.Unlock() - return err - } - c.mu.Unlock() - return c.notifyUpdated() -} - -func (c *BugCache) CommitAsNeeded() error { - c.mu.Lock() - err := c.bug.CommitAsNeeded(c.repoCache.repo) - if err != nil { - c.mu.Unlock() - return err - } - c.mu.Unlock() - return c.notifyUpdated() -} - -func (c *BugCache) NeedCommit() bool { - c.mu.RLock() - defer c.mu.RUnlock() - return c.bug.NeedCommit() -} diff --git a/cache/bug_excerpt.go b/cache/bug_excerpt.go index 7e3bcad4..26b7ec74 100644 --- a/cache/bug_excerpt.go +++ b/cache/bug_excerpt.go @@ -2,12 +2,10 @@ package cache import ( "encoding/gob" - "fmt" "time" "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entities/common" - "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/util/lamport" ) @@ -17,10 +15,12 @@ func init() { gob.Register(BugExcerpt{}) } +var _ Excerpt = &BugExcerpt{} + // BugExcerpt hold a subset of the bug values to be able to sort and filter bugs // efficiently without having to read and compile each raw bugs. type BugExcerpt struct { - Id entity.Id + id entity.Id CreateLamportTime lamport.Time EditLamportTime lamport.Time @@ -38,26 +38,8 @@ type BugExcerpt struct { CreateMetadata map[string]string } -// identity.Bare data are directly embedded in the bug excerpt -type LegacyAuthorExcerpt struct { - Name string - Login string -} - -func (l LegacyAuthorExcerpt) DisplayName() string { - switch { - case l.Name == "" && l.Login != "": - return l.Login - case l.Name != "" && l.Login == "": - return l.Name - case l.Name != "" && l.Login != "": - return fmt.Sprintf("%s (%s)", l.Name, l.Login) - } - - panic("invalid person data") -} - -func NewBugExcerpt(b bug.Interface, snap *bug.Snapshot) *BugExcerpt { +func NewBugExcerpt(b *BugCache) *BugExcerpt { + snap := b.Snapshot() participantsIds := make([]entity.Id, 0, len(snap.Participants)) for _, participant := range snap.Participants { participantsIds = append(participantsIds, participant.Id()) @@ -69,11 +51,12 @@ func NewBugExcerpt(b bug.Interface, snap *bug.Snapshot) *BugExcerpt { } e := &BugExcerpt{ - Id: b.Id(), + id: b.Id(), CreateLamportTime: b.CreateLamportTime(), EditLamportTime: b.EditLamportTime(), CreateUnixTime: b.FirstOp().Time().Unix(), EditUnixTime: snap.EditTime().Unix(), + AuthorId: snap.Author.Id(), Status: snap.Status, Labels: snap.Labels, Actors: actorsIds, @@ -83,16 +66,17 @@ func NewBugExcerpt(b bug.Interface, snap *bug.Snapshot) *BugExcerpt { CreateMetadata: b.FirstOp().AllMetadata(), } - switch snap.Author.(type) { - case *identity.Identity, *identity.IdentityStub, *IdentityCache: - e.AuthorId = snap.Author.Id() - default: - panic("unhandled identity type") - } - return e } +func (b *BugExcerpt) setId(id entity.Id) { + b.id = id +} + +func (b *BugExcerpt) Id() entity.Id { + return b.id +} + func (b *BugExcerpt) CreateTime() time.Time { return time.Unix(b.CreateUnixTime, 0) } @@ -112,7 +96,7 @@ func (b BugsById) Len() int { } func (b BugsById) Less(i, j int) bool { - return b[i].Id < b[j].Id + return b[i].id < b[j].id } func (b BugsById) Swap(i, j int) { diff --git a/cache/bug_subcache.go b/cache/bug_subcache.go new file mode 100644 index 00000000..920fe1dc --- /dev/null +++ b/cache/bug_subcache.go @@ -0,0 +1,254 @@ +package cache + +import ( + "errors" + "sort" + "time" + + "github.com/MichaelMure/git-bug/entities/bug" + "github.com/MichaelMure/git-bug/entities/identity" + "github.com/MichaelMure/git-bug/entity" + "github.com/MichaelMure/git-bug/query" + "github.com/MichaelMure/git-bug/repository" +) + +type RepoCacheBug struct { + *SubCache[*bug.Bug, *BugExcerpt, *BugCache] +} + +func NewRepoCacheBug(repo repository.ClockedRepo, + resolvers func() entity.Resolvers, + getUserIdentity getUserIdentityFunc) *RepoCacheBug { + + makeCached := func(b *bug.Bug, entityUpdated func(id entity.Id) error) *BugCache { + return NewBugCache(b, repo, getUserIdentity, entityUpdated) + } + + makeIndexData := func(b *BugCache) []string { + snap := b.Snapshot() + var res []string + for _, comment := range snap.Comments { + res = append(res, comment.Message) + } + res = append(res, snap.Title) + return res + } + + actions := Actions[*bug.Bug]{ + ReadWithResolver: bug.ReadWithResolver, + ReadAllWithResolver: bug.ReadAllWithResolver, + Remove: bug.Remove, + MergeAll: bug.MergeAll, + } + + sc := NewSubCache[*bug.Bug, *BugExcerpt, *BugCache]( + repo, resolvers, getUserIdentity, + makeCached, NewBugExcerpt, makeIndexData, actions, + bug.Typename, bug.Namespace, + formatVersion, defaultMaxLoadedBugs, + ) + + return &RepoCacheBug{SubCache: sc} +} + +// ResolveBugCreateMetadata retrieve a bug that has the exact given metadata on +// its Create operation, that is, the first operation. It fails if multiple bugs +// match. +func (c *RepoCacheBug) ResolveBugCreateMetadata(key string, value string) (*BugCache, error) { + return c.ResolveMatcher(func(excerpt *BugExcerpt) bool { + return excerpt.CreateMetadata[key] == value + }) +} + +// ResolveComment search for a Bug/Comment combination matching the merged +// bug/comment Id prefix. Returns the Bug containing the Comment and the Comment's +// Id. +func (c *RepoCacheBug) ResolveComment(prefix string) (*BugCache, entity.CombinedId, error) { + bugPrefix, _ := entity.SeparateIds(prefix) + bugCandidate := make([]entity.Id, 0, 5) + + // build a list of possible matching bugs + c.mu.RLock() + for _, excerpt := range c.excerpts { + if excerpt.Id().HasPrefix(bugPrefix) { + bugCandidate = append(bugCandidate, excerpt.Id()) + } + } + c.mu.RUnlock() + + matchingBugIds := make([]entity.Id, 0, 5) + matchingCommentId := entity.UnsetCombinedId + var matchingBug *BugCache + + // search for matching comments + // searching every bug candidate allow for some collision with the bug prefix only, + // before being refined with the full comment prefix + for _, bugId := range bugCandidate { + b, err := c.Resolve(bugId) + if err != nil { + return nil, entity.UnsetCombinedId, err + } + + for _, comment := range b.Snapshot().Comments { + if comment.CombinedId().HasPrefix(prefix) { + matchingBugIds = append(matchingBugIds, bugId) + matchingBug = b + matchingCommentId = comment.CombinedId() + } + } + } + + if len(matchingBugIds) > 1 { + return nil, entity.UnsetCombinedId, entity.NewErrMultipleMatch("bug/comment", matchingBugIds) + } else if len(matchingBugIds) == 0 { + return nil, entity.UnsetCombinedId, errors.New("comment doesn't exist") + } + + return matchingBug, matchingCommentId, nil +} + +// Query return the id of all Bug matching the given Query +func (c *RepoCacheBug) Query(q *query.Query) ([]entity.Id, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + if q == nil { + return c.AllIds(), nil + } + + matcher := compileMatcher(q.Filters) + + var filtered []*BugExcerpt + var foundBySearch map[entity.Id]*BugExcerpt + + if q.Search != nil { + foundBySearch = map[entity.Id]*BugExcerpt{} + + index, err := c.repo.GetIndex("bugs") + if err != nil { + return nil, err + } + + res, err := index.Search(q.Search) + if err != nil { + return nil, err + } + + for _, hit := range res { + id := entity.Id(hit) + foundBySearch[id] = c.excerpts[id] + } + } else { + foundBySearch = c.excerpts + } + + for _, excerpt := range foundBySearch { + if matcher.Match(excerpt, c.resolvers()) { + filtered = append(filtered, excerpt) + } + } + + var sorter sort.Interface + + switch q.OrderBy { + case query.OrderById: + sorter = BugsById(filtered) + case query.OrderByCreation: + sorter = BugsByCreationTime(filtered) + case query.OrderByEdit: + sorter = BugsByEditTime(filtered) + default: + return nil, errors.New("missing sort type") + } + + switch q.OrderDirection { + case query.OrderAscending: + // Nothing to do + case query.OrderDescending: + sorter = sort.Reverse(sorter) + default: + return nil, errors.New("missing sort direction") + } + + sort.Sort(sorter) + + result := make([]entity.Id, len(filtered)) + + for i, val := range filtered { + result[i] = val.Id() + } + + return result, nil +} + +// ValidLabels list valid labels +// +// Note: in the future, a proper label policy could be implemented where valid +// labels are defined in a configuration file. Until that, the default behavior +// is to return the list of labels already used. +func (c *RepoCacheBug) ValidLabels() []bug.Label { + c.mu.RLock() + defer c.mu.RUnlock() + + set := map[bug.Label]interface{}{} + + for _, excerpt := range c.excerpts { + for _, l := range excerpt.Labels { + set[l] = nil + } + } + + result := make([]bug.Label, len(set)) + + i := 0 + for l := range set { + result[i] = l + i++ + } + + // Sort + sort.Slice(result, func(i, j int) bool { + return string(result[i]) < string(result[j]) + }) + + return result +} + +// New create a new bug +// The new bug is written in the repository (commit) +func (c *RepoCacheBug) New(title string, message string) (*BugCache, *bug.CreateOperation, error) { + return c.NewWithFiles(title, message, nil) +} + +// NewWithFiles create a new bug with attached files for the message +// The new bug is written in the repository (commit) +func (c *RepoCacheBug) NewWithFiles(title string, message string, files []repository.Hash) (*BugCache, *bug.CreateOperation, error) { + author, err := c.getUserIdentity() + if err != nil { + return nil, nil, err + } + + return c.NewRaw(author, time.Now().Unix(), title, message, files, nil) +} + +// NewRaw create a new bug with attached files for the message, as +// well as metadata for the Create operation. +// The new bug is written in the repository (commit) +func (c *RepoCacheBug) NewRaw(author identity.Interface, unixTime int64, title string, message string, files []repository.Hash, metadata map[string]string) (*BugCache, *bug.CreateOperation, error) { + b, op, err := bug.Create(author, unixTime, title, message, files, metadata) + if err != nil { + return nil, nil, err + } + + err = b.Commit(c.repo) + if err != nil { + return nil, nil, err + } + + cached, err := c.add(b) + if err != nil { + return nil, nil, err + } + + return cached, op, nil +} diff --git a/cache/cached.go b/cache/cached.go new file mode 100644 index 00000000..9f9e170d --- /dev/null +++ b/cache/cached.go @@ -0,0 +1,111 @@ +package cache + +import ( + "sync" + + "github.com/MichaelMure/git-bug/entity" + "github.com/MichaelMure/git-bug/entity/dag" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/lamport" +) + +var _ CacheEntity = &CachedEntityBase[dag.Snapshot, dag.Operation]{} + +// CachedEntityBase provide the base function of an entity managed by the cache. +type CachedEntityBase[SnapT dag.Snapshot, OpT dag.Operation] struct { + repo repository.ClockedRepo + entityUpdated func(id entity.Id) error + getUserIdentity getUserIdentityFunc + + mu sync.RWMutex + entity dag.Interface[SnapT, OpT] +} + +func (e *CachedEntityBase[SnapT, OpT]) Id() entity.Id { + return e.entity.Id() +} + +func (e *CachedEntityBase[SnapT, OpT]) Snapshot() SnapT { + e.mu.RLock() + defer e.mu.RUnlock() + return e.entity.Compile() +} + +func (e *CachedEntityBase[SnapT, OpT]) notifyUpdated() error { + return e.entityUpdated(e.entity.Id()) +} + +// ResolveOperationWithMetadata will find an operation that has the matching metadata +func (e *CachedEntityBase[SnapT, OpT]) ResolveOperationWithMetadata(key string, value string) (entity.Id, error) { + e.mu.RLock() + defer e.mu.RUnlock() + // preallocate but empty + matching := make([]entity.Id, 0, 5) + + for _, op := range e.entity.Operations() { + opValue, ok := op.GetMetadata(key) + if ok && value == opValue { + matching = append(matching, op.Id()) + } + } + + if len(matching) == 0 { + return "", ErrNoMatchingOp + } + + if len(matching) > 1 { + return "", entity.NewErrMultipleMatch("operation", matching) + } + + return matching[0], nil +} + +func (e *CachedEntityBase[SnapT, OpT]) Validate() error { + e.mu.RLock() + defer e.mu.RUnlock() + return e.entity.Validate() +} + +func (e *CachedEntityBase[SnapT, OpT]) Commit() error { + e.mu.Lock() + err := e.entity.Commit(e.repo) + if err != nil { + e.mu.Unlock() + return err + } + e.mu.Unlock() + return e.notifyUpdated() +} + +func (e *CachedEntityBase[SnapT, OpT]) CommitAsNeeded() error { + e.mu.Lock() + err := e.entity.CommitAsNeeded(e.repo) + if err != nil { + e.mu.Unlock() + return err + } + e.mu.Unlock() + return e.notifyUpdated() +} + +func (e *CachedEntityBase[SnapT, OpT]) NeedCommit() bool { + e.mu.RLock() + defer e.mu.RUnlock() + return e.entity.NeedCommit() +} + +func (e *CachedEntityBase[SnapT, OpT]) Lock() { + e.mu.Lock() +} + +func (e *CachedEntityBase[SnapT, OpT]) CreateLamportTime() lamport.Time { + return e.entity.CreateLamportTime() +} + +func (e *CachedEntityBase[SnapT, OpT]) EditLamportTime() lamport.Time { + return e.entity.EditLamportTime() +} + +func (e *CachedEntityBase[SnapT, OpT]) FirstOp() OpT { + return e.entity.FirstOp() +} diff --git a/cache/filter.go b/cache/filter.go index 299e7c83..5a15e402 100644 --- a/cache/filter.go +++ b/cache/filter.go @@ -8,28 +8,22 @@ import ( "github.com/MichaelMure/git-bug/query" ) -// resolver has the resolving functions needed by filters. -// This exist mainly to go through the functions of the cache with proper locking. -type resolver interface { - ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) -} - // Filter is a predicate that match a subset of bugs -type Filter func(excerpt *BugExcerpt, resolver resolver) bool +type Filter func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool // StatusFilter return a Filter that match a bug status func StatusFilter(status common.Status) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return excerpt.Status == status } } // AuthorFilter return a Filter that match a bug author func AuthorFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) - author, err := resolver.ResolveIdentityExcerpt(excerpt.AuthorId) + author, err := entity.Resolve[*IdentityExcerpt](resolvers, excerpt.AuthorId) if err != nil { panic(err) } @@ -40,7 +34,7 @@ func AuthorFilter(query string) Filter { // MetadataFilter return a Filter that match a bug metadata at creation time func MetadataFilter(pair query.StringPair) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if value, ok := excerpt.CreateMetadata[pair.Key]; ok { return value == pair.Value } @@ -50,7 +44,7 @@ func MetadataFilter(pair query.StringPair) Filter { // LabelFilter return a Filter that match a label func LabelFilter(label string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { for _, l := range excerpt.Labels { if string(l) == label { return true @@ -62,11 +56,11 @@ func LabelFilter(label string) Filter { // ActorFilter return a Filter that match a bug actor func ActorFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) for _, id := range excerpt.Actors { - identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + identityExcerpt, err := entity.Resolve[*IdentityExcerpt](resolvers, id) if err != nil { panic(err) } @@ -81,11 +75,11 @@ func ActorFilter(query string) Filter { // ParticipantFilter return a Filter that match a bug participant func ParticipantFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { query = strings.ToLower(query) for _, id := range excerpt.Participants { - identityExcerpt, err := resolver.ResolveIdentityExcerpt(id) + identityExcerpt, err := entity.Resolve[*IdentityExcerpt](resolvers, id) if err != nil { panic(err) } @@ -100,7 +94,7 @@ func ParticipantFilter(query string) Filter { // TitleFilter return a Filter that match if the title contains the given query func TitleFilter(query string) Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return strings.Contains( strings.ToLower(excerpt.Title), strings.ToLower(query), @@ -110,7 +104,7 @@ func TitleFilter(query string) Filter { // NoLabelFilter return a Filter that match the absence of labels func NoLabelFilter() Filter { - return func(excerpt *BugExcerpt, resolver resolver) bool { + return func(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { return len(excerpt.Labels) == 0 } } @@ -161,36 +155,36 @@ func compileMatcher(filters query.Filters) *Matcher { } // Match check if a bug match the set of filters -func (f *Matcher) Match(excerpt *BugExcerpt, resolver resolver) bool { - if match := f.orMatch(f.Status, excerpt, resolver); !match { +func (f *Matcher) Match(excerpt *BugExcerpt, resolvers entity.Resolvers) bool { + if match := f.orMatch(f.Status, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Author, excerpt, resolver); !match { + if match := f.orMatch(f.Author, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Metadata, excerpt, resolver); !match { + if match := f.orMatch(f.Metadata, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Participant, excerpt, resolver); !match { + if match := f.orMatch(f.Participant, excerpt, resolvers); !match { return false } - if match := f.orMatch(f.Actor, excerpt, resolver); !match { + if match := f.orMatch(f.Actor, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.Label, excerpt, resolver); !match { + if match := f.andMatch(f.Label, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.NoFilters, excerpt, resolver); !match { + if match := f.andMatch(f.NoFilters, excerpt, resolvers); !match { return false } - if match := f.andMatch(f.Title, excerpt, resolver); !match { + if match := f.andMatch(f.Title, excerpt, resolvers); !match { return false } @@ -198,28 +192,28 @@ func (f *Matcher) Match(excerpt *BugExcerpt, resolver resolver) bool { } // Check if any of the filters provided match the bug -func (*Matcher) orMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { +func (*Matcher) orMatch(filters []Filter, excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if len(filters) == 0 { return true } match := false for _, f := range filters { - match = match || f(excerpt, resolver) + match = match || f(excerpt, resolvers) } return match } -// Check if all of the filters provided match the bug -func (*Matcher) andMatch(filters []Filter, excerpt *BugExcerpt, resolver resolver) bool { +// Check if all the filters provided match the bug +func (*Matcher) andMatch(filters []Filter, excerpt *BugExcerpt, resolvers entity.Resolvers) bool { if len(filters) == 0 { return true } match := true for _, f := range filters { - match = match && f(excerpt, resolver) + match = match && f(excerpt, resolvers) } return match diff --git a/cache/identity_cache.go b/cache/identity_cache.go index 3b7bb818..466b6150 100644 --- a/cache/identity_cache.go +++ b/cache/identity_cache.go @@ -1,31 +1,41 @@ package cache import ( + "sync" + "github.com/MichaelMure/git-bug/entities/identity" + "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" ) var _ identity.Interface = &IdentityCache{} +var _ CacheEntity = &IdentityCache{} // IdentityCache is a wrapper around an Identity for caching. type IdentityCache struct { + repo repository.ClockedRepo + entityUpdated func(id entity.Id) error + + mu sync.Mutex *identity.Identity - repoCache *RepoCache } -func NewIdentityCache(repoCache *RepoCache, id *identity.Identity) *IdentityCache { +func NewIdentityCache(i *identity.Identity, repo repository.ClockedRepo, entityUpdated func(id entity.Id) error) *IdentityCache { return &IdentityCache{ - Identity: id, - repoCache: repoCache, + repo: repo, + entityUpdated: entityUpdated, + Identity: i, } } func (i *IdentityCache) notifyUpdated() error { - return i.repoCache.identityUpdated(i.Identity.Id()) + return i.entityUpdated(i.Identity.Id()) } func (i *IdentityCache) Mutate(repo repository.RepoClock, f func(*identity.Mutator)) error { + i.mu.Lock() err := i.Identity.Mutate(repo, f) + i.mu.Unlock() if err != nil { return err } @@ -33,7 +43,9 @@ func (i *IdentityCache) Mutate(repo repository.RepoClock, f func(*identity.Mutat } func (i *IdentityCache) Commit() error { - err := i.Identity.Commit(i.repoCache.repo) + i.mu.Lock() + err := i.Identity.Commit(i.repo) + i.mu.Unlock() if err != nil { return err } @@ -41,9 +53,15 @@ func (i *IdentityCache) Commit() error { } func (i *IdentityCache) CommitAsNeeded() error { - err := i.Identity.CommitAsNeeded(i.repoCache.repo) + i.mu.Lock() + err := i.Identity.CommitAsNeeded(i.repo) + i.mu.Unlock() if err != nil { return err } return i.notifyUpdated() } + +func (i *IdentityCache) Lock() { + i.mu.Lock() +} diff --git a/cache/identity_excerpt.go b/cache/identity_excerpt.go index 0166f493..79d88537 100644 --- a/cache/identity_excerpt.go +++ b/cache/identity_excerpt.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" ) @@ -14,26 +13,36 @@ func init() { gob.Register(IdentityExcerpt{}) } +var _ Excerpt = &IdentityExcerpt{} + // IdentityExcerpt hold a subset of the identity values to be able to sort and // filter identities efficiently without having to read and compile each raw // identity. type IdentityExcerpt struct { - Id entity.Id + id entity.Id Name string Login string ImmutableMetadata map[string]string } -func NewIdentityExcerpt(i *identity.Identity) *IdentityExcerpt { +func NewIdentityExcerpt(i *IdentityCache) *IdentityExcerpt { return &IdentityExcerpt{ - Id: i.Id(), + id: i.Id(), Name: i.Name(), Login: i.Login(), ImmutableMetadata: i.ImmutableMetadata(), } } +func (i *IdentityExcerpt) setId(id entity.Id) { + i.id = id +} + +func (i *IdentityExcerpt) Id() entity.Id { + return i.id +} + // DisplayName return a non-empty string to display, representing the // identity, based on the non-empty values. func (i *IdentityExcerpt) DisplayName() string { @@ -51,7 +60,7 @@ func (i *IdentityExcerpt) DisplayName() string { // Match matches a query with the identity name, login and ID prefixes func (i *IdentityExcerpt) Match(query string) bool { - return i.Id.HasPrefix(query) || + return i.id.HasPrefix(query) || strings.Contains(strings.ToLower(i.Name), query) || strings.Contains(strings.ToLower(i.Login), query) } @@ -67,7 +76,7 @@ func (b IdentityById) Len() int { } func (b IdentityById) Less(i, j int) bool { - return b[i].Id < b[j].Id + return b[i].id < b[j].id } func (b IdentityById) Swap(i, j int) { diff --git a/cache/identity_subcache.go b/cache/identity_subcache.go new file mode 100644 index 00000000..f862ca8b --- /dev/null +++ b/cache/identity_subcache.go @@ -0,0 +1,124 @@ +package cache + +import ( + "fmt" + + "github.com/MichaelMure/git-bug/entities/identity" + "github.com/MichaelMure/git-bug/entity" + "github.com/MichaelMure/git-bug/repository" +) + +type RepoCacheIdentity struct { + *SubCache[*identity.Identity, *IdentityExcerpt, *IdentityCache] +} + +func NewRepoCacheIdentity(repo repository.ClockedRepo, + resolvers func() entity.Resolvers, + getUserIdentity getUserIdentityFunc) *RepoCacheIdentity { + + makeCached := func(i *identity.Identity, entityUpdated func(id entity.Id) error) *IdentityCache { + return NewIdentityCache(i, repo, entityUpdated) + } + + makeIndex := func(i *IdentityCache) []string { + // no indexing + return nil + } + + // TODO: this is terribly ugly, but we are currently stuck with the fact that identities are NOT using the fancy dag framework. + // This lead to various complication here and there to handle entities generically, and avoid large code duplication. + // TL;DR: something has to give, and this is the less ugly solution I found. This "normalize" identities as just another "dag framework" + // entity. Ideally identities would be converted to the dag framework, but right now that could lead to potential attack: if an old + // private key is leaked, it would be possible to craft a legal identity update that take over the most recent version. While this is + // meaningless in the case of a normal entity, it's really an issues for identities. + + actions := Actions[*identity.Identity]{ + ReadWithResolver: func(repo repository.ClockedRepo, resolvers entity.Resolvers, id entity.Id) (*identity.Identity, error) { + return identity.ReadLocal(repo, id) + }, + ReadAllWithResolver: func(repo repository.ClockedRepo, resolvers entity.Resolvers) <-chan entity.StreamedEntity[*identity.Identity] { + return identity.ReadAllLocal(repo) + }, + Remove: identity.RemoveIdentity, + MergeAll: func(repo repository.ClockedRepo, resolvers entity.Resolvers, remote string, mergeAuthor identity.Interface) <-chan entity.MergeResult { + return identity.MergeAll(repo, remote) + }, + } + + sc := NewSubCache[*identity.Identity, *IdentityExcerpt, *IdentityCache]( + repo, resolvers, getUserIdentity, + makeCached, NewIdentityExcerpt, makeIndex, actions, + identity.Typename, identity.Namespace, + formatVersion, defaultMaxLoadedBugs, + ) + + return &RepoCacheIdentity{SubCache: sc} +} + +// ResolveIdentityImmutableMetadata retrieve an Identity that has the exact given metadata on +// one of its version. If multiple version have the same key, the first defined take precedence. +func (c *RepoCacheIdentity) ResolveIdentityImmutableMetadata(key string, value string) (*IdentityCache, error) { + return c.ResolveMatcher(func(excerpt *IdentityExcerpt) bool { + return excerpt.ImmutableMetadata[key] == value + }) +} + +// New create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) New(name string, email string) (*IdentityCache, error) { + return c.NewRaw(name, email, "", "", nil, nil) +} + +// NewFull create a new identity +// The new identity is written in the repository (commit) +func (c *RepoCacheIdentity) NewFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { + return c.NewRaw(name, email, login, avatarUrl, keys, nil) +} + +func (c *RepoCacheIdentity) NewRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) + if err != nil { + return nil, err + } + return c.finishIdentity(i, metadata) +} + +func (c *RepoCacheIdentity) NewFromGitUser() (*IdentityCache, error) { + return c.NewFromGitUserRaw(nil) +} + +func (c *RepoCacheIdentity) NewFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { + i, err := identity.NewFromGitUser(c.repo) + if err != nil { + return nil, err + } + return c.finishIdentity(i, metadata) +} + +func (c *RepoCacheIdentity) finishIdentity(i *identity.Identity, metadata map[string]string) (*IdentityCache, error) { + for key, value := range metadata { + i.SetMetadata(key, value) + } + + err := i.Commit(c.repo) + if err != nil { + return nil, err + } + + c.mu.Lock() + if _, has := c.cached[i.Id()]; has { + return nil, fmt.Errorf("identity %s already exist in the cache", i.Id()) + } + + cached := NewIdentityCache(i, c.repo, c.entityUpdated) + c.cached[i.Id()] = cached + c.mu.Unlock() + + // force the write of the excerpt + err = c.entityUpdated(i.Id()) + if err != nil { + return nil, err + } + + return cached, nil +} diff --git a/cache/lru_id_cache.go b/cache/lru_id_cache.go index fda12ca6..0e5e31a7 100644 --- a/cache/lru_id_cache.go +++ b/cache/lru_id_cache.go @@ -8,49 +8,49 @@ import ( "github.com/MichaelMure/git-bug/entity" ) -type LRUIdCache struct { - parentCache *lru.Cache +type lruIdCache struct { + lru *lru.Cache } -func NewLRUIdCache() *LRUIdCache { +func newLRUIdCache() *lruIdCache { // we can ignore the error here as it would only fail if the size is negative. cache, _ := lru.New(math.MaxInt32) - return &LRUIdCache{ + return &lruIdCache{ cache, } } -func (c *LRUIdCache) Add(id entity.Id) bool { - return c.parentCache.Add(id, nil) +func (c *lruIdCache) Add(id entity.Id) bool { + return c.lru.Add(id, nil) } -func (c *LRUIdCache) Contains(id entity.Id) bool { - return c.parentCache.Contains(id) +func (c *lruIdCache) Contains(id entity.Id) bool { + return c.lru.Contains(id) } -func (c *LRUIdCache) Get(id entity.Id) bool { - _, present := c.parentCache.Get(id) +func (c *lruIdCache) Get(id entity.Id) bool { + _, present := c.lru.Get(id) return present } -func (c *LRUIdCache) GetOldest() (entity.Id, bool) { - id, _, present := c.parentCache.GetOldest() +func (c *lruIdCache) GetOldest() (entity.Id, bool) { + id, _, present := c.lru.GetOldest() return id.(entity.Id), present } -func (c *LRUIdCache) GetOldestToNewest() (ids []entity.Id) { - interfaceKeys := c.parentCache.Keys() +func (c *lruIdCache) GetOldestToNewest() (ids []entity.Id) { + interfaceKeys := c.lru.Keys() for _, id := range interfaceKeys { ids = append(ids, id.(entity.Id)) } return } -func (c *LRUIdCache) Len() int { - return c.parentCache.Len() +func (c *lruIdCache) Len() int { + return c.lru.Len() } -func (c *LRUIdCache) Remove(id entity.Id) bool { - return c.parentCache.Remove(id) +func (c *lruIdCache) Remove(id entity.Id) bool { + return c.lru.Remove(id) } diff --git a/cache/multi_repo_cache.go b/cache/multi_repo_cache.go index 659cd5e6..007737ad 100644 --- a/cache/multi_repo_cache.go +++ b/cache/multi_repo_cache.go @@ -21,25 +21,25 @@ func NewMultiRepoCache() *MultiRepoCache { } // RegisterRepository register a named repository. Use this for multi-repo setup -func (c *MultiRepoCache) RegisterRepository(ref string, repo repository.ClockedRepo) (*RepoCache, error) { - r, err := NewRepoCache(repo) +func (c *MultiRepoCache) RegisterRepository(name string, repo repository.ClockedRepo) (*RepoCache, chan BuildEvent, error) { + r, events, err := NewNamedRepoCache(repo, name) if err != nil { - return nil, err + return nil, nil, err } - c.repos[ref] = r - return r, nil + c.repos[name] = r + return r, events, nil } -// RegisterDefaultRepository register a unnamed repository. Use this for mono-repo setup -func (c *MultiRepoCache) RegisterDefaultRepository(repo repository.ClockedRepo) (*RepoCache, error) { - r, err := NewRepoCache(repo) +// RegisterDefaultRepository register an unnamed repository. Use this for mono-repo setup +func (c *MultiRepoCache) RegisterDefaultRepository(repo repository.ClockedRepo) (*RepoCache, chan BuildEvent, error) { + r, events, err := NewRepoCache(repo) if err != nil { - return nil, err + return nil, nil, err } c.repos[defaultRepoName] = r - return r, nil + return r, events, nil } // DefaultRepo retrieve the default repository @@ -55,9 +55,9 @@ func (c *MultiRepoCache) DefaultRepo() (*RepoCache, error) { panic("unreachable") } -// ResolveRepo retrieve a repository with a reference -func (c *MultiRepoCache) ResolveRepo(ref string) (*RepoCache, error) { - r, ok := c.repos[ref] +// ResolveRepo retrieve a repository by name +func (c *MultiRepoCache) ResolveRepo(name string) (*RepoCache, error) { + r, ok := c.repos[name] if !ok { return nil, fmt.Errorf("unknown repo") } diff --git a/cache/repo_cache.go b/cache/repo_cache.go index 71abf968..7852ec7d 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -8,10 +8,9 @@ import ( "strconv" "sync" - "github.com/MichaelMure/git-bug/entities/bug" - "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/multierr" "github.com/MichaelMure/git-bug/util/process" ) @@ -28,6 +27,17 @@ var _ repository.RepoCommon = &RepoCache{} var _ repository.RepoConfig = &RepoCache{} var _ repository.RepoKeyring = &RepoCache{} +// cacheMgmt is the expected interface for a sub-cache. +type cacheMgmt interface { + Typename() string + Load() error + Build() error + SetCacheSize(size int) + MergeAll(remote string) <-chan entity.MergeResult + GetNamespace() string + Close() error +} + // RepoCache is a cache for a Repository. This cache has multiple functions: // // 1. After being loaded, a Bug is kept in memory in the cache, allowing for fast @@ -49,88 +59,109 @@ type RepoCache struct { // the name of the repository, as defined in the MultiRepoCache name string - // resolvers for all known entities + // resolvers for all known entities and excerpts resolvers entity.Resolvers - // maximum number of loaded bugs - maxLoadedBugs int + bugs *RepoCacheBug + identities *RepoCacheIdentity - muBug sync.RWMutex - // excerpt of bugs data for all bugs - bugExcerpts map[entity.Id]*BugExcerpt - // bug loaded in memory - bugs map[entity.Id]*BugCache - // loadedBugs is an LRU cache that records which bugs the cache has loaded in - loadedBugs *LRUIdCache - - muIdentity sync.RWMutex - // excerpt of identities data for all identities - identitiesExcerpts map[entity.Id]*IdentityExcerpt - // identities loaded in memory - identities map[entity.Id]*IdentityCache + subcaches []cacheMgmt // the user identity's id, if known + muUserIdentity sync.RWMutex userIdentityId entity.Id } -func NewRepoCache(r repository.ClockedRepo) (*RepoCache, error) { +// NewRepoCache create or open an unnamed (aka default) cache on top of a raw repository. +// If the returned BuildEvent channel is not nil, the caller is expected to read all events before the cache is considered +// ready to use. +func NewRepoCache(r repository.ClockedRepo) (*RepoCache, chan BuildEvent, error) { return NewNamedRepoCache(r, "") } -func NewNamedRepoCache(r repository.ClockedRepo, name string) (*RepoCache, error) { +// NewNamedRepoCache create or open a named cache on top of a raw repository. +// If the returned BuildEvent channel is not nil, the caller is expected to read all events before the cache is considered +// ready to use. +func NewNamedRepoCache(r repository.ClockedRepo, name string) (*RepoCache, chan BuildEvent, error) { c := &RepoCache{ - repo: r, - name: name, - maxLoadedBugs: defaultMaxLoadedBugs, - bugs: make(map[entity.Id]*BugCache), - loadedBugs: NewLRUIdCache(), - identities: make(map[entity.Id]*IdentityCache), + repo: r, + name: name, } - c.resolvers = makeResolvers(c) + c.identities = NewRepoCacheIdentity(r, c.getResolvers, c.GetUserIdentity) + c.subcaches = append(c.subcaches, c.identities) + + c.bugs = NewRepoCacheBug(r, c.getResolvers, c.GetUserIdentity) + c.subcaches = append(c.subcaches, c.bugs) + + c.resolvers = entity.Resolvers{ + &IdentityCache{}: entity.ResolverFunc[*IdentityCache](c.identities.Resolve), + &IdentityExcerpt{}: entity.ResolverFunc[*IdentityExcerpt](c.identities.ResolveExcerpt), + &BugCache{}: entity.ResolverFunc[*BugCache](c.bugs.Resolve), + &BugExcerpt{}: entity.ResolverFunc[*BugExcerpt](c.bugs.ResolveExcerpt), + } err := c.lock() if err != nil { - return &RepoCache{}, err + return &RepoCache{}, nil, err } err = c.load() if err == nil { - return c, nil + return c, nil, nil } // Cache is either missing, broken or outdated. Rebuilding. - err = c.buildCache() + events := c.buildCache() + + return c, events, nil +} + +func NewRepoCacheNoEvents(r repository.ClockedRepo) (*RepoCache, error) { + cache, events, err := NewRepoCache(r) if err != nil { return nil, err } + if events != nil { + for event := range events { + if event.Err != nil { + for range events { + } + return nil, err + } + } + } + return cache, nil +} + +// Bugs gives access to the Bug entities +func (c *RepoCache) Bugs() *RepoCacheBug { + return c.bugs +} + +// Identities gives access to the Identity entities +func (c *RepoCache) Identities() *RepoCacheIdentity { + return c.identities +} - return c, c.write() +func (c *RepoCache) getResolvers() entity.Resolvers { + return c.resolvers } // setCacheSize change the maximum number of loaded bugs func (c *RepoCache) setCacheSize(size int) { - c.maxLoadedBugs = size - c.evictIfNeeded() + for _, subcache := range c.subcaches { + subcache.SetCacheSize(size) + } } // load will try to read from the disk all the cache files func (c *RepoCache) load() error { - err := c.loadBugCache() - if err != nil { - return err + var errWait multierr.ErrWaitGroup + for _, mgmt := range c.subcaches { + errWait.Go(mgmt.Load) } - - return c.loadIdentityCache() -} - -// write will serialize on disk all the cache files -func (c *RepoCache) write() error { - err := c.writeBugCache() - if err != nil { - return err - } - return c.writeIdentityCache() + return errWait.Wait() } func (c *RepoCache) lock() error { @@ -154,17 +185,16 @@ func (c *RepoCache) lock() error { } func (c *RepoCache) Close() error { - c.muBug.Lock() - defer c.muBug.Unlock() - c.muIdentity.Lock() - defer c.muIdentity.Unlock() - - c.identities = make(map[entity.Id]*IdentityCache) - c.identitiesExcerpts = nil - c.bugs = make(map[entity.Id]*BugCache) - c.bugExcerpts = nil + var errWait multierr.ErrWaitGroup + for _, mgmt := range c.subcaches { + errWait.Go(mgmt.Close) + } + err := errWait.Wait() + if err != nil { + return err + } - err := c.repo.Close() + err = c.repo.Close() if err != nil { return err } @@ -172,51 +202,59 @@ func (c *RepoCache) Close() error { return c.repo.LocalStorage().Remove(lockfile) } -func (c *RepoCache) buildCache() error { - _, _ = fmt.Fprintf(os.Stderr, "Building identity cache... ") - - c.identitiesExcerpts = make(map[entity.Id]*IdentityExcerpt) - - allIdentities := identity.ReadAllLocal(c.repo) - - for i := range allIdentities { - if i.Err != nil { - return i.Err - } - - c.identitiesExcerpts[i.Identity.Id()] = NewIdentityExcerpt(i.Identity) - } - - _, _ = fmt.Fprintln(os.Stderr, "Done.") - - _, _ = fmt.Fprintf(os.Stderr, "Building bug cache... ") - - c.bugExcerpts = make(map[entity.Id]*BugExcerpt) +type BuildEventType int - allBugs := bug.ReadAllWithResolver(c.repo, c.resolvers) - - // wipe the index just to be sure - err := c.repo.ClearBleveIndex("bug") - if err != nil { - return err - } - - for b := range allBugs { - if b.Err != nil { - return b.Err - } +const ( + _ BuildEventType = iota + BuildEventStarted + BuildEventFinished +) - snap := b.Bug.Compile() - c.bugExcerpts[b.Bug.Id()] = NewBugExcerpt(b.Bug, snap) +// BuildEvent carry an event happening during the cache build process. +type BuildEvent struct { + // Err carry an error if the build process failed. If set, no other field matter. + Err error + // Typename is the name of the entity of which the event relate to. + Typename string + // Event is the type of the event. + Event BuildEventType +} - if err := c.addBugToSearchIndex(snap); err != nil { - return err +func (c *RepoCache) buildCache() chan BuildEvent { + out := make(chan BuildEvent) + + go func() { + defer close(out) + + var wg sync.WaitGroup + for _, subcache := range c.subcaches { + wg.Add(1) + go func(subcache cacheMgmt) { + defer wg.Done() + out <- BuildEvent{ + Typename: subcache.Typename(), + Event: BuildEventStarted, + } + + err := subcache.Build() + if err != nil { + out <- BuildEvent{ + Typename: subcache.Typename(), + Err: err, + } + return + } + + out <- BuildEvent{ + Typename: subcache.Typename(), + Event: BuildEventFinished, + } + }(subcache) } - } - - _, _ = fmt.Fprintln(os.Stderr, "Done.") + wg.Wait() + }() - return nil + return out } // repoIsAvailable check is the given repository is locked by a Cache. diff --git a/cache/repo_cache_bug.go b/cache/repo_cache_bug.go deleted file mode 100644 index 2992421c..00000000 --- a/cache/repo_cache_bug.go +++ /dev/null @@ -1,556 +0,0 @@ -package cache - -import ( - "bytes" - "encoding/gob" - "errors" - "fmt" - "sort" - "strings" - "time" - "unicode/utf8" - - "github.com/blevesearch/bleve" - - "github.com/MichaelMure/git-bug/entities/bug" - "github.com/MichaelMure/git-bug/entity" - "github.com/MichaelMure/git-bug/query" - "github.com/MichaelMure/git-bug/repository" -) - -const bugCacheFile = "bug-cache" - -var errBugNotInCache = errors.New("bug missing from cache") - -// bugUpdated is a callback to trigger when the excerpt of a bug changed, -// that is each time a bug is updated -func (c *RepoCache) bugUpdated(id entity.Id) error { - c.muBug.Lock() - b, ok := c.bugs[id] - if !ok { - c.muBug.Unlock() - - // if the bug is not loaded at this point, it means it was loaded before - // but got evicted. Which means we potentially have multiple copies in - // memory and thus concurrent write. - // Failing immediately here is the simple and safe solution to avoid - // complicated data loss. - return errBugNotInCache - } - c.loadedBugs.Get(id) - c.bugExcerpts[id] = NewBugExcerpt(b.bug, b.Snapshot()) - c.muBug.Unlock() - - if err := c.addBugToSearchIndex(b.Snapshot()); err != nil { - return err - } - - // we only need to write the bug cache - return c.writeBugCache() -} - -// load will try to read from the disk the bug cache file -func (c *RepoCache) loadBugCache() error { - c.muBug.Lock() - defer c.muBug.Unlock() - - f, err := c.repo.LocalStorage().Open(bugCacheFile) - if err != nil { - return err - } - - decoder := gob.NewDecoder(f) - - aux := struct { - Version uint - Excerpts map[entity.Id]*BugExcerpt - }{} - - err = decoder.Decode(&aux) - if err != nil { - return err - } - - if aux.Version != formatVersion { - return fmt.Errorf("unknown cache format version %v", aux.Version) - } - - c.bugExcerpts = aux.Excerpts - - index, err := c.repo.GetBleveIndex("bug") - if err != nil { - return err - } - - // simple heuristic to detect a mismatch between the index and the bugs - count, err := index.DocCount() - if err != nil { - return err - } - if count != uint64(len(c.bugExcerpts)) { - return fmt.Errorf("count mismatch between bleve and bug excerpts") - } - - return nil -} - -// write will serialize on disk the bug cache file -func (c *RepoCache) writeBugCache() error { - c.muBug.RLock() - defer c.muBug.RUnlock() - - var data bytes.Buffer - - aux := struct { - Version uint - Excerpts map[entity.Id]*BugExcerpt - }{ - Version: formatVersion, - Excerpts: c.bugExcerpts, - } - - encoder := gob.NewEncoder(&data) - - err := encoder.Encode(aux) - if err != nil { - return err - } - - f, err := c.repo.LocalStorage().Create(bugCacheFile) - if err != nil { - return err - } - - _, err = f.Write(data.Bytes()) - if err != nil { - return err - } - - return f.Close() -} - -// ResolveBugExcerpt retrieve a BugExcerpt matching the exact given id -func (c *RepoCache) ResolveBugExcerpt(id entity.Id) (*BugExcerpt, error) { - c.muBug.RLock() - defer c.muBug.RUnlock() - - excerpt, ok := c.bugExcerpts[id] - if !ok { - return nil, bug.ErrBugNotExist - } - - return excerpt, nil -} - -// ResolveBug retrieve a bug matching the exact given id -func (c *RepoCache) ResolveBug(id entity.Id) (*BugCache, error) { - c.muBug.RLock() - cached, ok := c.bugs[id] - if ok { - c.loadedBugs.Get(id) - c.muBug.RUnlock() - return cached, nil - } - c.muBug.RUnlock() - - b, err := bug.ReadWithResolver(c.repo, c.resolvers, id) - if err != nil { - return nil, err - } - - cached = NewBugCache(c, b) - - c.muBug.Lock() - c.bugs[id] = cached - c.loadedBugs.Add(id) - c.muBug.Unlock() - - c.evictIfNeeded() - - return cached, nil -} - -// evictIfNeeded will evict a bug from the cache if needed -// it also removes references of the bug from the bugs -func (c *RepoCache) evictIfNeeded() { - c.muBug.Lock() - defer c.muBug.Unlock() - if c.loadedBugs.Len() <= c.maxLoadedBugs { - return - } - - for _, id := range c.loadedBugs.GetOldestToNewest() { - b := c.bugs[id] - if b.NeedCommit() { - continue - } - - b.mu.Lock() - c.loadedBugs.Remove(id) - delete(c.bugs, id) - - if c.loadedBugs.Len() <= c.maxLoadedBugs { - return - } - } -} - -// ResolveBugExcerptPrefix retrieve a BugExcerpt matching an id prefix. It fails if multiple -// bugs match. -func (c *RepoCache) ResolveBugExcerptPrefix(prefix string) (*BugExcerpt, error) { - return c.ResolveBugExcerptMatcher(func(excerpt *BugExcerpt) bool { - return excerpt.Id.HasPrefix(prefix) - }) -} - -// ResolveBugPrefix retrieve a bug matching an id prefix. It fails if multiple -// bugs match. -func (c *RepoCache) ResolveBugPrefix(prefix string) (*BugCache, error) { - return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool { - return excerpt.Id.HasPrefix(prefix) - }) -} - -// ResolveBugCreateMetadata retrieve a bug that has the exact given metadata on -// its Create operation, that is, the first operation. It fails if multiple bugs -// match. -func (c *RepoCache) ResolveBugCreateMetadata(key string, value string) (*BugCache, error) { - return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool { - return excerpt.CreateMetadata[key] == value - }) -} - -func (c *RepoCache) ResolveBugExcerptMatcher(f func(*BugExcerpt) bool) (*BugExcerpt, error) { - id, err := c.resolveBugMatcher(f) - if err != nil { - return nil, err - } - return c.ResolveBugExcerpt(id) -} - -func (c *RepoCache) ResolveBugMatcher(f func(*BugExcerpt) bool) (*BugCache, error) { - id, err := c.resolveBugMatcher(f) - if err != nil { - return nil, err - } - return c.ResolveBug(id) -} - -func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, error) { - c.muBug.RLock() - defer c.muBug.RUnlock() - - // preallocate but empty - matching := make([]entity.Id, 0, 5) - - for _, excerpt := range c.bugExcerpts { - if f(excerpt) { - matching = append(matching, excerpt.Id) - } - } - - if len(matching) > 1 { - return entity.UnsetId, bug.NewErrMultipleMatchBug(matching) - } - - if len(matching) == 0 { - return entity.UnsetId, bug.ErrBugNotExist - } - - return matching[0], nil -} - -// ResolveComment search for a Bug/Comment combination matching the merged -// bug/comment Id prefix. Returns the Bug containing the Comment and the Comment's -// Id. -func (c *RepoCache) ResolveComment(prefix string) (*BugCache, entity.CombinedId, error) { - bugPrefix, _ := entity.SeparateIds(prefix) - bugCandidate := make([]entity.Id, 0, 5) - - // build a list of possible matching bugs - c.muBug.RLock() - for _, excerpt := range c.bugExcerpts { - if excerpt.Id.HasPrefix(bugPrefix) { - bugCandidate = append(bugCandidate, excerpt.Id) - } - } - c.muBug.RUnlock() - - matchingBugIds := make([]entity.Id, 0, 5) - matchingCommentId := entity.UnsetCombinedId - var matchingBug *BugCache - - // search for matching comments - // searching every bug candidate allow for some collision with the bug prefix only, - // before being refined with the full comment prefix - for _, bugId := range bugCandidate { - b, err := c.ResolveBug(bugId) - if err != nil { - return nil, entity.UnsetCombinedId, err - } - - for _, comment := range b.Snapshot().Comments { - if comment.CombinedId().HasPrefix(prefix) { - matchingBugIds = append(matchingBugIds, bugId) - matchingBug = b - matchingCommentId = comment.CombinedId() - } - } - } - - if len(matchingBugIds) > 1 { - return nil, entity.UnsetCombinedId, entity.NewErrMultipleMatch("bug/comment", matchingBugIds) - } else if len(matchingBugIds) == 0 { - return nil, entity.UnsetCombinedId, errors.New("comment doesn't exist") - } - - return matchingBug, matchingCommentId, nil -} - -// QueryBugs return the id of all Bug matching the given Query -func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) { - c.muBug.RLock() - defer c.muBug.RUnlock() - - if q == nil { - return c.AllBugsIds(), nil - } - - matcher := compileMatcher(q.Filters) - - var filtered []*BugExcerpt - var foundBySearch map[entity.Id]*BugExcerpt - - if q.Search != nil { - foundBySearch = map[entity.Id]*BugExcerpt{} - - terms := make([]string, len(q.Search)) - copy(terms, q.Search) - for i, search := range q.Search { - if strings.Contains(search, " ") { - terms[i] = fmt.Sprintf("\"%s\"", search) - } - } - - bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " ")) - bleveSearch := bleve.NewSearchRequest(bleveQuery) - - index, err := c.repo.GetBleveIndex("bug") - if err != nil { - return nil, err - } - - searchResults, err := index.Search(bleveSearch) - if err != nil { - return nil, err - } - - for _, hit := range searchResults.Hits { - foundBySearch[entity.Id(hit.ID)] = c.bugExcerpts[entity.Id(hit.ID)] - } - } else { - foundBySearch = c.bugExcerpts - } - - for _, excerpt := range foundBySearch { - if matcher.Match(excerpt, c) { - filtered = append(filtered, excerpt) - } - } - - var sorter sort.Interface - - switch q.OrderBy { - case query.OrderById: - sorter = BugsById(filtered) - case query.OrderByCreation: - sorter = BugsByCreationTime(filtered) - case query.OrderByEdit: - sorter = BugsByEditTime(filtered) - default: - return nil, errors.New("missing sort type") - } - - switch q.OrderDirection { - case query.OrderAscending: - // Nothing to do - case query.OrderDescending: - sorter = sort.Reverse(sorter) - default: - return nil, errors.New("missing sort direction") - } - - sort.Sort(sorter) - - result := make([]entity.Id, len(filtered)) - - for i, val := range filtered { - result[i] = val.Id - } - - return result, nil -} - -// AllBugsIds return all known bug ids -func (c *RepoCache) AllBugsIds() []entity.Id { - c.muBug.RLock() - defer c.muBug.RUnlock() - - result := make([]entity.Id, len(c.bugExcerpts)) - - i := 0 - for _, excerpt := range c.bugExcerpts { - result[i] = excerpt.Id - i++ - } - - return result -} - -// ValidLabels list valid labels -// -// Note: in the future, a proper label policy could be implemented where valid -// labels are defined in a configuration file. Until that, the default behavior -// is to return the list of labels already used. -func (c *RepoCache) ValidLabels() []bug.Label { - c.muBug.RLock() - defer c.muBug.RUnlock() - - set := map[bug.Label]interface{}{} - - for _, excerpt := range c.bugExcerpts { - for _, l := range excerpt.Labels { - set[l] = nil - } - } - - result := make([]bug.Label, len(set)) - - i := 0 - for l := range set { - result[i] = l - i++ - } - - // Sort - sort.Slice(result, func(i, j int) bool { - return string(result[i]) < string(result[j]) - }) - - return result -} - -// NewBug create a new bug -// The new bug is written in the repository (commit) -func (c *RepoCache) NewBug(title string, message string) (*BugCache, *bug.CreateOperation, error) { - return c.NewBugWithFiles(title, message, nil) -} - -// NewBugWithFiles create a new bug with attached files for the message -// The new bug is written in the repository (commit) -func (c *RepoCache) NewBugWithFiles(title string, message string, files []repository.Hash) (*BugCache, *bug.CreateOperation, error) { - author, err := c.GetUserIdentity() - if err != nil { - return nil, nil, err - } - - return c.NewBugRaw(author, time.Now().Unix(), title, message, files, nil) -} - -// NewBugRaw create a new bug with attached files for the message, as -// well as metadata for the Create operation. -// The new bug is written in the repository (commit) -func (c *RepoCache) NewBugRaw(author *IdentityCache, unixTime int64, title string, message string, files []repository.Hash, metadata map[string]string) (*BugCache, *bug.CreateOperation, error) { - b, op, err := bug.Create(author.Identity, unixTime, title, message, files, metadata) - if err != nil { - return nil, nil, err - } - - err = b.Commit(c.repo) - if err != nil { - return nil, nil, err - } - - c.muBug.Lock() - if _, has := c.bugs[b.Id()]; has { - c.muBug.Unlock() - return nil, nil, fmt.Errorf("bug %s already exist in the cache", b.Id()) - } - - cached := NewBugCache(c, b) - c.bugs[b.Id()] = cached - c.loadedBugs.Add(b.Id()) - c.muBug.Unlock() - - c.evictIfNeeded() - - // force the write of the excerpt - err = c.bugUpdated(b.Id()) - if err != nil { - return nil, nil, err - } - - return cached, op, nil -} - -// RemoveBug removes a bug from the cache and repo given a bug id prefix -func (c *RepoCache) RemoveBug(prefix string) error { - b, err := c.ResolveBugPrefix(prefix) - if err != nil { - return err - } - - c.muBug.Lock() - - err = bug.Remove(c.repo, b.Id()) - if err != nil { - c.muBug.Unlock() - - return err - } - - delete(c.bugs, b.Id()) - delete(c.bugExcerpts, b.Id()) - c.loadedBugs.Remove(b.Id()) - - c.muBug.Unlock() - - return c.writeBugCache() -} - -func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error { - searchableBug := struct { - Text []string - }{} - - // See https://github.com/blevesearch/bleve/issues/1576 - var sb strings.Builder - normalize := func(text string) string { - sb.Reset() - for _, field := range strings.Fields(text) { - if utf8.RuneCountInString(field) < 100 { - sb.WriteString(field) - sb.WriteRune(' ') - } - } - return sb.String() - } - - for _, comment := range snap.Comments { - searchableBug.Text = append(searchableBug.Text, normalize(comment.Message)) - } - - searchableBug.Text = append(searchableBug.Text, normalize(snap.Title)) - - index, err := c.repo.GetBleveIndex("bug") - if err != nil { - return err - } - - err = index.Index(snap.Id().String(), searchableBug) - if err != nil { - return err - } - - return nil -} diff --git a/cache/repo_cache_common.go b/cache/repo_cache_common.go index 43ac6beb..f768b8e2 100644 --- a/cache/repo_cache_common.go +++ b/cache/repo_cache_common.go @@ -1,12 +1,11 @@ package cache import ( - "fmt" + "sync" "github.com/go-git/go-billy/v5" "github.com/pkg/errors" - "github.com/MichaelMure/git-bug/entities/bug" "github.com/MichaelMure/git-bug/entities/identity" "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/repository" @@ -74,72 +73,40 @@ func (c *RepoCache) StoreData(data []byte) (repository.Hash, error) { // Fetch retrieve updates from a remote // This does not change the local bugs or identities state func (c *RepoCache) Fetch(remote string) (string, error) { - stdout1, err := identity.Fetch(c.repo, remote) - if err != nil { - return stdout1, err - } - - stdout2, err := bug.Fetch(c.repo, remote) - if err != nil { - return stdout2, err + prefixes := make([]string, len(c.subcaches)) + for i, subcache := range c.subcaches { + prefixes[i] = subcache.GetNamespace() } - return stdout1 + stdout2, nil + // fetch everything at once, to have a single auth step if required. + return c.repo.FetchRefs(remote, prefixes...) } // MergeAll will merge all the available remote bug and identities func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { out := make(chan entity.MergeResult) - // Intercept merge results to update the cache properly + dependency := [][]cacheMgmt{ + {c.identities}, + {c.bugs}, + } + + // run MergeAll according to entities dependencies and merge the results go func() { defer close(out) - author, err := c.GetUserIdentity() - if err != nil { - out <- entity.NewMergeError(err, "") - return - } - - results := identity.MergeAll(c.repo, remote) - for result := range results { - out <- result - - if result.Err != nil { - continue - } - - switch result.Status { - case entity.MergeStatusNew, entity.MergeStatusUpdated: - i := result.Entity.(*identity.Identity) - c.muIdentity.Lock() - c.identitiesExcerpts[result.Id] = NewIdentityExcerpt(i) - c.muIdentity.Unlock() + for _, subcaches := range dependency { + var wg sync.WaitGroup + for _, subcache := range subcaches { + wg.Add(1) + go func(subcache cacheMgmt) { + for res := range subcache.MergeAll(remote) { + out <- res + } + wg.Done() + }(subcache) } - } - - results = bug.MergeAll(c.repo, c.resolvers, remote, author) - for result := range results { - out <- result - - if result.Err != nil { - continue - } - - switch result.Status { - case entity.MergeStatusNew, entity.MergeStatusUpdated: - b := result.Entity.(*bug.Bug) - snap := b.Compile() - c.muBug.Lock() - c.bugExcerpts[result.Id] = NewBugExcerpt(b, snap) - c.muBug.Unlock() - } - } - - err = c.write() - if err != nil { - out <- entity.NewMergeError(err, "") - return + wg.Wait() } }() @@ -148,17 +115,13 @@ func (c *RepoCache) MergeAll(remote string) <-chan entity.MergeResult { // Push update a remote with the local changes func (c *RepoCache) Push(remote string) (string, error) { - stdout1, err := identity.Push(c.repo, remote) - if err != nil { - return stdout1, err + prefixes := make([]string, len(c.subcaches)) + for i, subcache := range c.subcaches { + prefixes[i] = subcache.GetNamespace() } - stdout2, err := bug.Push(c.repo, remote) - if err != nil { - return stdout2, err - } - - return stdout1 + stdout2, nil + // push everything at once, to have a single auth step if required + return c.repo.PushRefs(remote, prefixes...) } // Pull will do a Fetch + MergeAll @@ -182,64 +145,64 @@ func (c *RepoCache) Pull(remote string) error { } func (c *RepoCache) SetUserIdentity(i *IdentityCache) error { - err := identity.SetUserIdentity(c.repo, i.Identity) - if err != nil { - return err - } - - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() + c.muUserIdentity.RLock() + defer c.muUserIdentity.RUnlock() // Make sure that everything is fine - if _, ok := c.identities[i.Id()]; !ok { + if _, err := c.identities.Resolve(i.Id()); err != nil { panic("SetUserIdentity while the identity is not from the cache, something is wrong") } + err := identity.SetUserIdentity(c.repo, i.Identity) + if err != nil { + return err + } + c.userIdentityId = i.Id() return nil } func (c *RepoCache) GetUserIdentity() (*IdentityCache, error) { + c.muUserIdentity.RLock() if c.userIdentityId != "" { - i, ok := c.identities[c.userIdentityId] - if ok { - return i, nil - } + defer c.muUserIdentity.RUnlock() + return c.identities.Resolve(c.userIdentityId) } + c.muUserIdentity.RUnlock() - c.muIdentity.Lock() - defer c.muIdentity.Unlock() + c.muUserIdentity.Lock() + defer c.muUserIdentity.Unlock() - i, err := identity.GetUserIdentity(c.repo) + i, err := identity.GetUserIdentityId(c.repo) if err != nil { return nil, err } - cached := NewIdentityCache(c, i) - c.identities[i.Id()] = cached - c.userIdentityId = i.Id() + c.userIdentityId = i - return cached, nil + return c.identities.Resolve(i) } func (c *RepoCache) GetUserIdentityExcerpt() (*IdentityExcerpt, error) { - if c.userIdentityId == "" { - id, err := identity.GetUserIdentityId(c.repo) - if err != nil { - return nil, err - } - c.userIdentityId = id + c.muUserIdentity.RLock() + if c.userIdentityId != "" { + defer c.muUserIdentity.RUnlock() + return c.identities.ResolveExcerpt(c.userIdentityId) } + c.muUserIdentity.RUnlock() - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() + c.muUserIdentity.Lock() + defer c.muUserIdentity.Unlock() - excerpt, ok := c.identitiesExcerpts[c.userIdentityId] - if !ok { - return nil, fmt.Errorf("cache: missing identity excerpt %v", c.userIdentityId) + i, err := identity.GetUserIdentityId(c.repo) + if err != nil { + return nil, err } - return excerpt, nil + + c.userIdentityId = i + + return c.identities.ResolveExcerpt(i) } func (c *RepoCache) IsUserIdentitySet() (bool, error) { diff --git a/cache/repo_cache_identity.go b/cache/repo_cache_identity.go deleted file mode 100644 index 4f612280..00000000 --- a/cache/repo_cache_identity.go +++ /dev/null @@ -1,271 +0,0 @@ -package cache - -import ( - "bytes" - "encoding/gob" - "fmt" - - "github.com/MichaelMure/git-bug/entities/identity" - "github.com/MichaelMure/git-bug/entity" -) - -const identityCacheFile = "identity-cache" - -// identityUpdated is a callback to trigger when the excerpt of an identity -// changed, that is each time an identity is updated -func (c *RepoCache) identityUpdated(id entity.Id) error { - c.muIdentity.Lock() - - i, ok := c.identities[id] - if !ok { - c.muIdentity.Unlock() - panic("missing identity in the cache") - } - - c.identitiesExcerpts[id] = NewIdentityExcerpt(i.Identity) - c.muIdentity.Unlock() - - // we only need to write the identity cache - return c.writeIdentityCache() -} - -// load will try to read from the disk the identity cache file -func (c *RepoCache) loadIdentityCache() error { - c.muIdentity.Lock() - defer c.muIdentity.Unlock() - - f, err := c.repo.LocalStorage().Open(identityCacheFile) - if err != nil { - return err - } - - decoder := gob.NewDecoder(f) - - aux := struct { - Version uint - Excerpts map[entity.Id]*IdentityExcerpt - }{} - - err = decoder.Decode(&aux) - if err != nil { - return err - } - - if aux.Version != formatVersion { - return fmt.Errorf("unknown cache format version %v", aux.Version) - } - - c.identitiesExcerpts = aux.Excerpts - return nil -} - -// write will serialize on disk the identity cache file -func (c *RepoCache) writeIdentityCache() error { - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() - - var data bytes.Buffer - - aux := struct { - Version uint - Excerpts map[entity.Id]*IdentityExcerpt - }{ - Version: formatVersion, - Excerpts: c.identitiesExcerpts, - } - - encoder := gob.NewEncoder(&data) - - err := encoder.Encode(aux) - if err != nil { - return err - } - - f, err := c.repo.LocalStorage().Create(identityCacheFile) - if err != nil { - return err - } - - _, err = f.Write(data.Bytes()) - if err != nil { - return err - } - - return f.Close() -} - -// ResolveIdentityExcerpt retrieve a IdentityExcerpt matching the exact given id -func (c *RepoCache) ResolveIdentityExcerpt(id entity.Id) (*IdentityExcerpt, error) { - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() - - e, ok := c.identitiesExcerpts[id] - if !ok { - return nil, identity.ErrIdentityNotExist - } - - return e, nil -} - -// ResolveIdentity retrieve an identity matching the exact given id -func (c *RepoCache) ResolveIdentity(id entity.Id) (*IdentityCache, error) { - c.muIdentity.RLock() - cached, ok := c.identities[id] - c.muIdentity.RUnlock() - if ok { - return cached, nil - } - - i, err := identity.ReadLocal(c.repo, id) - if err != nil { - return nil, err - } - - cached = NewIdentityCache(c, i) - - c.muIdentity.Lock() - c.identities[id] = cached - c.muIdentity.Unlock() - - return cached, nil -} - -// ResolveIdentityExcerptPrefix retrieve a IdentityExcerpt matching an id prefix. -// It fails if multiple identities match. -func (c *RepoCache) ResolveIdentityExcerptPrefix(prefix string) (*IdentityExcerpt, error) { - return c.ResolveIdentityExcerptMatcher(func(excerpt *IdentityExcerpt) bool { - return excerpt.Id.HasPrefix(prefix) - }) -} - -// ResolveIdentityPrefix retrieve an Identity matching an id prefix. -// It fails if multiple identities match. -func (c *RepoCache) ResolveIdentityPrefix(prefix string) (*IdentityCache, error) { - return c.ResolveIdentityMatcher(func(excerpt *IdentityExcerpt) bool { - return excerpt.Id.HasPrefix(prefix) - }) -} - -// ResolveIdentityImmutableMetadata retrieve an Identity that has the exact given metadata on -// one of its version. If multiple version have the same key, the first defined take precedence. -func (c *RepoCache) ResolveIdentityImmutableMetadata(key string, value string) (*IdentityCache, error) { - return c.ResolveIdentityMatcher(func(excerpt *IdentityExcerpt) bool { - return excerpt.ImmutableMetadata[key] == value - }) -} - -func (c *RepoCache) ResolveIdentityExcerptMatcher(f func(*IdentityExcerpt) bool) (*IdentityExcerpt, error) { - id, err := c.resolveIdentityMatcher(f) - if err != nil { - return nil, err - } - return c.ResolveIdentityExcerpt(id) -} - -func (c *RepoCache) ResolveIdentityMatcher(f func(*IdentityExcerpt) bool) (*IdentityCache, error) { - id, err := c.resolveIdentityMatcher(f) - if err != nil { - return nil, err - } - return c.ResolveIdentity(id) -} - -func (c *RepoCache) resolveIdentityMatcher(f func(*IdentityExcerpt) bool) (entity.Id, error) { - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() - - // preallocate but empty - matching := make([]entity.Id, 0, 5) - - for _, excerpt := range c.identitiesExcerpts { - if f(excerpt) { - matching = append(matching, excerpt.Id) - } - } - - if len(matching) > 1 { - return entity.UnsetId, identity.NewErrMultipleMatch(matching) - } - - if len(matching) == 0 { - return entity.UnsetId, identity.ErrIdentityNotExist - } - - return matching[0], nil -} - -// AllIdentityIds return all known identity ids -func (c *RepoCache) AllIdentityIds() []entity.Id { - c.muIdentity.RLock() - defer c.muIdentity.RUnlock() - - result := make([]entity.Id, len(c.identitiesExcerpts)) - - i := 0 - for _, excerpt := range c.identitiesExcerpts { - result[i] = excerpt.Id - i++ - } - - return result -} - -func (c *RepoCache) NewIdentityFromGitUser() (*IdentityCache, error) { - return c.NewIdentityFromGitUserRaw(nil) -} - -func (c *RepoCache) NewIdentityFromGitUserRaw(metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewFromGitUser(c.repo) - if err != nil { - return nil, err - } - return c.finishIdentity(i, metadata) -} - -// NewIdentity create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCache) NewIdentity(name string, email string) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, "", "", nil, nil) -} - -// NewIdentityFull create a new identity -// The new identity is written in the repository (commit) -func (c *RepoCache) NewIdentityFull(name string, email string, login string, avatarUrl string, keys []*identity.Key) (*IdentityCache, error) { - return c.NewIdentityRaw(name, email, login, avatarUrl, keys, nil) -} - -func (c *RepoCache) NewIdentityRaw(name string, email string, login string, avatarUrl string, keys []*identity.Key, metadata map[string]string) (*IdentityCache, error) { - i, err := identity.NewIdentityFull(c.repo, name, email, login, avatarUrl, keys) - if err != nil { - return nil, err - } - return c.finishIdentity(i, metadata) -} - -func (c *RepoCache) finishIdentity(i *identity.Identity, metadata map[string]string) (*IdentityCache, error) { - for key, value := range metadata { - i.SetMetadata(key, value) - } - - err := i.Commit(c.repo) - if err != nil { - return nil, err - } - - c.muIdentity.Lock() - if _, has := c.identities[i.Id()]; has { - return nil, fmt.Errorf("identity %s already exist in the cache", i.Id()) - } - - cached := NewIdentityCache(c, i) - c.identities[i.Id()] = cached - c.muIdentity.Unlock() - - // force the write of the excerpt - err = c.identityUpdated(i.Id()) - if err != nil { - return nil, err - } - - return cached, nil -} diff --git a/cache/repo_cache_test.go b/cache/repo_cache_test.go index a9557ff0..796b5db9 100644 --- a/cache/repo_cache_test.go +++ b/cache/repo_cache_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/MichaelMure/git-bug/entities/bug" + "github.com/MichaelMure/git-bug/entities/identity" + "github.com/MichaelMure/git-bug/entity" "github.com/MichaelMure/git-bug/query" "github.com/MichaelMure/git-bug/repository" ) @@ -16,11 +18,11 @@ import ( func TestCache(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - cache, err := NewRepoCache(repo) + cache, err := NewRepoCacheNoEvents(repo) require.NoError(t, err) // Create, set and get user identity - iden1, err := cache.NewIdentity("René Descartes", "rene@descartes.fr") + iden1, err := cache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = cache.SetUserIdentity(iden1) require.NoError(t, err) @@ -29,102 +31,122 @@ func TestCache(t *testing.T) { require.Equal(t, iden1.Id(), userIden.Id()) // it's possible to create two identical identities - iden2, err := cache.NewIdentity("René Descartes", "rene@descartes.fr") + iden2, err := cache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) // Two identical identities yield a different id require.NotEqual(t, iden1.Id(), iden2.Id()) + indexCount := func(name string) uint64 { + idx, err := repo.GetIndex(name) + require.NoError(t, err) + count, err := idx.DocCount() + require.NoError(t, err) + return count + } + // There is now two identities in the cache - require.Len(t, cache.AllIdentityIds(), 2) - require.Len(t, cache.identitiesExcerpts, 2) - require.Len(t, cache.identities, 2) + require.Len(t, cache.Identities().AllIds(), 2) + require.Len(t, cache.identities.excerpts, 2) + require.Len(t, cache.identities.cached, 2) + require.Equal(t, uint64(2), indexCount(identity.Namespace)) + require.Equal(t, uint64(0), indexCount(bug.Namespace)) // Create a bug - bug1, _, err := cache.NewBug("title", "message") + bug1, _, err := cache.Bugs().New("title", "message") require.NoError(t, err) // It's possible to create two identical bugs - bug2, _, err := cache.NewBug("title", "message") + bug2, _, err := cache.Bugs().New("title", "marker") require.NoError(t, err) // two identical bugs yield a different id require.NotEqual(t, bug1.Id(), bug2.Id()) // There is now two bugs in the cache - require.Len(t, cache.AllBugsIds(), 2) - require.Len(t, cache.bugExcerpts, 2) - require.Len(t, cache.bugs, 2) + require.Len(t, cache.Bugs().AllIds(), 2) + require.Len(t, cache.bugs.excerpts, 2) + require.Len(t, cache.bugs.cached, 2) + require.Equal(t, uint64(2), indexCount(identity.Namespace)) + require.Equal(t, uint64(2), indexCount(bug.Namespace)) // Resolving - _, err = cache.ResolveIdentity(iden1.Id()) + _, err = cache.Identities().Resolve(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityExcerpt(iden1.Id()) + _, err = cache.Identities().ResolveExcerpt(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityPrefix(iden1.Id().String()[:10]) + _, err = cache.Identities().ResolvePrefix(iden1.Id().String()[:10]) require.NoError(t, err) - _, err = cache.ResolveBug(bug1.Id()) + _, err = cache.Bugs().Resolve(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugExcerpt(bug1.Id()) + _, err = cache.Bugs().ResolveExcerpt(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugPrefix(bug1.Id().String()[:10]) + _, err = cache.Bugs().ResolvePrefix(bug1.Id().String()[:10]) require.NoError(t, err) // Querying q, err := query.Parse("status:open author:descartes sort:edit-asc") require.NoError(t, err) - res, err := cache.QueryBugs(q) + res, err := cache.Bugs().Query(q) require.NoError(t, err) require.Len(t, res, 2) + q, err = query.Parse("status:open marker") // full-text search + require.NoError(t, err) + res, err = cache.Bugs().Query(q) + require.NoError(t, err) + require.Len(t, res, 1) + // Close require.NoError(t, cache.Close()) - require.Empty(t, cache.bugs) - require.Empty(t, cache.bugExcerpts) - require.Empty(t, cache.identities) - require.Empty(t, cache.identitiesExcerpts) + require.Empty(t, cache.bugs.cached) + require.Empty(t, cache.bugs.excerpts) + require.Empty(t, cache.identities.cached) + require.Empty(t, cache.identities.excerpts) // Reload, only excerpt are loaded, but as we need to load the identities used in the bugs // to check the signatures, we also load the identity used above - cache, err = NewRepoCache(repo) + cache, err = NewRepoCacheNoEvents(repo) require.NoError(t, err) - require.Empty(t, cache.bugs) - require.Len(t, cache.identities, 1) - require.Len(t, cache.bugExcerpts, 2) - require.Len(t, cache.identitiesExcerpts, 2) + require.Len(t, cache.bugs.cached, 0) + require.Len(t, cache.bugs.excerpts, 2) + require.Len(t, cache.identities.cached, 0) + require.Len(t, cache.identities.excerpts, 2) + require.Equal(t, uint64(2), indexCount(identity.Namespace)) + require.Equal(t, uint64(2), indexCount(bug.Namespace)) // Resolving load from the disk - _, err = cache.ResolveIdentity(iden1.Id()) + _, err = cache.Identities().Resolve(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityExcerpt(iden1.Id()) + _, err = cache.Identities().ResolveExcerpt(iden1.Id()) require.NoError(t, err) - _, err = cache.ResolveIdentityPrefix(iden1.Id().String()[:10]) + _, err = cache.Identities().ResolvePrefix(iden1.Id().String()[:10]) require.NoError(t, err) - _, err = cache.ResolveBug(bug1.Id()) + _, err = cache.Bugs().Resolve(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugExcerpt(bug1.Id()) + _, err = cache.Bugs().ResolveExcerpt(bug1.Id()) require.NoError(t, err) - _, err = cache.ResolveBugPrefix(bug1.Id().String()[:10]) + _, err = cache.Bugs().ResolvePrefix(bug1.Id().String()[:10]) require.NoError(t, err) } func TestCachePushPull(t *testing.T) { repoA, repoB, _ := repository.SetupGoGitReposAndRemote(t) - cacheA, err := NewRepoCache(repoA) + cacheA, err := NewRepoCacheNoEvents(repoA) require.NoError(t, err) - cacheB, err := NewRepoCache(repoB) + cacheB, err := NewRepoCacheNoEvents(repoB) require.NoError(t, err) // Create, set and get user identity - reneA, err := cacheA.NewIdentity("René Descartes", "rene@descartes.fr") + reneA, err := cacheA.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = cacheA.SetUserIdentity(reneA) require.NoError(t, err) - isaacB, err := cacheB.NewIdentity("Isaac Newton", "isaac@newton.uk") + isaacB, err := cacheB.Identities().New("Isaac Newton", "isaac@newton.uk") require.NoError(t, err) err = cacheB.SetUserIdentity(isaacB) require.NoError(t, err) @@ -136,7 +158,7 @@ func TestCachePushPull(t *testing.T) { require.NoError(t, err) // Create a bug in A - _, _, err = cacheA.NewBug("bug1", "message") + _, _, err = cacheA.Bugs().New("bug1", "message") require.NoError(t, err) // A --> remote --> B @@ -146,17 +168,17 @@ func TestCachePushPull(t *testing.T) { err = cacheB.Pull("origin") require.NoError(t, err) - require.Len(t, cacheB.AllBugsIds(), 1) + require.Len(t, cacheB.Bugs().AllIds(), 1) // retrieve and set identity - reneB, err := cacheB.ResolveIdentity(reneA.Id()) + reneB, err := cacheB.Identities().Resolve(reneA.Id()) require.NoError(t, err) err = cacheB.SetUserIdentity(reneB) require.NoError(t, err) // B --> remote --> A - _, _, err = cacheB.NewBug("bug2", "message") + _, _, err = cacheB.Bugs().New("bug2", "message") require.NoError(t, err) _, err = cacheB.Push("origin") @@ -165,7 +187,7 @@ func TestCachePushPull(t *testing.T) { err = cacheA.Pull("origin") require.NoError(t, err) - require.Len(t, cacheA.AllBugsIds(), 2) + require.Len(t, cacheA.Bugs().AllIds(), 2) } func TestRemove(t *testing.T) { @@ -179,20 +201,20 @@ func TestRemove(t *testing.T) { err = repo.AddRemote("remoteB", remoteB.GetLocalRemote()) require.NoError(t, err) - repoCache, err := NewRepoCache(repo) + repoCache, err := NewRepoCacheNoEvents(repo) require.NoError(t, err) - rene, err := repoCache.NewIdentity("René Descartes", "rene@descartes.fr") + rene, err := repoCache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = repoCache.SetUserIdentity(rene) require.NoError(t, err) - _, _, err = repoCache.NewBug("title", "message") + _, _, err = repoCache.Bugs().New("title", "message") require.NoError(t, err) // and one more for testing - b1, _, err := repoCache.NewBug("title", "message") + b1, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) _, err = repoCache.Push("remoteA") @@ -207,72 +229,72 @@ func TestRemove(t *testing.T) { _, err = repoCache.Fetch("remoteB") require.NoError(t, err) - err = repoCache.RemoveBug(b1.Id().String()) + err = repoCache.Bugs().Remove(b1.Id().String()) require.NoError(t, err) - assert.Equal(t, 1, len(repoCache.bugs)) - assert.Equal(t, 1, len(repoCache.bugExcerpts)) + assert.Len(t, repoCache.bugs.cached, 1) + assert.Len(t, repoCache.bugs.excerpts, 1) - _, err = repoCache.ResolveBug(b1.Id()) - assert.Error(t, bug.ErrBugNotExist, err) + _, err = repoCache.Bugs().Resolve(b1.Id()) + assert.ErrorAs(t, entity.ErrNotFound{}, err) } func TestCacheEviction(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - repoCache, err := NewRepoCache(repo) + repoCache, err := NewRepoCacheNoEvents(repo) require.NoError(t, err) repoCache.setCacheSize(2) - require.Equal(t, 2, repoCache.maxLoadedBugs) - require.Equal(t, 0, repoCache.loadedBugs.Len()) - require.Equal(t, 0, len(repoCache.bugs)) + require.Equal(t, 2, repoCache.bugs.maxLoaded) + require.Len(t, repoCache.bugs.cached, 0) + require.Equal(t, repoCache.bugs.lru.Len(), 0) // Generating some bugs - rene, err := repoCache.NewIdentity("René Descartes", "rene@descartes.fr") + rene, err := repoCache.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) err = repoCache.SetUserIdentity(rene) require.NoError(t, err) - bug1, _, err := repoCache.NewBug("title", "message") + bug1, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) checkBugPresence(t, repoCache, bug1, true) - require.Equal(t, 1, repoCache.loadedBugs.Len()) - require.Equal(t, 1, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 1) + require.Equal(t, 1, repoCache.bugs.lru.Len()) - bug2, _, err := repoCache.NewBug("title", "message") + bug2, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) checkBugPresence(t, repoCache, bug1, true) checkBugPresence(t, repoCache, bug2, true) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) // Number of bugs should not exceed max size of lruCache, oldest one should be evicted - bug3, _, err := repoCache.NewBug("title", "message") + bug3, _, err := repoCache.Bugs().New("title", "message") require.NoError(t, err) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) checkBugPresence(t, repoCache, bug1, false) checkBugPresence(t, repoCache, bug2, true) checkBugPresence(t, repoCache, bug3, true) // Accessing bug should update position in lruCache and therefore it should not be evicted - repoCache.loadedBugs.Get(bug2.Id()) - oldestId, _ := repoCache.loadedBugs.GetOldest() + repoCache.bugs.lru.Get(bug2.Id()) + oldestId, _ := repoCache.bugs.lru.GetOldest() require.Equal(t, bug3.Id(), oldestId) checkBugPresence(t, repoCache, bug1, false) checkBugPresence(t, repoCache, bug2, true) checkBugPresence(t, repoCache, bug3, true) - require.Equal(t, 2, repoCache.loadedBugs.Len()) - require.Equal(t, 2, len(repoCache.bugs)) + require.Len(t, repoCache.bugs.cached, 2) + require.Equal(t, 2, repoCache.bugs.lru.Len()) } func checkBugPresence(t *testing.T, cache *RepoCache, bug *BugCache, presence bool) { id := bug.Id() - require.Equal(t, presence, cache.loadedBugs.Contains(id)) - b, ok := cache.bugs[id] + require.Equal(t, presence, cache.bugs.lru.Contains(id)) + b, ok := cache.bugs.cached[id] require.Equal(t, presence, ok) if ok { require.Equal(t, bug, b) @@ -286,12 +308,12 @@ func TestLongDescription(t *testing.T) { repo := repository.CreateGoGitTestRepo(t, false) - backend, err := NewRepoCache(repo) + backend, err := NewRepoCacheNoEvents(repo) require.NoError(t, err) - i, err := backend.NewIdentity("René Descartes", "rene@descartes.fr") + i, err := backend.Identities().New("René Descartes", "rene@descartes.fr") require.NoError(t, err) - _, _, err = backend.NewBugRaw(i, time.Now().Unix(), text, text, nil, nil) + _, _, err = backend.Bugs().NewRaw(i, time.Now().Unix(), text, text, nil, nil) require.NoError(t, err) } diff --git a/cache/resolvers.go b/cache/resolvers.go deleted file mode 100644 index 9ed2fa4c..00000000 --- a/cache/resolvers.go +++ /dev/null @@ -1,42 +0,0 @@ -package cache - -import ( - "github.com/MichaelMure/git-bug/entity" -) - -func makeResolvers(cache *RepoCache) entity.Resolvers { - return entity.Resolvers{ - &IdentityCache{}: newIdentityCacheResolver(cache), - &BugCache{}: newBugCacheResolver(cache), - } -} - -var _ entity.Resolver = &identityCacheResolver{} - -// identityCacheResolver is an identity Resolver that retrieve identities from -// the cache -type identityCacheResolver struct { - cache *RepoCache -} - -func newIdentityCacheResolver(cache *RepoCache) *identityCacheResolver { - return &identityCacheResolver{cache: cache} -} - -func (i *identityCacheResolver) Resolve(id entity.Id) (entity.Interface, error) { - return i.cache.ResolveIdentity(id) -} - -var _ entity.Resolver = &bugCacheResolver{} - -type bugCacheResolver struct { - cache *RepoCache -} - -func newBugCacheResolver(cache *RepoCache) *bugCacheResolver { - return &bugCacheResolver{cache: cache} -} - -func (b *bugCacheResolver) Resolve(id entity.Id) (entity.Interface, error) { - return b.cache.ResolveBug(id) -} diff --git a/cache/subcache.go b/cache/subcache.go new file mode 100644 index 00000000..7757ce82 --- /dev/null +++ b/cache/subcache.go @@ -0,0 +1,505 @@ +package cache + +import ( + "bytes" + "encoding/gob" + "fmt" + "path/filepath" + "sync" + + "github.com/pkg/errors" + + "github.com/MichaelMure/git-bug/entities/identity" + "github.com/MichaelMure/git-bug/entity" + "github.com/MichaelMure/git-bug/repository" +) + +type Excerpt interface { + Id() entity.Id + setId(id entity.Id) +} + +type CacheEntity interface { + Id() entity.Id + NeedCommit() bool + Lock() +} + +type getUserIdentityFunc func() (*IdentityCache, error) + +// Actions expose a number of action functions on Entities, to give upper layers (cache) a way to normalize interactions. +// Note: ideally this wouldn't exist, the cache layer would assume that everything is an entity/dag, and directly use the +// functions from this package, but right now identities are not using that framework. +type Actions[EntityT entity.Interface] struct { + ReadWithResolver func(repo repository.ClockedRepo, resolvers entity.Resolvers, id entity.Id) (EntityT, error) + ReadAllWithResolver func(repo repository.ClockedRepo, resolvers entity.Resolvers) <-chan entity.StreamedEntity[EntityT] + Remove func(repo repository.ClockedRepo, id entity.Id) error + MergeAll func(repo repository.ClockedRepo, resolvers entity.Resolvers, remote string, mergeAuthor identity.Interface) <-chan entity.MergeResult +} + +var _ cacheMgmt = &SubCache[entity.Interface, Excerpt, CacheEntity]{} + +type SubCache[EntityT entity.Interface, ExcerptT Excerpt, CacheT CacheEntity] struct { + repo repository.ClockedRepo + resolvers func() entity.Resolvers + + getUserIdentity getUserIdentityFunc + makeCached func(entity EntityT, entityUpdated func(id entity.Id) error) CacheT + makeExcerpt func(CacheT) ExcerptT + makeIndexData func(CacheT) []string + actions Actions[EntityT] + + typename string + namespace string + version uint + maxLoaded int + + mu sync.RWMutex + excerpts map[entity.Id]ExcerptT + cached map[entity.Id]CacheT + lru *lruIdCache +} + +func NewSubCache[EntityT entity.Interface, ExcerptT Excerpt, CacheT CacheEntity]( + repo repository.ClockedRepo, + resolvers func() entity.Resolvers, getUserIdentity getUserIdentityFunc, + makeCached func(entity EntityT, entityUpdated func(id entity.Id) error) CacheT, + makeExcerpt func(CacheT) ExcerptT, + makeIndexData func(CacheT) []string, + actions Actions[EntityT], + typename, namespace string, + version uint, maxLoaded int) *SubCache[EntityT, ExcerptT, CacheT] { + return &SubCache[EntityT, ExcerptT, CacheT]{ + repo: repo, + resolvers: resolvers, + getUserIdentity: getUserIdentity, + makeCached: makeCached, + makeExcerpt: makeExcerpt, + makeIndexData: makeIndexData, + actions: actions, + typename: typename, + namespace: namespace, + version: version, + maxLoaded: maxLoaded, + excerpts: make(map[entity.Id]ExcerptT), + cached: make(map[entity.Id]CacheT), + lru: newLRUIdCache(), + } +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Typename() string { + return sc.typename +} + +// Load will try to read from the disk the entity cache file +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Load() error { + sc.mu.Lock() + defer sc.mu.Unlock() + + f, err := sc.repo.LocalStorage().Open(filepath.Join("cache", sc.namespace)) + if err != nil { + return err + } + + decoder := gob.NewDecoder(f) + + aux := struct { + Version uint + Excerpts map[entity.Id]ExcerptT + }{} + + err = decoder.Decode(&aux) + if err != nil { + return err + } + + if aux.Version != sc.version { + return fmt.Errorf("unknown %s cache format version %v", sc.namespace, aux.Version) + } + + // the id is not serialized in the excerpt itself (non-exported field in go, long story ...), + // so we fix it here, which doubles as enforcing coherency. + for id, excerpt := range aux.Excerpts { + excerpt.setId(id) + } + + sc.excerpts = aux.Excerpts + + index, err := sc.repo.GetIndex(sc.namespace) + if err != nil { + return err + } + + // simple heuristic to detect a mismatch between the index and the entities + count, err := index.DocCount() + if err != nil { + return err + } + if count != uint64(len(sc.excerpts)) { + return fmt.Errorf("count mismatch between bleve and %s excerpts", sc.namespace) + } + + return nil +} + +// Write will serialize on disk the entity cache file +func (sc *SubCache[EntityT, ExcerptT, CacheT]) write() error { + sc.mu.RLock() + defer sc.mu.RUnlock() + + var data bytes.Buffer + + aux := struct { + Version uint + Excerpts map[entity.Id]ExcerptT + }{ + Version: sc.version, + Excerpts: sc.excerpts, + } + + encoder := gob.NewEncoder(&data) + + err := encoder.Encode(aux) + if err != nil { + return err + } + + f, err := sc.repo.LocalStorage().Create(filepath.Join("cache", sc.namespace)) + if err != nil { + return err + } + + _, err = f.Write(data.Bytes()) + if err != nil { + return err + } + + return f.Close() +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error { + sc.excerpts = make(map[entity.Id]ExcerptT) + + allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers()) + + index, err := sc.repo.GetIndex(sc.namespace) + if err != nil { + return err + } + + // wipe the index just to be sure + err = index.Clear() + if err != nil { + return err + } + + indexer, indexEnd := index.IndexBatch() + + for e := range allEntities { + if e.Err != nil { + return e.Err + } + + cached := sc.makeCached(e.Entity, sc.entityUpdated) + sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached) + // might as well keep them in memory + sc.cached[e.Entity.Id()] = cached + + indexData := sc.makeIndexData(cached) + if err := indexer(e.Entity.Id().String(), indexData); err != nil { + return err + } + } + + err = indexEnd() + if err != nil { + return err + } + + err = sc.write() + if err != nil { + return err + } + + return nil +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) SetCacheSize(size int) { + sc.maxLoaded = size + sc.evictIfNeeded() +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Close() error { + sc.mu.Lock() + defer sc.mu.Unlock() + sc.excerpts = nil + sc.cached = make(map[entity.Id]CacheT) + return nil +} + +// AllIds return all known bug ids +func (sc *SubCache[EntityT, ExcerptT, CacheT]) AllIds() []entity.Id { + sc.mu.RLock() + defer sc.mu.RUnlock() + + result := make([]entity.Id, len(sc.excerpts)) + + i := 0 + for _, excerpt := range sc.excerpts { + result[i] = excerpt.Id() + i++ + } + + return result +} + +// Resolve retrieve an entity matching the exact given id +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Resolve(id entity.Id) (CacheT, error) { + sc.mu.RLock() + cached, ok := sc.cached[id] + if ok { + sc.lru.Get(id) + sc.mu.RUnlock() + return cached, nil + } + sc.mu.RUnlock() + + e, err := sc.actions.ReadWithResolver(sc.repo, sc.resolvers(), id) + if err != nil { + return *new(CacheT), err + } + + cached = sc.makeCached(e, sc.entityUpdated) + + sc.mu.Lock() + sc.cached[id] = cached + sc.lru.Add(id) + sc.mu.Unlock() + + sc.evictIfNeeded() + + return cached, nil +} + +// ResolvePrefix retrieve an entity matching an id prefix. It fails if multiple +// entity match. +func (sc *SubCache[EntityT, ExcerptT, CacheT]) ResolvePrefix(prefix string) (CacheT, error) { + return sc.ResolveMatcher(func(excerpt ExcerptT) bool { + return excerpt.Id().HasPrefix(prefix) + }) +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) ResolveMatcher(f func(ExcerptT) bool) (CacheT, error) { + id, err := sc.resolveMatcher(f) + if err != nil { + return *new(CacheT), err + } + return sc.Resolve(id) +} + +// ResolveExcerpt retrieve an Excerpt matching the exact given id +func (sc *SubCache[EntityT, ExcerptT, CacheT]) ResolveExcerpt(id entity.Id) (ExcerptT, error) { + sc.mu.RLock() + defer sc.mu.RUnlock() + + excerpt, ok := sc.excerpts[id] + if !ok { + return *new(ExcerptT), entity.NewErrNotFound(sc.typename) + } + + return excerpt, nil +} + +// ResolveExcerptPrefix retrieve an Excerpt matching an id prefix. It fails if multiple +// entity match. +func (sc *SubCache[EntityT, ExcerptT, CacheT]) ResolveExcerptPrefix(prefix string) (ExcerptT, error) { + return sc.ResolveExcerptMatcher(func(excerpt ExcerptT) bool { + return excerpt.Id().HasPrefix(prefix) + }) +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) ResolveExcerptMatcher(f func(ExcerptT) bool) (ExcerptT, error) { + id, err := sc.resolveMatcher(f) + if err != nil { + return *new(ExcerptT), err + } + return sc.ResolveExcerpt(id) +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) resolveMatcher(f func(ExcerptT) bool) (entity.Id, error) { + sc.mu.RLock() + defer sc.mu.RUnlock() + + // preallocate but empty + matching := make([]entity.Id, 0, 5) + + for _, excerpt := range sc.excerpts { + if f(excerpt) { + matching = append(matching, excerpt.Id()) + } + } + + if len(matching) > 1 { + return entity.UnsetId, entity.NewErrMultipleMatch(sc.typename, matching) + } + + if len(matching) == 0 { + return entity.UnsetId, entity.NewErrNotFound(sc.typename) + } + + return matching[0], nil +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) add(e EntityT) (CacheT, error) { + sc.mu.Lock() + if _, has := sc.cached[e.Id()]; has { + sc.mu.Unlock() + return *new(CacheT), fmt.Errorf("entity %s already exist in the cache", e.Id()) + } + + cached := sc.makeCached(e, sc.entityUpdated) + sc.cached[e.Id()] = cached + sc.lru.Add(e.Id()) + sc.mu.Unlock() + + sc.evictIfNeeded() + + // force the write of the excerpt + err := sc.entityUpdated(e.Id()) + if err != nil { + return *new(CacheT), err + } + + return cached, nil +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Remove(prefix string) error { + e, err := sc.ResolvePrefix(prefix) + if err != nil { + return err + } + + sc.mu.Lock() + + err = sc.actions.Remove(sc.repo, e.Id()) + if err != nil { + sc.mu.Unlock() + return err + } + + delete(sc.cached, e.Id()) + delete(sc.excerpts, e.Id()) + sc.lru.Remove(e.Id()) + + sc.mu.Unlock() + + return sc.write() +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) MergeAll(remote string) <-chan entity.MergeResult { + out := make(chan entity.MergeResult) + + // Intercept merge results to update the cache properly + go func() { + defer close(out) + + author, err := sc.getUserIdentity() + if err != nil { + out <- entity.NewMergeError(err, "") + return + } + + results := sc.actions.MergeAll(sc.repo, sc.resolvers(), remote, author) + for result := range results { + out <- result + + if result.Err != nil { + continue + } + + switch result.Status { + case entity.MergeStatusNew, entity.MergeStatusUpdated: + e := result.Entity.(EntityT) + cached := sc.makeCached(e, sc.entityUpdated) + + sc.mu.Lock() + sc.excerpts[result.Id] = sc.makeExcerpt(cached) + // might as well keep them in memory + sc.cached[result.Id] = cached + sc.mu.Unlock() + } + } + + err = sc.write() + if err != nil { + out <- entity.NewMergeError(err, "") + return + } + }() + + return out + +} + +func (sc *SubCache[EntityT, ExcerptT, CacheT]) GetNamespace() string { + return sc.namespace +} + +// entityUpdated is a callback to trigger when the excerpt of an entity changed +func (sc *SubCache[EntityT, ExcerptT, CacheT]) entityUpdated(id entity.Id) error { + sc.mu.Lock() + e, ok := sc.cached[id] + if !ok { + sc.mu.Unlock() + + // if the bug is not loaded at this point, it means it was loaded before + // but got evicted. Which means we potentially have multiple copies in + // memory and thus concurrent write. + // Failing immediately here is the simple and safe solution to avoid + // complicated data loss. + return errors.New("entity missing from cache") + } + sc.lru.Get(id) + // sc.excerpts[id] = bug2.NewBugExcerpt(b.bug, b.Snapshot()) + sc.excerpts[id] = sc.makeExcerpt(e) + sc.mu.Unlock() + + index, err := sc.repo.GetIndex(sc.namespace) + if err != nil { + return err + } + + err = index.IndexOne(e.Id().String(), sc.makeIndexData(e)) + if err != nil { + return err + } + + return sc.write() +} + +// evictIfNeeded will evict an entity from the cache if needed +func (sc *SubCache[EntityT, ExcerptT, CacheT]) evictIfNeeded() { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.lru.Len() <= sc.maxLoaded { + return + } + + for _, id := range sc.lru.GetOldestToNewest() { + b := sc.cached[id] + if b.NeedCommit() { + continue + } + + // as a form of assurance that evicted entities don't get manipulated, we lock them here. + // if something try to do it anyway, it will lock the program and make it obvious. + b.Lock() + + sc.lru.Remove(id) + delete(sc.cached, id) + + if sc.lru.Len() <= sc.maxLoaded { + return + } + } +} diff --git a/cache/with_snapshot.go b/cache/with_snapshot.go new file mode 100644 index 00000000..674b6923 --- /dev/null +++ b/cache/with_snapshot.go @@ -0,0 +1,56 @@ +package cache + +import ( + "sync" + + "github.com/MichaelMure/git-bug/entity/dag" + "github.com/MichaelMure/git-bug/repository" +) + +var _ dag.Interface[dag.Snapshot, dag.OperationWithApply[dag.Snapshot]] = &withSnapshot[dag.Snapshot, dag.OperationWithApply[dag.Snapshot]]{} + +// withSnapshot encapsulate an entity and maintain a snapshot efficiently. +type withSnapshot[SnapT dag.Snapshot, OpT dag.OperationWithApply[SnapT]] struct { + dag.Interface[SnapT, OpT] + mu sync.Mutex + snap *SnapT +} + +func (ws *withSnapshot[SnapT, OpT]) Compile() SnapT { + ws.mu.Lock() + defer ws.mu.Unlock() + if ws.snap == nil { + snap := ws.Interface.Compile() + ws.snap = &snap + } + return *ws.snap +} + +// Append intercept Bug.Append() to update the snapshot efficiently +func (ws *withSnapshot[SnapT, OpT]) Append(op OpT) { + ws.mu.Lock() + defer ws.mu.Unlock() + + ws.Interface.Append(op) + + if ws.snap == nil { + return + } + + op.Apply(*ws.snap) + (*ws.snap).AppendOperation(op) +} + +// Commit intercept Bug.Commit() to update the snapshot efficiently +func (ws *withSnapshot[SnapT, OpT]) Commit(repo repository.ClockedRepo) error { + ws.mu.Lock() + defer ws.mu.Unlock() + + err := ws.Interface.Commit(repo) + if err != nil { + ws.snap = nil + return err + } + + return nil +} |