diff options
| -rw-r--r-- | .gitignore | 8 | ||||
| -rw-r--r-- | README.md | 12 | ||||
| -rw-r--r-- | api/graphql/models/lazy_bug.go | 6 | ||||
| -rw-r--r-- | api/graphql/models/lazy_identity.go | 6 | ||||
| -rw-r--r-- | bridge/github/client.go | 106 | ||||
| -rw-r--r-- | bridge/github/export.go | 15 | ||||
| -rw-r--r-- | bridge/github/import_events.go | 40 | ||||
| -rw-r--r-- | bridge/github/import_mediator.go | 134 | ||||
| -rw-r--r-- | doc/README.md | 15 | ||||
| -rw-r--r-- | doc/model.md | 2 | ||||
| -rw-r--r-- | git-bug.go | 2 | ||||
| -rw-r--r-- | misc/completion/bash/git-bug (renamed from misc/bash_completion/git-bug) | 2 | ||||
| -rw-r--r-- | misc/completion/fish/git-bug (renamed from misc/fish_completion/git-bug) | 0 | ||||
| -rw-r--r-- | misc/completion/gen_completion.go (renamed from misc/gen_completion.go) | 10 | ||||
| -rw-r--r-- | misc/completion/powershell/git-bug (renamed from misc/powershell_completion/git-bug) | 0 | ||||
| -rw-r--r-- | misc/completion/zsh/git-bug (renamed from misc/zsh_completion/git-bug) | 0 |
16 files changed, 187 insertions, 171 deletions
@@ -1,8 +1,8 @@ git-bug -!/misc/bash_completion/git-bug -!/misc/fish_completion/git-bug -!/misc/powershell_completion/git-bug -!/misc/zsh_completion/git-bug +!/misc/completion/bash/git-bug +!/misc/completion/fish/git-bug +!/misc/completion/powershell/git-bug +!/misc/completion/zsh/git-bug .gitkeep dist coverage.txt @@ -268,20 +268,22 @@ git bug bridge rm [<name>] ## Internals -Interested by how it works ? Have a look at the [data model](doc/model.md) and the [internal bird-view](doc/architecture.md). +Interested in how it works ? Have a look at the [data model](doc/model.md) and the [internal bird-view](doc/architecture.md). + +Or maybe you want to [make your own distributed data-structure in git](entity/dag/example_test.go) ? + +See also all the [docs](doc). ## Misc -- [Bash completion](misc/bash_completion) -- [Zsh completion](misc/zsh_completion) -- [PowerShell completion](misc/powershell_completion) +- [Bash, Zsh, fish, powershell completion](misc/completion) - [ManPages](doc/man) ## Planned features - media embedding - more bridges -- extendable data model to support arbitrary bug tracker +- webUI that can be used as a public portal to accept user's input - inflatable raptor ## Contribute diff --git a/api/graphql/models/lazy_bug.go b/api/graphql/models/lazy_bug.go index a7840df2..bc26aaca 100644 --- a/api/graphql/models/lazy_bug.go +++ b/api/graphql/models/lazy_bug.go @@ -49,13 +49,13 @@ func NewLazyBug(cache *cache.RepoCache, excerpt *cache.BugExcerpt) *lazyBug { } func (lb *lazyBug) load() error { + lb.mu.Lock() + defer lb.mu.Unlock() + if lb.snap != nil { return nil } - lb.mu.Lock() - defer lb.mu.Unlock() - b, err := lb.cache.ResolveBug(lb.excerpt.Id) if err != nil { return err diff --git a/api/graphql/models/lazy_identity.go b/api/graphql/models/lazy_identity.go index 002c38e4..451bdd54 100644 --- a/api/graphql/models/lazy_identity.go +++ b/api/graphql/models/lazy_identity.go @@ -41,13 +41,13 @@ func NewLazyIdentity(cache *cache.RepoCache, excerpt *cache.IdentityExcerpt) *la } func (li *lazyIdentity) load() (*cache.IdentityCache, error) { + li.mu.Lock() + defer li.mu.Unlock() + if li.id != nil { return li.id, nil } - li.mu.Lock() - defer li.mu.Unlock() - id, err := li.cache.ResolveIdentity(li.excerpt.Id) if err != nil { return nil, fmt.Errorf("cache: missing identity %v", li.excerpt.Id) diff --git a/bridge/github/client.go b/bridge/github/client.go index 10f0a03c..00981a56 100644 --- a/bridge/github/client.go +++ b/bridge/github/client.go @@ -7,8 +7,9 @@ import ( "strings" "time" - "github.com/MichaelMure/git-bug/bridge/core" "github.com/shurcooL/githubv4" + + "github.com/MichaelMure/git-bug/bridge/core" ) var _ Client = &githubv4.Client{} @@ -29,79 +30,69 @@ func newRateLimitHandlerClient(httpClient *http.Client) *rateLimitHandlerClient return &rateLimitHandlerClient{sc: githubv4.NewClient(httpClient)} } -type RateLimitingEvent struct { - msg string -} - -// mutate calls the github api with a graphql mutation and for each rate limiting event it sends an -// export result. +// mutate calls the github api with a graphql mutation and sends a core.ExportResult for each rate limiting event func (c *rateLimitHandlerClient) mutate(ctx context.Context, m interface{}, input githubv4.Input, vars map[string]interface{}, out chan<- core.ExportResult) error { // prepare a closure for the mutation mutFun := func(ctx context.Context) error { return c.sc.Mutate(ctx, m, input, vars) } - limitEvents := make(chan RateLimitingEvent) - defer close(limitEvents) - go func() { - for e := range limitEvents { - select { - case <-ctx.Done(): - return - case out <- core.NewExportRateLimiting(e.msg): - } + callback := func(msg string) { + select { + case <-ctx.Done(): + case out <- core.NewExportRateLimiting(msg): } - }() - return c.callAPIAndRetry(mutFun, ctx, limitEvents) + } + return c.callAPIAndRetry(ctx, mutFun, callback) } -// queryWithLimitEvents calls the github api with a graphql query and it sends rate limiting events -// to a given channel of type RateLimitingEvent. -func (c *rateLimitHandlerClient) queryWithLimitEvents(ctx context.Context, query interface{}, vars map[string]interface{}, limitEvents chan<- RateLimitingEvent) error { - // prepare a closure fot the query +// queryImport calls the github api with a graphql query, and sends an ImportEvent for each rate limiting event +func (c *rateLimitHandlerClient) queryImport(ctx context.Context, query interface{}, vars map[string]interface{}, importEvents chan<- ImportEvent) error { + // prepare a closure for the query queryFun := func(ctx context.Context) error { return c.sc.Query(ctx, query, vars) } - return c.callAPIAndRetry(queryFun, ctx, limitEvents) + callback := func(msg string) { + select { + case <-ctx.Done(): + case importEvents <- RateLimitingEvent{msg}: + } + } + return c.callAPIAndRetry(ctx, queryFun, callback) } -// queryWithImportEvents calls the github api with a graphql query and it sends rate limiting events -// to a given channel of type ImportEvent. -func (c *rateLimitHandlerClient) queryWithImportEvents(ctx context.Context, query interface{}, vars map[string]interface{}, importEvents chan<- ImportEvent) error { - // forward rate limiting events to channel of import events - limitEvents := make(chan RateLimitingEvent) - defer close(limitEvents) - go func() { - for e := range limitEvents { - select { - case <-ctx.Done(): - return - case importEvents <- e: - } +// queryImport calls the github api with a graphql query, and sends a core.ExportResult for each rate limiting event +func (c *rateLimitHandlerClient) queryExport(ctx context.Context, query interface{}, vars map[string]interface{}, out chan<- core.ExportResult) error { + // prepare a closure for the query + queryFun := func(ctx context.Context) error { + return c.sc.Query(ctx, query, vars) + } + callback := func(msg string) { + select { + case <-ctx.Done(): + case out <- core.NewExportRateLimiting(msg): } - }() - return c.queryWithLimitEvents(ctx, query, vars, limitEvents) + } + return c.callAPIAndRetry(ctx, queryFun, callback) } -// queryPrintMsgs calls the github api with a graphql query and it prints for ever rate limiting -// event a message to stdout. +// queryPrintMsgs calls the github api with a graphql query, and prints a message to stdout for every rate limiting event . func (c *rateLimitHandlerClient) queryPrintMsgs(ctx context.Context, query interface{}, vars map[string]interface{}) error { - // print rate limiting events directly to stdout. - limitEvents := make(chan RateLimitingEvent) - defer close(limitEvents) - go func() { - for e := range limitEvents { - fmt.Println(e.msg) - } - }() - return c.queryWithLimitEvents(ctx, query, vars, limitEvents) + // prepare a closure for the query + queryFun := func(ctx context.Context) error { + return c.sc.Query(ctx, query, vars) + } + callback := func(msg string) { + fmt.Println(msg) + } + return c.callAPIAndRetry(ctx, queryFun, callback) } // callAPIAndRetry calls the Github GraphQL API (inderectely through callAPIDealWithLimit) and in // case of error it repeats the request to the Github API. The parameter `apiCall` is intended to be // a closure containing a query or a mutation to the Github GraphQL API. -func (c *rateLimitHandlerClient) callAPIAndRetry(apiCall func(context.Context) error, ctx context.Context, events chan<- RateLimitingEvent) error { +func (c *rateLimitHandlerClient) callAPIAndRetry(ctx context.Context, apiCall func(context.Context) error, rateLimitEvent func(msg string)) error { var err error - if err = c.callAPIDealWithLimit(apiCall, ctx, events); err == nil { + if err = c.callAPIDealWithLimit(ctx, apiCall, rateLimitEvent); err == nil { return nil } // failure; the reason may be temporary network problems or internal errors @@ -117,7 +108,7 @@ func (c *rateLimitHandlerClient) callAPIAndRetry(apiCall func(context.Context) e stop(timer) return ctx.Err() case <-timer.C: - err = c.callAPIDealWithLimit(apiCall, ctx, events) + err = c.callAPIDealWithLimit(ctx, apiCall, rateLimitEvent) if err == nil { return nil } @@ -127,10 +118,10 @@ func (c *rateLimitHandlerClient) callAPIAndRetry(apiCall func(context.Context) e } // callAPIDealWithLimit calls the Github GraphQL API and if the Github API returns a rate limiting -// error, then it waits until the rate limit is reset and it repeats the request to the API. The +// error, then it waits until the rate limit is reset, and it repeats the request to the API. The // parameter `apiCall` is intended to be a closure containing a query or a mutation to the Github // GraphQL API. -func (c *rateLimitHandlerClient) callAPIDealWithLimit(apiCall func(context.Context) error, ctx context.Context, events chan<- RateLimitingEvent) error { +func (c *rateLimitHandlerClient) callAPIDealWithLimit(ctx context.Context, apiCall func(context.Context) error, rateLimitCallback func(msg string)) error { qctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() // call the function fun() @@ -155,11 +146,8 @@ func (c *rateLimitHandlerClient) callAPIDealWithLimit(apiCall func(context.Conte resetTime.String(), ) // Send message about rate limiting event. - select { - case <-ctx.Done(): - return ctx.Err() - case events <- RateLimitingEvent{msg}: - } + rateLimitCallback(msg) + // Pause current goroutine timer := time.NewTimer(time.Until(resetTime)) select { diff --git a/bridge/github/export.go b/bridge/github/export.go index 8c40eb74..35d456c2 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -486,23 +486,10 @@ func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *rateLimitHa } q := labelsQuery{} - // When performing the queries we have to forward rate limiting events to the - // current channel of export results. - events := make(chan RateLimitingEvent) - defer close(events) - go func() { - for e := range events { - select { - case <-ctx.Done(): - return - case ge.out <- core.NewExportRateLimiting(e.msg): - } - } - }() hasNextPage := true for hasNextPage { - if err := gc.queryWithLimitEvents(ctx, &q, variables, events); err != nil { + if err := gc.queryExport(ctx, &q, variables, ge.out); err != nil { return err } diff --git a/bridge/github/import_events.go b/bridge/github/import_events.go new file mode 100644 index 00000000..7ae86d75 --- /dev/null +++ b/bridge/github/import_events.go @@ -0,0 +1,40 @@ +package github + +import "github.com/shurcooL/githubv4" + +type ImportEvent interface { + isImportEvent() +} + +type RateLimitingEvent struct { + msg string +} + +func (RateLimitingEvent) isImportEvent() {} + +type IssueEvent struct { + issue +} + +func (IssueEvent) isImportEvent() {} + +type IssueEditEvent struct { + issueId githubv4.ID + userContentEdit +} + +func (IssueEditEvent) isImportEvent() {} + +type TimelineEvent struct { + issueId githubv4.ID + timelineItem +} + +func (TimelineEvent) isImportEvent() {} + +type CommentEditEvent struct { + commentId githubv4.ID + userContentEdit +} + +func (CommentEditEvent) isImportEvent() {} diff --git a/bridge/github/import_mediator.go b/bridge/github/import_mediator.go index db9f877c..be4e3880 100644 --- a/bridge/github/import_mediator.go +++ b/bridge/github/import_mediator.go @@ -9,6 +9,7 @@ import ( const ( // These values influence how fast the github graphql rate limit is exhausted. + NumIssues = 40 NumIssueEdits = 100 NumTimelineItems = 100 @@ -41,43 +42,6 @@ type importMediator struct { err error } -type ImportEvent interface { - isImportEvent() -} - -func (RateLimitingEvent) isImportEvent() {} - -type IssueEvent struct { - issue -} - -func (IssueEvent) isImportEvent() {} - -type IssueEditEvent struct { - issueId githubv4.ID - userContentEdit -} - -func (IssueEditEvent) isImportEvent() {} - -type TimelineEvent struct { - issueId githubv4.ID - timelineItem -} - -func (TimelineEvent) isImportEvent() {} - -type CommentEditEvent struct { - commentId githubv4.ID - userContentEdit -} - -func (CommentEditEvent) isImportEvent() {} - -func (mm *importMediator) NextImportEvent() ImportEvent { - return <-mm.importEvents -} - func NewImportMediator(ctx context.Context, client *rateLimitHandlerClient, owner, project string, since time.Time) *importMediator { mm := importMediator{ gh: client, @@ -87,48 +51,24 @@ func NewImportMediator(ctx context.Context, client *rateLimitHandlerClient, owne importEvents: make(chan ImportEvent, ChanCapacity), err: nil, } - go func() { - mm.fillImportEvents(ctx) - close(mm.importEvents) - }() - return &mm -} -type varmap map[string]interface{} + go mm.start(ctx) -func newIssueVars(owner, project string, since time.Time) varmap { - return varmap{ - "owner": githubv4.String(owner), - "name": githubv4.String(project), - "issueSince": githubv4.DateTime{Time: since}, - "issueFirst": githubv4.Int(NumIssues), - "issueEditLast": githubv4.Int(NumIssueEdits), - "issueEditBefore": (*githubv4.String)(nil), - "timelineFirst": githubv4.Int(NumTimelineItems), - "timelineAfter": (*githubv4.String)(nil), - "commentEditLast": githubv4.Int(NumCommentEdits), - "commentEditBefore": (*githubv4.String)(nil), - } -} - -func newIssueEditVars() varmap { - return varmap{ - "issueEditLast": githubv4.Int(NumIssueEdits), - } + return &mm } -func newTimelineVars() varmap { - return varmap{ - "timelineFirst": githubv4.Int(NumTimelineItems), - "commentEditLast": githubv4.Int(NumCommentEdits), - "commentEditBefore": (*githubv4.String)(nil), - } +func (mm *importMediator) start(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + mm.fillImportEvents(ctx) + // Make sure we cancel everything when we are done, instead of relying on the parent context + // This should unblock pending send to the channel if the capacity was reached and avoid a panic/race when closing. + cancel() + close(mm.importEvents) } -func newCommentEditVars() varmap { - return varmap{ - "commentEditLast": githubv4.Int(NumCommentEdits), - } +// NextImportEvent returns the next ImportEvent, or nil if done. +func (mm *importMediator) NextImportEvent() ImportEvent { + return <-mm.importEvents } func (mm *importMediator) Error() error { @@ -138,7 +78,7 @@ func (mm *importMediator) Error() error { func (mm *importMediator) User(ctx context.Context, loginName string) (*user, error) { query := userQuery{} vars := varmap{"login": githubv4.String(loginName)} - if err := mm.gh.queryWithImportEvents(ctx, &query, vars, mm.importEvents); err != nil { + if err := mm.gh.queryImport(ctx, &query, vars, mm.importEvents); err != nil { return nil, err } return &query.User, nil @@ -200,7 +140,7 @@ func (mm *importMediator) queryIssueEdits(ctx context.Context, nid githubv4.ID, vars["issueEditBefore"] = cursor } query := issueEditQuery{} - if err := mm.gh.queryWithImportEvents(ctx, &query, vars, mm.importEvents); err != nil { + if err := mm.gh.queryImport(ctx, &query, vars, mm.importEvents); err != nil { mm.err = err return nil, false } @@ -244,7 +184,7 @@ func (mm *importMediator) queryTimeline(ctx context.Context, nid githubv4.ID, cu vars["timelineAfter"] = cursor } query := timelineQuery{} - if err := mm.gh.queryWithImportEvents(ctx, &query, vars, mm.importEvents); err != nil { + if err := mm.gh.queryImport(ctx, &query, vars, mm.importEvents); err != nil { mm.err = err return nil, false } @@ -294,7 +234,7 @@ func (mm *importMediator) queryCommentEdits(ctx context.Context, nid githubv4.ID vars["commentEditBefore"] = cursor } query := commentEditQuery{} - if err := mm.gh.queryWithImportEvents(ctx, &query, vars, mm.importEvents); err != nil { + if err := mm.gh.queryImport(ctx, &query, vars, mm.importEvents); err != nil { mm.err = err return nil, false } @@ -313,7 +253,7 @@ func (mm *importMediator) queryIssue(ctx context.Context, cursor githubv4.String vars["issueAfter"] = cursor } query := issueQuery{} - if err := mm.gh.queryWithImportEvents(ctx, &query, vars, mm.importEvents); err != nil { + if err := mm.gh.queryImport(ctx, &query, vars, mm.importEvents); err != nil { mm.err = err return nil, false } @@ -334,3 +274,41 @@ func reverse(eds []userContentEdit) chan userContentEdit { }() return ret } + +// varmap is a container for Github API's pagination variables +type varmap map[string]interface{} + +func newIssueVars(owner, project string, since time.Time) varmap { + return varmap{ + "owner": githubv4.String(owner), + "name": githubv4.String(project), + "issueSince": githubv4.DateTime{Time: since}, + "issueFirst": githubv4.Int(NumIssues), + "issueEditLast": githubv4.Int(NumIssueEdits), + "issueEditBefore": (*githubv4.String)(nil), + "timelineFirst": githubv4.Int(NumTimelineItems), + "timelineAfter": (*githubv4.String)(nil), + "commentEditLast": githubv4.Int(NumCommentEdits), + "commentEditBefore": (*githubv4.String)(nil), + } +} + +func newIssueEditVars() varmap { + return varmap{ + "issueEditLast": githubv4.Int(NumIssueEdits), + } +} + +func newTimelineVars() varmap { + return varmap{ + "timelineFirst": githubv4.Int(NumTimelineItems), + "commentEditLast": githubv4.Int(NumCommentEdits), + "commentEditBefore": (*githubv4.String)(nil), + } +} + +func newCommentEditVars() varmap { + return varmap{ + "commentEditLast": githubv4.Int(NumCommentEdits), + } +} diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..cf9fd845 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,15 @@ +# Documentation + +## For users + +- [data model](model.md) describe how the data model works and why. +- [query language](queries.md) describe git-bug's query language. +- [How-to: Read and edit offline your Github/Gitlab/Jira issues with git-bug](howto-github.md) + +## For developers + +- :exclamation: [data model](model.md) describe how the data model works and why. +- :exclamation: [internal bird-view](architecture.md) gives an overview of the project architecture. +- :exclamation: [Entity/DAG](../entity/dag/example_test.go) explain how to easily make your own distributed entity in git. +- [query language](queries.md) describe git-bug's query language. +- [JIRA bridge de v notes](jira_bridge.md)
\ No newline at end of file diff --git a/doc/model.md b/doc/model.md index de0de42a..0aad4789 100644 --- a/doc/model.md +++ b/doc/model.md @@ -3,6 +3,8 @@ Entities data model If you are not familiar with [git internals](https://git-scm.com/book/en/v1/Git-Internals), you might first want to read about them, as the `git-bug` data model is built on top of them. +In a different format, see how you can easily make your own [distributed data structure](../entity/dag/example_test.go). + ## Entities (bug, author, ...) are a series of edit operations As entities are stored and edited in multiple processes at the same time, it's not possible to store the current state like it would be done in a normal application. If two processes change the same entity and later try to merge the states, we wouldn't know which change takes precedence or how to merge those states. @@ -1,5 +1,5 @@ //go:generate go run doc/gen_docs.go -//go:generate go run misc/gen_completion.go +//go:generate go run misc/completion/gen_completion.go package main diff --git a/misc/bash_completion/git-bug b/misc/completion/bash/git-bug index 85d2a15b..92b90d3c 100644 --- a/misc/bash_completion/git-bug +++ b/misc/completion/bash/git-bug @@ -287,6 +287,8 @@ fi # ex: ts=4 sw=4 et filetype=sh +# Custom bash code to connect the git completion for "git bug" to the +# git-bug completion for "git-bug" _git_bug() { local cur prev words cword split diff --git a/misc/fish_completion/git-bug b/misc/completion/fish/git-bug index a3c2d008..a3c2d008 100644 --- a/misc/fish_completion/git-bug +++ b/misc/completion/fish/git-bug diff --git a/misc/gen_completion.go b/misc/completion/gen_completion.go index 1f86124d..5f643832 100644 --- a/misc/gen_completion.go +++ b/misc/completion/gen_completion.go @@ -44,13 +44,15 @@ func genBash(root *cobra.Command) error { if err != nil { return err } - f, err := os.Create(filepath.Join(cwd, "misc", "bash_completion", "git-bug")) + f, err := os.Create(filepath.Join(cwd, "misc", "completion", "bash", "git-bug")) if err != nil { return err } defer f.Close() const patch = ` +# Custom bash code to connect the git completion for "git bug" to the +# git-bug completion for "git-bug" _git_bug() { local cur prev words cword split @@ -102,7 +104,7 @@ func genFish(root *cobra.Command) error { if err != nil { return err } - dir := filepath.Join(cwd, "misc", "fish_completion", "git-bug") + dir := filepath.Join(cwd, "misc", "completion", "fish", "git-bug") return root.GenFishCompletionFile(dir, true) } @@ -111,7 +113,7 @@ func genPowerShell(root *cobra.Command) error { if err != nil { return err } - path := filepath.Join(cwd, "misc", "powershell_completion", "git-bug") + path := filepath.Join(cwd, "misc", "completion", "powershell", "git-bug") return root.GenPowerShellCompletionFile(path) } @@ -120,6 +122,6 @@ func genZsh(root *cobra.Command) error { if err != nil { return err } - path := filepath.Join(cwd, "misc", "zsh_completion", "git-bug") + path := filepath.Join(cwd, "misc", "completion", "zsh", "git-bug") return root.GenZshCompletionFile(path) } diff --git a/misc/powershell_completion/git-bug b/misc/completion/powershell/git-bug index b8280eea..b8280eea 100644 --- a/misc/powershell_completion/git-bug +++ b/misc/completion/powershell/git-bug diff --git a/misc/zsh_completion/git-bug b/misc/completion/zsh/git-bug index e7cbe9a9..e7cbe9a9 100644 --- a/misc/zsh_completion/git-bug +++ b/misc/completion/zsh/git-bug |