diff options
Diffstat (limited to 'bridge')
| -rw-r--r-- | bridge/core/bridge.go | 47 | ||||
| -rw-r--r-- | bridge/core/interfaces.go | 8 | ||||
| -rw-r--r-- | bridge/github/config.go | 10 | ||||
| -rw-r--r-- | bridge/github/github.go | 4 | ||||
| -rw-r--r-- | bridge/github/import.go | 479 | ||||
| -rw-r--r-- | bridge/github/import_query.go | 6 | ||||
| -rw-r--r-- | bridge/github/import_test.go | 197 | ||||
| -rw-r--r-- | bridge/github/iterator.go | 409 | ||||
| -rw-r--r-- | bridge/launchpad/import.go | 7 |
9 files changed, 779 insertions, 388 deletions
diff --git a/bridge/core/bridge.go b/bridge/core/bridge.go index b849bec6..aa02ceb5 100644 --- a/bridge/core/bridge.go +++ b/bridge/core/bridge.go @@ -6,6 +6,7 @@ import ( "reflect" "regexp" "strings" + "time" "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/repository" @@ -265,7 +266,7 @@ func (b *Bridge) ensureInit() error { return nil } -func (b *Bridge) ImportAll() error { +func (b *Bridge) ImportAll(since time.Time) error { importer := b.getImporter() if importer == nil { return ErrImportNotSupported @@ -281,48 +282,10 @@ func (b *Bridge) ImportAll() error { return err } - return importer.ImportAll(b.repo) + return importer.ImportAll(b.repo, since) } -func (b *Bridge) Import(id string) error { - importer := b.getImporter() - if importer == nil { - return ErrImportNotSupported - } - - err := b.ensureConfig() - if err != nil { - return err - } - - err = b.ensureInit() - if err != nil { - return err - } - - return importer.Import(b.repo, id) -} - -func (b *Bridge) ExportAll() error { - exporter := b.getExporter() - if exporter == nil { - return ErrExportNotSupported - } - - err := b.ensureConfig() - if err != nil { - return err - } - - err = b.ensureInit() - if err != nil { - return err - } - - return exporter.ExportAll(b.repo) -} - -func (b *Bridge) Export(id string) error { +func (b *Bridge) ExportAll(since time.Time) error { exporter := b.getExporter() if exporter == nil { return ErrExportNotSupported @@ -338,5 +301,5 @@ func (b *Bridge) Export(id string) error { return err } - return exporter.Export(b.repo, id) + return exporter.ExportAll(b.repo, since) } diff --git a/bridge/core/interfaces.go b/bridge/core/interfaces.go index 4836dab3..be5afa62 100644 --- a/bridge/core/interfaces.go +++ b/bridge/core/interfaces.go @@ -1,6 +1,8 @@ package core import ( + "time" + "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/repository" ) @@ -27,12 +29,10 @@ type BridgeImpl interface { type Importer interface { Init(conf Configuration) error - ImportAll(repo *cache.RepoCache) error - Import(repo *cache.RepoCache, id string) error + ImportAll(repo *cache.RepoCache, since time.Time) error } type Exporter interface { Init(conf Configuration) error - ExportAll(repo *cache.RepoCache) error - Export(repo *cache.RepoCache, id string) error + ExportAll(repo *cache.RepoCache, since time.Time) error } diff --git a/bridge/github/config.go b/bridge/github/config.go index b881c585..2a3119a6 100644 --- a/bridge/github/config.go +++ b/bridge/github/config.go @@ -20,10 +20,12 @@ import ( "golang.org/x/crypto/ssh/terminal" ) -const githubV3Url = "https://api.github.com" -const keyUser = "user" -const keyProject = "project" -const keyToken = "token" +const ( + githubV3Url = "https://api.github.com" + keyUser = "user" + keyProject = "project" + keyToken = "token" +) func (*Github) Configure(repo repository.RepoCommon) (core.Configuration, error) { conf := make(core.Configuration) diff --git a/bridge/github/github.go b/bridge/github/github.go index b3f8d763..5fee7487 100644 --- a/bridge/github/github.go +++ b/bridge/github/github.go @@ -27,9 +27,9 @@ func (*Github) NewExporter() core.Exporter { return nil } -func buildClient(conf core.Configuration) *githubv4.Client { +func buildClient(token string) *githubv4.Client { src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: conf[keyToken]}, + &oauth2.Token{AccessToken: token}, ) httpClient := oauth2.NewClient(context.TODO(), src) diff --git a/bridge/github/import.go b/bridge/github/import.go index d641b192..0c5468d8 100644 --- a/bridge/github/import.go +++ b/bridge/github/import.go @@ -3,273 +3,173 @@ package github import ( "context" "fmt" - "strings" + "time" "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/util/git" + "github.com/MichaelMure/git-bug/util/text" "github.com/shurcooL/githubv4" ) -const keyGithubId = "github-id" -const keyGithubUrl = "github-url" -const keyGithubLogin = "github-login" +const ( + keyGithubId = "github-id" + keyGithubUrl = "github-url" + keyGithubLogin = "github-login" +) // githubImporter implement the Importer interface type githubImporter struct { - client *githubv4.Client - conf core.Configuration + conf core.Configuration } func (gi *githubImporter) Init(conf core.Configuration) error { gi.conf = conf - gi.client = buildClient(conf) - return nil } -func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error { - q := &issueTimelineQuery{} - variables := map[string]interface{}{ - "owner": githubv4.String(gi.conf[keyUser]), - "name": githubv4.String(gi.conf[keyProject]), - "issueFirst": githubv4.Int(1), - "issueAfter": (*githubv4.String)(nil), - "timelineFirst": githubv4.Int(10), - "timelineAfter": (*githubv4.String)(nil), - - // Fun fact, github provide the comment edition in reverse chronological - // order, because haha. Look at me, I'm dying of laughter. - "issueEditLast": githubv4.Int(10), - "issueEditBefore": (*githubv4.String)(nil), - "commentEditLast": githubv4.Int(10), - "commentEditBefore": (*githubv4.String)(nil), - } - - var b *cache.BugCache +// ImportAll . +func (gi *githubImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { + iterator := NewIterator(gi.conf[keyUser], gi.conf[keyProject], gi.conf[keyToken], since) + + // Loop over all matching issues + for iterator.NextIssue() { + issue := iterator.IssueValue() + + fmt.Printf("importing issue: %v %v\n", iterator.importedIssues, issue.Title) + // get issue edits + issueEdits := []userContentEdit{} + for iterator.NextIssueEdit() { + // issueEdit.Diff == nil happen if the event is older than early 2018, Github doesn't have the data before that. + // Best we can do is to ignore the event. + if issueEdit := iterator.IssueEditValue(); issueEdit.Diff != nil && string(*issueEdit.Diff) != "" { + issueEdits = append(issueEdits, issueEdit) + } + } - for { - err := gi.client.Query(context.TODO(), &q, variables) + // create issue + b, err := gi.ensureIssue(repo, issue, issueEdits) if err != nil { - return err + return fmt.Errorf("issue creation: %v", err) } - if len(q.Repository.Issues.Nodes) == 0 { - return nil - } + // loop over timeline items + for iterator.NextTimeline() { + item := iterator.TimelineValue() - issue := q.Repository.Issues.Nodes[0] + // if item is comment + if item.Typename == "IssueComment" { + // collect all edits + commentEdits := []userContentEdit{} + for iterator.NextCommentEdit() { + if commentEdit := iterator.CommentEditValue(); commentEdit.Diff != nil && string(*commentEdit.Diff) != "" { + commentEdits = append(commentEdits, commentEdit) + } + } - if b == nil { - b, err = gi.ensureIssue(repo, issue, variables) - if err != nil { - return err - } - } + err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits) + if err != nil { + return fmt.Errorf("timeline comment creation: %v", err) + } - for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges { - err = gi.ensureTimelineItem(repo, b, itemEdge.Cursor, itemEdge.Node, variables) - if err != nil { - return err + } else { + if err := gi.ensureTimelineItem(repo, b, item); err != nil { + return fmt.Errorf("timeline event creation: %v", err) + } } } - if !issue.Timeline.PageInfo.HasNextPage { - err = b.CommitAsNeeded() - if err != nil { - return err - } - - b = nil - - if !q.Repository.Issues.PageInfo.HasNextPage { - break - } - - variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor) - variables["timelineAfter"] = (*githubv4.String)(nil) - continue + // commit bug state + if err := b.CommitAsNeeded(); err != nil { + return fmt.Errorf("bug commit: %v", err) } - - variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor) } - return nil -} - -func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error { - fmt.Println("IMPORT") + if err := iterator.Error(); err != nil { + fmt.Printf("import error: %v\n", err) + return err + } + fmt.Printf("Successfully imported %v issues from Github\n", iterator.ImportedIssues()) return nil } -func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, rootVariables map[string]interface{}) (*cache.BugCache, error) { - fmt.Printf("import issue: %s\n", issue.Title) - +func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, issueEdits []userContentEdit) (*cache.BugCache, error) { + // ensure issue author author, err := gi.ensurePerson(repo, issue.Author) if err != nil { return nil, err } - b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id)) + // resolve bug + b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String()) if err != nil && err != bug.ErrBugNotExist { return nil, err } - // if there is no edit, the UserContentEdits given by github is empty. That - // means that the original message is given by the issue message. - // - // if there is edits, the UserContentEdits given by github contains both the - // original message and the following edits. The issue message give the last - // version so we don't care about that. - // - // the tricky part: for an issue older than the UserContentEdits API, github - // doesn't have the previous message version anymore and give an edition - // with .Diff == nil. We have to filter them. - - if len(issue.UserContentEdits.Nodes) == 0 { + // if issueEdits is empty + if len(issueEdits) == 0 { if err == bug.ErrBugNotExist { + cleanText, err := text.Cleanup(string(issue.Body)) + if err != nil { + return nil, err + } + + // create bug b, err = repo.NewBugRaw( author, issue.CreatedAt.Unix(), - // Todo: this might not be the initial title, we need to query the - // timeline to be sure issue.Title, - cleanupText(string(issue.Body)), + cleanText, nil, map[string]string{ keyGithubId: parseId(issue.Id), keyGithubUrl: issue.Url.String(), - }, - ) + }) if err != nil { return nil, err } } - return b, nil - } - - // reverse the order, because github - reverseEdits(issue.UserContentEdits.Nodes) - - for i, edit := range issue.UserContentEdits.Nodes { - if b != nil && i == 0 { - // The first edit in the github result is the creation itself, we already have that - continue - } - - if b == nil { - if edit.Diff == nil { - // not enough data given by github for old edit, ignore them + } else { + // create bug from given issueEdits + for i, edit := range issueEdits { + if i == 0 && b != nil { + // The first edit in the github result is the issue creation itself, we already have that continue } - // we create the bug as soon as we have a legit first edition - b, err = repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - // Todo: this might not be the initial title, we need to query the - // timeline to be sure - issue.Title, - cleanupText(string(*edit.Diff)), - nil, - map[string]string{ - keyGithubId: parseId(issue.Id), - keyGithubUrl: issue.Url.String(), - }, - ) + cleanText, err := text.Cleanup(string(*edit.Diff)) if err != nil { return nil, err } - continue - } - - target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id)) - if err != nil { - return nil, err - } - - err = gi.ensureCommentEdit(repo, b, target, edit) - if err != nil { - return nil, err - } - } - - if !issue.UserContentEdits.PageInfo.HasNextPage { - // if we still didn't get a legit edit, create the bug from the issue data - if b == nil { - return repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - // Todo: this might not be the initial title, we need to query the - // timeline to be sure - issue.Title, - cleanupText(string(issue.Body)), - nil, - map[string]string{ - keyGithubId: parseId(issue.Id), - keyGithubUrl: issue.Url.String(), - }, - ) - } - return b, nil - } - - // We have more edit, querying them - q := &issueEditQuery{} - variables := map[string]interface{}{ - "owner": rootVariables["owner"], - "name": rootVariables["name"], - "issueFirst": rootVariables["issueFirst"], - "issueAfter": rootVariables["issueAfter"], - "issueEditLast": githubv4.Int(10), - "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor, - } - - for { - err := gi.client.Query(context.TODO(), &q, variables) - if err != nil { - return nil, err - } - - edits := q.Repository.Issues.Nodes[0].UserContentEdits - - if len(edits.Nodes) == 0 { - return b, nil - } - - for _, edit := range edits.Nodes { + // if the bug doesn't exist if b == nil { - if edit.Diff == nil { - // not enough data given by github for old edit, ignore them - continue - } - // we create the bug as soon as we have a legit first edition b, err = repo.NewBugRaw( author, issue.CreatedAt.Unix(), - // Todo: this might not be the initial title, we need to query the - // timeline to be sure issue.Title, - cleanupText(string(*edit.Diff)), + cleanText, nil, map[string]string{ keyGithubId: parseId(issue.Id), keyGithubUrl: issue.Url.String(), }, ) + if err != nil { return nil, err } + continue } - target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id)) + // other edits will be added as CommentEdit operations + target, err := b.ResolveOperationWithMetadata(keyGithubUrl, issue.Url.String()) if err != nil { return nil, err } @@ -279,42 +179,16 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline return nil, err } } - - if !edits.PageInfo.HasNextPage { - break - } - - variables["issueEditBefore"] = edits.PageInfo.StartCursor - } - - // TODO: check + import files - - // if we still didn't get a legit edit, create the bug from the issue data - if b == nil { - return repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - // Todo: this might not be the initial title, we need to query the - // timeline to be sure - issue.Title, - cleanupText(string(issue.Body)), - nil, - map[string]string{ - keyGithubId: parseId(issue.Id), - keyGithubUrl: issue.Url.String(), - }, - ) } return b, nil } -func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, item timelineItem, rootVariables map[string]interface{}) error { - fmt.Printf("import %s\n", item.Typename) +func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error { + fmt.Printf("import event item: %s\n", item.Typename) switch item.Typename { case "IssueComment": - return gi.ensureComment(repo, b, cursor, item.IssueComment, rootVariables) case "LabeledEvent": id := parseId(item.LabeledEvent.Id) @@ -326,7 +200,7 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug if err != nil { return err } - _, _, err = b.ChangeLabelsRaw( + _, err = b.ForceChangeLabelsRaw( author, item.LabeledEvent.CreatedAt.Unix(), []string{ @@ -335,6 +209,7 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug nil, map[string]string{keyGithubId: id}, ) + return err case "UnlabeledEvent": @@ -347,7 +222,8 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug if err != nil { return err } - _, _, err = b.ChangeLabelsRaw( + + _, err = b.ForceChangeLabelsRaw( author, item.UnlabeledEvent.CreatedAt.Unix(), nil, @@ -411,162 +287,109 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug return err default: - fmt.Println("ignore event ", item.Typename) + fmt.Printf("ignore event: %v\n", item.Typename) } return nil } -func (gi *githubImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, comment issueComment, rootVariables map[string]interface{}) error { - author, err := gi.ensurePerson(repo, comment.Author) +func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error { + // ensure person + author, err := gi.ensurePerson(repo, item.Author) if err != nil { return err } var target git.Hash - target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(comment.Id)) + target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id)) if err != nil && err != cache.ErrNoMatchingOp { // real error return err } + // if no edits are given we create the comment + if len(edits) == 0 { - // if there is no edit, the UserContentEdits given by github is empty. That - // means that the original message is given by the comment message. - // - // if there is edits, the UserContentEdits given by github contains both the - // original message and the following edits. The comment message give the last - // version so we don't care about that. - // - // the tricky part: for a comment older than the UserContentEdits API, github - // doesn't have the previous message version anymore and give an edition - // with .Diff == nil. We have to filter them. - - if len(comment.UserContentEdits.Nodes) == 0 { + // if comment doesn't exist if err == cache.ErrNoMatchingOp { + cleanText, err := text.Cleanup(string(item.Body)) + if err != nil { + return err + } + + // add comment operation op, err := b.AddCommentRaw( author, - comment.CreatedAt.Unix(), - cleanupText(string(comment.Body)), + item.CreatedAt.Unix(), + cleanText, nil, map[string]string{ - keyGithubId: parseId(comment.Id), + keyGithubId: parseId(item.Id), + keyGithubUrl: parseId(item.Url.String()), }, ) if err != nil { return err } + // set hash target, err = op.Hash() if err != nil { return err } } - - return nil - } - - // reverse the order, because github - reverseEdits(comment.UserContentEdits.Nodes) - - for i, edit := range comment.UserContentEdits.Nodes { - if target != "" && i == 0 { - // The first edit in the github result is the comment creation itself, we already have that - continue - } - - if target == "" { - if edit.Diff == nil { - // not enough data given by github for old edit, ignore them + } else { + for i, edit := range edits { + if i == 0 && target != "" { + // The first edit in the github result is the comment creation itself, we already have that continue } - op, err := b.AddCommentRaw( - author, - comment.CreatedAt.Unix(), - cleanupText(string(*edit.Diff)), - nil, - map[string]string{ - keyGithubId: parseId(comment.Id), - keyGithubUrl: comment.Url.String(), - }, - ) - if err != nil { - return err - } - - target, err = op.Hash() + // ensure editor identity + editor, err := gi.ensurePerson(repo, edit.Editor) if err != nil { return err } - } - - err := gi.ensureCommentEdit(repo, b, target, edit) - if err != nil { - return err - } - } - - if !comment.UserContentEdits.PageInfo.HasNextPage { - return nil - } - - // We have more edit, querying them - q := &commentEditQuery{} - variables := map[string]interface{}{ - "owner": rootVariables["owner"], - "name": rootVariables["name"], - "issueFirst": rootVariables["issueFirst"], - "issueAfter": rootVariables["issueAfter"], - "timelineFirst": githubv4.Int(1), - "timelineAfter": cursor, - "commentEditLast": githubv4.Int(10), - "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor, - } - - for { - err := gi.client.Query(context.TODO(), &q, variables) - if err != nil { - return err - } + // create comment when target is empty + if target == "" { + cleanText, err := text.Cleanup(string(*edit.Diff)) + if err != nil { + return err + } - edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits + op, err := b.AddCommentRaw( + editor, + edit.CreatedAt.Unix(), + cleanText, + nil, + map[string]string{ + keyGithubId: parseId(item.Id), + keyGithubUrl: item.Url.String(), + }, + ) + if err != nil { + return err + } - if len(edits.Nodes) == 0 { - return nil - } + // set hash + target, err = op.Hash() + if err != nil { + return err + } - for i, edit := range edits.Nodes { - if i == 0 { - // The first edit in the github result is the creation itself, we already have that continue } - err := gi.ensureCommentEdit(repo, b, target, edit) + err = gi.ensureCommentEdit(repo, b, target, edit) if err != nil { return err } } - - if !edits.PageInfo.HasNextPage { - break - } - - variables["commentEditBefore"] = edits.PageInfo.StartCursor } - - // TODO: check + import files - return nil } func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error { - if edit.Diff == nil { - // this happen if the event is older than early 2018, Github doesn't have the data before that. - // Best we can do is to ignore the event. - return nil - } - _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id)) if err == nil { // already imported @@ -587,18 +410,26 @@ func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugC switch { case edit.DeletedAt != nil: // comment deletion, not supported yet + fmt.Println("comment deletion is not supported yet") case edit.DeletedAt == nil: + + cleanText, err := text.Cleanup(string(*edit.Diff)) + if err != nil { + return err + } + // comment edition - _, err := b.EditCommentRaw( + _, err = b.EditCommentRaw( editor, edit.CreatedAt.Unix(), target, - cleanupText(string(*edit.Diff)), + cleanText, map[string]string{ keyGithubId: parseId(edit.Id), }, ) + if err != nil { return err } @@ -670,7 +501,9 @@ func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, "login": githubv4.String("ghost"), } - err = gi.client.Query(context.TODO(), &q, variables) + gc := buildClient(gi.conf[keyToken]) + + err = gc.Query(context.TODO(), &q, variables) if err != nil { return nil, err } @@ -696,14 +529,6 @@ func parseId(id githubv4.ID) string { return fmt.Sprintf("%v", id) } -func cleanupText(text string) string { - // windows new line, Github, really ? - text = strings.Replace(text, "\r\n", "\n", -1) - - // trim extra new line not displayed in the github UI but still present in the data - return strings.TrimSpace(text) -} - func reverseEdits(edits []userContentEdit) []userContentEdit { for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 { edits[i], edits[j] = edits[j], edits[i] diff --git a/bridge/github/import_query.go b/bridge/github/import_query.go index 59799f6a..4d5886f6 100644 --- a/bridge/github/import_query.go +++ b/bridge/github/import_query.go @@ -128,7 +128,7 @@ type issueTimelineQuery struct { Issues struct { Nodes []issueTimeline PageInfo pageInfo - } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"` + } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC}, filterBy: {since: $issueSince})"` } `graphql:"repository(owner: $owner, name: $name)"` } @@ -137,7 +137,7 @@ type issueEditQuery struct { Issues struct { Nodes []issueEdit PageInfo pageInfo - } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"` + } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC}, filterBy: {since: $issueSince})"` } `graphql:"repository(owner: $owner, name: $name)"` } @@ -156,7 +156,7 @@ type commentEditQuery struct { } } `graphql:"timeline(first: $timelineFirst, after: $timelineAfter)"` } - } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"` + } `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC}, filterBy: {since: $issueSince})"` } `graphql:"repository(owner: $owner, name: $name)"` } diff --git a/bridge/github/import_test.go b/bridge/github/import_test.go new file mode 100644 index 00000000..48283b7a --- /dev/null +++ b/bridge/github/import_test.go @@ -0,0 +1,197 @@ +package github + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/identity" + "github.com/MichaelMure/git-bug/util/interrupt" + "github.com/MichaelMure/git-bug/util/test" +) + +func Test_Importer(t *testing.T) { + author := identity.NewIdentity("Michael Muré", "batolettre@gmail.com") + tests := []struct { + name string + url string + bug *bug.Snapshot + }{ + { + name: "simple issue", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/1", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewAddCommentOp(author, 0, "second comment", nil), + }, + }, + }, + { + name: "empty issue", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/2", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "empty issue", "", nil), + }, + }, + }, + { + name: "complex issue", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/3", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "complex issue", "initial comment", nil), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"bug"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"duplicate"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{}, []bug.Label{"duplicate"}), + bug.NewAddCommentOp(author, 0, "### header\n\n**bold**\n\n_italic_\n\n> with quote\n\n`inline code`\n\n```\nmultiline code\n```\n\n- bulleted\n- list\n\n1. numbered\n1. list\n\n- [ ] task\n- [x] list\n\n@MichaelMure mention\n\n#2 reference issue\n#3 auto-reference issue\n\n", nil), + bug.NewSetTitleOp(author, 0, "complex issue edited", "complex issue"), + bug.NewSetTitleOp(author, 0, "complex issue", "complex issue edited"), + bug.NewSetStatusOp(author, 0, bug.ClosedStatus), + bug.NewSetStatusOp(author, 0, bug.OpenStatus), + }, + }, + }, + { + name: "editions", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/4", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "editions", "initial comment edited", nil), + bug.NewEditCommentOp(author, 0, "", "erased then edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited", nil), + }, + }, + }, + { + name: "comment deletion", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/5", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "comment deletion", "", nil), + }, + }, + }, + { + name: "edition deletion", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/6", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "edition deletion", "initial comment", nil), + bug.NewEditCommentOp(author, 0, "", "initial comment edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited again", nil), + }, + }, + }, + { + name: "hidden comment", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/7", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "hidden comment", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + }, + }, + }, + { + name: "transfered issue", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/8", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "transfered issue", "", nil), + }, + }, + }, + { + name: "unicode control characters", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/10", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "unicode control characters", "u0000: \nu0001: \nu0002: \nu0003: \nu0004: \nu0005: \nu0006: \nu0007: \nu0008: \nu0009: \t\nu0010: \nu0011: \nu0012: \nu0013: \nu0014: \nu0015: \nu0016: \nu0017: \nu0018: \nu0019:", nil), + }, + }, + }, + } + + repo := test.CreateRepo(false) + + backend, err := cache.NewRepoCache(repo) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + t.Skip("Env var GITHUB_TOKEN missing") + } + + importer := &githubImporter{} + err = importer.Init(core.Configuration{ + "user": "MichaelMure", + "project": "git-bug-test-github-bridge", + "token": token, + }) + require.NoError(t, err) + + start := time.Now() + + err = importer.ImportAll(backend, time.Time{}) + require.NoError(t, err) + + fmt.Printf("test repository imported in %f seconds\n", time.Since(start).Seconds()) + + require.Len(t, backend.AllBugsIds(), 9) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := backend.ResolveBugCreateMetadata(keyGithubUrl, tt.url) + require.NoError(t, err) + + ops := b.Snapshot().Operations + assert.Len(t, tt.bug.Operations, len(b.Snapshot().Operations)) + + for i, op := range tt.bug.Operations { + require.IsType(t, ops[i], op) + + switch op.(type) { + case *bug.CreateOperation: + assert.Equal(t, op.(*bug.CreateOperation).Title, ops[i].(*bug.CreateOperation).Title) + assert.Equal(t, op.(*bug.CreateOperation).Message, ops[i].(*bug.CreateOperation).Message) + assert.Equal(t, op.(*bug.CreateOperation).Author.Name(), ops[i].(*bug.CreateOperation).Author.Name()) + case *bug.SetStatusOperation: + assert.Equal(t, op.(*bug.SetStatusOperation).Status, ops[i].(*bug.SetStatusOperation).Status) + assert.Equal(t, op.(*bug.SetStatusOperation).Author.Name(), ops[i].(*bug.SetStatusOperation).Author.Name()) + case *bug.SetTitleOperation: + assert.Equal(t, op.(*bug.SetTitleOperation).Was, ops[i].(*bug.SetTitleOperation).Was) + assert.Equal(t, op.(*bug.SetTitleOperation).Title, ops[i].(*bug.SetTitleOperation).Title) + assert.Equal(t, op.(*bug.SetTitleOperation).Author.Name(), ops[i].(*bug.SetTitleOperation).Author.Name()) + case *bug.LabelChangeOperation: + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Added, ops[i].(*bug.LabelChangeOperation).Added) + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Removed, ops[i].(*bug.LabelChangeOperation).Removed) + assert.Equal(t, op.(*bug.LabelChangeOperation).Author.Name(), ops[i].(*bug.LabelChangeOperation).Author.Name()) + case *bug.AddCommentOperation: + assert.Equal(t, op.(*bug.AddCommentOperation).Message, ops[i].(*bug.AddCommentOperation).Message) + assert.Equal(t, op.(*bug.AddCommentOperation).Author.Name(), ops[i].(*bug.AddCommentOperation).Author.Name()) + case *bug.EditCommentOperation: + assert.Equal(t, op.(*bug.EditCommentOperation).Message, ops[i].(*bug.EditCommentOperation).Message) + assert.Equal(t, op.(*bug.EditCommentOperation).Author.Name(), ops[i].(*bug.EditCommentOperation).Author.Name()) + + default: + panic("Unknown operation type") + } + } + }) + } +} diff --git a/bridge/github/iterator.go b/bridge/github/iterator.go new file mode 100644 index 00000000..48e98f17 --- /dev/null +++ b/bridge/github/iterator.go @@ -0,0 +1,409 @@ +package github + +import ( + "context" + "time" + + "github.com/shurcooL/githubv4" +) + +type indexer struct{ index int } + +type issueEditIterator struct { + index int + query issueEditQuery + variables map[string]interface{} +} + +type commentEditIterator struct { + index int + query commentEditQuery + variables map[string]interface{} +} + +type timelineIterator struct { + index int + query issueTimelineQuery + variables map[string]interface{} + + issueEdit indexer + commentEdit indexer + + // lastEndCursor cache the timeline end cursor for one iteration + lastEndCursor githubv4.String +} + +type iterator struct { + // github graphql client + gc *githubv4.Client + + // if since is given the iterator will query only the updated + // and created issues after this date + since time.Time + + // number of timelines/userEditcontent/issueEdit to query + // at a time, more capacity = more used memory = less queries + // to make + capacity int + + // sticky error + err error + + // number of imported issues + importedIssues int + + // timeline iterator + timeline timelineIterator + + // issue edit iterator + issueEdit issueEditIterator + + // comment edit iterator + commentEdit commentEditIterator +} + +func NewIterator(user, project, token string, since time.Time) *iterator { + return &iterator{ + gc: buildClient(token), + since: since, + capacity: 10, + timeline: timelineIterator{ + index: -1, + issueEdit: indexer{-1}, + commentEdit: indexer{-1}, + variables: map[string]interface{}{ + "owner": githubv4.String(user), + "name": githubv4.String(project), + }, + }, + commentEdit: commentEditIterator{ + index: -1, + variables: map[string]interface{}{ + "owner": githubv4.String(user), + "name": githubv4.String(project), + }, + }, + issueEdit: issueEditIterator{ + index: -1, + variables: map[string]interface{}{ + "owner": githubv4.String(user), + "name": githubv4.String(project), + }, + }, + } +} + +// init issue timeline variables +func (i *iterator) initTimelineQueryVariables() { + i.timeline.variables["issueFirst"] = githubv4.Int(1) + i.timeline.variables["issueAfter"] = (*githubv4.String)(nil) + i.timeline.variables["issueSince"] = githubv4.DateTime{Time: i.since} + i.timeline.variables["timelineFirst"] = githubv4.Int(i.capacity) + i.timeline.variables["timelineAfter"] = (*githubv4.String)(nil) + // Fun fact, github provide the comment edition in reverse chronological + // order, because haha. Look at me, I'm dying of laughter. + i.timeline.variables["issueEditLast"] = githubv4.Int(i.capacity) + i.timeline.variables["issueEditBefore"] = (*githubv4.String)(nil) + i.timeline.variables["commentEditLast"] = githubv4.Int(i.capacity) + i.timeline.variables["commentEditBefore"] = (*githubv4.String)(nil) +} + +// init issue edit variables +func (i *iterator) initIssueEditQueryVariables() { + i.issueEdit.variables["issueFirst"] = githubv4.Int(1) + i.issueEdit.variables["issueAfter"] = i.timeline.variables["issueAfter"] + i.issueEdit.variables["issueSince"] = githubv4.DateTime{Time: i.since} + i.issueEdit.variables["issueEditLast"] = githubv4.Int(i.capacity) + i.issueEdit.variables["issueEditBefore"] = (*githubv4.String)(nil) +} + +// init issue comment variables +func (i *iterator) initCommentEditQueryVariables() { + i.commentEdit.variables["issueFirst"] = githubv4.Int(1) + i.commentEdit.variables["issueAfter"] = i.timeline.variables["issueAfter"] + i.commentEdit.variables["issueSince"] = githubv4.DateTime{Time: i.since} + i.commentEdit.variables["timelineFirst"] = githubv4.Int(1) + i.commentEdit.variables["timelineAfter"] = (*githubv4.String)(nil) + i.commentEdit.variables["commentEditLast"] = githubv4.Int(i.capacity) + i.commentEdit.variables["commentEditBefore"] = (*githubv4.String)(nil) +} + +// reverse UserContentEdits arrays in both of the issue and +// comment timelines +func (i *iterator) reverseTimelineEditNodes() { + node := i.timeline.query.Repository.Issues.Nodes[0] + reverseEdits(node.UserContentEdits.Nodes) + for index, ce := range node.Timeline.Edges { + if ce.Node.Typename == "IssueComment" && len(node.Timeline.Edges) != 0 { + reverseEdits(node.Timeline.Edges[index].Node.IssueComment.UserContentEdits.Nodes) + } + } +} + +// Error return last encountered error +func (i *iterator) Error() error { + return i.err +} + +// ImportedIssues return the number of issues we iterated over +func (i *iterator) ImportedIssues() int { + return i.importedIssues +} + +func (i *iterator) queryIssue() bool { + if err := i.gc.Query(context.TODO(), &i.timeline.query, i.timeline.variables); err != nil { + i.err = err + return false + } + + if len(i.timeline.query.Repository.Issues.Nodes) == 0 { + return false + } + + i.reverseTimelineEditNodes() + i.importedIssues++ + return true +} + +// Next issue +func (i *iterator) NextIssue() bool { + // we make the first move + if i.importedIssues == 0 { + + // init variables and goto queryIssue block + i.initTimelineQueryVariables() + return i.queryIssue() + } + + if i.err != nil { + return false + } + + if !i.timeline.query.Repository.Issues.PageInfo.HasNextPage { + return false + } + + // if we have more issues, query them + i.timeline.variables["timelineAfter"] = (*githubv4.String)(nil) + i.timeline.variables["issueAfter"] = i.timeline.query.Repository.Issues.PageInfo.EndCursor + i.timeline.index = -1 + + // store cursor for future use + i.timeline.lastEndCursor = i.timeline.query.Repository.Issues.Nodes[0].Timeline.PageInfo.EndCursor + + // query issue block + return i.queryIssue() +} + +func (i *iterator) IssueValue() issueTimeline { + return i.timeline.query.Repository.Issues.Nodes[0] +} + +func (i *iterator) NextTimeline() bool { + if i.err != nil { + return false + } + + if len(i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges) == 0 { + return false + } + + if i.timeline.index < min(i.capacity, len(i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges))-1 { + i.timeline.index++ + return true + } + + if !i.timeline.query.Repository.Issues.Nodes[0].Timeline.PageInfo.HasNextPage { + return false + } + + i.timeline.lastEndCursor = i.timeline.query.Repository.Issues.Nodes[0].Timeline.PageInfo.EndCursor + + // more timelines, query them + i.timeline.variables["timelineAfter"] = i.timeline.query.Repository.Issues.Nodes[0].Timeline.PageInfo.EndCursor + if err := i.gc.Query(context.TODO(), &i.timeline.query, i.timeline.variables); err != nil { + i.err = err + return false + } + + i.reverseTimelineEditNodes() + i.timeline.index = 0 + return true +} + +func (i *iterator) TimelineValue() timelineItem { + return i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node +} + +func (i *iterator) queryIssueEdit() bool { + if err := i.gc.Query(context.TODO(), &i.issueEdit.query, i.issueEdit.variables); err != nil { + i.err = err + //i.timeline.issueEdit.index = -1 + return false + } + + // reverse issue edits because github + reverseEdits(i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes) + + // this is not supposed to happen + if len(i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes) == 0 { + i.timeline.issueEdit.index = -1 + return false + } + + i.issueEdit.index = 0 + i.timeline.issueEdit.index = -2 + return true +} + +func (i *iterator) NextIssueEdit() bool { + if i.err != nil { + return false + } + + // this mean we looped over all available issue edits in the timeline. + // now we have to use i.issueEditQuery + if i.timeline.issueEdit.index == -2 { + if i.issueEdit.index < min(i.capacity, len(i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes))-1 { + i.issueEdit.index++ + return true + } + + if !i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.PageInfo.HasPreviousPage { + i.timeline.issueEdit.index = -1 + i.issueEdit.index = -1 + return false + } + + // if there is more edits, query them + i.issueEdit.variables["issueEditBefore"] = i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.PageInfo.StartCursor + return i.queryIssueEdit() + } + + // if there is no edit, the UserContentEdits given by github is empty. That + // means that the original message is given by the issue message. + // + // if there is edits, the UserContentEdits given by github contains both the + // original message and the following edits. The issue message give the last + // version so we don't care about that. + // + // the tricky part: for an issue older than the UserContentEdits API, github + // doesn't have the previous message version anymore and give an edition + // with .Diff == nil. We have to filter them. + if len(i.timeline.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes) == 0 { + return false + } + + // loop over them timeline comment edits + if i.timeline.issueEdit.index < min(i.capacity, len(i.timeline.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes))-1 { + i.timeline.issueEdit.index++ + return true + } + + if !i.timeline.query.Repository.Issues.Nodes[0].UserContentEdits.PageInfo.HasPreviousPage { + i.timeline.issueEdit.index = -1 + return false + } + + // if there is more edits, query them + i.initIssueEditQueryVariables() + i.issueEdit.variables["issueEditBefore"] = i.timeline.query.Repository.Issues.Nodes[0].UserContentEdits.PageInfo.StartCursor + return i.queryIssueEdit() +} + +func (i *iterator) IssueEditValue() userContentEdit { + // if we are using issue edit query + if i.timeline.issueEdit.index == -2 { + return i.issueEdit.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes[i.issueEdit.index] + } + + // else get it from timeline issue edit query + return i.timeline.query.Repository.Issues.Nodes[0].UserContentEdits.Nodes[i.timeline.issueEdit.index] +} + +func (i *iterator) queryCommentEdit() bool { + if err := i.gc.Query(context.TODO(), &i.commentEdit.query, i.commentEdit.variables); err != nil { + i.err = err + return false + } + + // this is not supposed to happen + if len(i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.Nodes) == 0 { + i.timeline.commentEdit.index = -1 + return false + } + + reverseEdits(i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.Nodes) + + i.commentEdit.index = 0 + i.timeline.commentEdit.index = -2 + return true +} + +func (i *iterator) NextCommentEdit() bool { + if i.err != nil { + return false + } + + // same as NextIssueEdit + if i.timeline.commentEdit.index == -2 { + + if i.commentEdit.index < min(i.capacity, len(i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.Nodes))-1 { + i.commentEdit.index++ + return true + } + + if !i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.PageInfo.HasPreviousPage { + i.timeline.commentEdit.index = -1 + i.commentEdit.index = -1 + return false + } + + // if there is more comment edits, query them + i.commentEdit.variables["commentEditBefore"] = i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.PageInfo.StartCursor + return i.queryCommentEdit() + } + + // if there is no comment edits + if len(i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node.IssueComment.UserContentEdits.Nodes) == 0 { + return false + } + + // loop over them timeline comment edits + if i.timeline.commentEdit.index < min(i.capacity, len(i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node.IssueComment.UserContentEdits.Nodes))-1 { + i.timeline.commentEdit.index++ + return true + } + + if !i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node.IssueComment.UserContentEdits.PageInfo.HasPreviousPage { + i.timeline.commentEdit.index = -1 + return false + } + + i.initCommentEditQueryVariables() + if i.timeline.index == 0 { + i.commentEdit.variables["timelineAfter"] = i.timeline.lastEndCursor + } else { + i.commentEdit.variables["timelineAfter"] = i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index-1].Cursor + } + + i.commentEdit.variables["commentEditBefore"] = i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node.IssueComment.UserContentEdits.PageInfo.StartCursor + + return i.queryCommentEdit() +} + +func (i *iterator) CommentEditValue() userContentEdit { + if i.timeline.commentEdit.index == -2 { + return i.commentEdit.query.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits.Nodes[i.commentEdit.index] + } + + return i.timeline.query.Repository.Issues.Nodes[0].Timeline.Edges[i.timeline.index].Node.IssueComment.UserContentEdits.Nodes[i.timeline.commentEdit.index] +} + +func min(a, b int) int { + if a > b { + return b + } + + return a +} diff --git a/bridge/launchpad/import.go b/bridge/launchpad/import.go index 30ec5c3f..177ff3fc 100644 --- a/bridge/launchpad/import.go +++ b/bridge/launchpad/import.go @@ -44,7 +44,7 @@ func (li *launchpadImporter) ensurePerson(repo *cache.RepoCache, owner LPPerson) ) } -func (li *launchpadImporter) ImportAll(repo *cache.RepoCache) error { +func (li *launchpadImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { lpAPI := new(launchpadAPI) err := lpAPI.Init() @@ -139,8 +139,3 @@ func (li *launchpadImporter) ImportAll(repo *cache.RepoCache) error { } return nil } - -func (li *launchpadImporter) Import(repo *cache.RepoCache, id string) error { - fmt.Println("IMPORT") - return nil -} |