diff options
Diffstat (limited to 'bridge/github')
| -rw-r--r-- | bridge/github/import.go | 376 | ||||
| -rw-r--r-- | bridge/github/import_test.go | 223 | ||||
| -rw-r--r-- | bridge/github/iterator.go | 25 | ||||
| -rw-r--r-- | bridge/github/iterator_test.go | 48 |
4 files changed, 420 insertions, 252 deletions
diff --git a/bridge/github/import.go b/bridge/github/import.go index 74ccb776..4960117a 100644 --- a/bridge/github/import.go +++ b/bridge/github/import.go @@ -27,237 +27,157 @@ type githubImporter struct { } func (gi *githubImporter) Init(conf core.Configuration) error { - var since time.Time - - // parse since value from configuration - if value, ok := conf["since"]; ok && value != "" { - s, err := time.Parse(time.RFC3339, value) - if err != nil { - return err - } - - since = s - } - - gi.iterator = newIterator(conf, since) + gi.conf = conf + gi.iterator = newIterator(conf) return nil } -func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error { - // Loop over all available issues +// ImportAll . +func (gi *githubImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { + gi.iterator.since = since + + // Loop over all matching issues for gi.iterator.NextIssue() { issue := gi.iterator.IssueValue() - fmt.Printf("importing issue: %v\n", issue.Title) - - // In each iteration create a new bug - var b *cache.BugCache - - // ensure issue author - author, err := gi.ensurePerson(repo, issue.Author) - if err != nil { - return err - } - - // resolve bug - b, err = repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id)) - if err != nil && err != bug.ErrBugNotExist { - return err - } + fmt.Printf("importing issue: %v\n", gi.iterator.count) // get issue edits issueEdits := []userContentEdit{} for gi.iterator.NextIssueEdit() { - // append only edits with non empty diff - if issueEdit := gi.iterator.IssueEditValue(); issueEdit.Diff != nil { + if issueEdit := gi.iterator.IssueEditValue(); issueEdit.Diff != nil && string(*issueEdit.Diff) != "" { issueEdits = append(issueEdits, issueEdit) } } - // if issueEdits is empty - if len(issueEdits) == 0 { - if err == bug.ErrBugNotExist { - // create bug - b, err = repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - issue.Title, - cleanupText(string(issue.Body)), - nil, - map[string]string{ - keyGithubId: parseId(issue.Id), - keyGithubUrl: issue.Url.String(), - }) - if err != nil { - return err - } - } - } else { - // create bug from given issueEdits - for _, edit := range issueEdits { - // if the bug doesn't exist - if b == nil { - // we create the bug as soon as we have a legit first edition - b, err = repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - issue.Title, - cleanupText(string(*edit.Diff)), - nil, - map[string]string{ - keyGithubId: parseId(issue.Id), - keyGithubUrl: issue.Url.String(), - }, - ) - - if err != nil { - return err - } - - continue - } - - // other edits will be added as CommentEdit operations - - target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id)) - if err != nil { - return err - } - - err = gi.ensureCommentEdit(repo, b, target, edit) - if err != nil { - return err - } - } + // create issue + b, err := gi.ensureIssue(repo, issue, issueEdits) + if err != nil { + return fmt.Errorf("issue creation: %v", err) } - // check timeline items + // loop over timeline items for gi.iterator.NextTimeline() { item := gi.iterator.TimelineValue() - // if item is not a comment (label, unlabel, rename, close, open ...) - if item.Typename != "IssueComment" { - if err := gi.ensureTimelineItem(repo, b, item); err != nil { - return err - } - } else { // if item is comment - - // ensure person - author, err := gi.ensurePerson(repo, item.IssueComment.Author) - if err != nil { - return err - } - - var target git.Hash - target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(item.IssueComment.Id)) - if err != nil && err != cache.ErrNoMatchingOp { - // real error - return err - } - + // if item is comment + if item.Typename == "IssueComment" { // collect all edits commentEdits := []userContentEdit{} for gi.iterator.NextCommentEdit() { - if commentEdit := gi.iterator.CommentEditValue(); commentEdit.Diff != nil { + if commentEdit := gi.iterator.CommentEditValue(); commentEdit.Diff != nil && string(*commentEdit.Diff) != "" { commentEdits = append(commentEdits, commentEdit) } } - // if no edits are given we create the comment - if len(commentEdits) == 0 { - - // if comment doesn't exist - if err == cache.ErrNoMatchingOp { - - // add comment operation - op, err := b.AddCommentRaw( - author, - item.IssueComment.CreatedAt.Unix(), - cleanupText(string(item.IssueComment.Body)), - nil, - map[string]string{ - keyGithubId: parseId(item.IssueComment.Id), - }, - ) - if err != nil { - return err - } - - // set hash - target, err = op.Hash() - if err != nil { - return err - } - } - } else { - // if we have some edits - for _, edit := range item.IssueComment.UserContentEdits.Nodes { - - // create comment when target is an empty string - if target == "" { - op, err := b.AddCommentRaw( - author, - item.IssueComment.CreatedAt.Unix(), - cleanupText(string(*edit.Diff)), - nil, - map[string]string{ - keyGithubId: parseId(item.IssueComment.Id), - keyGithubUrl: item.IssueComment.Url.String(), - }, - ) - if err != nil { - return err - } - - // set hash - target, err = op.Hash() - if err != nil { - return err - } - } - - err := gi.ensureCommentEdit(repo, b, target, edit) - if err != nil { - return err - } - - } + err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits) + if err != nil { + return fmt.Errorf("timeline event creation: %v", err) } + } else { + if err := gi.ensureTimelineItem(repo, b, item); err != nil { + return fmt.Errorf("timeline comment creation: %v", err) + } } - - } - - if err := gi.iterator.Error(); err != nil { - fmt.Printf("error importing issue %v\n", issue.Id) - return err } // commit bug state - err = b.CommitAsNeeded() - if err != nil { - return err + if err := b.CommitAsNeeded(); err != nil { + return fmt.Errorf("bug commit: %v", err) } } if err := gi.iterator.Error(); err != nil { fmt.Printf("import error: %v\n", err) + return err } fmt.Printf("Successfully imported %v issues from Github\n", gi.iterator.Count()) return nil } -func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error { - fmt.Println("IMPORT") - return nil +func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, issueEdits []userContentEdit) (*cache.BugCache, error) { + // ensure issue author + author, err := gi.ensurePerson(repo, issue.Author) + if err != nil { + return nil, err + } + + // resolve bug + b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String()) + if err != nil && err != bug.ErrBugNotExist { + return nil, err + } + + // if issueEdits is empty + if len(issueEdits) == 0 { + if err == bug.ErrBugNotExist { + // create bug + b, err = repo.NewBugRaw( + author, + issue.CreatedAt.Unix(), + issue.Title, + cleanupText(string(issue.Body)), + nil, + map[string]string{ + keyGithubId: parseId(issue.Id), + keyGithubUrl: issue.Url.String(), + }) + if err != nil { + return nil, err + } + } + + } else { + // create bug from given issueEdits + for i, edit := range issueEdits { + if i == 0 && b != nil { + continue + } + + // if the bug doesn't exist + if b == nil { + // we create the bug as soon as we have a legit first edition + b, err = repo.NewBugRaw( + author, + issue.CreatedAt.Unix(), + issue.Title, + cleanupText(string(*edit.Diff)), + nil, + map[string]string{ + keyGithubId: parseId(issue.Id), + keyGithubUrl: issue.Url.String(), + }, + ) + + if err != nil { + return nil, err + } + + continue + } + + // other edits will be added as CommentEdit operations + target, err := b.ResolveOperationWithMetadata(keyGithubUrl, issue.Url.String()) + if err != nil { + return nil, err + } + + err = gi.ensureCommentEdit(repo, b, target, edit) + if err != nil { + return nil, err + } + } + } + + return b, nil } func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error { - fmt.Printf("import item: %s\n", item.Typename) + fmt.Printf("import event item: %s\n", item.Typename) switch item.Typename { case "IssueComment": - //return gi.ensureComment(repo, b, cursor, item.IssueComment, rootVariables) case "LabeledEvent": id := parseId(item.LabeledEvent.Id) @@ -290,6 +210,7 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug if err != nil { return err } + _, _, err = b.ChangeLabelsRaw( author, item.UnlabeledEvent.CreatedAt.Unix(), @@ -360,6 +281,92 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug return nil } +func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error { + // ensure person + author, err := gi.ensurePerson(repo, item.Author) + if err != nil { + return err + } + + var target git.Hash + target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id)) + if err != nil && err != cache.ErrNoMatchingOp { + // real error + return err + } + // if no edits are given we create the comment + if len(edits) == 0 { + + // if comment doesn't exist + if err == cache.ErrNoMatchingOp { + + // add comment operation + op, err := b.AddCommentRaw( + author, + item.CreatedAt.Unix(), + cleanupText(string(item.Body)), + nil, + map[string]string{ + keyGithubId: parseId(item.Id), + keyGithubUrl: parseId(item.Url.String()), + }, + ) + if err != nil { + return err + } + + // set hash + target, err = op.Hash() + if err != nil { + return err + } + } + } else { + for i, edit := range item.UserContentEdits.Nodes { + if i == 0 && target != "" { + continue + } + + // ensure editor identity + editor, err := gi.ensurePerson(repo, edit.Editor) + if err != nil { + return err + } + + // create comment when target is empty + if target == "" { + op, err := b.AddCommentRaw( + editor, + edit.CreatedAt.Unix(), + cleanupText(string(*edit.Diff)), + nil, + map[string]string{ + keyGithubId: parseId(item.Id), + keyGithubUrl: item.Url.String(), + }, + ) + if err != nil { + return err + } + + // set hash + target, err = op.Hash() + if err != nil { + return err + } + + continue + } + + err = gi.ensureCommentEdit(repo, b, target, edit) + if err != nil { + return err + } + } + } + return nil +} + func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error { _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id)) if err == nil { @@ -381,8 +388,10 @@ func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugC switch { case edit.DeletedAt != nil: // comment deletion, not supported yet + fmt.Println("comment deletion ....") case edit.DeletedAt == nil: + // comment edition _, err := b.EditCommentRaw( editor, @@ -393,6 +402,7 @@ func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugC keyGithubId: parseId(edit.Id), }, ) + if err != nil { return err } diff --git a/bridge/github/import_test.go b/bridge/github/import_test.go new file mode 100644 index 00000000..d64f0b4b --- /dev/null +++ b/bridge/github/import_test.go @@ -0,0 +1,223 @@ +package github + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/identity" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/interrupt" +) + +func Test_Importer(t *testing.T) { + author := identity.NewIdentity("Michael Muré", "batolettre@gmail.com") + tests := []struct { + name string + exist bool + url string + bug *bug.Snapshot + }{ + { + name: "simple issue", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/1", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewAddCommentOp(author, 0, "second comment", nil)}, + }, + }, + { + name: "empty issue", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/2", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "empty issue", "", nil), + }, + }, + }, + { + name: "complex issue", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/3", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "complex issue", "initial comment", nil), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"bug"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"duplicate"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{}, []bug.Label{"duplicate"}), + bug.NewAddCommentOp(author, 0, "### header\n\n**bold**\n\n_italic_\n\n> with quote\n\n`inline code`\n\n```\nmultiline code\n```\n\n- bulleted\n- list\n\n1. numbered\n1. list\n\n- [ ] task\n- [x] list\n\n@MichaelMure mention\n\n#2 reference issue\n#3 auto-reference issue\n\n", nil), + bug.NewSetTitleOp(author, 0, "complex issue edited", "complex issue"), + bug.NewSetTitleOp(author, 0, "complex issue", "complex issue edited"), + bug.NewSetStatusOp(author, 0, bug.ClosedStatus), + bug.NewSetStatusOp(author, 0, bug.OpenStatus), + }, + }, + }, + { + name: "editions", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/4", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "editions", "initial comment edited", nil), + bug.NewEditCommentOp(author, 0, "", "erased then edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited", nil), + }, + }, + }, + { + name: "comment deletion", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/5", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "comment deletion", "", nil), + }, + }, + }, + { + name: "edition deletion", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/6", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "edition deletion", "initial comment", nil), + bug.NewEditCommentOp(author, 0, "", "initial comment edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited again", nil), + }, + }, + }, + { + name: "hidden comment", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/7", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "hidden comment", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + }, + }, + }, + { + name: "transfered issue", + exist: true, + url: "https://github.com/MichaelMure/git-but-test-github-bridge/issues/8", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "transfered issue", "", nil), + }, + }, + }, + } + + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + repo, err := repository.NewGitRepo(cwd, bug.Witnesser) + if err != nil { + t.Fatal(err) + } + + backend, err := cache.NewRepoCache(repo) + if err != nil { + t.Fatal(err) + } + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + importer := &githubImporter{} + err = importer.Init(core.Configuration{ + "user": "MichaelMure", + "project": "git-but-test-github-bridge", + "token": os.Getenv("GITHUB_TOKEN"), + }) + if err != nil { + t.Fatal(err) + } + + err = importer.ImportAll(backend, time.Time{}) + if err != nil { + t.Fatal(err) + } + + ids := backend.AllBugsIds() + assert.Equal(t, len(ids), 8) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := backend.ResolveBugCreateMetadata(keyGithubUrl, tt.url) + if err != nil { + t.Fatal(err) + } + + ops := b.Snapshot().Operations + if tt.exist { + assert.Equal(t, len(tt.bug.Operations), len(b.Snapshot().Operations)) + + for i, op := range tt.bug.Operations { + switch op.(type) { + case *bug.CreateOperation: + if op2, ok := ops[i].(*bug.CreateOperation); ok { + assert.Equal(t, op2.Title, op.(*bug.CreateOperation).Title) + assert.Equal(t, op2.Message, op.(*bug.CreateOperation).Message) + continue + } + t.Errorf("bad operation type index = %d expected = CreationOperation", i) + case *bug.SetStatusOperation: + if op2, ok := ops[i].(*bug.SetStatusOperation); ok { + assert.Equal(t, op2.Status, op.(*bug.SetStatusOperation).Status) + continue + } + t.Errorf("bad operation type index = %d expected = SetStatusOperation", i) + case *bug.SetTitleOperation: + if op2, ok := ops[i].(*bug.SetTitleOperation); ok { + assert.Equal(t, op.(*bug.SetTitleOperation).Was, op2.Was) + assert.Equal(t, op.(*bug.SetTitleOperation).Title, op2.Title) + continue + } + t.Errorf("bad operation type index = %d expected = SetTitleOperation", i) + case *bug.LabelChangeOperation: + if op2, ok := ops[i].(*bug.LabelChangeOperation); ok { + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Added, op2.Added) + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Removed, op2.Removed) + continue + } + t.Errorf("bad operation type index = %d expected = ChangeLabelOperation", i) + case *bug.AddCommentOperation: + if op2, ok := ops[i].(*bug.AddCommentOperation); ok { + assert.Equal(t, op.(*bug.AddCommentOperation).Message, op2.Message) + continue + } + t.Errorf("bad operation type index = %d expected = AddCommentOperation", i) + case *bug.EditCommentOperation: + if op2, ok := ops[i].(*bug.EditCommentOperation); ok { + assert.Equal(t, op.(*bug.EditCommentOperation).Message, op2.Message) + continue + } + t.Errorf("bad operation type index = %d expected = EditCommentOperation", i) + default: + + } + } + + } else { + assert.Equal(t, b, nil) + } + }) + } + +} diff --git a/bridge/github/iterator.go b/bridge/github/iterator.go index 9e1ff30e..281f8a6b 100644 --- a/bridge/github/iterator.go +++ b/bridge/github/iterator.go @@ -8,23 +8,6 @@ import ( "github.com/shurcooL/githubv4" ) -/** -type iterator interface { - Count() int - Error() error - - NextIssue() bool - NextIssueEdit() bool - NextTimeline() bool - NextCommentEdit() bool - - IssueValue() issueTimeline - IssueEditValue() userContentEdit - TimelineValue() timelineItem - CommentEditValue() userContentEdit -} -*/ - type indexer struct{ index int } type issueEditIterator struct { @@ -47,7 +30,8 @@ type timelineIterator struct { issueEdit indexer commentEdit indexer - lastEndCursor githubv4.String // storing timeline end cursor for future use + // lastEndCursor cache the timeline end cursor for one iteration + lastEndCursor githubv4.String } type iterator struct { @@ -59,7 +43,7 @@ type iterator struct { since time.Time // number of timelines/userEditcontent/issueEdit to query - // at a time more capacity = more used memory = less queries + // at a time, more capacity = more used memory = less queries // to make capacity int @@ -79,9 +63,8 @@ type iterator struct { commentEdit commentEditIterator } -func newIterator(conf core.Configuration, since time.Time) *iterator { +func newIterator(conf core.Configuration) *iterator { return &iterator{ - since: since, gc: buildClient(conf), capacity: 10, count: 0, diff --git a/bridge/github/iterator_test.go b/bridge/github/iterator_test.go deleted file mode 100644 index c5fad349..00000000 --- a/bridge/github/iterator_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package github - -import ( - "fmt" - "os" - "testing" - "time" -) - -func Test_Iterator(t *testing.T) { - token := os.Getenv("GITHUB_TOKEN") - user := os.Getenv("GITHUB_USER") - project := os.Getenv("GITHUB_PROJECT") - - i := newIterator(map[string]string{ - keyToken: token, - "user": user, - "project": project, - }, time.Time{}) - //time.Now().Add(-14*24*time.Hour)) - - for i.NextIssue() { - v := i.IssueValue() - fmt.Printf(" issue = id:%v title:%v\n", v.Id, v.Title) - - for i.NextIssueEdit() { - v := i.IssueEditValue() - fmt.Printf("issue edit = %v\n", string(*v.Diff)) - } - - for i.NextTimeline() { - v := i.TimelineValue() - fmt.Printf("timeline = type:%v\n", v.Typename) - - if v.Typename == "IssueComment" { - for i.NextCommentEdit() { - - _ = i.CommentEditValue() - - fmt.Printf("comment edit\n") - } - } - } - } - - fmt.Println(i.Error()) - fmt.Println(i.Count()) -} |