aboutsummaryrefslogtreecommitdiffstats
path: root/bridge/github/import.go
diff options
context:
space:
mode:
Diffstat (limited to 'bridge/github/import.go')
-rw-r--r--bridge/github/import.go363
1 files changed, 147 insertions, 216 deletions
diff --git a/bridge/github/import.go b/bridge/github/import.go
index e8a4d3cb..2e36f5fe 100644
--- a/bridge/github/import.go
+++ b/bridge/github/import.go
@@ -3,6 +3,7 @@ package github
import (
"context"
"fmt"
+ "strconv"
"time"
"github.com/shurcooL/githubv4"
@@ -19,41 +20,40 @@ import (
type githubImporter struct {
conf core.Configuration
- // default client
- client *githubv4.Client
-
- // iterator
- iterator *iterator
+ // mediator to access the Github API
+ mediator *importMediator
// send only channel
out chan<- core.ImportResult
+
+ // closure to get the username from github without any additional parameters
+ ghUser func(string) (*user, error)
}
-func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
+func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
gi.conf = conf
+ return nil
+}
+// ImportAll iterate over all the configured repository issues and ensure the creation of the
+// missing issues / timeline items / edits / label events ...
+func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
creds, err := auth.List(repo,
auth.WithTarget(target),
auth.WithKind(auth.KindToken),
- auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
+ auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
)
if err != nil {
- return err
+ return nil, err
}
-
- if len(creds) == 0 {
- return ErrMissingIdentityToken
+ if len(creds) <= 0 {
+ return nil, ErrMissingIdentityToken
+ }
+ client := buildClient(creds[0].(*auth.Token))
+ gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
+ gi.ghUser = func(login string) (*user, error) {
+ return gi.mediator.User(ctx, login)
}
-
- gi.client = buildClient(creds[0].(*auth.Token))
-
- return nil
-}
-
-// ImportAll iterate over all the configured repository issues and ensure the creation of the
-// missing issues / timeline items / edits / label events ...
-func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
- gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
out := make(chan core.ImportResult)
gi.out = out
@@ -61,19 +61,19 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
defer close(gi.out)
// Loop over all matching issues
- for gi.iterator.NextIssue() {
- issue := gi.iterator.IssueValue()
+ for issue := range gi.mediator.Issues() {
+ // fmt.Println("issue loop")
// create issue
- b, err := gi.ensureIssue(repo, issue)
+ b, err := gi.ensureIssue(repo, &issue)
if err != nil {
err := fmt.Errorf("issue creation: %v", err)
out <- core.NewImportError(err, "")
return
}
+ // fmt.Println("Just before timeline items loop")
// loop over timeline items
- for gi.iterator.NextTimelineItem() {
- item := gi.iterator.TimelineItemValue()
+ for item := range gi.mediator.TimelineItems(&issue) {
err := gi.ensureTimelineItem(repo, b, item)
if err != nil {
err = fmt.Errorf("timeline item creation: %v", err)
@@ -92,7 +92,7 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
}
}
- if err := gi.iterator.Error(); err != nil {
+ if err := gi.mediator.Error(); err != nil {
gi.out <- core.NewImportError(err, "")
}
}()
@@ -100,8 +100,8 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
return out, nil
}
-func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issue) (*cache.BugCache, error) {
- // ensure issue author
+func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue *issue) (*cache.BugCache, error) {
+ // fmt.Printf("ensureIssue()\n")
author, err := gi.ensurePerson(repo, issue.Author)
if err != nil {
return nil, err
@@ -116,94 +116,73 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issue) (*cach
return nil, err
}
- // get issue edits
- var issueEdits []userContentEdit
- for gi.iterator.NextIssueEdit() {
- issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
+ // get first issue edit
+ // if it exists, then it holds the bug creation
+ firstEdit, hasEdit := <-gi.mediator.IssueEdits(issue)
+ // fmt.Printf("hasEdit == %v\n", hasEdit)
+ //fmt.Printf("%v\n", firstEdit)
+
+ title := string(issue.Title)
+ if title == "" {
+ fmt.Printf("%v\n", issue)
+ fmt.Println("title == \"\" holds")
+ title = "#" + strconv.Itoa(int(issue.Number))
+ fmt.Println("setting title := ", title)
}
- // if issueEdits is empty
- if len(issueEdits) == 0 {
- if err == bug.ErrBugNotExist {
- cleanText, err := text.Cleanup(string(issue.Body))
- if err != nil {
- return nil, err
- }
-
- // create bug
- b, _, err = repo.NewBugRaw(
- author,
- issue.CreatedAt.Unix(),
- issue.Title,
- cleanText,
- nil,
- map[string]string{
- core.MetaKeyOrigin: target,
- metaKeyGithubId: parseId(issue.Id),
- metaKeyGithubUrl: issue.Url.String(),
- })
- if err != nil {
- return nil, err
- }
-
- // importing a new bug
- gi.out <- core.NewImportBug(b.Id())
+ if err == bug.ErrBugNotExist {
+ var textInput string
+ if hasEdit {
+ // use the first issue edit: it represents the bug creation itself
+ textInput = string(*firstEdit.Diff)
+ } else {
+ // if there are no issue edits then the issue struct holds the bug creation
+ textInput = string(issue.Body)
}
- } else {
- // create bug from given issueEdits
- for i, edit := range issueEdits {
- if i == 0 && b != nil {
- // The first edit in the github result is the issue creation itself, we already have that
- continue
- }
-
- cleanText, err := text.Cleanup(string(*edit.Diff))
- if err != nil {
- return nil, err
- }
-
- // if the bug doesn't exist
- if b == nil {
- // we create the bug as soon as we have a legit first edition
- b, _, err = repo.NewBugRaw(
- author,
- issue.CreatedAt.Unix(),
- issue.Title, // TODO: this is the *current* title, not the original one
- cleanText,
- nil,
- map[string]string{
- core.MetaKeyOrigin: target,
- metaKeyGithubId: parseId(issue.Id),
- metaKeyGithubUrl: issue.Url.String(),
- },
- )
-
- if err != nil {
- return nil, err
- }
- // importing a new bug
- gi.out <- core.NewImportBug(b.Id())
- continue
- }
-
- // other edits will be added as CommentEdit operations
- target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
- if err == cache.ErrNoMatchingOp {
- // original comment is missing somehow, issuing a warning
- gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
- continue
- }
- if err != nil {
- return nil, err
- }
-
- err = gi.ensureCommentEdit(repo, b, target, edit)
- if err != nil {
- return nil, err
- }
+ cleanText, err := text.Cleanup(textInput)
+ if err != nil {
+ return nil, err
}
+ // create bug
+ b, _, err = repo.NewBugRaw(
+ author,
+ issue.CreatedAt.Unix(),
+ title, // TODO: this is the *current* title, not the original one
+ cleanText,
+ nil,
+ map[string]string{
+ core.MetaKeyOrigin: target,
+ metaKeyGithubId: parseId(issue.Id),
+ metaKeyGithubUrl: issue.Url.String(),
+ })
+ if err != nil {
+ fmt.Printf("%v\n", issue)
+ return nil, err
+ }
+ // importing a new bug
+ gi.out <- core.NewImportBug(b.Id())
+ }
+ if b == nil {
+ return nil, fmt.Errorf("finding or creating issue")
}
+ // process remaining issue edits, if they exist
+ for edit := range gi.mediator.IssueEdits(issue) {
+ // other edits will be added as CommentEdit operations
+ target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
+ if err == cache.ErrNoMatchingOp {
+ // original comment is missing somehow, issuing a warning
+ gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+ err = gi.ensureCommentEdit(repo, b, target, edit)
+ if err != nil {
+ return nil, err
+ }
+ }
return b, nil
}
@@ -211,14 +190,7 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug
switch item.Typename {
case "IssueComment":
- // collect all comment edits
- var commentEdits []userContentEdit
- for gi.iterator.NextCommentEdit() {
- commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
- }
-
- // ensureTimelineComment send import events over out chanel
- err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
+ err := gi.ensureComment(repo, b, &item.IssueComment)
if err != nil {
return fmt.Errorf("timeline comment creation: %v", err)
}
@@ -366,90 +338,64 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug
return nil
}
-func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
- // ensure person
- author, err := gi.ensurePerson(repo, item.Author)
+func (gi *githubImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, comment *issueComment) error {
+ author, err := gi.ensurePerson(repo, comment.Author)
if err != nil {
return err
}
- targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
+ targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
if err != nil && err != cache.ErrNoMatchingOp {
// real error
return err
}
-
- // if no edits are given we create the comment
- if len(edits) == 0 {
- if err == cache.ErrNoMatchingOp {
- cleanText, err := text.Cleanup(string(item.Body))
- if err != nil {
- return err
- }
-
- // add comment operation
- op, err := b.AddCommentRaw(
- author,
- item.CreatedAt.Unix(),
- cleanText,
- nil,
- map[string]string{
- metaKeyGithubId: parseId(item.Id),
- metaKeyGithubUrl: parseId(item.Url.String()),
- },
- )
- if err != nil {
- return err
- }
-
- gi.out <- core.NewImportComment(op.Id())
- return nil
+ firstEdit, hasEdit := <-gi.mediator.CommentEdits(comment)
+ if err == cache.ErrNoMatchingOp {
+ var textInput string
+ if hasEdit {
+ // use the first comment edit: it represents the comment creation itself
+ textInput = string(*firstEdit.Diff)
+ } else {
+ // if there are not comment edits, then the comment struct holds the comment creation
+ textInput = string(comment.Body)
+ }
+ cleanText, err := text.Cleanup(textInput)
+ if err != nil {
+ return err
}
- } else {
- for i, edit := range edits {
- if i == 0 && targetOpID != "" {
- // The first edit in the github result is the comment creation itself, we already have that
- continue
- }
-
- // ensure editor identity
- editor, err := gi.ensurePerson(repo, edit.Editor)
- if err != nil {
- return err
- }
-
- // create comment when target is empty
- if targetOpID == "" {
- cleanText, err := text.Cleanup(string(*edit.Diff))
- if err != nil {
- return err
- }
-
- op, err := b.AddCommentRaw(
- editor,
- edit.CreatedAt.Unix(),
- cleanText,
- nil,
- map[string]string{
- metaKeyGithubId: parseId(item.Id),
- metaKeyGithubUrl: item.Url.String(),
- },
- )
- if err != nil {
- return err
- }
- gi.out <- core.NewImportComment(op.Id())
+ // add comment operation
+ op, err := b.AddCommentRaw(
+ author,
+ comment.CreatedAt.Unix(),
+ cleanText,
+ nil,
+ map[string]string{
+ metaKeyGithubId: parseId(comment.Id),
+ metaKeyGithubUrl: comment.Url.String(),
+ },
+ )
+ if err != nil {
+ return err
+ }
- // set target for the next edit now that the comment is created
- targetOpID = op.Id()
- continue
- }
+ gi.out <- core.NewImportComment(op.Id())
+ targetOpID = op.Id()
+ }
+ if targetOpID == "" {
+ return fmt.Errorf("finding or creating issue comment")
+ }
+ // process remaining comment edits, if they exist
+ for edit := range gi.mediator.CommentEdits(comment) {
+ // ensure editor identity
+ _, err := gi.ensurePerson(repo, edit.Editor)
+ if err != nil {
+ return err
+ }
- err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
- if err != nil {
- return err
- }
+ err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
+ if err != nil {
+ return err
}
}
return nil
@@ -521,7 +467,6 @@ func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*ca
}
// importing a new identity
-
var name string
var email string
@@ -565,41 +510,27 @@ func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*ca
}
func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
+ loginName := "ghost"
// Look first in the cache
- i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
+ i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
if err == nil {
return i, nil
}
if entity.IsErrMultipleMatch(err) {
return nil, err
}
-
- var q ghostQuery
-
- variables := map[string]interface{}{
- "login": githubv4.String("ghost"),
+ user, err := gi.ghUser(loginName)
+ userName := ""
+ if user.Name != nil {
+ userName = string(*user.Name)
}
-
- ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
- defer cancel()
-
- err = gi.client.Query(ctx, &q, variables)
- if err != nil {
- return nil, err
- }
-
- var name string
- if q.User.Name != nil {
- name = string(*q.User.Name)
- }
-
return repo.NewIdentityRaw(
- name,
+ userName,
"",
- string(q.User.Login),
- string(q.User.AvatarUrl),
+ string(user.Login),
+ string(user.AvatarUrl),
map[string]string{
- metaKeyGithubLogin: string(q.User.Login),
+ metaKeyGithubLogin: string(user.Login),
},
)
}
generated by cgit (git 2.34.1) at 2024-11-13 01:14:26 +0000