Deal with github bridge import rate limit

1 files changed, 147 insertions, 216 deletions

diff --git a/bridge/github/import.go b/bridge/github/import.go
index e8a4d3cb..2e36f5fe 100644
--- a/bridge/github/import.go
+++ b/bridge/github/import.go
@@ -3,6 +3,7 @@ package github
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/shurcooL/githubv4"
@@ -19,41 +20,40 @@ import (
 type githubImporter struct {
 	conf core.Configuration
 
-	// default client
-	client *githubv4.Client
-
-	// iterator
-	iterator *iterator
+	// mediator to access the Github API
+	mediator *importMediator
 
 	// send only channel
 	out chan<- core.ImportResult
+
+	// closure to get the username from github without any additional parameters
+	ghUser func(string) (*user, error)
 }
 
-func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
+func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
 	gi.conf = conf
+	return nil
+}
 
+// ImportAll iterate over all the configured repository issues and ensure the creation of the
+// missing issues / timeline items / edits / label events ...
+func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
 	creds, err := auth.List(repo,
 		auth.WithTarget(target),
 		auth.WithKind(auth.KindToken),
-		auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
+		auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
 	)
 	if err != nil {
-		return err
+		return nil, err
 	}
-
-	if len(creds) == 0 {
-		return ErrMissingIdentityToken
+	if len(creds) <= 0 {
+		return nil, ErrMissingIdentityToken
+	}
+	client := buildClient(creds[0].(*auth.Token))
+	gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
+	gi.ghUser = func(login string) (*user, error) {
+		return gi.mediator.User(ctx, login)
 	}
-
-	gi.client = buildClient(creds[0].(*auth.Token))
-
-	return nil
-}
-
-// ImportAll iterate over all the configured repository issues and ensure the creation of the
-// missing issues / timeline items / edits / label events ...
-func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
-	gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
 	out := make(chan core.ImportResult)
 	gi.out = out
 
@@ -61,19 +61,19 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
 		defer close(gi.out)
 
 		// Loop over all matching issues
-		for gi.iterator.NextIssue() {
-			issue := gi.iterator.IssueValue()
+		for issue := range gi.mediator.Issues() {
+			// fmt.Println("issue loop")
 			// create issue
-			b, err := gi.ensureIssue(repo, issue)
+			b, err := gi.ensureIssue(repo, &issue)
 			if err != nil {
 				err := fmt.Errorf("issue creation: %v", err)
 				out <- core.NewImportError(err, "")
 				return
 			}
 
+			// fmt.Println("Just before timeline items loop")
 			// loop over timeline items
-			for gi.iterator.NextTimelineItem() {
-				item := gi.iterator.TimelineItemValue()
+			for item := range gi.mediator.TimelineItems(&issue) {
 				err := gi.ensureTimelineItem(repo, b, item)
 				if err != nil {
 					err = fmt.Errorf("timeline item creation: %v", err)
@@ -92,7 +92,7 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
 			}
 		}
 
-		if err := gi.iterator.Error(); err != nil {
+		if err := gi.mediator.Error(); err != nil {
 			gi.out <- core.NewImportError(err, "")
 		}
 	}()
@@ -100,8 +100,8 @@ func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache,
 	return out, nil
 }
 
-func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issue) (*cache.BugCache, error) {
-	// ensure issue author
+func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue *issue) (*cache.BugCache, error) {
+	// fmt.Printf("ensureIssue()\n")
 	author, err := gi.ensurePerson(repo, issue.Author)
 	if err != nil {
 		return nil, err
@@ -116,94 +116,73 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issue) (*cach
 		return nil, err
 	}
 
-	// get issue edits
-	var issueEdits []userContentEdit
-	for gi.iterator.NextIssueEdit() {
-		issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
+	// get first issue edit
+	// if it exists, then it holds the bug creation
+	firstEdit, hasEdit := <-gi.mediator.IssueEdits(issue)
+	// fmt.Printf("hasEdit == %v\n", hasEdit)
+	//fmt.Printf("%v\n", firstEdit)
+
+	title := string(issue.Title)
+	if title == "" {
+		fmt.Printf("%v\n", issue)
+		fmt.Println("title == \"\" holds")
+		title = "#" + strconv.Itoa(int(issue.Number))
+		fmt.Println("setting title := ", title)
 	}
 
-	// if issueEdits is empty
-	if len(issueEdits) == 0 {
-		if err == bug.ErrBugNotExist {
-			cleanText, err := text.Cleanup(string(issue.Body))
-			if err != nil {
-				return nil, err
-			}
-
-			// create bug
-			b, _, err = repo.NewBugRaw(
-				author,
-				issue.CreatedAt.Unix(),
-				issue.Title,
-				cleanText,
-				nil,
-				map[string]string{
-					core.MetaKeyOrigin: target,
-					metaKeyGithubId:    parseId(issue.Id),
-					metaKeyGithubUrl:   issue.Url.String(),
-				})
-			if err != nil {
-				return nil, err
-			}
-
-			// importing a new bug
-			gi.out <- core.NewImportBug(b.Id())
+	if err == bug.ErrBugNotExist {
+		var textInput string
+		if hasEdit {
+			// use the first issue edit: it represents the bug creation itself
+			textInput = string(*firstEdit.Diff)
+		} else {
+			// if there are no issue edits then the issue struct holds the bug creation
+			textInput = string(issue.Body)
 		}
-	} else {
-		// create bug from given issueEdits
-		for i, edit := range issueEdits {
-			if i == 0 && b != nil {
-				// The first edit in the github result is the issue creation itself, we already have that
-				continue
-			}
-
-			cleanText, err := text.Cleanup(string(*edit.Diff))
-			if err != nil {
-				return nil, err
-			}
-
-			// if the bug doesn't exist
-			if b == nil {
-				// we create the bug as soon as we have a legit first edition
-				b, _, err = repo.NewBugRaw(
-					author,
-					issue.CreatedAt.Unix(),
-					issue.Title, // TODO: this is the *current* title, not the original one
-					cleanText,
-					nil,
-					map[string]string{
-						core.MetaKeyOrigin: target,
-						metaKeyGithubId:    parseId(issue.Id),
-						metaKeyGithubUrl:   issue.Url.String(),
-					},
-				)
-
-				if err != nil {
-					return nil, err
-				}
-				// importing a new bug
-				gi.out <- core.NewImportBug(b.Id())
-				continue
-			}
-
-			// other edits will be added as CommentEdit operations
-			target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
-			if err == cache.ErrNoMatchingOp {
-				// original comment is missing somehow, issuing a warning
-				gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
-				continue
-			}
-			if err != nil {
-				return nil, err
-			}
-
-			err = gi.ensureCommentEdit(repo, b, target, edit)
-			if err != nil {
-				return nil, err
-			}
+		cleanText, err := text.Cleanup(textInput)
+		if err != nil {
+			return nil, err
 		}
+		// create bug
+		b, _, err = repo.NewBugRaw(
+			author,
+			issue.CreatedAt.Unix(),
+			title, // TODO: this is the *current* title, not the original one
+			cleanText,
+			nil,
+			map[string]string{
+				core.MetaKeyOrigin: target,
+				metaKeyGithubId:    parseId(issue.Id),
+				metaKeyGithubUrl:   issue.Url.String(),
+			})
+		if err != nil {
+			fmt.Printf("%v\n", issue)
+			return nil, err
+		}
+		// importing a new bug
+		gi.out <- core.NewImportBug(b.Id())
+	}
+	if b == nil {
+		return nil, fmt.Errorf("finding or creating issue")
 	}
+	// process remaining issue edits, if they exist
+	for edit := range gi.mediator.IssueEdits(issue) {
+		// other edits will be added as CommentEdit operations
+		target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
+		if err == cache.ErrNoMatchingOp {
+			// original comment is missing somehow, issuing a warning
+			gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
 
+		err = gi.ensureCommentEdit(repo, b, target, edit)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return b, nil
 }
 
@@ -211,14 +190,7 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug
 
 	switch item.Typename {
 	case "IssueComment":
-		// collect all comment edits
-		var commentEdits []userContentEdit
-		for gi.iterator.NextCommentEdit() {
-			commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
-		}
-
-		// ensureTimelineComment send import events over out chanel
-		err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
+		err := gi.ensureComment(repo, b, &item.IssueComment)
 		if err != nil {
 			return fmt.Errorf("timeline comment creation: %v", err)
 		}
@@ -366,90 +338,64 @@ func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.Bug
 	return nil
 }
 
-func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
-	// ensure person
-	author, err := gi.ensurePerson(repo, item.Author)
+func (gi *githubImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, comment *issueComment) error {
+	author, err := gi.ensurePerson(repo, comment.Author)
 	if err != nil {
 		return err
 	}
 
-	targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
+	targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
 	if err != nil && err != cache.ErrNoMatchingOp {
 		// real error
 		return err
 	}
-
-	// if no edits are given we create the comment
-	if len(edits) == 0 {
-		if err == cache.ErrNoMatchingOp {
-			cleanText, err := text.Cleanup(string(item.Body))
-			if err != nil {
-				return err
-			}
-
-			// add comment operation
-			op, err := b.AddCommentRaw(
-				author,
-				item.CreatedAt.Unix(),
-				cleanText,
-				nil,
-				map[string]string{
-					metaKeyGithubId:  parseId(item.Id),
-					metaKeyGithubUrl: parseId(item.Url.String()),
-				},
-			)
-			if err != nil {
-				return err
-			}
-
-			gi.out <- core.NewImportComment(op.Id())
-			return nil
+	firstEdit, hasEdit := <-gi.mediator.CommentEdits(comment)
+	if err == cache.ErrNoMatchingOp {
+		var textInput string
+		if hasEdit {
+			// use the first comment edit: it represents the comment creation itself
+			textInput = string(*firstEdit.Diff)
+		} else {
+			// if there are not comment edits, then the comment struct holds the comment creation
+			textInput = string(comment.Body)
+		}
+		cleanText, err := text.Cleanup(textInput)
+		if err != nil {
+			return err
 		}
 
-	} else {
-		for i, edit := range edits {
-			if i == 0 && targetOpID != "" {
-				// The first edit in the github result is the comment creation itself, we already have that
-				continue
-			}
-
-			// ensure editor identity
-			editor, err := gi.ensurePerson(repo, edit.Editor)
-			if err != nil {
-				return err
-			}
-
-			// create comment when target is empty
-			if targetOpID == "" {
-				cleanText, err := text.Cleanup(string(*edit.Diff))
-				if err != nil {
-					return err
-				}
-
-				op, err := b.AddCommentRaw(
-					editor,
-					edit.CreatedAt.Unix(),
-					cleanText,
-					nil,
-					map[string]string{
-						metaKeyGithubId:  parseId(item.Id),
-						metaKeyGithubUrl: item.Url.String(),
-					},
-				)
-				if err != nil {
-					return err
-				}
-				gi.out <- core.NewImportComment(op.Id())
+		// add comment operation
+		op, err := b.AddCommentRaw(
+			author,
+			comment.CreatedAt.Unix(),
+			cleanText,
+			nil,
+			map[string]string{
+				metaKeyGithubId:  parseId(comment.Id),
+				metaKeyGithubUrl: comment.Url.String(),
+			},
+		)
+		if err != nil {
+			return err
+		}
 
-				// set target for the next edit now that the comment is created
-				targetOpID = op.Id()
-				continue
-			}
+		gi.out <- core.NewImportComment(op.Id())
+		targetOpID = op.Id()
+	}
+	if targetOpID == "" {
+		return fmt.Errorf("finding or creating issue comment")
+	}
+	// process remaining comment edits, if they exist
+	for edit := range gi.mediator.CommentEdits(comment) {
+		// ensure editor identity
+		_, err := gi.ensurePerson(repo, edit.Editor)
+		if err != nil {
+			return err
+		}
 
-			err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
-			if err != nil {
-				return err
-			}
+		err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -521,7 +467,6 @@ func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*ca
 	}
 
 	// importing a new identity
-
 	var name string
 	var email string
 
@@ -565,41 +510,27 @@ func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*ca
 }
 
 func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
+	loginName := "ghost"
 	// Look first in the cache
-	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
+	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
 	if err == nil {
 		return i, nil
 	}
 	if entity.IsErrMultipleMatch(err) {
 		return nil, err
 	}
-
-	var q ghostQuery
-
-	variables := map[string]interface{}{
-		"login": githubv4.String("ghost"),
+	user, err := gi.ghUser(loginName)
+	userName := ""
+	if user.Name != nil {
+		userName = string(*user.Name)
 	}
-
-	ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
-	defer cancel()
-
-	err = gi.client.Query(ctx, &q, variables)
-	if err != nil {
-		return nil, err
-	}
-
-	var name string
-	if q.User.Name != nil {
-		name = string(*q.User.Name)
-	}
-
 	return repo.NewIdentityRaw(
-		name,
+		userName,
 		"",
-		string(q.User.Login),
-		string(q.User.AvatarUrl),
+		string(user.Login),
+		string(user.AvatarUrl),
 		map[string]string{
-			metaKeyGithubLogin: string(q.User.Login),
+			metaKeyGithubLogin: string(user.Login),
 		},
 	)
 }