github: working incremental + comment history for the first comment

2 files changed, 334 insertions, 115 deletions

diff --git a/bridge/github/import.go b/bridge/github/import.go
index 03ce2746..41011082 100644
--- a/bridge/github/import.go
+++ b/bridge/github/import.go
@@ -17,104 +17,24 @@ const keyGithubUrl = "github-url"
 // githubImporter implement the Importer interface
 type githubImporter struct{}
 
-type Actor struct {
-	Login     githubv4.String
-	AvatarUrl githubv4.String
-}
-
-type ActorEvent struct {
-	Id        githubv4.ID
-	CreatedAt githubv4.DateTime
-	Actor     Actor
-}
-
-type AuthorEvent struct {
-	Id        githubv4.ID
-	CreatedAt githubv4.DateTime
-	Author    Actor
-}
-
-type TimelineItem struct {
-	Typename githubv4.String `graphql:"__typename"`
-
-	// Issue
-	IssueComment struct {
-		AuthorEvent
-		Body githubv4.String
-		Url  githubv4.URI
-		// TODO: edition
-	} `graphql:"... on IssueComment"`
-
-	// Label
-	LabeledEvent struct {
-		ActorEvent
-		Label struct {
-			// Color githubv4.String
-			Name githubv4.String
-		}
-	} `graphql:"... on LabeledEvent"`
-	UnlabeledEvent struct {
-		ActorEvent
-		Label struct {
-			// Color githubv4.String
-			Name githubv4.String
-		}
-	} `graphql:"... on UnlabeledEvent"`
-
-	// Status
-	ClosedEvent struct {
-		ActorEvent
-		// Url githubv4.URI
-	} `graphql:"... on  ClosedEvent"`
-	ReopenedEvent struct {
-		ActorEvent
-	} `graphql:"... on  ReopenedEvent"`
-
-	// Title
-	RenamedTitleEvent struct {
-		ActorEvent
-		CurrentTitle  githubv4.String
-		PreviousTitle githubv4.String
-	} `graphql:"... on RenamedTitleEvent"`
-}
-
-type Issue struct {
-	AuthorEvent
-	Title string
-	Body  githubv4.String
-	Url   githubv4.URI
-
-	Timeline struct {
-		Nodes    []TimelineItem
-		PageInfo struct {
-			EndCursor   githubv4.String
-			HasNextPage bool
-		}
-	} `graphql:"timeline(first: $timelineFirst, after: $timelineAfter)"`
-}
-
-var q struct {
-	Repository struct {
-		Issues struct {
-			Nodes    []Issue
-			PageInfo struct {
-				EndCursor   githubv4.String
-				HasNextPage bool
-			}
-		} `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"`
-	} `graphql:"repository(owner: $owner, name: $name)"`
-}
-
 func (*githubImporter) ImportAll(repo *cache.RepoCache, conf core.Configuration) error {
 	client := buildClient(conf)
 
+	q := &issueTimelineQuery{}
 	variables := map[string]interface{}{
-		"owner":         githubv4.String(conf[keyUser]),
-		"name":          githubv4.String(conf[keyProject]),
-		"issueFirst":    githubv4.Int(1),
-		"issueAfter":    (*githubv4.String)(nil),
-		"timelineFirst": githubv4.Int(10),
-		"timelineAfter": (*githubv4.String)(nil),
+		"owner":            githubv4.String(conf[keyUser]),
+		"name":             githubv4.String(conf[keyProject]),
+		"issueFirst":       githubv4.Int(1),
+		"issueAfter":       (*githubv4.String)(nil),
+		"timelineFirst":    githubv4.Int(10),
+		"timelineAfter":    (*githubv4.String)(nil),
+		"commentEditFirst": githubv4.Int(10),
+		"commentEditAfter": (*githubv4.String)(nil),
+
+		// Fun fact, github provide the comment edition in reverse chronological
+		// order, because haha. Look at me, I'm dying of laughter.
+		"issueEditLast":   githubv4.Int(10),
+		"issueEditBefore": (*githubv4.String)(nil),
 	}
 
 	var b *cache.BugCache
@@ -125,22 +45,22 @@ func (*githubImporter) ImportAll(repo *cache.RepoCache, conf core.Configuration)
 			return err
 		}
 
-		if len(q.Repository.Issues.Nodes) != 1 {
-			return fmt.Errorf("Something went wrong when iterating issues, len is %d", len(q.Repository.Issues.Nodes))
+		if len(q.Repository.Issues.Nodes) == 0 {
+			return nil
 		}
 
 		issue := q.Repository.Issues.Nodes[0]
 
 		if b == nil {
-			b, err = importIssue(repo, issue)
+			b, err = ensureIssue(repo, issue, client, variables)
 			if err != nil {
 				return err
 			}
 		}
 
-		for _, item := range q.Repository.Issues.Nodes[0].Timeline.Nodes {
-			importTimelineItem(b, item)
-		}
+		// for _, item := range q.Repository.Issues.Nodes[0].Timeline.Nodes {
+		// 	importTimelineItem(b, item)
+		// }
 
 		if !issue.Timeline.PageInfo.HasNextPage {
 			err = b.CommitAsNeeded()
@@ -172,28 +92,138 @@ func (*githubImporter) Import(repo *cache.RepoCache, conf core.Configuration, id
 	return nil
 }
 
-func importIssue(repo *cache.RepoCache, issue Issue) (*cache.BugCache, error) {
+func ensureIssue(repo *cache.RepoCache, issue issueTimeline, client *githubv4.Client, rootVariables map[string]interface{}) (*cache.BugCache, error) {
 	fmt.Printf("import issue: %s\n", issue.Title)
 
+	b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
+	if err != nil && err != bug.ErrBugNotExist {
+		return nil, err
+	}
+
+	// if there is no edit, the UserContentEdits given by github is empty. That
+	// means that the original message is given by the issue message.
+
+	// if there is edits, the UserContentEdits given by github contains both the
+	// original message and the following edits. The issue message give the last
+	// version so we don't care about that.
+
+	if len(issue.UserContentEdits.Nodes) == 0 {
+		if err == bug.ErrBugNotExist {
+			b, err = repo.NewBugRaw(
+				makePerson(issue.Author),
+				issue.CreatedAt.Unix(),
+				// Todo: this might not be the initial title, we need to query the
+				// timeline to be sure
+				issue.Title,
+				cleanupText(string(issue.Body)),
+				nil,
+				map[string]string{
+					keyGithubId:  parseId(issue.Id),
+					keyGithubUrl: issue.Url.String(),
+				},
+			)
+
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return b, nil
+	}
+
+	// reverse the order, because github
+	reverseEdits(issue.UserContentEdits.Nodes)
+
+	if err == bug.ErrBugNotExist {
+		firstEdit := issue.UserContentEdits.Nodes[0]
+
+		if firstEdit.Diff == nil {
+			return nil, fmt.Errorf("no diff")
+		}
+
+		b, err = repo.NewBugRaw(
+			makePerson(issue.Author),
+			issue.CreatedAt.Unix(),
+			// Todo: this might not be the initial title, we need to query the
+			// timeline to be sure
+			issue.Title,
+			cleanupText(string(*issue.UserContentEdits.Nodes[0].Diff)),
+			nil,
+			map[string]string{
+				keyGithubId:  parseId(issue.Id),
+				keyGithubUrl: issue.Url.String(),
+			},
+		)
+	}
+
+	for i, edit := range issue.UserContentEdits.Nodes {
+		if i == 0 {
+			// The first edit in the github result is the creation itself, we already have that
+			continue
+		}
+
+		err := ensureCommentEdit(b, parseId(issue.Id), edit)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if !issue.UserContentEdits.PageInfo.HasNextPage {
+		return b, nil
+	}
+
+	// We have more edit, querying them
+
+	q := &issueEditQuery{}
+	variables := map[string]interface{}{
+		"owner":          rootVariables["owner"],
+		"name":           rootVariables["name"],
+		"issueFirst":     rootVariables["issueFirst"],
+		"issueAfter":     rootVariables["issueAfter"],
+		"issueEditFirst": githubv4.Int(10),
+		"issueEditAfter": issue.UserContentEdits.PageInfo.EndCursor,
+	}
+
+	for {
+		err := client.Query(context.TODO(), &q, variables)
+		if err != nil {
+			return nil, err
+		}
+
+		edits := q.Repository.Issues.Nodes[0].UserContentEdits
+
+		if len(edits.Nodes) == 0 {
+			return b, nil
+		}
+
+		for i, edit := range edits.Nodes {
+			if i == 0 {
+				// The first edit in the github result is the creation itself, we already have that
+				continue
+			}
+
+			err := ensureCommentEdit(b, parseId(issue.Id), edit)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		if !edits.PageInfo.HasNextPage {
+			break
+		}
+
+		variables["issueEditAfter"] = edits.PageInfo.EndCursor
+	}
+
 	// TODO: check + import files
 
-	return repo.NewBugRaw(
-		makePerson(issue.Author),
-		issue.CreatedAt.Unix(),
-		issue.Title,
-		cleanupText(string(issue.Body)),
-		nil,
-		map[string]string{
-			keyGithubId:  parseId(issue.Id),
-			keyGithubUrl: issue.Url.String(),
-		},
-	)
+	return b, nil
 }
 
-func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
+func importTimelineItem(b *cache.BugCache, item timelineItem) error {
 	switch item.Typename {
 	case "IssueComment":
-		// fmt.Printf("import %s: %s\n", item.Typename, item.IssueComment)
+		// fmt.Printf("import %s: %s\n", item.Typename, item.issueComment)
 		return b.AddCommentRaw(
 			makePerson(item.IssueComment.Author),
 			item.IssueComment.CreatedAt.Unix(),
@@ -214,6 +244,7 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 				string(item.LabeledEvent.Label.Name),
 			},
 			nil,
+			nil,
 		)
 		return err
 
@@ -226,6 +257,7 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 			[]string{
 				string(item.UnlabeledEvent.Label.Name),
 			},
+			nil,
 		)
 		return err
 
@@ -234,6 +266,7 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 		return b.CloseRaw(
 			makePerson(item.ClosedEvent.Actor),
 			item.ClosedEvent.CreatedAt.Unix(),
+			nil,
 		)
 
 	case "ReopenedEvent":
@@ -241,6 +274,7 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 		return b.OpenRaw(
 			makePerson(item.ReopenedEvent.Actor),
 			item.ReopenedEvent.CreatedAt.Unix(),
+			nil,
 		)
 
 	case "RenamedTitleEvent":
@@ -249,6 +283,7 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 			makePerson(item.RenamedTitleEvent.Actor),
 			item.RenamedTitleEvent.CreatedAt.Unix(),
 			string(item.RenamedTitleEvent.CurrentTitle),
+			nil,
 		)
 
 	default:
@@ -258,8 +293,57 @@ func importTimelineItem(b *cache.BugCache, item TimelineItem) error {
 	return nil
 }
 
+func ensureCommentEdit(b *cache.BugCache, target string, edit userContentEdit) error {
+	if edit.Editor == nil {
+		return fmt.Errorf("no editor")
+	}
+
+	if edit.Diff == nil {
+		return fmt.Errorf("no diff")
+	}
+
+	_, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(edit.Id))
+	if err == nil {
+		// already imported
+		return nil
+	}
+	if err != cache.ErrNoMatchingOp {
+		// real error
+		return err
+	}
+
+	fmt.Printf("import edition\n")
+
+	targetHash, err := b.ResolveTargetWithMetadata(keyGithubId, target)
+	if err != nil {
+		return err
+	}
+
+	switch {
+	case edit.DeletedAt != nil:
+		// comment deletion, not supported yet
+
+	case edit.DeletedAt == nil:
+		// comment edition
+		err := b.EditCommentRaw(
+			makePerson(*edit.Editor),
+			edit.CreatedAt.Unix(),
+			targetHash,
+			cleanupText(string(*edit.Diff)),
+			map[string]string{
+				keyGithubId: parseId(edit.Id),
+			},
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // makePerson create a bug.Person from the Github data
-func makePerson(actor Actor) bug.Person {
+func makePerson(actor actor) bug.Person {
 	return bug.Person{
 		Name:      string(actor.Login),
 		AvatarUrl: string(actor.AvatarUrl),
@@ -275,3 +359,10 @@ func cleanupText(text string) string {
 	// windows new line, Github, really ?
 	return strings.Replace(text, "\r\n", "\n", -1)
 }
+
+func reverseEdits(edits []userContentEdit) []userContentEdit {
+	for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
+		edits[i], edits[j] = edits[j], edits[i]
+	}
+	return edits
+}
diff --git a/bridge/github/import_query.go b/bridge/github/import_query.go
new file mode 100644
index 00000000..0eb8ad34
--- /dev/null
+++ b/bridge/github/import_query.go
@@ -0,0 +1,128 @@
+package github
+
+import "github.com/shurcooL/githubv4"
+
+type pageInfo struct {
+	EndCursor   githubv4.String
+	HasNextPage bool
+}
+
+type actor struct {
+	Login     githubv4.String
+	AvatarUrl githubv4.String
+}
+
+type actorEvent struct {
+	Id        githubv4.ID
+	CreatedAt githubv4.DateTime
+	Actor     actor
+}
+
+type authorEvent struct {
+	Id        githubv4.ID
+	CreatedAt githubv4.DateTime
+	Author    actor
+}
+
+type userContentEdit struct {
+	Id        githubv4.ID
+	CreatedAt githubv4.DateTime
+	UpdatedAt githubv4.DateTime
+	EditedAt  githubv4.DateTime
+	Editor    *actor
+	DeletedAt *githubv4.DateTime
+	DeletedBy *actor
+	Diff      *githubv4.String
+}
+
+type issueComment struct {
+	authorEvent
+	Body githubv4.String
+	Url  githubv4.URI
+
+	UserContentEdits struct {
+		Nodes    []userContentEdit
+		PageInfo pageInfo
+	} `graphql:"userContentEdits(first: $commentEditFirst, after: $commentEditAfter)"`
+}
+
+type timelineItem struct {
+	Typename githubv4.String `graphql:"__typename"`
+
+	// issue
+	IssueComment issueComment `graphql:"... on IssueComment"`
+
+	// Label
+	LabeledEvent struct {
+		actorEvent
+		Label struct {
+			// Color githubv4.String
+			Name githubv4.String
+		}
+	} `graphql:"... on LabeledEvent"`
+	UnlabeledEvent struct {
+		actorEvent
+		Label struct {
+			// Color githubv4.String
+			Name githubv4.String
+		}
+	} `graphql:"... on UnlabeledEvent"`
+
+	// Status
+	ClosedEvent struct {
+		actorEvent
+		// Url githubv4.URI
+	} `graphql:"... on  ClosedEvent"`
+	ReopenedEvent struct {
+		actorEvent
+	} `graphql:"... on  ReopenedEvent"`
+
+	// Title
+	RenamedTitleEvent struct {
+		actorEvent
+		CurrentTitle  githubv4.String
+		PreviousTitle githubv4.String
+	} `graphql:"... on RenamedTitleEvent"`
+}
+
+type issueTimeline struct {
+	authorEvent
+	Title string
+	Body  githubv4.String
+	Url   githubv4.URI
+
+	Timeline struct {
+		Nodes    []timelineItem
+		PageInfo pageInfo
+	} `graphql:"timeline(first: $timelineFirst, after: $timelineAfter)"`
+
+	UserContentEdits struct {
+		Nodes    []userContentEdit
+		PageInfo pageInfo
+	} `graphql:"userContentEdits(last: $issueEditLast, before: $issueEditBefore)"`
+}
+
+type issueEdit struct {
+	UserContentEdits struct {
+		Nodes    []userContentEdit
+		PageInfo pageInfo
+	} `graphql:"userContentEdits(last: $issueEditLast, before: $issueEditBefore)"`
+}
+
+type issueTimelineQuery struct {
+	Repository struct {
+		Issues struct {
+			Nodes    []issueTimeline
+			PageInfo pageInfo
+		} `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"`
+	} `graphql:"repository(owner: $owner, name: $name)"`
+}
+
+type issueEditQuery struct {
+	Repository struct {
+		Issues struct {
+			Nodes    []issueEdit
+			PageInfo pageInfo
+		} `graphql:"issues(first: $issueFirst, after: $issueAfter, orderBy: {field: CREATED_AT, direction: ASC})"`
+	} `graphql:"repository(owner: $owner, name: $name)"`
+}