From f1c65a9f8655fd1810b8e26c122ce875a1e9af2f Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 26 Jul 2019 03:22:07 +0200 Subject: bridge/gitlab: add exporter implementation --- bridge/gitlab/export.go | 520 +++++++++++++++++++++++++++++++++++++++++++++++- bridge/gitlab/gitlab.go | 2 +- 2 files changed, 519 insertions(+), 3 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 85b5ee2e..723b7b6a 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -1,12 +1,17 @@ package gitlab import ( + "fmt" + "strconv" "time" "github.com/pkg/errors" + "github.com/xanzy/go-gitlab" "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/util/git" ) var ( @@ -14,14 +19,525 @@ var ( ) // gitlabExporter implement the Exporter interface -type gitlabExporter struct{} +type gitlabExporter struct { + conf core.Configuration + + // cache identities clients + identityClient map[string]*gitlab.Client + + // map identities with their tokens + identityToken map[string]string + + // gitlab repository ID + repositoryID string + + // cache identifiers used to speed up exporting operations + // cleared for each bug + cachedOperationIDs map[string]string + + // cache labels used to speed up exporting labels events + cachedLabels map[string]string +} // Init . func (ge *gitlabExporter) Init(conf core.Configuration) error { + ge.conf = conf + //TODO: initialize with multiple tokens + ge.identityToken = make(map[string]string) + ge.identityClient = make(map[string]*gitlab.Client) + ge.cachedOperationIDs = make(map[string]string) + ge.cachedLabels = make(map[string]string) + return nil } +// getIdentityClient return a gitlab v4 API client configured with the access token of the given identity. +// if no client were found it will initialize it from the known tokens map and cache it for next use +func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { + client, ok := ge.identityClient[id] + if ok { + return client, nil + } + + // get token + token, ok := ge.identityToken[id] + if !ok { + return nil, ErrMissingIdentityToken + } + + // create client + client = buildClient(token) + // cache client + ge.identityClient[id] = client + + return client, nil +} + // ExportAll export all event made by the current user to Gitlab func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { - return nil, nil + out := make(chan core.ExportResult) + + user, err := repo.GetUserIdentity() + if err != nil { + return nil, err + } + + ge.identityToken[user.Id()] = ge.conf[keyToken] + + // get repository node id + ge.repositoryID = ge.conf[keyProjectID] + + if err != nil { + return nil, err + } + + go func() { + defer close(out) + + var allIdentitiesIds []string + for id := range ge.identityToken { + allIdentitiesIds = append(allIdentitiesIds, id) + } + + allBugsIds := repo.AllBugsIds() + + for _, id := range allBugsIds { + b, err := repo.ResolveBug(id) + if err != nil { + out <- core.NewExportError(err, id) + return + } + + snapshot := b.Snapshot() + + // ignore issues created before since date + // TODO: compare the Lamport time instead of using the unix time + if snapshot.CreatedAt.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } + + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + ge.exportBug(b, since, out) + } else { + out <- core.NewExportNothing(id, "not an actor") + } + } + }() + + return out, nil +} + +// exportBug publish bugs and related events +func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { + snapshot := b.Snapshot() + + var err error + var bugGitlabID int + var bugGitlabIDString string + var bugGitlabURL string + var bugCreationHash string + //labels := make([]string, 0) + + // Special case: + // if a user try to export a bug that is not already exported to Gitlab (or imported + // from Gitlab) and we do not have the token of the bug author, there is nothing we can do. + + // skip bug if origin is not allowed + origin, ok := snapshot.GetCreateMetadata(core.KeyOrigin) + if ok && origin != target { + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) + return + } + + // first operation is always createOp + createOp := snapshot.Operations[0].(*bug.CreateOperation) + author := snapshot.Author + + // get gitlab bug ID + gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) + if ok { + gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl) + if !ok { + // if we find gitlab ID, gitlab URL must be found too + err := fmt.Errorf("expected to find gitlab issue URL") + out <- core.NewExportError(err, b.Id()) + } + + //FIXME: + // ignore issue comming from other repositories + + out <- core.NewExportNothing(b.Id(), "bug already exported") + // will be used to mark operation related to a bug as exported + bugGitlabIDString = gitlabID + bugGitlabID, err = strconv.Atoi(bugGitlabIDString) + if err != nil { + panic("unexpected gitlab id format") + } + + bugGitlabURL = gitlabURL + + } else { + // check that we have a token for operation author + client, err := ge.getIdentityClient(author.Id()) + if err != nil { + // if bug is still not exported and we do not have the author stop the execution + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) + return + } + + // create bug + id, url, err := createGitlabIssue(client, ge.repositoryID, createOp.Title, createOp.Message) + if err != nil { + err := errors.Wrap(err, "exporting gitlab issue") + out <- core.NewExportError(err, b.Id()) + return + } + + idString := strconv.Itoa(id) + + out <- core.NewExportBug(b.Id()) + + hash, err := createOp.Hash() + if err != nil { + err := errors.Wrap(err, "comment hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // mark bug creation operation as exported + if err := markOperationAsExported(b, hash, idString, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit operation to avoid creating multiple issues with multiple pushes + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + + // cache bug gitlab ID and URL + bugGitlabID = id + bugGitlabIDString = idString + bugGitlabURL = url + } + + // get createOp hash + hash, err := createOp.Hash() + if err != nil { + out <- core.NewExportError(err, b.Id()) + return + } + + bugCreationHash = hash.String() + + // cache operation gitlab id + ge.cachedOperationIDs[bugCreationHash] = bugGitlabIDString + + for _, op := range snapshot.Operations[1:] { + // ignore SetMetadata operations + if _, ok := op.(*bug.SetMetadataOperation); ok { + continue + } + + // get operation hash + hash, err := op.Hash() + if err != nil { + err := errors.Wrap(err, "operation hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // ignore operations already existing in gitlab (due to import or export) + // cache the ID of already exported or imported issues and events from Gitlab + if id, ok := op.GetMetadata(keyGitlabId); ok { + ge.cachedOperationIDs[hash.String()] = id + out <- core.NewExportNothing(hash.String(), "already exported operation") + continue + } + + opAuthor := op.GetAuthor() + client, err := ge.getIdentityClient(opAuthor.Id()) + if err != nil { + out <- core.NewExportNothing(hash.String(), "missing operation author token") + continue + } + + var id int + var idString, url string + switch op.(type) { + case *bug.AddCommentOperation: + opr := op.(*bug.AddCommentOperation) + + // send operation to gitlab + id, err = addCommentGitlabIssue(client, ge.repositoryID, bugGitlabID, opr.Message) + if err != nil { + err := errors.Wrap(err, "adding comment") + out <- core.NewExportError(err, b.Id()) + return + } + + idString = strconv.Itoa(id) + out <- core.NewExportComment(hash.String()) + + // cache comment id + ge.cachedOperationIDs[hash.String()] = idString + + case *bug.EditCommentOperation: + + opr := op.(*bug.EditCommentOperation) + targetHash := opr.Target.String() + + // Since gitlab doesn't consider the issue body as a comment + if targetHash == bugCreationHash { + + // case bug creation operation: we need to edit the Gitlab issue + if err := updateGitlabIssueBody(client, ge.repositoryID, id, opr.Message); err != nil { + err := errors.Wrap(err, "editing issue") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + } else { + + // case comment edition operation: we need to edit the Gitlab comment + _, ok := ge.cachedOperationIDs[targetHash] + if !ok { + panic("unexpected error: comment id not found") + } + + err := editCommentGitlabIssue(client, ge.repositoryID, id, id, opr.Message) + if err != nil { + err := errors.Wrap(err, "editing comment") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + // use comment id/url instead of issue id/url + //id = eid + //url = eurl + } + + case *bug.SetStatusOperation: + opr := op.(*bug.SetStatusOperation) + if err := updateGitlabIssueStatus(client, idString, id, opr.Status); err != nil { + err := errors.Wrap(err, "editing status") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportStatusChange(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + case *bug.SetTitleOperation: + opr := op.(*bug.SetTitleOperation) + if err := updateGitlabIssueTitle(client, ge.repositoryID, id, opr.Title); err != nil { + err := errors.Wrap(err, "editing title") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportTitleEdition(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + case *bug.LabelChangeOperation: + _ = op.(*bug.LabelChangeOperation) + if err := updateGitlabIssueLabels(client, ge.repositoryID, bugGitlabID, []string{}); err != nil { + err := errors.Wrap(err, "updating labels") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportLabelChange(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + default: + panic("unhandled operation type case") + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, idString, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit at each operation export to avoid exporting same events multiple times + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + } +} + +func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { + _, err := b.SetMetadata( + target, + map[string]string{ + keyGitlabId: gitlabID, + keyGitlabUrl: gitlabURL, + }, + ) + + return err +} + +func (ge *gitlabExporter) getGitlabLabelID(label string) (string, error) { + id, ok := ge.cachedLabels[label] + if !ok { + return "", fmt.Errorf("non cached label") + } + + return id, nil +} + +// get label from gitlab +func (ge *gitlabExporter) loadLabelsFromGitlab(gc *gitlab.Client) error { + labels, _, err := gc.Labels.ListLabels( + ge.repositoryID, + &gitlab.ListLabelsOptions{ + Page: 0, + }, + ) + + if err != nil { + return err + } + + for _, label := range labels { + ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) + } + + for page := 2; len(labels) != 0; page++ { + + labels, _, err = gc.Labels.ListLabels( + ge.repositoryID, + &gitlab.ListLabelsOptions{ + Page: page, + }, + ) + + if err != nil { + return err + } + + for _, label := range labels { + ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) + } + } + + return nil +} + +func (ge *gitlabExporter) createGitlabLabel(gc *gitlab.Client, label bug.Label) error { + client := buildClient(ge.conf[keyToken]) + + // RGBA to hex color + rgba := label.RGBA() + hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + name := label.String() + + _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ + Name: &name, + Color: &hexColor, + }) + + return err +} + +// create a gitlab. issue and return it ID +func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (int, string, error) { + issue, _, err := gc.Issues.CreateIssue(repositoryID, &gitlab.CreateIssueOptions{ + Title: &title, + Description: &body, + }) + + if err != nil { + return 0, "", err + } + + return issue.IID, issue.WebURL, nil +} + +// add a comment to an issue and return it ID +func addCommentGitlabIssue(gc *gitlab.Client, repositoryID string, issueID int, body string) (int, error) { + note, _, err := gc.Notes.CreateIssueNote(repositoryID, issueID, &gitlab.CreateIssueNoteOptions{ + Body: &body, + }) + + if err != nil { + return 0, err + } + + return note.ID, nil +} + +func editCommentGitlabIssue(gc *gitlab.Client, repositoryID string, issueID, noteID int, body string) error { + _, _, err := gc.Notes.UpdateIssueNote(repositoryID, issueID, noteID, &gitlab.UpdateIssueNoteOptions{ + Body: &body, + }) + + return err +} + +func updateGitlabIssueStatus(gc *gitlab.Client, repositoryID string, issueID int, status bug.Status) error { + var state string + + switch status { + case bug.OpenStatus: + state = "reopen" + case bug.ClosedStatus: + state = "close" + default: + panic("unknown bug state") + } + + _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ + StateEvent: &state, + }) + + return err +} + +func updateGitlabIssueBody(gc *gitlab.Client, repositoryID string, issueID int, body string) error { + _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ + Description: &body, + }) + + return err +} + +func updateGitlabIssueTitle(gc *gitlab.Client, repositoryID string, issueID int, title string) error { + _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ + Title: &title, + }) + + return err +} + +// update gitlab. issue labels +func updateGitlabIssueLabels(gc *gitlab.Client, repositoryID string, issueID int, labels []string) error { + _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ + Labels: labels, + }) + + return err } diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index 63c212ed..f4b980ac 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -38,7 +38,7 @@ func (*Gitlab) NewImporter() core.Importer { } func (*Gitlab) NewExporter() core.Exporter { - return nil + return &gitlabExporter{} } func buildClient(token string) *gitlab.Client { -- cgit From c7e932e5a95816402248ba8c36ab8d631fdc9232 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 13 Aug 2019 20:51:39 +0200 Subject: bridge/gitlab: add exporter e2e tests bridge/gitlab: fix export bugs and rebase --- bridge/gitlab/export.go | 118 ++++++------------ bridge/gitlab/export_test.go | 276 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+), 79 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 723b7b6a..3f339354 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -11,7 +11,7 @@ import ( "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" - "github.com/MichaelMure/git-bug/util/git" + "github.com/MichaelMure/git-bug/entity" ) var ( @@ -53,14 +53,14 @@ func (ge *gitlabExporter) Init(conf core.Configuration) error { // getIdentityClient return a gitlab v4 API client configured with the access token of the given identity. // if no client were found it will initialize it from the known tokens map and cache it for next use -func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { - client, ok := ge.identityClient[id] +func (ge *gitlabExporter) getIdentityClient(id entity.Id) (*gitlab.Client, error) { + client, ok := ge.identityClient[id.String()] if ok { return client, nil } // get token - token, ok := ge.identityToken[id] + token, ok := ge.identityToken[id.String()] if !ok { return nil, ErrMissingIdentityToken } @@ -68,7 +68,7 @@ func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { // create client client = buildClient(token) // cache client - ge.identityClient[id] = client + ge.identityClient[id.String()] = client return client, nil } @@ -82,21 +82,17 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c return nil, err } - ge.identityToken[user.Id()] = ge.conf[keyToken] + ge.identityToken[user.Id().String()] = ge.conf[keyToken] // get repository node id ge.repositoryID = ge.conf[keyProjectID] - if err != nil { - return nil, err - } - go func() { defer close(out) - var allIdentitiesIds []string + var allIdentitiesIds []entity.Id for id := range ge.identityToken { - allIdentitiesIds = append(allIdentitiesIds, id) + allIdentitiesIds = append(allIdentitiesIds, entity.Id(id)) } allBugsIds := repo.AllBugsIds() @@ -136,8 +132,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan var err error var bugGitlabID int var bugGitlabIDString string - var bugGitlabURL string - var bugCreationHash string + var bugCreationId string + //labels := make([]string, 0) // Special case: @@ -158,17 +154,16 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan // get gitlab bug ID gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) if ok { - gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl) + _, ok := snapshot.GetCreateMetadata(keyGitlabUrl) if !ok { // if we find gitlab ID, gitlab URL must be found too err := fmt.Errorf("expected to find gitlab issue URL") out <- core.NewExportError(err, b.Id()) + return } - //FIXME: - // ignore issue comming from other repositories - out <- core.NewExportNothing(b.Id(), "bug already exported") + // will be used to mark operation related to a bug as exported bugGitlabIDString = gitlabID bugGitlabID, err = strconv.Atoi(bugGitlabIDString) @@ -176,8 +171,6 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan panic("unexpected gitlab id format") } - bugGitlabURL = gitlabURL - } else { // check that we have a token for operation author client, err := ge.getIdentityClient(author.Id()) @@ -199,15 +192,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan out <- core.NewExportBug(b.Id()) - hash, err := createOp.Hash() - if err != nil { - err := errors.Wrap(err, "comment hash") - out <- core.NewExportError(err, b.Id()) - return - } - // mark bug creation operation as exported - if err := markOperationAsExported(b, hash, idString, url); err != nil { + if err := markOperationAsExported(b, createOp.Id(), idString, url); err != nil { err := errors.Wrap(err, "marking operation as exported") out <- core.NewExportError(err, b.Id()) return @@ -223,20 +209,11 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan // cache bug gitlab ID and URL bugGitlabID = id bugGitlabIDString = idString - bugGitlabURL = url - } - - // get createOp hash - hash, err := createOp.Hash() - if err != nil { - out <- core.NewExportError(err, b.Id()) - return } - bugCreationHash = hash.String() - + bugCreationId = createOp.Id().String() // cache operation gitlab id - ge.cachedOperationIDs[bugCreationHash] = bugGitlabIDString + ge.cachedOperationIDs[bugCreationId] = bugGitlabIDString for _, op := range snapshot.Operations[1:] { // ignore SetMetadata operations @@ -244,26 +221,18 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan continue } - // get operation hash - hash, err := op.Hash() - if err != nil { - err := errors.Wrap(err, "operation hash") - out <- core.NewExportError(err, b.Id()) - return - } - // ignore operations already existing in gitlab (due to import or export) // cache the ID of already exported or imported issues and events from Gitlab if id, ok := op.GetMetadata(keyGitlabId); ok { - ge.cachedOperationIDs[hash.String()] = id - out <- core.NewExportNothing(hash.String(), "already exported operation") + ge.cachedOperationIDs[op.Id().String()] = id + out <- core.NewExportNothing(op.Id(), "already exported operation") continue } opAuthor := op.GetAuthor() client, err := ge.getIdentityClient(opAuthor.Id()) if err != nil { - out <- core.NewExportNothing(hash.String(), "missing operation author token") + out <- core.NewExportNothing(op.Id(), "missing operation author token") continue } @@ -281,52 +250,50 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - idString = strconv.Itoa(id) - out <- core.NewExportComment(hash.String()) + out <- core.NewExportComment(op.Id()) + idString = strconv.Itoa(id) // cache comment id - ge.cachedOperationIDs[hash.String()] = idString + ge.cachedOperationIDs[op.Id().String()] = idString case *bug.EditCommentOperation: - opr := op.(*bug.EditCommentOperation) - targetHash := opr.Target.String() + targetId := opr.Target.String() // Since gitlab doesn't consider the issue body as a comment - if targetHash == bugCreationHash { + if targetId == bugCreationId { // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(client, ge.repositoryID, id, opr.Message); err != nil { + if err := updateGitlabIssueBody(client, ge.repositoryID, bugGitlabID, opr.Message); err != nil { err := errors.Wrap(err, "editing issue") out <- core.NewExportError(err, b.Id()) return } - out <- core.NewExportCommentEdition(hash.String()) - + out <- core.NewExportCommentEdition(op.Id()) id = bugGitlabID - url = bugGitlabURL } else { // case comment edition operation: we need to edit the Gitlab comment - _, ok := ge.cachedOperationIDs[targetHash] + commentID, ok := ge.cachedOperationIDs[targetId] if !ok { panic("unexpected error: comment id not found") } - err := editCommentGitlabIssue(client, ge.repositoryID, id, id, opr.Message) + commentIDint, err := strconv.Atoi(commentID) if err != nil { + panic("unexpected comment id format") + } + + if err := editCommentGitlabIssue(client, ge.repositoryID, commentIDint, id, opr.Message); err != nil { err := errors.Wrap(err, "editing comment") out <- core.NewExportError(err, b.Id()) return } - out <- core.NewExportCommentEdition(hash.String()) - - // use comment id/url instead of issue id/url - //id = eid - //url = eurl + out <- core.NewExportCommentEdition(op.Id()) + id = commentIDint } case *bug.SetStatusOperation: @@ -337,10 +304,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportStatusChange(hash.String()) - + out <- core.NewExportStatusChange(op.Id()) id = bugGitlabID - url = bugGitlabURL case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) @@ -350,10 +315,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportTitleEdition(hash.String()) - + out <- core.NewExportTitleEdition(op.Id()) id = bugGitlabID - url = bugGitlabURL case *bug.LabelChangeOperation: _ = op.(*bug.LabelChangeOperation) @@ -363,17 +326,14 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportLabelChange(hash.String()) - + out <- core.NewExportLabelChange(op.Id()) id = bugGitlabID - url = bugGitlabURL - default: panic("unhandled operation type case") } // mark operation as exported - if err := markOperationAsExported(b, hash, idString, url); err != nil { + if err := markOperationAsExported(b, op.Id(), idString, url); err != nil { err := errors.Wrap(err, "marking operation as exported") out <- core.NewExportError(err, b.Id()) return @@ -388,7 +348,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan } } -func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { +func markOperationAsExported(b *cache.BugCache, target entity.Id, gitlabID, gitlabURL string) error { _, err := b.SetMetadata( target, map[string]string{ diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go index 4d4b35af..43403361 100644 --- a/bridge/gitlab/export_test.go +++ b/bridge/gitlab/export_test.go @@ -1 +1,277 @@ package gitlab + +import ( + "context" + "fmt" + "math/rand" + "os" + "strconv" + "testing" + "time" + + "github.com/xanzy/go-gitlab" + + "github.com/stretchr/testify/require" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/interrupt" +) + +const ( + testRepoBaseName = "git-bug-test-gitlab-exporter" +) + +type testCase struct { + name string + bug *cache.BugCache + numOrOp int // number of original operations +} + +func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCache) []*testCase { + // simple bug + simpleBug, _, err := repo.NewBug("simple bug", "new bug") + require.NoError(t, err) + + // bug with comments + bugWithComments, _, err := repo.NewBug("bug with comments", "new bug") + require.NoError(t, err) + + _, err = bugWithComments.AddComment("new comment") + require.NoError(t, err) + + // bug with label changes + bugLabelChange, _, err := repo.NewBug("bug label change", "new bug") + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"bug"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"core"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels(nil, []string{"bug"}) + require.NoError(t, err) + + // bug with comments editions + bugWithCommentEditions, createOp, err := repo.NewBug("bug with comments editions", "new bug") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(createOp.Id(), "first comment edited") + require.NoError(t, err) + + commentOp, err := bugWithCommentEditions.AddComment("first comment") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(commentOp.Id(), "first comment edited") + require.NoError(t, err) + + // bug status changed + bugStatusChanged, _, err := repo.NewBug("bug status changed", "new bug") + require.NoError(t, err) + + _, err = bugStatusChanged.Close() + require.NoError(t, err) + + _, err = bugStatusChanged.Open() + require.NoError(t, err) + + // bug title changed + bugTitleEdited, _, err := repo.NewBug("bug title edited", "new bug") + require.NoError(t, err) + + _, err = bugTitleEdited.SetTitle("bug title edited again") + require.NoError(t, err) + + return []*testCase{ + &testCase{ + name: "simple bug", + bug: simpleBug, + numOrOp: 1, + }, + &testCase{ + name: "bug with comments", + bug: bugWithComments, + numOrOp: 2, + }, + &testCase{ + name: "bug label change", + bug: bugLabelChange, + numOrOp: 4, + }, + &testCase{ + name: "bug with comment editions", + bug: bugWithCommentEditions, + numOrOp: 4, + }, + &testCase{ + name: "bug changed status", + bug: bugStatusChanged, + numOrOp: 3, + }, + &testCase{ + name: "bug title edited", + bug: bugTitleEdited, + numOrOp: 2, + }, + } +} + +func TestPushPull(t *testing.T) { + // token must have 'repo' and 'delete_repo' scopes + token := os.Getenv("GITLAB_API_TOKEN") + if token == "" { + t.Skip("Env var GITLAB_API_TOKEN missing") + } + + // create repo backend + repo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repo) + + backend, err := cache.NewRepoCache(repo) + require.NoError(t, err) + + // set author identity + author, err := backend.NewIdentity("test identity", "test@test.org") + require.NoError(t, err) + + err = backend.SetUserIdentity(author) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + tests := testCases(t, backend, author) + + // generate project name + projectName := generateRepoName() + + // create target Gitlab repository + projectID, err := createRepository(context.TODO(), projectName, token) + require.NoError(t, err) + + fmt.Println("created repository", projectName) + + // Make sure to remove the Gitlab repository when the test end + defer func(t *testing.T) { + if err := deleteRepository(context.TODO(), projectID, token); err != nil { + t.Fatal(err) + } + fmt.Println("deleted repository:", projectName) + }(t) + + interrupt.RegisterCleaner(func() error { + return deleteRepository(context.TODO(), projectID, token) + }) + + // initialize exporter + exporter := &gitlabExporter{} + err = exporter.Init(core.Configuration{ + keyProjectID: strconv.Itoa(projectID), + keyToken: token, + }) + require.NoError(t, err) + + start := time.Now() + + // export all bugs + events, err := exporter.ExportAll(backend, time.Time{}) + require.NoError(t, err) + + for result := range events { + require.NoError(t, result.Err) + } + require.NoError(t, err) + + fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds()) + + repoTwo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repoTwo) + + // create a second backend + backendTwo, err := cache.NewRepoCache(repoTwo) + require.NoError(t, err) + + importer := &gitlabImporter{} + err = importer.Init(core.Configuration{ + keyProjectID: strconv.Itoa(projectID), + keyToken: token, + }) + require.NoError(t, err) + + // import all exported bugs to the second backend + err = importer.ImportAll(backendTwo, time.Time{}) + require.NoError(t, err) + + require.Len(t, backendTwo.AllBugsIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // for each operation a SetMetadataOperation will be added + // so number of operations should double + require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2) + + // verify operation have correct metadata + for _, op := range tt.bug.Snapshot().Operations { + // Check if the originals operations (*not* SetMetadata) are tagged properly + if _, ok := op.(*bug.SetMetadataOperation); !ok { + _, haveIDMetadata := op.GetMetadata(keyGitlabId) + require.True(t, haveIDMetadata) + + _, haveURLMetada := op.GetMetadata(keyGitlabUrl) + require.True(t, haveURLMetada) + } + } + + // get bug gitlab ID + bugGitlabID, ok := tt.bug.Snapshot().GetCreateMetadata(keyGitlabId) + require.True(t, ok) + + // retrieve bug from backendTwo + importedBug, err := backendTwo.ResolveBugCreateMetadata(keyGitlabId, bugGitlabID) + require.NoError(t, err) + + // verify bug have same number of original operations + require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp) + + // verify bugs are taged with origin=gitlab + issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.KeyOrigin) + require.True(t, ok) + require.Equal(t, issueOrigin, target) + + //TODO: maybe more tests to ensure bug final state + }) + } +} + +func generateRepoName() string { + rand.Seed(time.Now().UnixNano()) + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, 8) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return fmt.Sprintf("%s-%s", testRepoBaseName, string(b)) +} + +// create repository need a token with scope 'repo' +func createRepository(ctx context.Context, name, token string) (int, error) { + client := buildClient(token) + project, _, err := client.Projects.CreateProject( + &gitlab.CreateProjectOptions{ + Name: gitlab.String(name), + }, + gitlab.WithContext(ctx), + ) + + return project.ID, err +} + +// delete repository need a token with scope 'delete_repo' +func deleteRepository(ctx context.Context, project int, token string) error { + client := buildClient(token) + _, err := client.Projects.DeleteProject(project, gitlab.WithContext(ctx)) + return err +} -- cgit From f1be129d75c56a168a1519674f7c3ccd429af01b Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Mon, 19 Aug 2019 13:50:48 +0200 Subject: bridge/gitlab: rebase and correct exporter --- bridge/gitlab/export.go | 195 ++++++++++++++++++++++++++++--------------- bridge/gitlab/export_test.go | 11 ++- 2 files changed, 135 insertions(+), 71 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 3f339354..5b7c35d0 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "fmt" "strconv" "time" @@ -74,7 +75,7 @@ func (ge *gitlabExporter) getIdentityClient(id entity.Id) (*gitlab.Client, error } // ExportAll export all event made by the current user to Gitlab -func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { +func (ge *gitlabExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { out := make(chan core.ExportResult) user, err := repo.GetUserIdentity() @@ -115,7 +116,7 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c if snapshot.HasAnyActor(allIdentitiesIds...) { // try to export the bug and it associated events - ge.exportBug(b, since, out) + ge.exportBug(ctx, b, since, out) } else { out <- core.NewExportNothing(id, "not an actor") } @@ -126,7 +127,7 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c } // exportBug publish bugs and related events -func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { +func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { snapshot := b.Snapshot() var err error @@ -181,7 +182,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan } // create bug - id, url, err := createGitlabIssue(client, ge.repositoryID, createOp.Title, createOp.Message) + id, url, err := createGitlabIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { err := errors.Wrap(err, "exporting gitlab issue") out <- core.NewExportError(err, b.Id()) @@ -189,7 +190,6 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan } idString := strconv.Itoa(id) - out <- core.NewExportBug(b.Id()) // mark bug creation operation as exported @@ -215,6 +215,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan // cache operation gitlab id ge.cachedOperationIDs[bugCreationId] = bugGitlabIDString + var actualLabels []string for _, op := range snapshot.Operations[1:] { // ignore SetMetadata operations if _, ok := op.(*bug.SetMetadataOperation); ok { @@ -243,7 +244,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan opr := op.(*bug.AddCommentOperation) // send operation to gitlab - id, err = addCommentGitlabIssue(client, ge.repositoryID, bugGitlabID, opr.Message) + id, err = addCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, opr.Message) if err != nil { err := errors.Wrap(err, "adding comment") out <- core.NewExportError(err, b.Id()) @@ -264,7 +265,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan if targetId == bugCreationId { // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(client, ge.repositoryID, bugGitlabID, opr.Message); err != nil { + if err := updateGitlabIssueBody(ctx, client, ge.repositoryID, bugGitlabID, opr.Message); err != nil { err := errors.Wrap(err, "editing issue") out <- core.NewExportError(err, b.Id()) return @@ -286,7 +287,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan panic("unexpected comment id format") } - if err := editCommentGitlabIssue(client, ge.repositoryID, commentIDint, id, opr.Message); err != nil { + if err := editCommentGitlabIssue(ctx, client, ge.repositoryID, commentIDint, id, opr.Message); err != nil { err := errors.Wrap(err, "editing comment") out <- core.NewExportError(err, b.Id()) return @@ -298,7 +299,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) - if err := updateGitlabIssueStatus(client, idString, id, opr.Status); err != nil { + if err := updateGitlabIssueStatus(ctx, client, idString, id, opr.Status); err != nil { err := errors.Wrap(err, "editing status") out <- core.NewExportError(err, b.Id()) return @@ -309,7 +310,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) - if err := updateGitlabIssueTitle(client, ge.repositoryID, id, opr.Title); err != nil { + if err := updateGitlabIssueTitle(ctx, client, ge.repositoryID, id, opr.Title); err != nil { err := errors.Wrap(err, "editing title") out <- core.NewExportError(err, b.Id()) return @@ -319,8 +320,31 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan id = bugGitlabID case *bug.LabelChangeOperation: - _ = op.(*bug.LabelChangeOperation) - if err := updateGitlabIssueLabels(client, ge.repositoryID, bugGitlabID, []string{}); err != nil { + opr := op.(*bug.LabelChangeOperation) + // we need to set the actual list of labels at each label change operation + // because gitlab update issue requests need directly the latest list of the verison + + if len(opr.Added) != 0 { + for _, label := range opr.Added { + actualLabels = append(actualLabels, string(label)) + } + } + + if len(opr.Removed) != 0 { + var newActualLabels []string + for _, label := range actualLabels { + for _, l := range opr.Removed { + if label == string(l) { + continue + } + + newActualLabels = append(newActualLabels, label) + } + } + actualLabels = newActualLabels + } + + if err := updateGitlabIssueLabels(ctx, client, ge.repositoryID, bugGitlabID, actualLabels); err != nil { err := errors.Wrap(err, "updating labels") out <- core.NewExportError(err, b.Id()) return @@ -370,35 +394,25 @@ func (ge *gitlabExporter) getGitlabLabelID(label string) (string, error) { } // get label from gitlab -func (ge *gitlabExporter) loadLabelsFromGitlab(gc *gitlab.Client) error { - labels, _, err := gc.Labels.ListLabels( - ge.repositoryID, - &gitlab.ListLabelsOptions{ - Page: 0, - }, - ) - - if err != nil { - return err - } - - for _, label := range labels { - ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) - } - - for page := 2; len(labels) != 0; page++ { - - labels, _, err = gc.Labels.ListLabels( +func (ge *gitlabExporter) loadLabelsFromGitlab(ctx context.Context, gc *gitlab.Client) error { + page := 1 + for ; ; page++ { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + labels, _, err := gc.Labels.ListLabels( ge.repositoryID, &gitlab.ListLabelsOptions{ Page: page, }, + gitlab.WithContext(ctx), ) - if err != nil { return err } + if len(labels) == 0 { + break + } for _, label := range labels { ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) } @@ -407,7 +421,7 @@ func (ge *gitlabExporter) loadLabelsFromGitlab(gc *gitlab.Client) error { return nil } -func (ge *gitlabExporter) createGitlabLabel(gc *gitlab.Client, label bug.Label) error { +func (ge *gitlabExporter) createGitlabLabel(ctx context.Context, gc *gitlab.Client, label bug.Label) error { client := buildClient(ge.conf[keyToken]) // RGBA to hex color @@ -415,21 +429,31 @@ func (ge *gitlabExporter) createGitlabLabel(gc *gitlab.Client, label bug.Label) hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) name := label.String() - _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ - Name: &name, - Color: &hexColor, - }) + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := client.Labels.CreateLabel(ge.repositoryID, + &gitlab.CreateLabelOptions{ + Name: &name, + Color: &hexColor, + }, + gitlab.WithContext(ctx), + ) return err } // create a gitlab. issue and return it ID -func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (int, string, error) { - issue, _, err := gc.Issues.CreateIssue(repositoryID, &gitlab.CreateIssueOptions{ - Title: &title, - Description: &body, - }) - +func createGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID, title, body string) (int, string, error) { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + issue, _, err := gc.Issues.CreateIssue( + repositoryID, + &gitlab.CreateIssueOptions{ + Title: &title, + Description: &body, + }, + gitlab.WithContext(ctx), + ) if err != nil { return 0, "", err } @@ -438,11 +462,16 @@ func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (int } // add a comment to an issue and return it ID -func addCommentGitlabIssue(gc *gitlab.Client, repositoryID string, issueID int, body string) (int, error) { - note, _, err := gc.Notes.CreateIssueNote(repositoryID, issueID, &gitlab.CreateIssueNoteOptions{ - Body: &body, - }) - +func addCommentGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID int, body string) (int, error) { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + note, _, err := gc.Notes.CreateIssueNote( + repositoryID, issueID, + &gitlab.CreateIssueNoteOptions{ + Body: &body, + }, + gitlab.WithContext(ctx), + ) if err != nil { return 0, err } @@ -450,15 +479,21 @@ func addCommentGitlabIssue(gc *gitlab.Client, repositoryID string, issueID int, return note.ID, nil } -func editCommentGitlabIssue(gc *gitlab.Client, repositoryID string, issueID, noteID int, body string) error { - _, _, err := gc.Notes.UpdateIssueNote(repositoryID, issueID, noteID, &gitlab.UpdateIssueNoteOptions{ - Body: &body, - }) +func editCommentGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID, noteID int, body string) error { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := gc.Notes.UpdateIssueNote( + repositoryID, issueID, noteID, + &gitlab.UpdateIssueNoteOptions{ + Body: &body, + }, + gitlab.WithContext(ctx), + ) return err } -func updateGitlabIssueStatus(gc *gitlab.Client, repositoryID string, issueID int, status bug.Status) error { +func updateGitlabIssueStatus(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID int, status bug.Status) error { var state string switch status { @@ -470,34 +505,58 @@ func updateGitlabIssueStatus(gc *gitlab.Client, repositoryID string, issueID int panic("unknown bug state") } - _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ - StateEvent: &state, - }) + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := gc.Issues.UpdateIssue( + repositoryID, issueID, + &gitlab.UpdateIssueOptions{ + StateEvent: &state, + }, + gitlab.WithContext(ctx), + ) return err } -func updateGitlabIssueBody(gc *gitlab.Client, repositoryID string, issueID int, body string) error { - _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ - Description: &body, - }) +func updateGitlabIssueBody(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID int, body string) error { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := gc.Issues.UpdateIssue( + repositoryID, issueID, + &gitlab.UpdateIssueOptions{ + Description: &body, + }, + gitlab.WithContext(ctx), + ) return err } -func updateGitlabIssueTitle(gc *gitlab.Client, repositoryID string, issueID int, title string) error { - _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ - Title: &title, - }) +func updateGitlabIssueTitle(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID int, title string) error { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := gc.Issues.UpdateIssue( + repositoryID, issueID, + &gitlab.UpdateIssueOptions{ + Title: &title, + }, + gitlab.WithContext(ctx), + ) return err } // update gitlab. issue labels -func updateGitlabIssueLabels(gc *gitlab.Client, repositoryID string, issueID int, labels []string) error { - _, _, err := gc.Issues.UpdateIssue(repositoryID, issueID, &gitlab.UpdateIssueOptions{ - Labels: labels, - }) +func updateGitlabIssueLabels(ctx context.Context, gc *gitlab.Client, repositoryID string, issueID int, labels []string) error { + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + _, _, err := gc.Issues.UpdateIssue( + repositoryID, issueID, + &gitlab.UpdateIssueOptions{ + Labels: labels, + }, + gitlab.WithContext(ctx), + ) return err } diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go index 43403361..f7a99610 100644 --- a/bridge/gitlab/export_test.go +++ b/bridge/gitlab/export_test.go @@ -174,13 +174,14 @@ func TestPushPull(t *testing.T) { }) require.NoError(t, err) + ctx := context.Background() start := time.Now() // export all bugs - events, err := exporter.ExportAll(backend, time.Time{}) + exportEvents, err := exporter.ExportAll(ctx, backend, time.Time{}) require.NoError(t, err) - for result := range events { + for result := range exportEvents { require.NoError(t, result.Err) } require.NoError(t, err) @@ -202,9 +203,13 @@ func TestPushPull(t *testing.T) { require.NoError(t, err) // import all exported bugs to the second backend - err = importer.ImportAll(backendTwo, time.Time{}) + importEvents, err := importer.ImportAll(ctx, backendTwo, time.Time{}) require.NoError(t, err) + for result := range importEvents { + require.NoError(t, result.Err) + } + require.Len(t, backendTwo.AllBugsIds(), len(tests)) for _, tt := range tests { -- cgit From 514dc30c20c8a9a068c68f4d9d4935abc56dbda7 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Mon, 19 Aug 2019 14:09:31 +0200 Subject: bridge/gitlab: fix edit comment request and remove label functionalities --- bridge/core/export.go | 5 +- bridge/gitlab/export.go | 116 +++++++++++-------------------------------- bridge/gitlab/export_test.go | 60 +++++++++++++--------- bridge/gitlab/import.go | 2 +- 4 files changed, 70 insertions(+), 113 deletions(-) (limited to 'bridge') diff --git a/bridge/core/export.go b/bridge/core/export.go index ceae1401..558b3d78 100644 --- a/bridge/core/export.go +++ b/bridge/core/export.go @@ -62,8 +62,9 @@ func (er ExportResult) String() string { func NewExportError(err error, id entity.Id) ExportResult { return ExportResult{ - ID: id, - Err: err, + ID: id, + Err: err, + Event: ExportEventError, } } diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 5b7c35d0..f6b402fc 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -35,9 +35,6 @@ type gitlabExporter struct { // cache identifiers used to speed up exporting operations // cleared for each bug cachedOperationIDs map[string]string - - // cache labels used to speed up exporting labels events - cachedLabels map[string]string } // Init . @@ -47,7 +44,6 @@ func (ge *gitlabExporter) Init(conf core.Configuration) error { ge.identityToken = make(map[string]string) ge.identityClient = make(map[string]*gitlab.Client) ge.cachedOperationIDs = make(map[string]string) - ge.cachedLabels = make(map[string]string) return nil } @@ -99,26 +95,31 @@ func (ge *gitlabExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, allBugsIds := repo.AllBugsIds() for _, id := range allBugsIds { - b, err := repo.ResolveBug(id) - if err != nil { - out <- core.NewExportError(err, id) + select { + case <-ctx.Done(): return - } + default: + b, err := repo.ResolveBug(id) + if err != nil { + out <- core.NewExportError(err, id) + return + } - snapshot := b.Snapshot() + snapshot := b.Snapshot() - // ignore issues created before since date - // TODO: compare the Lamport time instead of using the unix time - if snapshot.CreatedAt.Before(since) { - out <- core.NewExportNothing(b.Id(), "bug created before the since date") - continue - } + // ignore issues created before since date + // TODO: compare the Lamport time instead of using the unix time + if snapshot.CreatedAt.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } - if snapshot.HasAnyActor(allIdentitiesIds...) { - // try to export the bug and it associated events - ge.exportBug(ctx, b, since, out) - } else { - out <- core.NewExportNothing(id, "not an actor") + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + ge.exportBug(ctx, b, since, out) + } else { + out <- core.NewExportNothing(id, "not an actor") + } } } }() @@ -135,8 +136,6 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc var bugGitlabIDString string var bugCreationId string - //labels := make([]string, 0) - // Special case: // if a user try to export a bug that is not already exported to Gitlab (or imported // from Gitlab) and we do not have the token of the bug author, there is nothing we can do. @@ -182,7 +181,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc } // create bug - id, url, err := createGitlabIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message) + _, id, url, err := createGitlabIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { err := errors.Wrap(err, "exporting gitlab issue") out <- core.NewExportError(err, b.Id()) @@ -287,7 +286,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc panic("unexpected comment id format") } - if err := editCommentGitlabIssue(ctx, client, ge.repositoryID, commentIDint, id, opr.Message); err != nil { + if err := editCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, commentIDint, opr.Message); err != nil { err := errors.Wrap(err, "editing comment") out <- core.NewExportError(err, b.Id()) return @@ -299,7 +298,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) - if err := updateGitlabIssueStatus(ctx, client, idString, id, opr.Status); err != nil { + if err := updateGitlabIssueStatus(ctx, client, ge.repositoryID, bugGitlabID, opr.Status); err != nil { err := errors.Wrap(err, "editing status") out <- core.NewExportError(err, b.Id()) return @@ -310,7 +309,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) - if err := updateGitlabIssueTitle(ctx, client, ge.repositoryID, id, opr.Title); err != nil { + if err := updateGitlabIssueTitle(ctx, client, ge.repositoryID, bugGitlabID, opr.Title); err != nil { err := errors.Wrap(err, "editing title") out <- core.NewExportError(err, b.Id()) return @@ -356,6 +355,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc panic("unhandled operation type case") } + idString = strconv.Itoa(id) // mark operation as exported if err := markOperationAsExported(b, op.Id(), idString, url); err != nil { err := errors.Wrap(err, "marking operation as exported") @@ -384,66 +384,8 @@ func markOperationAsExported(b *cache.BugCache, target entity.Id, gitlabID, gitl return err } -func (ge *gitlabExporter) getGitlabLabelID(label string) (string, error) { - id, ok := ge.cachedLabels[label] - if !ok { - return "", fmt.Errorf("non cached label") - } - - return id, nil -} - -// get label from gitlab -func (ge *gitlabExporter) loadLabelsFromGitlab(ctx context.Context, gc *gitlab.Client) error { - page := 1 - for ; ; page++ { - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - labels, _, err := gc.Labels.ListLabels( - ge.repositoryID, - &gitlab.ListLabelsOptions{ - Page: page, - }, - gitlab.WithContext(ctx), - ) - if err != nil { - return err - } - - if len(labels) == 0 { - break - } - for _, label := range labels { - ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) - } - } - - return nil -} - -func (ge *gitlabExporter) createGitlabLabel(ctx context.Context, gc *gitlab.Client, label bug.Label) error { - client := buildClient(ge.conf[keyToken]) - - // RGBA to hex color - rgba := label.RGBA() - hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) - name := label.String() - - ctx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - _, _, err := client.Labels.CreateLabel(ge.repositoryID, - &gitlab.CreateLabelOptions{ - Name: &name, - Color: &hexColor, - }, - gitlab.WithContext(ctx), - ) - - return err -} - // create a gitlab. issue and return it ID -func createGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID, title, body string) (int, string, error) { +func createGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID, title, body string) (int, int, string, error) { ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() issue, _, err := gc.Issues.CreateIssue( @@ -455,10 +397,10 @@ func createGitlabIssue(ctx context.Context, gc *gitlab.Client, repositoryID, tit gitlab.WithContext(ctx), ) if err != nil { - return 0, "", err + return 0, 0, "", err } - return issue.IID, issue.WebURL, nil + return issue.ID, issue.IID, issue.WebURL, nil } // add a comment to an issue and return it ID diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go index f7a99610..02e06efd 100644 --- a/bridge/gitlab/export_test.go +++ b/bridge/gitlab/export_test.go @@ -25,9 +25,11 @@ const ( ) type testCase struct { - name string - bug *cache.BugCache - numOrOp int // number of original operations + name string + bug *cache.BugCache + numOp int // number of original operations + numOpExp int // number of operations after export + numOpImp int // number of operations after import } func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCache) []*testCase { @@ -87,34 +89,46 @@ func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCach return []*testCase{ &testCase{ - name: "simple bug", - bug: simpleBug, - numOrOp: 1, + name: "simple bug", + bug: simpleBug, + numOp: 1, + numOpExp: 2, + numOpImp: 1, }, &testCase{ - name: "bug with comments", - bug: bugWithComments, - numOrOp: 2, + name: "bug with comments", + bug: bugWithComments, + numOp: 2, + numOpExp: 4, + numOpImp: 2, }, &testCase{ - name: "bug label change", - bug: bugLabelChange, - numOrOp: 4, + name: "bug label change", + bug: bugLabelChange, + numOp: 4, + numOpExp: 8, + numOpImp: 4, }, &testCase{ - name: "bug with comment editions", - bug: bugWithCommentEditions, - numOrOp: 4, + name: "bug with comment editions", + bug: bugWithCommentEditions, + numOp: 4, + numOpExp: 8, + numOpImp: 2, }, &testCase{ - name: "bug changed status", - bug: bugStatusChanged, - numOrOp: 3, + name: "bug changed status", + bug: bugStatusChanged, + numOp: 3, + numOpExp: 6, + numOpImp: 3, }, &testCase{ - name: "bug title edited", - bug: bugTitleEdited, - numOrOp: 2, + name: "bug title edited", + bug: bugTitleEdited, + numOp: 2, + numOpExp: 4, + numOpImp: 2, }, } } @@ -216,7 +230,7 @@ func TestPushPull(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // for each operation a SetMetadataOperation will be added // so number of operations should double - require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2) + require.Len(t, tt.bug.Snapshot().Operations, tt.numOpExp) // verify operation have correct metadata for _, op := range tt.bug.Snapshot().Operations { @@ -239,7 +253,7 @@ func TestPushPull(t *testing.T) { require.NoError(t, err) // verify bug have same number of original operations - require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp) + require.Len(t, importedBug.Snapshot().Operations, tt.numOpImp) // verify bugs are taged with origin=gitlab issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.KeyOrigin) diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 1391bf88..e2015773 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -121,7 +121,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue nil, map[string]string{ core.KeyOrigin: target, - keyGitlabId: parseID(issue.ID), + keyGitlabId: parseID(issue.IID), keyGitlabUrl: issue.WebURL, keyGitlabProject: gi.conf[keyProjectID], }, -- cgit From 63e7b08628d7cd6bdad31991eec1521639174c00 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 23 Aug 2019 01:54:02 +0200 Subject: bridge/gitlab: improve exporter error handling and label change operations --- bridge/gitlab/export.go | 56 ++++++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index f6b402fc..2b861dbf 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -87,7 +87,7 @@ func (ge *gitlabExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, go func() { defer close(out) - var allIdentitiesIds []entity.Id + allIdentitiesIds := make([]entity.Id, 0, len(ge.identityToken)) for id := range ge.identityToken { allIdentitiesIds = append(allIdentitiesIds, entity.Id(id)) } @@ -168,7 +168,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc bugGitlabIDString = gitlabID bugGitlabID, err = strconv.Atoi(bugGitlabIDString) if err != nil { - panic("unexpected gitlab id format") + panic(fmt.Sprintf("unexpected gitlab id format: %s", bugGitlabIDString)) } } else { @@ -214,7 +214,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc // cache operation gitlab id ge.cachedOperationIDs[bugCreationId] = bugGitlabIDString - var actualLabels []string + labelSet := make(map[string]struct{}) for _, op := range snapshot.Operations[1:] { // ignore SetMetadata operations if _, ok := op.(*bug.SetMetadataOperation); ok { @@ -238,12 +238,11 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc var id int var idString, url string - switch op.(type) { + switch op := op.(type) { case *bug.AddCommentOperation: - opr := op.(*bug.AddCommentOperation) // send operation to gitlab - id, err = addCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, opr.Message) + id, err = addCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, op.Message) if err != nil { err := errors.Wrap(err, "adding comment") out <- core.NewExportError(err, b.Id()) @@ -257,14 +256,13 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc ge.cachedOperationIDs[op.Id().String()] = idString case *bug.EditCommentOperation: - opr := op.(*bug.EditCommentOperation) - targetId := opr.Target.String() + targetId := op.Target.String() // Since gitlab doesn't consider the issue body as a comment if targetId == bugCreationId { // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(ctx, client, ge.repositoryID, bugGitlabID, opr.Message); err != nil { + if err := updateGitlabIssueBody(ctx, client, ge.repositoryID, bugGitlabID, op.Message); err != nil { err := errors.Wrap(err, "editing issue") out <- core.NewExportError(err, b.Id()) return @@ -278,15 +276,17 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc // case comment edition operation: we need to edit the Gitlab comment commentID, ok := ge.cachedOperationIDs[targetId] if !ok { - panic("unexpected error: comment id not found") + out <- core.NewExportError(fmt.Errorf("unexpected error: comment id not found"), op.Target) + return } commentIDint, err := strconv.Atoi(commentID) if err != nil { - panic("unexpected comment id format") + out <- core.NewExportError(fmt.Errorf("unexpected comment id format"), op.Target) + return } - if err := editCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, commentIDint, opr.Message); err != nil { + if err := editCommentGitlabIssue(ctx, client, ge.repositoryID, bugGitlabID, commentIDint, op.Message); err != nil { err := errors.Wrap(err, "editing comment") out <- core.NewExportError(err, b.Id()) return @@ -297,8 +297,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc } case *bug.SetStatusOperation: - opr := op.(*bug.SetStatusOperation) - if err := updateGitlabIssueStatus(ctx, client, ge.repositoryID, bugGitlabID, opr.Status); err != nil { + if err := updateGitlabIssueStatus(ctx, client, ge.repositoryID, bugGitlabID, op.Status); err != nil { err := errors.Wrap(err, "editing status") out <- core.NewExportError(err, b.Id()) return @@ -308,8 +307,7 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc id = bugGitlabID case *bug.SetTitleOperation: - opr := op.(*bug.SetTitleOperation) - if err := updateGitlabIssueTitle(ctx, client, ge.repositoryID, bugGitlabID, opr.Title); err != nil { + if err := updateGitlabIssueTitle(ctx, client, ge.repositoryID, bugGitlabID, op.Title); err != nil { err := errors.Wrap(err, "editing title") out <- core.NewExportError(err, b.Id()) return @@ -319,31 +317,23 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc id = bugGitlabID case *bug.LabelChangeOperation: - opr := op.(*bug.LabelChangeOperation) // we need to set the actual list of labels at each label change operation // because gitlab update issue requests need directly the latest list of the verison - if len(opr.Added) != 0 { - for _, label := range opr.Added { - actualLabels = append(actualLabels, string(label)) - } + for _, label := range op.Added { + labelSet[label.String()] = struct{}{} } - if len(opr.Removed) != 0 { - var newActualLabels []string - for _, label := range actualLabels { - for _, l := range opr.Removed { - if label == string(l) { - continue - } + for _, label := range op.Removed { + delete(labelSet, label.String()) + } - newActualLabels = append(newActualLabels, label) - } - } - actualLabels = newActualLabels + labels := make([]string, 0, len(labelSet)) + for key := range labelSet { + labels = append(labels, key) } - if err := updateGitlabIssueLabels(ctx, client, ge.repositoryID, bugGitlabID, actualLabels); err != nil { + if err := updateGitlabIssueLabels(ctx, client, ge.repositoryID, bugGitlabID, labels); err != nil { err := errors.Wrap(err, "updating labels") out <- core.NewExportError(err, b.Id()) return -- cgit From 22960159e9ea97b7eb7dd98611f854622da020f7 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 23 Aug 2019 14:57:34 +0200 Subject: bridge/gitlab: exporter ignore issues imported from or exported to different projects --- bridge/gitlab/export.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 2b861dbf..ee45c41b 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -154,7 +154,19 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc // get gitlab bug ID gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) if ok { - _, ok := snapshot.GetCreateMetadata(keyGitlabUrl) + projectID, ok := snapshot.GetCreateMetadata(keyGitlabProject) + if !ok { + err := fmt.Errorf("expected to find gitlab project id") + out <- core.NewExportError(err, b.Id()) + return + } + + if projectID != ge.conf[keyProjectID] { + out <- core.NewExportNothing(b.Id(), "skipping issue imported from another repository") + return + } + + _, ok = snapshot.GetCreateMetadata(keyGitlabUrl) if !ok { // if we find gitlab ID, gitlab URL must be found too err := fmt.Errorf("expected to find gitlab issue URL") @@ -168,7 +180,8 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc bugGitlabIDString = gitlabID bugGitlabID, err = strconv.Atoi(bugGitlabIDString) if err != nil { - panic(fmt.Sprintf("unexpected gitlab id format: %s", bugGitlabIDString)) + out <- core.NewExportError(fmt.Errorf("unexpected gitlab id format: %s", bugGitlabIDString), b.Id()) + return } } else { @@ -191,8 +204,15 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc idString := strconv.Itoa(id) out <- core.NewExportBug(b.Id()) - // mark bug creation operation as exported - if err := markOperationAsExported(b, createOp.Id(), idString, url); err != nil { + _, err = b.SetMetadata( + createOp.Id(), + map[string]string{ + keyGitlabId: idString, + keyGitlabUrl: url, + keyGitlabProject: ge.repositoryID, + }, + ) + if err != nil { err := errors.Wrap(err, "marking operation as exported") out <- core.NewExportError(err, b.Id()) return -- cgit From c8fdaab50ad022c34ac7926f9f30ced28ba4c682 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 24 Aug 2019 13:22:23 +0200 Subject: bridge/gitlab: remove gitlab url checking before export --- bridge/gitlab/export.go | 8 -------- 1 file changed, 8 deletions(-) (limited to 'bridge') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index ee45c41b..7cbbf903 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -166,14 +166,6 @@ func (ge *gitlabExporter) exportBug(ctx context.Context, b *cache.BugCache, sinc return } - _, ok = snapshot.GetCreateMetadata(keyGitlabUrl) - if !ok { - // if we find gitlab ID, gitlab URL must be found too - err := fmt.Errorf("expected to find gitlab issue URL") - out <- core.NewExportError(err, b.Id()) - return - } - out <- core.NewExportNothing(b.Id(), "bug already exported") // will be used to mark operation related to a bug as exported -- cgit