diff options
| -rw-r--r-- | bridge/gitlab/export.go | 118 | ||||
| -rw-r--r-- | bridge/gitlab/export_test.go | 276 |
2 files changed, 315 insertions, 79 deletions
diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 723b7b6a..3f339354 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -11,7 +11,7 @@ import ( "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" - "github.com/MichaelMure/git-bug/util/git" + "github.com/MichaelMure/git-bug/entity" ) var ( @@ -53,14 +53,14 @@ func (ge *gitlabExporter) Init(conf core.Configuration) error { // getIdentityClient return a gitlab v4 API client configured with the access token of the given identity. // if no client were found it will initialize it from the known tokens map and cache it for next use -func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { - client, ok := ge.identityClient[id] +func (ge *gitlabExporter) getIdentityClient(id entity.Id) (*gitlab.Client, error) { + client, ok := ge.identityClient[id.String()] if ok { return client, nil } // get token - token, ok := ge.identityToken[id] + token, ok := ge.identityToken[id.String()] if !ok { return nil, ErrMissingIdentityToken } @@ -68,7 +68,7 @@ func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { // create client client = buildClient(token) // cache client - ge.identityClient[id] = client + ge.identityClient[id.String()] = client return client, nil } @@ -82,21 +82,17 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c return nil, err } - ge.identityToken[user.Id()] = ge.conf[keyToken] + ge.identityToken[user.Id().String()] = ge.conf[keyToken] // get repository node id ge.repositoryID = ge.conf[keyProjectID] - if err != nil { - return nil, err - } - go func() { defer close(out) - var allIdentitiesIds []string + var allIdentitiesIds []entity.Id for id := range ge.identityToken { - allIdentitiesIds = append(allIdentitiesIds, id) + allIdentitiesIds = append(allIdentitiesIds, entity.Id(id)) } allBugsIds := repo.AllBugsIds() @@ -136,8 +132,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan var err error var bugGitlabID int var bugGitlabIDString string - var bugGitlabURL string - var bugCreationHash string + var bugCreationId string + //labels := make([]string, 0) // Special case: @@ -158,17 +154,16 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan // get gitlab bug ID gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) if ok { - gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl) + _, ok := snapshot.GetCreateMetadata(keyGitlabUrl) if !ok { // if we find gitlab ID, gitlab URL must be found too err := fmt.Errorf("expected to find gitlab issue URL") out <- core.NewExportError(err, b.Id()) + return } - //FIXME: - // ignore issue comming from other repositories - out <- core.NewExportNothing(b.Id(), "bug already exported") + // will be used to mark operation related to a bug as exported bugGitlabIDString = gitlabID bugGitlabID, err = strconv.Atoi(bugGitlabIDString) @@ -176,8 +171,6 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan panic("unexpected gitlab id format") } - bugGitlabURL = gitlabURL - } else { // check that we have a token for operation author client, err := ge.getIdentityClient(author.Id()) @@ -199,15 +192,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan out <- core.NewExportBug(b.Id()) - hash, err := createOp.Hash() - if err != nil { - err := errors.Wrap(err, "comment hash") - out <- core.NewExportError(err, b.Id()) - return - } - // mark bug creation operation as exported - if err := markOperationAsExported(b, hash, idString, url); err != nil { + if err := markOperationAsExported(b, createOp.Id(), idString, url); err != nil { err := errors.Wrap(err, "marking operation as exported") out <- core.NewExportError(err, b.Id()) return @@ -223,20 +209,11 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan // cache bug gitlab ID and URL bugGitlabID = id bugGitlabIDString = idString - bugGitlabURL = url - } - - // get createOp hash - hash, err := createOp.Hash() - if err != nil { - out <- core.NewExportError(err, b.Id()) - return } - bugCreationHash = hash.String() - + bugCreationId = createOp.Id().String() // cache operation gitlab id - ge.cachedOperationIDs[bugCreationHash] = bugGitlabIDString + ge.cachedOperationIDs[bugCreationId] = bugGitlabIDString for _, op := range snapshot.Operations[1:] { // ignore SetMetadata operations @@ -244,26 +221,18 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan continue } - // get operation hash - hash, err := op.Hash() - if err != nil { - err := errors.Wrap(err, "operation hash") - out <- core.NewExportError(err, b.Id()) - return - } - // ignore operations already existing in gitlab (due to import or export) // cache the ID of already exported or imported issues and events from Gitlab if id, ok := op.GetMetadata(keyGitlabId); ok { - ge.cachedOperationIDs[hash.String()] = id - out <- core.NewExportNothing(hash.String(), "already exported operation") + ge.cachedOperationIDs[op.Id().String()] = id + out <- core.NewExportNothing(op.Id(), "already exported operation") continue } opAuthor := op.GetAuthor() client, err := ge.getIdentityClient(opAuthor.Id()) if err != nil { - out <- core.NewExportNothing(hash.String(), "missing operation author token") + out <- core.NewExportNothing(op.Id(), "missing operation author token") continue } @@ -281,52 +250,50 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - idString = strconv.Itoa(id) - out <- core.NewExportComment(hash.String()) + out <- core.NewExportComment(op.Id()) + idString = strconv.Itoa(id) // cache comment id - ge.cachedOperationIDs[hash.String()] = idString + ge.cachedOperationIDs[op.Id().String()] = idString case *bug.EditCommentOperation: - opr := op.(*bug.EditCommentOperation) - targetHash := opr.Target.String() + targetId := opr.Target.String() // Since gitlab doesn't consider the issue body as a comment - if targetHash == bugCreationHash { + if targetId == bugCreationId { // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(client, ge.repositoryID, id, opr.Message); err != nil { + if err := updateGitlabIssueBody(client, ge.repositoryID, bugGitlabID, opr.Message); err != nil { err := errors.Wrap(err, "editing issue") out <- core.NewExportError(err, b.Id()) return } - out <- core.NewExportCommentEdition(hash.String()) - + out <- core.NewExportCommentEdition(op.Id()) id = bugGitlabID - url = bugGitlabURL } else { // case comment edition operation: we need to edit the Gitlab comment - _, ok := ge.cachedOperationIDs[targetHash] + commentID, ok := ge.cachedOperationIDs[targetId] if !ok { panic("unexpected error: comment id not found") } - err := editCommentGitlabIssue(client, ge.repositoryID, id, id, opr.Message) + commentIDint, err := strconv.Atoi(commentID) if err != nil { + panic("unexpected comment id format") + } + + if err := editCommentGitlabIssue(client, ge.repositoryID, commentIDint, id, opr.Message); err != nil { err := errors.Wrap(err, "editing comment") out <- core.NewExportError(err, b.Id()) return } - out <- core.NewExportCommentEdition(hash.String()) - - // use comment id/url instead of issue id/url - //id = eid - //url = eurl + out <- core.NewExportCommentEdition(op.Id()) + id = commentIDint } case *bug.SetStatusOperation: @@ -337,10 +304,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportStatusChange(hash.String()) - + out <- core.NewExportStatusChange(op.Id()) id = bugGitlabID - url = bugGitlabURL case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) @@ -350,10 +315,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportTitleEdition(hash.String()) - + out <- core.NewExportTitleEdition(op.Id()) id = bugGitlabID - url = bugGitlabURL case *bug.LabelChangeOperation: _ = op.(*bug.LabelChangeOperation) @@ -363,17 +326,14 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - out <- core.NewExportLabelChange(hash.String()) - + out <- core.NewExportLabelChange(op.Id()) id = bugGitlabID - url = bugGitlabURL - default: panic("unhandled operation type case") } // mark operation as exported - if err := markOperationAsExported(b, hash, idString, url); err != nil { + if err := markOperationAsExported(b, op.Id(), idString, url); err != nil { err := errors.Wrap(err, "marking operation as exported") out <- core.NewExportError(err, b.Id()) return @@ -388,7 +348,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan } } -func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { +func markOperationAsExported(b *cache.BugCache, target entity.Id, gitlabID, gitlabURL string) error { _, err := b.SetMetadata( target, map[string]string{ diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go index 4d4b35af..43403361 100644 --- a/bridge/gitlab/export_test.go +++ b/bridge/gitlab/export_test.go @@ -1 +1,277 @@ package gitlab + +import ( + "context" + "fmt" + "math/rand" + "os" + "strconv" + "testing" + "time" + + "github.com/xanzy/go-gitlab" + + "github.com/stretchr/testify/require" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/interrupt" +) + +const ( + testRepoBaseName = "git-bug-test-gitlab-exporter" +) + +type testCase struct { + name string + bug *cache.BugCache + numOrOp int // number of original operations +} + +func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCache) []*testCase { + // simple bug + simpleBug, _, err := repo.NewBug("simple bug", "new bug") + require.NoError(t, err) + + // bug with comments + bugWithComments, _, err := repo.NewBug("bug with comments", "new bug") + require.NoError(t, err) + + _, err = bugWithComments.AddComment("new comment") + require.NoError(t, err) + + // bug with label changes + bugLabelChange, _, err := repo.NewBug("bug label change", "new bug") + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"bug"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"core"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels(nil, []string{"bug"}) + require.NoError(t, err) + + // bug with comments editions + bugWithCommentEditions, createOp, err := repo.NewBug("bug with comments editions", "new bug") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(createOp.Id(), "first comment edited") + require.NoError(t, err) + + commentOp, err := bugWithCommentEditions.AddComment("first comment") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(commentOp.Id(), "first comment edited") + require.NoError(t, err) + + // bug status changed + bugStatusChanged, _, err := repo.NewBug("bug status changed", "new bug") + require.NoError(t, err) + + _, err = bugStatusChanged.Close() + require.NoError(t, err) + + _, err = bugStatusChanged.Open() + require.NoError(t, err) + + // bug title changed + bugTitleEdited, _, err := repo.NewBug("bug title edited", "new bug") + require.NoError(t, err) + + _, err = bugTitleEdited.SetTitle("bug title edited again") + require.NoError(t, err) + + return []*testCase{ + &testCase{ + name: "simple bug", + bug: simpleBug, + numOrOp: 1, + }, + &testCase{ + name: "bug with comments", + bug: bugWithComments, + numOrOp: 2, + }, + &testCase{ + name: "bug label change", + bug: bugLabelChange, + numOrOp: 4, + }, + &testCase{ + name: "bug with comment editions", + bug: bugWithCommentEditions, + numOrOp: 4, + }, + &testCase{ + name: "bug changed status", + bug: bugStatusChanged, + numOrOp: 3, + }, + &testCase{ + name: "bug title edited", + bug: bugTitleEdited, + numOrOp: 2, + }, + } +} + +func TestPushPull(t *testing.T) { + // token must have 'repo' and 'delete_repo' scopes + token := os.Getenv("GITLAB_API_TOKEN") + if token == "" { + t.Skip("Env var GITLAB_API_TOKEN missing") + } + + // create repo backend + repo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repo) + + backend, err := cache.NewRepoCache(repo) + require.NoError(t, err) + + // set author identity + author, err := backend.NewIdentity("test identity", "test@test.org") + require.NoError(t, err) + + err = backend.SetUserIdentity(author) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + tests := testCases(t, backend, author) + + // generate project name + projectName := generateRepoName() + + // create target Gitlab repository + projectID, err := createRepository(context.TODO(), projectName, token) + require.NoError(t, err) + + fmt.Println("created repository", projectName) + + // Make sure to remove the Gitlab repository when the test end + defer func(t *testing.T) { + if err := deleteRepository(context.TODO(), projectID, token); err != nil { + t.Fatal(err) + } + fmt.Println("deleted repository:", projectName) + }(t) + + interrupt.RegisterCleaner(func() error { + return deleteRepository(context.TODO(), projectID, token) + }) + + // initialize exporter + exporter := &gitlabExporter{} + err = exporter.Init(core.Configuration{ + keyProjectID: strconv.Itoa(projectID), + keyToken: token, + }) + require.NoError(t, err) + + start := time.Now() + + // export all bugs + events, err := exporter.ExportAll(backend, time.Time{}) + require.NoError(t, err) + + for result := range events { + require.NoError(t, result.Err) + } + require.NoError(t, err) + + fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds()) + + repoTwo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repoTwo) + + // create a second backend + backendTwo, err := cache.NewRepoCache(repoTwo) + require.NoError(t, err) + + importer := &gitlabImporter{} + err = importer.Init(core.Configuration{ + keyProjectID: strconv.Itoa(projectID), + keyToken: token, + }) + require.NoError(t, err) + + // import all exported bugs to the second backend + err = importer.ImportAll(backendTwo, time.Time{}) + require.NoError(t, err) + + require.Len(t, backendTwo.AllBugsIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // for each operation a SetMetadataOperation will be added + // so number of operations should double + require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2) + + // verify operation have correct metadata + for _, op := range tt.bug.Snapshot().Operations { + // Check if the originals operations (*not* SetMetadata) are tagged properly + if _, ok := op.(*bug.SetMetadataOperation); !ok { + _, haveIDMetadata := op.GetMetadata(keyGitlabId) + require.True(t, haveIDMetadata) + + _, haveURLMetada := op.GetMetadata(keyGitlabUrl) + require.True(t, haveURLMetada) + } + } + + // get bug gitlab ID + bugGitlabID, ok := tt.bug.Snapshot().GetCreateMetadata(keyGitlabId) + require.True(t, ok) + + // retrieve bug from backendTwo + importedBug, err := backendTwo.ResolveBugCreateMetadata(keyGitlabId, bugGitlabID) + require.NoError(t, err) + + // verify bug have same number of original operations + require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp) + + // verify bugs are taged with origin=gitlab + issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.KeyOrigin) + require.True(t, ok) + require.Equal(t, issueOrigin, target) + + //TODO: maybe more tests to ensure bug final state + }) + } +} + +func generateRepoName() string { + rand.Seed(time.Now().UnixNano()) + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, 8) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return fmt.Sprintf("%s-%s", testRepoBaseName, string(b)) +} + +// create repository need a token with scope 'repo' +func createRepository(ctx context.Context, name, token string) (int, error) { + client := buildClient(token) + project, _, err := client.Projects.CreateProject( + &gitlab.CreateProjectOptions{ + Name: gitlab.String(name), + }, + gitlab.WithContext(ctx), + ) + + return project.ID, err +} + +// delete repository need a token with scope 'delete_repo' +func deleteRepository(ctx context.Context, project int, token string) error { + client := buildClient(token) + _, err := client.Projects.DeleteProject(project, gitlab.WithContext(ctx)) + return err +} |