bridge/gitlab: add exporter e2e tests

bridge/gitlab: fix export bugs and rebase

2 files changed, 315 insertions, 79 deletions

diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go
index 723b7b6a..3f339354 100644
--- a/bridge/gitlab/export.go
+++ b/bridge/gitlab/export.go
@@ -11,7 +11,7 @@ import (
 	"github.com/MichaelMure/git-bug/bridge/core"
 	"github.com/MichaelMure/git-bug/bug"
 	"github.com/MichaelMure/git-bug/cache"
-	"github.com/MichaelMure/git-bug/util/git"
+	"github.com/MichaelMure/git-bug/entity"
 )
 
 var (
@@ -53,14 +53,14 @@ func (ge *gitlabExporter) Init(conf core.Configuration) error {
 
 // getIdentityClient return a gitlab v4 API client configured with the access token of the given identity.
 // if no client were found it will initialize it from the known tokens map and cache it for next use
-func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) {
-	client, ok := ge.identityClient[id]
+func (ge *gitlabExporter) getIdentityClient(id entity.Id) (*gitlab.Client, error) {
+	client, ok := ge.identityClient[id.String()]
 	if ok {
 		return client, nil
 	}
 
 	// get token
-	token, ok := ge.identityToken[id]
+	token, ok := ge.identityToken[id.String()]
 	if !ok {
 		return nil, ErrMissingIdentityToken
 	}
@@ -68,7 +68,7 @@ func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) {
 	// create client
 	client = buildClient(token)
 	// cache client
-	ge.identityClient[id] = client
+	ge.identityClient[id.String()] = client
 
 	return client, nil
 }
@@ -82,21 +82,17 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c
 		return nil, err
 	}
 
-	ge.identityToken[user.Id()] = ge.conf[keyToken]
+	ge.identityToken[user.Id().String()] = ge.conf[keyToken]
 
 	// get repository node id
 	ge.repositoryID = ge.conf[keyProjectID]
 
-	if err != nil {
-		return nil, err
-	}
-
 	go func() {
 		defer close(out)
 
-		var allIdentitiesIds []string
+		var allIdentitiesIds []entity.Id
 		for id := range ge.identityToken {
-			allIdentitiesIds = append(allIdentitiesIds, id)
+			allIdentitiesIds = append(allIdentitiesIds, entity.Id(id))
 		}
 
 		allBugsIds := repo.AllBugsIds()
@@ -136,8 +132,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 	var err error
 	var bugGitlabID int
 	var bugGitlabIDString string
-	var bugGitlabURL string
-	var bugCreationHash string
+	var bugCreationId string
+
 	//labels := make([]string, 0)
 
 	// Special case:
@@ -158,17 +154,16 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 	// get gitlab bug ID
 	gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId)
 	if ok {
-		gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl)
+		_, ok := snapshot.GetCreateMetadata(keyGitlabUrl)
 		if !ok {
 			// if we find gitlab ID, gitlab URL must be found too
 			err := fmt.Errorf("expected to find gitlab issue URL")
 			out <- core.NewExportError(err, b.Id())
+			return
 		}
 
-		//FIXME:
-		// ignore issue comming from other repositories
-
 		out <- core.NewExportNothing(b.Id(), "bug already exported")
+
 		// will be used to mark operation related to a bug as exported
 		bugGitlabIDString = gitlabID
 		bugGitlabID, err = strconv.Atoi(bugGitlabIDString)
@@ -176,8 +171,6 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 			panic("unexpected gitlab id format")
 		}
 
-		bugGitlabURL = gitlabURL
-
 	} else {
 		// check that we have a token for operation author
 		client, err := ge.getIdentityClient(author.Id())
@@ -199,15 +192,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 
 		out <- core.NewExportBug(b.Id())
 
-		hash, err := createOp.Hash()
-		if err != nil {
-			err := errors.Wrap(err, "comment hash")
-			out <- core.NewExportError(err, b.Id())
-			return
-		}
-
 		// mark bug creation operation as exported
-		if err := markOperationAsExported(b, hash, idString, url); err != nil {
+		if err := markOperationAsExported(b, createOp.Id(), idString, url); err != nil {
 			err := errors.Wrap(err, "marking operation as exported")
 			out <- core.NewExportError(err, b.Id())
 			return
@@ -223,20 +209,11 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 		// cache bug gitlab ID and URL
 		bugGitlabID = id
 		bugGitlabIDString = idString
-		bugGitlabURL = url
-	}
-
-	// get createOp hash
-	hash, err := createOp.Hash()
-	if err != nil {
-		out <- core.NewExportError(err, b.Id())
-		return
 	}
 
-	bugCreationHash = hash.String()
-
+	bugCreationId = createOp.Id().String()
 	// cache operation gitlab id
-	ge.cachedOperationIDs[bugCreationHash] = bugGitlabIDString
+	ge.cachedOperationIDs[bugCreationId] = bugGitlabIDString
 
 	for _, op := range snapshot.Operations[1:] {
 		// ignore SetMetadata operations
@@ -244,26 +221,18 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 			continue
 		}
 
-		// get operation hash
-		hash, err := op.Hash()
-		if err != nil {
-			err := errors.Wrap(err, "operation hash")
-			out <- core.NewExportError(err, b.Id())
-			return
-		}
-
 		// ignore operations already existing in gitlab (due to import or export)
 		// cache the ID of already exported or imported issues and events from Gitlab
 		if id, ok := op.GetMetadata(keyGitlabId); ok {
-			ge.cachedOperationIDs[hash.String()] = id
-			out <- core.NewExportNothing(hash.String(), "already exported operation")
+			ge.cachedOperationIDs[op.Id().String()] = id
+			out <- core.NewExportNothing(op.Id(), "already exported operation")
 			continue
 		}
 
 		opAuthor := op.GetAuthor()
 		client, err := ge.getIdentityClient(opAuthor.Id())
 		if err != nil {
-			out <- core.NewExportNothing(hash.String(), "missing operation author token")
+			out <- core.NewExportNothing(op.Id(), "missing operation author token")
 			continue
 		}
 
@@ -281,52 +250,50 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 				return
 			}
 
-			idString = strconv.Itoa(id)
-			out <- core.NewExportComment(hash.String())
+			out <- core.NewExportComment(op.Id())
 
+			idString = strconv.Itoa(id)
 			// cache comment id
-			ge.cachedOperationIDs[hash.String()] = idString
+			ge.cachedOperationIDs[op.Id().String()] = idString
 
 		case *bug.EditCommentOperation:
-
 			opr := op.(*bug.EditCommentOperation)
-			targetHash := opr.Target.String()
+			targetId := opr.Target.String()
 
 			// Since gitlab doesn't consider the issue body as a comment
-			if targetHash == bugCreationHash {
+			if targetId == bugCreationId {
 
 				// case bug creation operation: we need to edit the Gitlab issue
-				if err := updateGitlabIssueBody(client, ge.repositoryID, id, opr.Message); err != nil {
+				if err := updateGitlabIssueBody(client, ge.repositoryID, bugGitlabID, opr.Message); err != nil {
 					err := errors.Wrap(err, "editing issue")
 					out <- core.NewExportError(err, b.Id())
 					return
 				}
 
-				out <- core.NewExportCommentEdition(hash.String())
-
+				out <- core.NewExportCommentEdition(op.Id())
 				id = bugGitlabID
-				url = bugGitlabURL
 
 			} else {
 
 				// case comment edition operation: we need to edit the Gitlab comment
-				_, ok := ge.cachedOperationIDs[targetHash]
+				commentID, ok := ge.cachedOperationIDs[targetId]
 				if !ok {
 					panic("unexpected error: comment id not found")
 				}
 
-				err := editCommentGitlabIssue(client, ge.repositoryID, id, id, opr.Message)
+				commentIDint, err := strconv.Atoi(commentID)
 				if err != nil {
+					panic("unexpected comment id format")
+				}
+
+				if err := editCommentGitlabIssue(client, ge.repositoryID, commentIDint, id, opr.Message); err != nil {
 					err := errors.Wrap(err, "editing comment")
 					out <- core.NewExportError(err, b.Id())
 					return
 				}
 
-				out <- core.NewExportCommentEdition(hash.String())
-
-				// use comment id/url instead of issue id/url
-				//id = eid
-				//url = eurl
+				out <- core.NewExportCommentEdition(op.Id())
+				id = commentIDint
 			}
 
 		case *bug.SetStatusOperation:
@@ -337,10 +304,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 				return
 			}
 
-			out <- core.NewExportStatusChange(hash.String())
-
+			out <- core.NewExportStatusChange(op.Id())
 			id = bugGitlabID
-			url = bugGitlabURL
 
 		case *bug.SetTitleOperation:
 			opr := op.(*bug.SetTitleOperation)
@@ -350,10 +315,8 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 				return
 			}
 
-			out <- core.NewExportTitleEdition(hash.String())
-
+			out <- core.NewExportTitleEdition(op.Id())
 			id = bugGitlabID
-			url = bugGitlabURL
 
 		case *bug.LabelChangeOperation:
 			_ = op.(*bug.LabelChangeOperation)
@@ -363,17 +326,14 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 				return
 			}
 
-			out <- core.NewExportLabelChange(hash.String())
-
+			out <- core.NewExportLabelChange(op.Id())
 			id = bugGitlabID
-			url = bugGitlabURL
-
 		default:
 			panic("unhandled operation type case")
 		}
 
 		// mark operation as exported
-		if err := markOperationAsExported(b, hash, idString, url); err != nil {
+		if err := markOperationAsExported(b, op.Id(), idString, url); err != nil {
 			err := errors.Wrap(err, "marking operation as exported")
 			out <- core.NewExportError(err, b.Id())
 			return
@@ -388,7 +348,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan
 	}
 }
 
-func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error {
+func markOperationAsExported(b *cache.BugCache, target entity.Id, gitlabID, gitlabURL string) error {
 	_, err := b.SetMetadata(
 		target,
 		map[string]string{
diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go
index 4d4b35af..43403361 100644
--- a/bridge/gitlab/export_test.go
+++ b/bridge/gitlab/export_test.go
@@ -1 +1,277 @@
 package gitlab
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/xanzy/go-gitlab"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/MichaelMure/git-bug/bridge/core"
+	"github.com/MichaelMure/git-bug/bug"
+	"github.com/MichaelMure/git-bug/cache"
+	"github.com/MichaelMure/git-bug/repository"
+	"github.com/MichaelMure/git-bug/util/interrupt"
+)
+
+const (
+	testRepoBaseName = "git-bug-test-gitlab-exporter"
+)
+
+type testCase struct {
+	name    string
+	bug     *cache.BugCache
+	numOrOp int // number of original operations
+}
+
+func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCache) []*testCase {
+	// simple bug
+	simpleBug, _, err := repo.NewBug("simple bug", "new bug")
+	require.NoError(t, err)
+
+	// bug with comments
+	bugWithComments, _, err := repo.NewBug("bug with comments", "new bug")
+	require.NoError(t, err)
+
+	_, err = bugWithComments.AddComment("new comment")
+	require.NoError(t, err)
+
+	// bug with label changes
+	bugLabelChange, _, err := repo.NewBug("bug label change", "new bug")
+	require.NoError(t, err)
+
+	_, _, err = bugLabelChange.ChangeLabels([]string{"bug"}, nil)
+	require.NoError(t, err)
+
+	_, _, err = bugLabelChange.ChangeLabels([]string{"core"}, nil)
+	require.NoError(t, err)
+
+	_, _, err = bugLabelChange.ChangeLabels(nil, []string{"bug"})
+	require.NoError(t, err)
+
+	// bug with comments editions
+	bugWithCommentEditions, createOp, err := repo.NewBug("bug with comments editions", "new bug")
+	require.NoError(t, err)
+
+	_, err = bugWithCommentEditions.EditComment(createOp.Id(), "first comment edited")
+	require.NoError(t, err)
+
+	commentOp, err := bugWithCommentEditions.AddComment("first comment")
+	require.NoError(t, err)
+
+	_, err = bugWithCommentEditions.EditComment(commentOp.Id(), "first comment edited")
+	require.NoError(t, err)
+
+	// bug status changed
+	bugStatusChanged, _, err := repo.NewBug("bug status changed", "new bug")
+	require.NoError(t, err)
+
+	_, err = bugStatusChanged.Close()
+	require.NoError(t, err)
+
+	_, err = bugStatusChanged.Open()
+	require.NoError(t, err)
+
+	// bug title changed
+	bugTitleEdited, _, err := repo.NewBug("bug title edited", "new bug")
+	require.NoError(t, err)
+
+	_, err = bugTitleEdited.SetTitle("bug title edited again")
+	require.NoError(t, err)
+
+	return []*testCase{
+		&testCase{
+			name:    "simple bug",
+			bug:     simpleBug,
+			numOrOp: 1,
+		},
+		&testCase{
+			name:    "bug with comments",
+			bug:     bugWithComments,
+			numOrOp: 2,
+		},
+		&testCase{
+			name:    "bug label change",
+			bug:     bugLabelChange,
+			numOrOp: 4,
+		},
+		&testCase{
+			name:    "bug with comment editions",
+			bug:     bugWithCommentEditions,
+			numOrOp: 4,
+		},
+		&testCase{
+			name:    "bug changed status",
+			bug:     bugStatusChanged,
+			numOrOp: 3,
+		},
+		&testCase{
+			name:    "bug title edited",
+			bug:     bugTitleEdited,
+			numOrOp: 2,
+		},
+	}
+}
+
+func TestPushPull(t *testing.T) {
+	// token must have 'repo' and 'delete_repo' scopes
+	token := os.Getenv("GITLAB_API_TOKEN")
+	if token == "" {
+		t.Skip("Env var GITLAB_API_TOKEN missing")
+	}
+
+	// create repo backend
+	repo := repository.CreateTestRepo(false)
+	defer repository.CleanupTestRepos(t, repo)
+
+	backend, err := cache.NewRepoCache(repo)
+	require.NoError(t, err)
+
+	// set author identity
+	author, err := backend.NewIdentity("test identity", "test@test.org")
+	require.NoError(t, err)
+
+	err = backend.SetUserIdentity(author)
+	require.NoError(t, err)
+
+	defer backend.Close()
+	interrupt.RegisterCleaner(backend.Close)
+
+	tests := testCases(t, backend, author)
+
+	// generate project name
+	projectName := generateRepoName()
+
+	// create target Gitlab repository
+	projectID, err := createRepository(context.TODO(), projectName, token)
+	require.NoError(t, err)
+
+	fmt.Println("created repository", projectName)
+
+	// Make sure to remove the Gitlab repository when the test end
+	defer func(t *testing.T) {
+		if err := deleteRepository(context.TODO(), projectID, token); err != nil {
+			t.Fatal(err)
+		}
+		fmt.Println("deleted repository:", projectName)
+	}(t)
+
+	interrupt.RegisterCleaner(func() error {
+		return deleteRepository(context.TODO(), projectID, token)
+	})
+
+	// initialize exporter
+	exporter := &gitlabExporter{}
+	err = exporter.Init(core.Configuration{
+		keyProjectID: strconv.Itoa(projectID),
+		keyToken:     token,
+	})
+	require.NoError(t, err)
+
+	start := time.Now()
+
+	// export all bugs
+	events, err := exporter.ExportAll(backend, time.Time{})
+	require.NoError(t, err)
+
+	for result := range events {
+		require.NoError(t, result.Err)
+	}
+	require.NoError(t, err)
+
+	fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds())
+
+	repoTwo := repository.CreateTestRepo(false)
+	defer repository.CleanupTestRepos(t, repoTwo)
+
+	// create a second backend
+	backendTwo, err := cache.NewRepoCache(repoTwo)
+	require.NoError(t, err)
+
+	importer := &gitlabImporter{}
+	err = importer.Init(core.Configuration{
+		keyProjectID: strconv.Itoa(projectID),
+		keyToken:     token,
+	})
+	require.NoError(t, err)
+
+	// import all exported bugs to the second backend
+	err = importer.ImportAll(backendTwo, time.Time{})
+	require.NoError(t, err)
+
+	require.Len(t, backendTwo.AllBugsIds(), len(tests))
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// for each operation a SetMetadataOperation will be added
+			// so number of operations should double
+			require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2)
+
+			// verify operation have correct metadata
+			for _, op := range tt.bug.Snapshot().Operations {
+				// Check if the originals operations (*not* SetMetadata) are tagged properly
+				if _, ok := op.(*bug.SetMetadataOperation); !ok {
+					_, haveIDMetadata := op.GetMetadata(keyGitlabId)
+					require.True(t, haveIDMetadata)
+
+					_, haveURLMetada := op.GetMetadata(keyGitlabUrl)
+					require.True(t, haveURLMetada)
+				}
+			}
+
+			// get bug gitlab ID
+			bugGitlabID, ok := tt.bug.Snapshot().GetCreateMetadata(keyGitlabId)
+			require.True(t, ok)
+
+			// retrieve bug from backendTwo
+			importedBug, err := backendTwo.ResolveBugCreateMetadata(keyGitlabId, bugGitlabID)
+			require.NoError(t, err)
+
+			// verify bug have same number of original operations
+			require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp)
+
+			// verify bugs are taged with origin=gitlab
+			issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.KeyOrigin)
+			require.True(t, ok)
+			require.Equal(t, issueOrigin, target)
+
+			//TODO: maybe more tests to ensure bug final state
+		})
+	}
+}
+
+func generateRepoName() string {
+	rand.Seed(time.Now().UnixNano())
+	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]rune, 8)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return fmt.Sprintf("%s-%s", testRepoBaseName, string(b))
+}
+
+// create repository need a token with scope 'repo'
+func createRepository(ctx context.Context, name, token string) (int, error) {
+	client := buildClient(token)
+	project, _, err := client.Projects.CreateProject(
+		&gitlab.CreateProjectOptions{
+			Name: gitlab.String(name),
+		},
+		gitlab.WithContext(ctx),
+	)
+
+	return project.ID, err
+}
+
+// delete repository need a token with scope 'delete_repo'
+func deleteRepository(ctx context.Context, project int, token string) error {
+	client := buildClient(token)
+	_, err := client.Projects.DeleteProject(project, gitlab.WithContext(ctx))
+	return err
+}