aboutsummaryrefslogtreecommitdiffstats
path: root/bridge
diff options
context:
space:
mode:
Diffstat (limited to 'bridge')
-rw-r--r--bridge/core/bridge.go10
-rw-r--r--bridge/core/export.go101
-rw-r--r--bridge/core/interfaces.go2
-rw-r--r--bridge/github/export.go185
-rw-r--r--bridge/github/export_test.go24
5 files changed, 231 insertions, 91 deletions
diff --git a/bridge/core/bridge.go b/bridge/core/bridge.go
index 1b960e0e..30e051be 100644
--- a/bridge/core/bridge.go
+++ b/bridge/core/bridge.go
@@ -297,21 +297,21 @@ func (b *Bridge) ImportAll(since time.Time) error {
return importer.ImportAll(b.repo, since)
}
-func (b *Bridge) ExportAll(since time.Time) error {
+func (b *Bridge) ExportAll(since time.Time) (<-chan ExportResult, error) {
exporter := b.getExporter()
if exporter == nil {
- return ErrExportNotSupported
+ return nil, ErrExportNotSupported
}
err := b.ensureConfig()
if err != nil {
- return err
+ return nil, err
}
err = b.ensureInit()
if err != nil {
- return err
+ return nil, err
}
- return exporter.ExportAll(b.repo, since)
+ return exporter.ExportAll(b.repo, since), nil
}
diff --git a/bridge/core/export.go b/bridge/core/export.go
new file mode 100644
index 00000000..51149430
--- /dev/null
+++ b/bridge/core/export.go
@@ -0,0 +1,101 @@
+package core
+
+import "fmt"
+
+type EventStatus int
+
+const (
+ _ EventStatus = iota
+ EventStatusBug
+ EventStatusComment
+ EventStatusCommentEdition
+ EventStatusStatusChange
+ EventStatusTitleEdition
+ EventStatusLabelChange
+ EventStatusNothing
+)
+
+type ExportResult struct {
+ Err error
+ Event EventStatus
+ ID string
+ Reason string
+}
+
+func (er ExportResult) String() string {
+ switch er.Event {
+ case EventStatusBug:
+ return "new issue"
+ case EventStatusComment:
+ return "new comment"
+ case EventStatusCommentEdition:
+ return "updated comment"
+ case EventStatusStatusChange:
+ return "changed status"
+ case EventStatusTitleEdition:
+ return "changed title"
+ case EventStatusLabelChange:
+ return "changed label"
+ case EventStatusNothing:
+ return fmt.Sprintf("no event: %v", er.Reason)
+ default:
+ panic("unknown export result")
+ }
+}
+
+func NewExportError(err error, reason string) ExportResult {
+ return ExportResult{
+ Err: err,
+ Reason: reason,
+ }
+}
+
+func NewExportNothing(id string, reason string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Reason: reason,
+ Event: EventStatusNothing,
+ }
+}
+
+func NewExportBug(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusBug,
+ }
+}
+
+func NewExportComment(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusComment,
+ }
+}
+
+func NewExportCommentEdition(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusCommentEdition,
+ }
+}
+
+func NewExportStatusChange(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusStatusChange,
+ }
+}
+
+func NewExportLabelChange(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusLabelChange,
+ }
+}
+
+func NewExportTitleEdition(id string) ExportResult {
+ return ExportResult{
+ ID: id,
+ Event: EventStatusTitleEdition,
+ }
+}
diff --git a/bridge/core/interfaces.go b/bridge/core/interfaces.go
index 37fdb3d7..5cc7006a 100644
--- a/bridge/core/interfaces.go
+++ b/bridge/core/interfaces.go
@@ -34,5 +34,5 @@ type Importer interface {
type Exporter interface {
Init(conf Configuration) error
- ExportAll(repo *cache.RepoCache, since time.Time) error
+ ExportAll(repo *cache.RepoCache, since time.Time) <-chan ExportResult
}
diff --git a/bridge/github/export.go b/bridge/github/export.go
index 00ba1585..40044ca1 100644
--- a/bridge/github/export.go
+++ b/bridge/github/export.go
@@ -26,13 +26,7 @@ var (
type githubExporter struct {
conf core.Configuration
- // number of exported bugs
- exportedBugs int
-
- // number of exported labels
- exportedLabels int
-
- // export only bugs taged with one of these origins
+ // export only bugs tagged with one of these origins
onlyOrigins []string
// cache identities clients
@@ -81,7 +75,7 @@ func (ge *githubExporter) allowOrigin(origin string) bool {
}
// getIdentityClient return an identity github api v4 client
-// if no client were found it will initilize it from the known tokens map and cache it for next use
+// if no client were found it will initialize it from the known tokens map and cache it for next use
func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) {
client, ok := ge.identityClient[id]
if ok {
@@ -103,60 +97,68 @@ func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error)
}
// ExportAll export all event made by the current user to Github
-func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) error {
- user, err := repo.GetUserIdentity()
- if err != nil {
- return err
- }
+func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) <-chan core.ExportResult {
+ out := make(chan core.ExportResult)
- ge.identityToken[user.Id()] = ge.conf[keyToken]
+ go func(out chan<- core.ExportResult) {
+ defer close(out)
- // get repository node id
- ge.repositoryID, err = getRepositoryNodeID(
- ge.conf[keyOwner],
- ge.conf[keyProject],
- ge.conf[keyToken],
- )
-
- if err != nil {
- return err
- }
+ user, err := repo.GetUserIdentity()
+ if err != nil {
+ out <- core.NewExportError(err, "")
+ return
+ }
- allIdentitiesIds := []string{}
- for id := range ge.identityToken {
- allIdentitiesIds = append(allIdentitiesIds, id)
- }
+ ge.identityToken[user.Id()] = ge.conf[keyToken]
- allBugsIds := repo.AllBugsIds()
+ // get repository node id
+ ge.repositoryID, err = getRepositoryNodeID(
+ ge.conf[keyOwner],
+ ge.conf[keyProject],
+ ge.conf[keyToken],
+ )
- for _, id := range allBugsIds {
- b, err := repo.ResolveBug(id)
if err != nil {
- return err
+ out <- core.NewExportError(err, ge.repositoryID)
+ return
}
- snapshot := b.Snapshot()
-
- // ignore issues created before since date
- if snapshot.CreatedAt.Before(since) {
- continue
+ var allIdentitiesIds []string
+ for id := range ge.identityToken {
+ allIdentitiesIds = append(allIdentitiesIds, id)
}
- if snapshot.HasAnyParticipant(allIdentitiesIds...) {
- // try to export the bug and it associated events
- if err := ge.exportBug(b, since); err != nil {
- return err
+ allBugsIds := repo.AllBugsIds()
+
+ for _, id := range allBugsIds {
+ b, err := repo.ResolveBug(id)
+ if err != nil {
+ out <- core.NewExportError(err, id)
+ return
}
- }
- }
+ snapshot := b.Snapshot()
- fmt.Printf("Successfully exported %d issues and %d labels to Github\n", ge.exportedBugs, ge.exportedLabels)
- return nil
+ // ignore issues created before since date
+ if snapshot.CreatedAt.Before(since) {
+ out <- core.NewExportNothing(b.Id(), "bug created before the since date")
+ continue
+ }
+
+ if snapshot.HasAnyActor(allIdentitiesIds...) {
+ // try to export the bug and it associated events
+ ge.exportBug(b, since, out)
+ } else {
+ out <- core.NewExportNothing(id, "not an actor")
+ }
+ }
+ }(out)
+
+ return out
}
// exportBug publish bugs and related events
-func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
+func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) {
snapshot := b.Snapshot()
var bugGithubID string
@@ -174,8 +176,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// skip bug if origin is not allowed
origin, ok := createOp.GetMetadata(keyOrigin)
if ok && !ge.allowOrigin(origin) {
- // TODO print a warn ?
- return nil
+ out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue taged with origin: %s", origin))
+ return
}
// get github bug ID
@@ -187,6 +189,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
panic("expected to find github issue URL")
}
+ out <- core.NewExportNothing(b.Id(), "bug already exported")
// will be used to mark operation related to a bug as exported
bugGithubID = githubID
bugGithubURL = githubURL
@@ -197,33 +200,41 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
if err != nil {
// if bug is still not exported and we do not have the author stop the execution
- fmt.Println("warning: skipping issue due to missing token for bug creator")
+ // fmt.Println("warning: skipping issue due to missing token for bug creator")
// this is not an error, don't export bug
- return nil
+ out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
+ return
}
// create bug
id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message)
if err != nil {
- return errors.Wrap(err, "exporting github issue")
+ err := errors.Wrap(err, "exporting github issue")
+ out <- core.NewExportError(err, b.Id())
+ return
}
- // incr exported bugs
- ge.exportedBugs++
+ out <- core.NewExportBug(b.Id())
hash, err := createOp.Hash()
if err != nil {
- return errors.Wrap(err, "comment hash")
+ err := errors.Wrap(err, "comment hash")
+ out <- core.NewExportError(err, b.Id())
+ return
}
// mark bug creation operation as exported
if err := markOperationAsExported(b, hash, id, url); err != nil {
- return errors.Wrap(err, "marking operation as exported")
+ err := errors.Wrap(err, "marking operation as exported")
+ out <- core.NewExportError(err, b.Id())
+ return
}
// commit operation to avoid creating multiple issues with multiple pushes
if err := b.CommitAsNeeded(); err != nil {
- return errors.Wrap(err, "bug commit")
+ err := errors.Wrap(err, "bug commit")
+ out <- core.NewExportError(err, b.Id())
+ return
}
// cache bug github ID and URL
@@ -234,7 +245,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// get createOp hash
hash, err := createOp.Hash()
if err != nil {
- return err
+ out <- core.NewExportError(err, b.Id())
+ return
}
bugCreationHash = hash.String()
@@ -251,20 +263,23 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// get operation hash
hash, err := op.Hash()
if err != nil {
- return errors.Wrap(err, "operation hash")
+ err := errors.Wrap(err, "operation hash")
+ out <- core.NewExportError(err, b.Id())
+ return
}
// ignore imported (or exported) operations from github
// cache the ID of already exported or imported issues and events from Github
if id, ok := op.GetMetadata(keyGithubId); ok {
ge.cachedIDs[hash.String()] = id
+ out <- core.NewExportNothing(hash.String(), "already exported operation")
continue
}
opAuthor := op.GetAuthor()
client, err := ge.getIdentityClient(opAuthor.Id())
if err != nil {
- // don't export operation
+ out <- core.NewExportNothing(hash.String(), "missing operation author token")
continue
}
@@ -276,9 +291,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// send operation to github
id, url, err = addCommentGithubIssue(client, bugGithubID, opr.Message)
if err != nil {
- return errors.Wrap(err, "adding comment")
+ err := errors.Wrap(err, "adding comment")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportComment(hash.String())
+
// cache comment id
ge.cachedIDs[hash.String()] = id
@@ -292,9 +311,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// case bug creation operation: we need to edit the Github issue
if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil {
- return errors.Wrap(err, "editing issue")
+ err := errors.Wrap(err, "editing issue")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportCommentEdition(hash.String())
+
id = bugGithubID
url = bugGithubURL
@@ -308,9 +331,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
eid, eurl, err := editCommentGithubIssue(client, commentID, opr.Message)
if err != nil {
- return errors.Wrap(err, "editing comment")
+ err := errors.Wrap(err, "editing comment")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportCommentEdition(hash.String())
+
// use comment id/url instead of issue id/url
id = eid
url = eurl
@@ -319,27 +346,39 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
case *bug.SetStatusOperation:
opr := op.(*bug.SetStatusOperation)
if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil {
- return errors.Wrap(err, "editing status")
+ err := errors.Wrap(err, "editing status")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportStatusChange(hash.String())
+
id = bugGithubID
url = bugGithubURL
case *bug.SetTitleOperation:
opr := op.(*bug.SetTitleOperation)
if err := updateGithubIssueTitle(client, bugGithubID, opr.Title); err != nil {
- return errors.Wrap(err, "editing title")
+ err := errors.Wrap(err, "editing title")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportTitleEdition(hash.String())
+
id = bugGithubID
url = bugGithubURL
case *bug.LabelChangeOperation:
opr := op.(*bug.LabelChangeOperation)
if err := ge.updateGithubIssueLabels(client, bugGithubID, opr.Added, opr.Removed); err != nil {
- return errors.Wrap(err, "updating labels")
+ err := errors.Wrap(err, "updating labels")
+ out <- core.NewExportError(err, b.Id())
+ return
}
+ out <- core.NewExportLabelChange(hash.String())
+
id = bugGithubID
url = bugGithubURL
@@ -349,16 +388,18 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error {
// mark operation as exported
if err := markOperationAsExported(b, hash, id, url); err != nil {
- return errors.Wrap(err, "marking operation as exported")
+ err := errors.Wrap(err, "marking operation as exported")
+ out <- core.NewExportError(err, b.Id())
+ return
}
// commit at each operation export to avoid exporting same events multiple times
if err := b.CommitAsNeeded(); err != nil {
- return errors.Wrap(err, "bug commit")
+ err := errors.Wrap(err, "bug commit")
+ out <- core.NewExportError(err, b.Id())
+ return
}
}
-
- return nil
}
// getRepositoryNodeID request github api v3 to get repository node id
@@ -391,7 +432,10 @@ func getRepositoryNodeID(owner, project, token string) (string, error) {
}{}
data, _ := ioutil.ReadAll(resp.Body)
- defer resp.Body.Close()
+ err = resp.Body.Close()
+ if err != nil {
+ return "", err
+ }
err = json.Unmarshal(data, &aux)
if err != nil {
@@ -534,7 +578,6 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito
return "", err
}
- ge.exportedLabels++
return labelID, nil
}
diff --git a/bridge/github/export_test.go b/bridge/github/export_test.go
index ff8b70b8..827152b7 100644
--- a/bridge/github/export_test.go
+++ b/bridge/github/export_test.go
@@ -163,7 +163,7 @@ func testCases(repo *cache.RepoCache, identity *cache.IdentityCache) ([]*testCas
func TestPushPull(t *testing.T) {
// repo owner
- user := os.Getenv("TEST_USER")
+ user := os.Getenv("GITHUB_TEST_USER")
// token must have 'repo' and 'delete_repo' scopes
token := os.Getenv("GITHUB_TOKEN_ADMIN")
@@ -180,30 +180,24 @@ func TestPushPull(t *testing.T) {
// set author identity
author, err := backend.NewIdentity("test identity", "test@test.org")
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
err = backend.SetUserIdentity(author)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
defer backend.Close()
interrupt.RegisterCleaner(backend.Close)
tests, err := testCases(backend, author)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
// generate project name
projectName := generateRepoName()
// create target Github repository
- if err := createRepository(projectName, token); err != nil {
- t.Fatal(err)
- }
+ err = createRepository(projectName, token)
+ require.NoError(t, err)
+
fmt.Println("created repository", projectName)
// Make sure to remove the Github repository when the test end
@@ -230,7 +224,9 @@ func TestPushPull(t *testing.T) {
start := time.Now()
// export all bugs
- err = exporter.ExportAll(backend, time.Time{})
+ for result := range exporter.ExportAll(backend, time.Time{}) {
+ require.NoError(t, result.Err)
+ }
require.NoError(t, err)
fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds())
generated by cgit (git 2.34.1) at 2024-11-06 08:53:05 +0000