From fbb0e836d3d23e5a3e47029c92426090e21615f7 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 8 Jun 2019 23:14:59 +0200 Subject: [bridge/github] Add exporter implementation --- bridge/github/export.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 bridge/github/export.go (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go new file mode 100644 index 00000000..3c07d682 --- /dev/null +++ b/bridge/github/export.go @@ -0,0 +1,49 @@ +package github + +import ( + "time" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/cache" +) + +// githubImporter implement the Importer interface +type githubExporter struct { + conf core.Configuration +} + +func (ge *githubExporter) Init(conf core.Configuration) error { + ge.conf = conf + return nil +} + +// ExportAll export all event made by the current user to Github +func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) error { + identity, err := repo.GetUserIdentity() + if err != nil { + return err + } + + allBugsIds := repo.AllBugsIds() + + // + bugs := make([]*cache.BugCache, 0) + for _, id := range allBugsIds { + b, err := repo.ResolveBug(id) + if err != nil { + return err + } + + // check if user participated in the issue + participants := b.Snapshot().Participants + for _, p := range participants { + if p.Id() == identity.Id() { + bugs = append(bugs, b) + } + } + } + + //TODO: Export bugs/events/editions + + return nil +} -- cgit From 2f620e65b8edeb761a161cea0dda11eb1cdf8fec Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 14 Jun 2019 20:28:53 +0200 Subject: [bridge/github] exporter: add graphql functionalities [bridge/github] graphql mutation objects [bridge/github] github create and get labels [bridge/github] repository node_id query [bridge/github] exporter: ignore old bugs and old operations [bridge/github] add update labels/status functionalities [bridge/github] exporter: cache labels while exporting --- bridge/github/export.go | 442 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 436 insertions(+), 6 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 3c07d682..8fcbd9e8 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -1,19 +1,38 @@ package github import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" "time" "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" + "github.com/shurcooL/githubv4" +) + +const ( + keyGithubIdExport = "github-id" + keyGithubUrlExport = "github-url" ) // githubImporter implement the Importer interface type githubExporter struct { - conf core.Configuration + gc *githubv4.Client + conf core.Configuration + cachedLabels map[string]githubv4.ID } +// Init . func (ge *githubExporter) Init(conf core.Configuration) error { + ge.gc = buildClient(conf["token"]) ge.conf = conf + ge.cachedLabels = make(map[string]githubv4.ID) return nil } @@ -26,7 +45,7 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro allBugsIds := repo.AllBugsIds() - // + // collect bugs bugs := make([]*cache.BugCache, 0) for _, id := range allBugsIds { b, err := repo.ResolveBug(id) @@ -34,16 +53,427 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro return err } - // check if user participated in the issue - participants := b.Snapshot().Participants - for _, p := range participants { + snapshot := b.Snapshot() + + // ignore issues edited before since date + if snapshot.LastEditTime().Before(since) { + continue + } + + // if identity participated in a bug + for _, p := range snapshot.Participants { if p.Id() == identity.Id() { bugs = append(bugs, b) } } } - //TODO: Export bugs/events/editions + // get repository node id + repositoryID, err := getRepositoryNodeID( + ge.conf[keyOwner], + ge.conf[keyProject], + ge.conf[keyToken], + ) + if err != nil { + return err + } + + for _, b := range bugs { + snapshot := b.Snapshot() + bugGithubID := "" + + for _, op := range snapshot.Operations { + // treat only operations after since date + if op.Time().Before(since) { + continue + } + + // ignore SetMetadata operations + if _, ok := op.(*bug.SetMetadataOperation); ok { + continue + } + + // ignore imported issues and operations from github + if _, ok := op.GetMetadata(keyGithubId); ok { + continue + } + + // get operation hash + hash, err := op.Hash() + if err != nil { + return fmt.Errorf("reading operation hash: %v", err) + } + + // ignore already exported issues and operations + if _, err := b.ResolveOperationWithMetadata("github-exported-op", hash.String()); err != nil { + continue + } + + switch op.(type) { + case *bug.CreateOperation: + opr := op.(*bug.CreateOperation) + //TODO export files + bugGithubID, err = ge.createGithubIssue(repositoryID, opr.Title, opr.Message) + if err != nil { + return fmt.Errorf("exporting bug %v: %v", b.HumanId(), err) + } + + case *bug.AddCommentOperation: + opr := op.(*bug.AddCommentOperation) + bugGithubID, err = ge.addCommentGithubIssue(bugGithubID, opr.Message) + if err != nil { + return fmt.Errorf("adding comment %v: %v", "", err) + } + + case *bug.EditCommentOperation: + opr := op.(*bug.EditCommentOperation) + if err := ge.editCommentGithubIssue(bugGithubID, opr.Message); err != nil { + return fmt.Errorf("editing comment %v: %v", "", err) + } + + case *bug.SetStatusOperation: + opr := op.(*bug.SetStatusOperation) + if err := ge.updateGithubIssueStatus(bugGithubID, opr.Status); err != nil { + return fmt.Errorf("updating status %v: %v", bugGithubID, err) + } + + case *bug.SetTitleOperation: + opr := op.(*bug.SetTitleOperation) + if err := ge.updateGithubIssueTitle(bugGithubID, opr.Title); err != nil { + return fmt.Errorf("editing comment %v: %v", bugGithubID, err) + } + + case *bug.LabelChangeOperation: + opr := op.(*bug.LabelChangeOperation) + if err := ge.updateGithubIssueLabels(bugGithubID, opr.Added, opr.Removed); err != nil { + return fmt.Errorf("updating labels %v: %v", bugGithubID, err) + } + + default: + // ignore other type of operations + } + + } + + if err := b.CommitAsNeeded(); err != nil { + return fmt.Errorf("bug commit: %v", err) + } + + fmt.Printf("debug: %v", bugGithubID) + } + + return nil +} + +// getRepositoryNodeID request github api v3 to get repository node id +func getRepositoryNodeID(owner, project, token string) (string, error) { + url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("error retrieving repository node id %v", resp.StatusCode) + } + + aux := struct { + NodeID string `json:"node_id"` + }{} + + data, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + err = json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + return aux.NodeID, nil +} + +func (ge *githubExporter) markOperationAsExported(b *cache.BugCache, opHash string) error { + return nil +} + +// get label from github +func (ge *githubExporter) getGithubLabelID(label string) (string, error) { + url := fmt.Sprintf("%s/repos/%s/%s/labels/%s", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject], label) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[keyToken])) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusFound { + return "", fmt.Errorf("error getting label: status code: %v", resp.StatusCode) + } + + aux := struct { + ID string `json:"id"` + NodeID string `json:"node_id"` + Color string `json:"color"` + }{} + + data, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + err = json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + return aux.NodeID, nil +} + +// create github label using api v3 +func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, error) { + url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return "", err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[keyToken])) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode) + } + + aux := struct { + ID string `json:"id"` + NodeID string `json:"node_id"` + Color string `json:"color"` + }{} + + data, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + err = json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + return aux.NodeID, nil +} + +// randomHexColor return a random hex color code +func randomHexColor() string { + bytes := make([]byte, 6) + if _, err := rand.Read(bytes); err != nil { + return "fffff" + } + + return hex.EncodeToString(bytes) +} + +func (ge *githubExporter) getOrCreateGithubLabelID(repositoryID, label string) (string, error) { + // try to get label id + labelID, err := ge.getGithubLabelID(label) + if err == nil { + return labelID, nil + } + + // random color + color := randomHexColor() + + // create label and return id + labelID, err = ge.createGithubLabel(label, color) + if err != nil { + return "", err + } + + return labelID, nil +} + +func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) ([]githubv4.ID, error) { + ids := make([]githubv4.ID, 0, len(labels)) + var err error + + // check labels ids + for _, l := range labels { + label := string(l) + + id, ok := ge.cachedLabels[label] + if !ok { + // try to query label id + id, err = ge.getOrCreateGithubLabelID(repositoryID, label) + if err != nil { + return nil, fmt.Errorf("get or create github label: %v", err) + } + + // cache label id + ge.cachedLabels[label] = id + } + + ids = append(ids, githubv4.ID(id)) + } + + return ids, nil +} + +// create a github issue and return it ID +func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (string, error) { + m := &createIssueMutation{} + input := &githubv4.CreateIssueInput{ + RepositoryID: repositoryID, + Title: githubv4.String(title), + Body: (*githubv4.String)(&body), + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return "", err + } + + return m.CreateIssue.Issue.ID, nil +} + +// add a comment to an issue and return it ID +func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) (string, error) { + m := &addCommentToIssueMutation{} + input := &githubv4.AddCommentInput{ + SubjectID: subjectID, + Body: githubv4.String(body), + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return "", err + } + + return m.AddComment.CommentEdge.Node.ID, nil +} + +func (ge *githubExporter) editCommentGithubIssue(commentID, body string) error { + m := &updateIssueCommentMutation{} + input := &githubv4.UpdateIssueCommentInput{ + ID: commentID, + Body: githubv4.String(body), + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return err + } + + return nil +} + +func (ge *githubExporter) updateGithubIssueStatus(id string, status bug.Status) error { + m := &updateIssueMutation{} + + // set state + state := githubv4.IssueStateClosed + if status == bug.OpenStatus { + state = githubv4.IssueStateOpen + } + + input := &githubv4.UpdateIssueInput{ + ID: id, + State: &state, + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return err + } + + return nil +} + +func (ge *githubExporter) updateGithubIssueBody(id string, body string) error { + m := &updateIssueMutation{} + input := &githubv4.UpdateIssueInput{ + ID: id, + Body: (*githubv4.String)(&body), + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return err + } + + return nil +} + +func (ge *githubExporter) updateGithubIssueTitle(id, title string) error { + m := &updateIssueMutation{} + input := &githubv4.UpdateIssueInput{ + ID: id, + Title: (*githubv4.String)(&title), + } + + if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + return err + } + + return nil +} + +// update github issue labels +func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, removed []bug.Label) error { + addedIDs, err := ge.getLabelsIDs(labelableID, added) + if err != nil { + return fmt.Errorf("getting added labels ids: %v", err) + } + + m := &updateIssueMutation{} + inputAdd := &githubv4.AddLabelsToLabelableInput{ + LabelableID: labelableID, + LabelIDs: addedIDs, + } + + // add labels + if err := ge.gc.Mutate(context.TODO(), m, inputAdd, nil); err != nil { + return err + } + + removedIDs, err := ge.getLabelsIDs(labelableID, added) + if err != nil { + return fmt.Errorf("getting added labels ids: %v", err) + } + + inputRemove := &githubv4.RemoveLabelsFromLabelableInput{ + LabelableID: labelableID, + LabelIDs: removedIDs, + } + + // remove label labels + if err := ge.gc.Mutate(context.TODO(), m, inputRemove, nil); err != nil { + return err + } return nil } -- cgit From 6451dd2928d7299dbcc3c34b3e6b8a8050f4c796 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 15 Jun 2019 02:50:09 +0200 Subject: [cache] BugCache: Add set metadata methods [bridge/github] fix add/remove mutation requests [bridge/github] iterator: fix typo --- bridge/github/export.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 8fcbd9e8..37207df7 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -10,15 +10,11 @@ import ( "net/http" "time" + "github.com/shurcooL/githubv4" + "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" - "github.com/shurcooL/githubv4" -) - -const ( - keyGithubIdExport = "github-id" - keyGithubUrlExport = "github-url" ) // githubImporter implement the Importer interface @@ -449,7 +445,7 @@ func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, rem return fmt.Errorf("getting added labels ids: %v", err) } - m := &updateIssueMutation{} + m := &addLabelsToLabelableMutation{} inputAdd := &githubv4.AddLabelsToLabelableInput{ LabelableID: labelableID, LabelIDs: addedIDs, @@ -465,13 +461,14 @@ func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, rem return fmt.Errorf("getting added labels ids: %v", err) } + m2 := &removeLabelsFromLabelableMutation{} inputRemove := &githubv4.RemoveLabelsFromLabelableInput{ LabelableID: labelableID, LabelIDs: removedIDs, } // remove label labels - if err := ge.gc.Mutate(context.TODO(), m, inputRemove, nil); err != nil { + if err := ge.gc.Mutate(context.TODO(), m2, inputRemove, nil); err != nil { return err } -- cgit From cbac48277000b6d6c85418f099c1daf0e4d64956 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Thu, 20 Jun 2019 18:08:29 +0200 Subject: [bridge/github] update github bug exporter - cache repository ID - cache all comment IDs --- bridge/github/export.go | 328 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 240 insertions(+), 88 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 37207df7..a703e1e2 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -15,34 +15,54 @@ import ( "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/identity" + "github.com/MichaelMure/git-bug/util/git" ) // githubImporter implement the Importer interface type githubExporter struct { - gc *githubv4.Client - conf core.Configuration - cachedLabels map[string]githubv4.ID + gc *githubv4.Client + conf core.Configuration + + // github repository ID + repositoryID string + + // cache identifiers used to speed up exporting operations + // cleared for each bug + cachedIDs map[string]string + + // cache labels used to speed up exporting labels events + cachedLabels map[string]string } // Init . func (ge *githubExporter) Init(conf core.Configuration) error { - ge.gc = buildClient(conf["token"]) ge.conf = conf - ge.cachedLabels = make(map[string]githubv4.ID) + ge.gc = buildClient(conf["token"]) + ge.cachedIDs = make(map[string]string) + ge.cachedLabels = make(map[string]string) return nil } // ExportAll export all event made by the current user to Github func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) error { - identity, err := repo.GetUserIdentity() + user, err := repo.GetUserIdentity() if err != nil { return err } - allBugsIds := repo.AllBugsIds() + // get repository node id + ge.repositoryID, err = getRepositoryNodeID( + ge.conf[keyOwner], + ge.conf[keyProject], + ge.conf[keyToken], + ) - // collect bugs - bugs := make([]*cache.BugCache, 0) + if err != nil { + return err + } + + allBugsIds := repo.AllBugsIds() for _, id := range allBugsIds { b, err := repo.ResolveBug(id) if err != nil { @@ -51,111 +71,229 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro snapshot := b.Snapshot() - // ignore issues edited before since date - if snapshot.LastEditTime().Before(since) { + // ignore issues created before since date + if snapshot.CreatedAt.Before(since) { continue } // if identity participated in a bug for _, p := range snapshot.Participants { - if p.Id() == identity.Id() { - bugs = append(bugs, b) + if p.Id() == user.Id() { + // try to export the bug and it associated events + if err := ge.exportBug(b, user.Identity, since); err != nil { + return err + } } } } - // get repository node id - repositoryID, err := getRepositoryNodeID( - ge.conf[keyOwner], - ge.conf[keyProject], - ge.conf[keyToken], - ) + return nil +} + +// exportBug publish bugs and related events +func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, since time.Time) error { + snapshot := b.Snapshot() + + var bugGithubID string + var bugGithubURL string + var bugCreationHash string + + // Special case: + // if a user try to export a bug that is not already exported to Github (or imported + // from Github) and he is not the author of the bug. There is nothing we can do. + + // first operation is always createOp + createOp := snapshot.Operations[0].(*bug.CreateOperation) + bugAuthorID := createOp.OpBase.Author.Id() + + // get github bug ID + githubID, ok := createOp.GetMetadata(keyGithubId) + if ok { + githubURL, ok := createOp.GetMetadata(keyGithubId) + if !ok { + // if we find github ID, github URL must be found too + panic("expected to find github issue URL") + } + + // will be used to mark operation related to a bug as exported + bugGithubID = githubID + bugGithubURL = githubURL + + } else if !ok && bugAuthorID == user.Id() { + // create bug + id, url, err := ge.createGithubIssue(ge.repositoryID, createOp.Title, createOp.Message) + if err != nil { + return fmt.Errorf("creating exporting github issue %v", err) + } + + hash, err := createOp.Hash() + if err != nil { + return fmt.Errorf("comment hash: %v", err) + } + + // mark bug creation operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) + } + + // cache bug github ID and URL + bugGithubID = id + bugGithubURL = url + } else { + // if bug is still not exported and user cannot author bug stop the execution + + //TODO: maybe print a warning ? + // this is not an error + return nil + } + + // get createOp hash + hash, err := createOp.Hash() if err != nil { return err } - for _, b := range bugs { - snapshot := b.Snapshot() - bugGithubID := "" + bugCreationHash = hash.String() - for _, op := range snapshot.Operations { - // treat only operations after since date - if op.Time().Before(since) { - continue - } + // cache operation github id + ge.cachedIDs[bugCreationHash] = bugGithubID - // ignore SetMetadata operations - if _, ok := op.(*bug.SetMetadataOperation); ok { - continue - } + for _, op := range snapshot.Operations[1:] { + // ignore SetMetadata operations + if _, ok := op.(*bug.SetMetadataOperation); ok { + continue + } - // ignore imported issues and operations from github - if _, ok := op.GetMetadata(keyGithubId); ok { - continue + // get operation hash + hash, err := op.Hash() + if err != nil { + return fmt.Errorf("reading operation hash: %v", err) + } + + // ignore imported (or exported) operations from github + // cache the ID of already exported or imported issues and events from Github + if id, ok := op.GetMetadata(keyGithubId); ok { + ge.cachedIDs[hash.String()] = id + continue + } + + switch op.(type) { + case *bug.AddCommentOperation: + opr := op.(*bug.AddCommentOperation) + + // send operation to github + id, url, err := ge.addCommentGithubIssue(bugGithubID, opr.Message) + if err != nil { + return fmt.Errorf("adding comment: %v", err) } - // get operation hash - hash, err := op.Hash() + hash, err := opr.Hash() if err != nil { - return fmt.Errorf("reading operation hash: %v", err) + return fmt.Errorf("comment hash: %v", err) } - // ignore already exported issues and operations - if _, err := b.ResolveOperationWithMetadata("github-exported-op", hash.String()); err != nil { - continue + // mark operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) } - switch op.(type) { - case *bug.CreateOperation: - opr := op.(*bug.CreateOperation) - //TODO export files - bugGithubID, err = ge.createGithubIssue(repositoryID, opr.Title, opr.Message) - if err != nil { - return fmt.Errorf("exporting bug %v: %v", b.HumanId(), err) - } + case *bug.EditCommentOperation: - case *bug.AddCommentOperation: - opr := op.(*bug.AddCommentOperation) - bugGithubID, err = ge.addCommentGithubIssue(bugGithubID, opr.Message) - if err != nil { - return fmt.Errorf("adding comment %v: %v", "", err) - } + id := bugGithubID + url := bugGithubURL + opr := op.(*bug.EditCommentOperation) + targetHash := opr.Target.String() - case *bug.EditCommentOperation: - opr := op.(*bug.EditCommentOperation) - if err := ge.editCommentGithubIssue(bugGithubID, opr.Message); err != nil { - return fmt.Errorf("editing comment %v: %v", "", err) + // Since github doesn't consider the issue body as a comment + if targetHash == bugCreationHash { + // case bug creation operation: we need to edit the Github issue + if err := ge.updateGithubIssueBody(bugGithubID, opr.Message); err != nil { + return fmt.Errorf("editing issue: %v", err) } - case *bug.SetStatusOperation: - opr := op.(*bug.SetStatusOperation) - if err := ge.updateGithubIssueStatus(bugGithubID, opr.Status); err != nil { - return fmt.Errorf("updating status %v: %v", bugGithubID, err) + } else { + // case comment edition operation: we need to edit the Github comment + commentID, ok := ge.cachedIDs[targetHash] + if !ok { + panic("unexpected error: comment id not found") } - case *bug.SetTitleOperation: - opr := op.(*bug.SetTitleOperation) - if err := ge.updateGithubIssueTitle(bugGithubID, opr.Title); err != nil { - return fmt.Errorf("editing comment %v: %v", bugGithubID, err) + eid, eurl, err := ge.editCommentGithubIssue(commentID, opr.Message) + if err != nil { + return fmt.Errorf("editing comment: %v", err) } - case *bug.LabelChangeOperation: - opr := op.(*bug.LabelChangeOperation) - if err := ge.updateGithubIssueLabels(bugGithubID, opr.Added, opr.Removed); err != nil { - return fmt.Errorf("updating labels %v: %v", bugGithubID, err) - } + // use comment id/url instead of issue id/url + id = eid + url = eurl + } + + hash, err := opr.Hash() + if err != nil { + return fmt.Errorf("comment hash: %v", err) + } - default: - // ignore other type of operations + // mark operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) } - } + case *bug.SetStatusOperation: + opr := op.(*bug.SetStatusOperation) + if err := ge.updateGithubIssueStatus(bugGithubID, opr.Status); err != nil { + return fmt.Errorf("updating status %v: %v", bugGithubID, err) + } + + hash, err := opr.Hash() + if err != nil { + return fmt.Errorf("comment hash: %v", err) + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) + } - if err := b.CommitAsNeeded(); err != nil { - return fmt.Errorf("bug commit: %v", err) + case *bug.SetTitleOperation: + opr := op.(*bug.SetTitleOperation) + if err := ge.updateGithubIssueTitle(bugGithubID, opr.Title); err != nil { + return fmt.Errorf("editing comment %v: %v", bugGithubID, err) + } + + hash, err := opr.Hash() + if err != nil { + return fmt.Errorf("comment hash: %v", err) + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) + } + + case *bug.LabelChangeOperation: + opr := op.(*bug.LabelChangeOperation) + if err := ge.updateGithubIssueLabels(bugGithubID, opr.Added, opr.Removed); err != nil { + return fmt.Errorf("updating labels %v: %v", bugGithubID, err) + } + + hash, err := opr.Hash() + if err != nil { + return fmt.Errorf("comment hash: %v", err) + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) + } + + default: + panic("unhandled operation type case") } - fmt.Printf("debug: %v", bugGithubID) + } + + if err := b.CommitAsNeeded(); err != nil { + return fmt.Errorf("bug commit: %v", err) } return nil @@ -201,8 +339,16 @@ func getRepositoryNodeID(owner, project, token string) (string, error) { return aux.NodeID, nil } -func (ge *githubExporter) markOperationAsExported(b *cache.BugCache, opHash string) error { - return nil +func markOperationAsExported(b *cache.BugCache, target git.Hash, githubID, githubURL string) error { + _, err := b.SetMetadata( + target, + map[string]string{ + keyGithubId: githubID, + keyGithubUrl: githubURL, + }, + ) + + return err } // get label from github @@ -296,6 +442,8 @@ func randomHexColor() string { return "fffff" } + // fmt.Sprintf("#%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + return hex.EncodeToString(bytes) } @@ -307,6 +455,7 @@ func (ge *githubExporter) getOrCreateGithubLabelID(repositoryID, label string) ( } // random color + //TODO: no random color := randomHexColor() // create label and return id @@ -345,7 +494,7 @@ func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) } // create a github issue and return it ID -func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (string, error) { +func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (string, string, error) { m := &createIssueMutation{} input := &githubv4.CreateIssueInput{ RepositoryID: repositoryID, @@ -354,14 +503,15 @@ func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (s } if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { - return "", err + return "", "", err } - return m.CreateIssue.Issue.ID, nil + issue := m.CreateIssue.Issue + return issue.ID, issue.URL, nil } // add a comment to an issue and return it ID -func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) (string, error) { +func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) (string, string, error) { m := &addCommentToIssueMutation{} input := &githubv4.AddCommentInput{ SubjectID: subjectID, @@ -369,13 +519,14 @@ func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) ( } if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { - return "", err + return "", "", err } - return m.AddComment.CommentEdge.Node.ID, nil + node := m.AddComment.CommentEdge.Node + return node.ID, node.URL, nil } -func (ge *githubExporter) editCommentGithubIssue(commentID, body string) error { +func (ge *githubExporter) editCommentGithubIssue(commentID, body string) (string, string, error) { m := &updateIssueCommentMutation{} input := &githubv4.UpdateIssueCommentInput{ ID: commentID, @@ -383,10 +534,11 @@ func (ge *githubExporter) editCommentGithubIssue(commentID, body string) error { } if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { - return err + return "", "", err } - return nil + comment := m.IssueComment + return commentID, comment.URL, nil } func (ge *githubExporter) updateGithubIssueStatus(id string, status bug.Status) error { -- cgit From a121991f556f5b2a362d769b9304956bcedb3805 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 21 Jun 2019 00:25:43 +0200 Subject: [bridge/github] exporter: support multiple tokens/clients --- bridge/github/export.go | 145 +++++++++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 62 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index a703e1e2..32df8240 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -24,6 +24,12 @@ type githubExporter struct { gc *githubv4.Client conf core.Configuration + // a map containing + tokens map[string]string + + // clients + clients map[string]*githubv4.Client + // github repository ID repositoryID string @@ -37,13 +43,30 @@ type githubExporter struct { // Init . func (ge *githubExporter) Init(conf core.Configuration) error { + //TODO: initialize with multiple tokens ge.conf = conf ge.gc = buildClient(conf["token"]) + ge.tokens = make(map[string]string) + ge.clients = make(map[string]*githubv4.Client) ge.cachedIDs = make(map[string]string) ge.cachedLabels = make(map[string]string) return nil } +func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { + client, ok := ge.clients[id] + if ok { + return client, nil + } + + token, ok := ge.tokens[id] + if !ok { + return nil, fmt.Errorf("unknown identity token") + } + + return buildClient(token), nil +} + // ExportAll export all event made by the current user to Github func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) error { user, err := repo.GetUserIdentity() @@ -104,7 +127,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, // first operation is always createOp createOp := snapshot.Operations[0].(*bug.CreateOperation) - bugAuthorID := createOp.OpBase.Author.Id() + author := createOp.GetAuthor() // get github bug ID githubID, ok := createOp.GetMetadata(keyGithubId) @@ -119,9 +142,19 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, bugGithubID = githubID bugGithubURL = githubURL - } else if !ok && bugAuthorID == user.Id() { + } else { + client, err := ge.getIdentityClient(author.Id()) + if err != nil { + // if bug is still not exported and user cannot author bug stop the execution + + // TODO: maybe print a warning ? + // this is not an error + // don't export bug + return nil + } + // create bug - id, url, err := ge.createGithubIssue(ge.repositoryID, createOp.Title, createOp.Message) + id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { return fmt.Errorf("creating exporting github issue %v", err) } @@ -139,12 +172,6 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, // cache bug github ID and URL bugGithubID = id bugGithubURL = url - } else { - // if bug is still not exported and user cannot author bug stop the execution - - //TODO: maybe print a warning ? - // this is not an error - return nil } // get createOp hash @@ -177,40 +204,43 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, continue } + opAuthor := op.GetAuthor() + client, err := ge.getIdentityClient(opAuthor.Id()) + if err != nil { + // don't export operation + continue + } + + var id, url string switch op.(type) { case *bug.AddCommentOperation: opr := op.(*bug.AddCommentOperation) // send operation to github - id, url, err := ge.addCommentGithubIssue(bugGithubID, opr.Message) + id, url, err = addCommentGithubIssue(client, bugGithubID, opr.Message) if err != nil { return fmt.Errorf("adding comment: %v", err) } - hash, err := opr.Hash() + hash, err = opr.Hash() if err != nil { return fmt.Errorf("comment hash: %v", err) } - // mark operation as exported - if err := markOperationAsExported(b, hash, id, url); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) - } - case *bug.EditCommentOperation: - - id := bugGithubID - url := bugGithubURL opr := op.(*bug.EditCommentOperation) targetHash := opr.Target.String() // Since github doesn't consider the issue body as a comment if targetHash == bugCreationHash { // case bug creation operation: we need to edit the Github issue - if err := ge.updateGithubIssueBody(bugGithubID, opr.Message); err != nil { + if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil { return fmt.Errorf("editing issue: %v", err) } + id = bugGithubID + url = bugGithubURL + } else { // case comment edition operation: we need to edit the Github comment commentID, ok := ge.cachedIDs[targetHash] @@ -218,7 +248,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, panic("unexpected error: comment id not found") } - eid, eurl, err := ge.editCommentGithubIssue(commentID, opr.Message) + eid, eurl, err := editCommentGithubIssue(client, commentID, opr.Message) if err != nil { return fmt.Errorf("editing comment: %v", err) } @@ -228,68 +258,61 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, url = eurl } - hash, err := opr.Hash() + hash, err = opr.Hash() if err != nil { return fmt.Errorf("comment hash: %v", err) } - // mark operation as exported - if err := markOperationAsExported(b, hash, id, url); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) - } - case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) - if err := ge.updateGithubIssueStatus(bugGithubID, opr.Status); err != nil { + if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil { return fmt.Errorf("updating status %v: %v", bugGithubID, err) } - hash, err := opr.Hash() + hash, err = opr.Hash() if err != nil { return fmt.Errorf("comment hash: %v", err) } - // mark operation as exported - if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) - } + id = bugGithubID + url = bugGithubURL case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) - if err := ge.updateGithubIssueTitle(bugGithubID, opr.Title); err != nil { + if err := updateGithubIssueTitle(client, bugGithubID, opr.Title); err != nil { return fmt.Errorf("editing comment %v: %v", bugGithubID, err) } - hash, err := opr.Hash() + hash, err = opr.Hash() if err != nil { return fmt.Errorf("comment hash: %v", err) } - // mark operation as exported - if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) - } + id = bugGithubID + url = bugGithubURL case *bug.LabelChangeOperation: opr := op.(*bug.LabelChangeOperation) - if err := ge.updateGithubIssueLabels(bugGithubID, opr.Added, opr.Removed); err != nil { + if err := ge.updateGithubIssueLabels(client, bugGithubID, opr.Added, opr.Removed); err != nil { return fmt.Errorf("updating labels %v: %v", bugGithubID, err) } - hash, err := opr.Hash() + hash, err = opr.Hash() if err != nil { return fmt.Errorf("comment hash: %v", err) } - // mark operation as exported - if err := markOperationAsExported(b, hash, bugGithubID, bugGithubURL); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) - } + id = bugGithubID + url = bugGithubURL default: panic("unhandled operation type case") } + // mark operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + return fmt.Errorf("marking operation as exported: %v", err) + } } if err := b.CommitAsNeeded(); err != nil { @@ -442,8 +465,6 @@ func randomHexColor() string { return "fffff" } - // fmt.Sprintf("#%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) - return hex.EncodeToString(bytes) } @@ -494,7 +515,7 @@ func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) } // create a github issue and return it ID -func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (string, string, error) { +func createGithubIssue(gc *githubv4.Client, repositoryID, title, body string) (string, string, error) { m := &createIssueMutation{} input := &githubv4.CreateIssueInput{ RepositoryID: repositoryID, @@ -502,7 +523,7 @@ func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (s Body: (*githubv4.String)(&body), } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return "", "", err } @@ -511,14 +532,14 @@ func (ge *githubExporter) createGithubIssue(repositoryID, title, body string) (s } // add a comment to an issue and return it ID -func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) (string, string, error) { +func addCommentGithubIssue(gc *githubv4.Client, subjectID string, body string) (string, string, error) { m := &addCommentToIssueMutation{} input := &githubv4.AddCommentInput{ SubjectID: subjectID, Body: githubv4.String(body), } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return "", "", err } @@ -526,14 +547,14 @@ func (ge *githubExporter) addCommentGithubIssue(subjectID string, body string) ( return node.ID, node.URL, nil } -func (ge *githubExporter) editCommentGithubIssue(commentID, body string) (string, string, error) { +func editCommentGithubIssue(gc *githubv4.Client, commentID, body string) (string, string, error) { m := &updateIssueCommentMutation{} input := &githubv4.UpdateIssueCommentInput{ ID: commentID, Body: githubv4.String(body), } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return "", "", err } @@ -541,7 +562,7 @@ func (ge *githubExporter) editCommentGithubIssue(commentID, body string) (string return commentID, comment.URL, nil } -func (ge *githubExporter) updateGithubIssueStatus(id string, status bug.Status) error { +func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) error { m := &updateIssueMutation{} // set state @@ -555,35 +576,35 @@ func (ge *githubExporter) updateGithubIssueStatus(id string, status bug.Status) State: &state, } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return err } return nil } -func (ge *githubExporter) updateGithubIssueBody(id string, body string) error { +func updateGithubIssueBody(gc *githubv4.Client, id string, body string) error { m := &updateIssueMutation{} input := &githubv4.UpdateIssueInput{ ID: id, Body: (*githubv4.String)(&body), } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return err } return nil } -func (ge *githubExporter) updateGithubIssueTitle(id, title string) error { +func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { m := &updateIssueMutation{} input := &githubv4.UpdateIssueInput{ ID: id, Title: (*githubv4.String)(&title), } - if err := ge.gc.Mutate(context.TODO(), m, input, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { return err } @@ -591,7 +612,7 @@ func (ge *githubExporter) updateGithubIssueTitle(id, title string) error { } // update github issue labels -func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, removed []bug.Label) error { +func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { addedIDs, err := ge.getLabelsIDs(labelableID, added) if err != nil { return fmt.Errorf("getting added labels ids: %v", err) @@ -604,7 +625,7 @@ func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, rem } // add labels - if err := ge.gc.Mutate(context.TODO(), m, inputAdd, nil); err != nil { + if err := gc.Mutate(context.TODO(), m, inputAdd, nil); err != nil { return err } @@ -620,7 +641,7 @@ func (ge *githubExporter) updateGithubIssueLabels(labelableID string, added, rem } // remove label labels - if err := ge.gc.Mutate(context.TODO(), m2, inputRemove, nil); err != nil { + if err := gc.Mutate(context.TODO(), m2, inputRemove, nil); err != nil { return err } -- cgit From 1d42814166f783766a1b81e926c93c21244289dc Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 21 Jun 2019 01:09:05 +0200 Subject: [bridge/github] cache user token fix import typo init tests verify issue --- bridge/github/export.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 32df8240..e7611360 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -21,7 +21,6 @@ import ( // githubImporter implement the Importer interface type githubExporter struct { - gc *githubv4.Client conf core.Configuration // a map containing @@ -45,7 +44,6 @@ type githubExporter struct { func (ge *githubExporter) Init(conf core.Configuration) error { //TODO: initialize with multiple tokens ge.conf = conf - ge.gc = buildClient(conf["token"]) ge.tokens = make(map[string]string) ge.clients = make(map[string]*githubv4.Client) ge.cachedIDs = make(map[string]string) @@ -74,6 +72,8 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro return err } + ge.tokens[user.Id()] = ge.conf[keyToken] + // get repository node id ge.repositoryID, err = getRepositoryNodeID( ge.conf[keyOwner], @@ -86,6 +86,7 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro } allBugsIds := repo.AllBugsIds() +bugLoop: for _, id := range allBugsIds { b, err := repo.ResolveBug(id) if err != nil { @@ -101,10 +102,14 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro // if identity participated in a bug for _, p := range snapshot.Participants { - if p.Id() == user.Id() { - // try to export the bug and it associated events - if err := ge.exportBug(b, user.Identity, since); err != nil { - return err + for userId := range ge.tokens { + if p.Id() == userId { + // try to export the bug and it associated events + if err := ge.exportBug(b, user.Identity, since); err != nil { + return err + } + + continue bugLoop } } } -- cgit From 07492fb72a4af565d607432c681143d77f102515 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 21 Jun 2019 23:25:23 +0200 Subject: [bridge/github] importer: tag imported issues with origin metadata [bridge/github] exporter: correct export signature and cache maps --- bridge/github/export.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index e7611360..658ef484 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -15,7 +15,6 @@ import ( "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" - "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/util/git" ) @@ -23,11 +22,11 @@ import ( type githubExporter struct { conf core.Configuration - // a map containing - tokens map[string]string + // cache identities clients + identityClient map[string]*githubv4.Client - // clients - clients map[string]*githubv4.Client + // map identity with their tokens + identityToken map[string]string // github repository ID repositoryID string @@ -44,20 +43,20 @@ type githubExporter struct { func (ge *githubExporter) Init(conf core.Configuration) error { //TODO: initialize with multiple tokens ge.conf = conf - ge.tokens = make(map[string]string) - ge.clients = make(map[string]*githubv4.Client) + ge.identityToken = make(map[string]string) + ge.identityClient = make(map[string]*githubv4.Client) ge.cachedIDs = make(map[string]string) ge.cachedLabels = make(map[string]string) return nil } func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { - client, ok := ge.clients[id] + client, ok := ge.identityClient[id] if ok { return client, nil } - token, ok := ge.tokens[id] + token, ok := ge.identityToken[id] if !ok { return nil, fmt.Errorf("unknown identity token") } @@ -72,7 +71,7 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro return err } - ge.tokens[user.Id()] = ge.conf[keyToken] + ge.identityToken[user.Id()] = ge.conf[keyToken] // get repository node id ge.repositoryID, err = getRepositoryNodeID( @@ -102,10 +101,10 @@ bugLoop: // if identity participated in a bug for _, p := range snapshot.Participants { - for userId := range ge.tokens { + for userId := range ge.identityToken { if p.Id() == userId { // try to export the bug and it associated events - if err := ge.exportBug(b, user.Identity, since); err != nil { + if err := ge.exportBug(b, since); err != nil { return err } @@ -119,7 +118,7 @@ bugLoop: } // exportBug publish bugs and related events -func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, since time.Time) error { +func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { snapshot := b.Snapshot() var bugGithubID string @@ -148,6 +147,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, user identity.Interface, bugGithubURL = githubURL } else { + // check that we have a token for operation author client, err := ge.getIdentityClient(author.Id()) if err != nil { // if bug is still not exported and user cannot author bug stop the execution -- cgit From a58ac69c1f9246a1e544f0032f8dee8e8a769b04 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 13:06:51 +0200 Subject: [bridge/github] exporter: Check bug import origin [bridge/github] export only allowed bugs --- bridge/github/export.go | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 658ef484..71790031 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -22,6 +22,9 @@ import ( type githubExporter struct { conf core.Configuration + // export only bugs taged with one of these origins + onlyOrigins []string + // cache identities clients identityClient map[string]*githubv4.Client @@ -41,8 +44,8 @@ type githubExporter struct { // Init . func (ge *githubExporter) Init(conf core.Configuration) error { - //TODO: initialize with multiple tokens ge.conf = conf + //TODO: initialize with multiple tokens ge.identityToken = make(map[string]string) ge.identityClient = make(map[string]*githubv4.Client) ge.cachedIDs = make(map[string]string) @@ -50,18 +53,38 @@ func (ge *githubExporter) Init(conf core.Configuration) error { return nil } +func (ge *githubExporter) allowOrigin(origin string) bool { + if ge.onlyOrigins == nil { + return true + } + + for _, o := range ge.onlyOrigins { + if origin == o { + return true + } + } + + return false +} + func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { client, ok := ge.identityClient[id] if ok { return client, nil } + // get token token, ok := ge.identityToken[id] if !ok { return nil, fmt.Errorf("unknown identity token") } - return buildClient(token), nil + // create client + client = buildClient(token) + // cache client + ge.identityClient[id] = client + + return client, nil } // ExportAll export all event made by the current user to Github @@ -99,8 +122,8 @@ bugLoop: continue } - // if identity participated in a bug for _, p := range snapshot.Participants { + // if we have a token for one of the participants for userId := range ge.identityToken { if p.Id() == userId { // try to export the bug and it associated events @@ -133,6 +156,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { createOp := snapshot.Operations[0].(*bug.CreateOperation) author := createOp.GetAuthor() + // skip bug if origin is not allowed + origin, ok := createOp.GetMetadata(keyOrigin) + if ok && !ge.allowOrigin(origin) { + // TODO print a warn ? + return nil + } + // get github bug ID githubID, ok := createOp.GetMetadata(keyGithubId) if ok { -- cgit From e511f3d885e5e7f15d3b7f8e500380f0c63bc377 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 13:40:17 +0200 Subject: [misc] update man docs and completion scripts [cache] BugCache: Fix set metadata raw [bridge/github] Fix graphql input objects --- bridge/github/export.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 71790031..38df9caf 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -22,6 +22,9 @@ import ( type githubExporter struct { conf core.Configuration + // number of exported bugs + exportedBugs int + // export only bugs taged with one of these origins onlyOrigins []string @@ -166,12 +169,11 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // get github bug ID githubID, ok := createOp.GetMetadata(keyGithubId) if ok { - githubURL, ok := createOp.GetMetadata(keyGithubId) + githubURL, ok := createOp.GetMetadata(keyGithubUrl) if !ok { // if we find github ID, github URL must be found too panic("expected to find github issue URL") } - // will be used to mark operation related to a bug as exported bugGithubID = githubID bugGithubURL = githubURL @@ -191,7 +193,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // create bug id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { - return fmt.Errorf("creating exporting github issue %v", err) + return fmt.Errorf("exporting github issue: %v", err) } hash, err := createOp.Hash() @@ -263,11 +265,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { } case *bug.EditCommentOperation: + opr := op.(*bug.EditCommentOperation) targetHash := opr.Target.String() // Since github doesn't consider the issue body as a comment if targetHash == bugCreationHash { + // case bug creation operation: we need to edit the Github issue if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil { return fmt.Errorf("editing issue: %v", err) @@ -277,6 +281,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { url = bugGithubURL } else { + // case comment edition operation: we need to edit the Github comment commentID, ok := ge.cachedIDs[targetHash] if !ok { @@ -552,7 +557,7 @@ func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) // create a github issue and return it ID func createGithubIssue(gc *githubv4.Client, repositoryID, title, body string) (string, string, error) { m := &createIssueMutation{} - input := &githubv4.CreateIssueInput{ + input := githubv4.CreateIssueInput{ RepositoryID: repositoryID, Title: githubv4.String(title), Body: (*githubv4.String)(&body), @@ -569,7 +574,7 @@ func createGithubIssue(gc *githubv4.Client, repositoryID, title, body string) (s // add a comment to an issue and return it ID func addCommentGithubIssue(gc *githubv4.Client, subjectID string, body string) (string, string, error) { m := &addCommentToIssueMutation{} - input := &githubv4.AddCommentInput{ + input := githubv4.AddCommentInput{ SubjectID: subjectID, Body: githubv4.String(body), } @@ -584,7 +589,7 @@ func addCommentGithubIssue(gc *githubv4.Client, subjectID string, body string) ( func editCommentGithubIssue(gc *githubv4.Client, commentID, body string) (string, string, error) { m := &updateIssueCommentMutation{} - input := &githubv4.UpdateIssueCommentInput{ + input := githubv4.UpdateIssueCommentInput{ ID: commentID, Body: githubv4.String(body), } @@ -606,7 +611,7 @@ func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) state = githubv4.IssueStateOpen } - input := &githubv4.UpdateIssueInput{ + input := githubv4.UpdateIssueInput{ ID: id, State: &state, } @@ -620,7 +625,7 @@ func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) func updateGithubIssueBody(gc *githubv4.Client, id string, body string) error { m := &updateIssueMutation{} - input := &githubv4.UpdateIssueInput{ + input := githubv4.UpdateIssueInput{ ID: id, Body: (*githubv4.String)(&body), } @@ -634,7 +639,7 @@ func updateGithubIssueBody(gc *githubv4.Client, id string, body string) error { func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { m := &updateIssueMutation{} - input := &githubv4.UpdateIssueInput{ + input := githubv4.UpdateIssueInput{ ID: id, Title: (*githubv4.String)(&title), } @@ -654,7 +659,7 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable } m := &addLabelsToLabelableMutation{} - inputAdd := &githubv4.AddLabelsToLabelableInput{ + inputAdd := githubv4.AddLabelsToLabelableInput{ LabelableID: labelableID, LabelIDs: addedIDs, } @@ -670,7 +675,7 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable } m2 := &removeLabelsFromLabelableMutation{} - inputRemove := &githubv4.RemoveLabelsFromLabelableInput{ + inputRemove := githubv4.RemoveLabelsFromLabelableInput{ LabelableID: labelableID, LabelIDs: removedIDs, } -- cgit From f70c279c1e8dc7f73e08679f8789c35c1dfdd59d Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 17:10:23 +0200 Subject: [bridge/github] exporter: Improve error handling [bridge/github] queries: use api v4 for getLabel / createLabel [bridge/github] add comments to getIdentityClient --- bridge/github/export.go | 139 +++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 72 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 38df9caf..a41efc99 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -10,6 +10,7 @@ import ( "net/http" "time" + "github.com/pkg/errors" "github.com/shurcooL/githubv4" "github.com/MichaelMure/git-bug/bridge/core" @@ -18,6 +19,10 @@ import ( "github.com/MichaelMure/git-bug/util/git" ) +var ( + ErrMissingIdentityToken = errors.New("missing identity token") +) + // githubImporter implement the Importer interface type githubExporter struct { conf core.Configuration @@ -70,6 +75,8 @@ func (ge *githubExporter) allowOrigin(origin string) bool { return false } +// getIdentityClient return an identity github api v4 client +// if no client were found it will initilize it from the known tokens map and cache it for next it use func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { client, ok := ge.identityClient[id] if ok { @@ -79,7 +86,7 @@ func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) // get token token, ok := ge.identityToken[id] if !ok { - return nil, fmt.Errorf("unknown identity token") + return nil, ErrMissingIdentityToken } // create client @@ -182,28 +189,32 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // check that we have a token for operation author client, err := ge.getIdentityClient(author.Id()) if err != nil { - // if bug is still not exported and user cannot author bug stop the execution + // if bug is still not exported and we do not have the author stop the execution - // TODO: maybe print a warning ? - // this is not an error - // don't export bug + fmt.Println("warning: skipping issue due to missing token for bug creator") + // this is not an error, don't export bug return nil } // create bug id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { - return fmt.Errorf("exporting github issue: %v", err) + return errors.Wrap(err, "exporting github issue") } hash, err := createOp.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "comment hash") } // mark bug creation operation as exported if err := markOperationAsExported(b, hash, id, url); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) + return errors.Wrap(err, "marking operation as exported") + } + + // commit operation to avoid creating multiple issues with multiple pushes + if err := b.CommitAsNeeded(); err != nil { + return errors.Wrap(err, "bug commit") } // cache bug github ID and URL @@ -231,7 +242,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // get operation hash hash, err := op.Hash() if err != nil { - return fmt.Errorf("reading operation hash: %v", err) + return errors.Wrap(err, "operation hash") } // ignore imported (or exported) operations from github @@ -256,12 +267,12 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // send operation to github id, url, err = addCommentGithubIssue(client, bugGithubID, opr.Message) if err != nil { - return fmt.Errorf("adding comment: %v", err) + return errors.Wrap(err, "adding comment") } hash, err = opr.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "comment hash") } case *bug.EditCommentOperation: @@ -274,7 +285,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // case bug creation operation: we need to edit the Github issue if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil { - return fmt.Errorf("editing issue: %v", err) + return errors.Wrap(err, "editing issue") } id = bugGithubID @@ -290,7 +301,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { eid, eurl, err := editCommentGithubIssue(client, commentID, opr.Message) if err != nil { - return fmt.Errorf("editing comment: %v", err) + return errors.Wrap(err, "editing comment") } // use comment id/url instead of issue id/url @@ -300,18 +311,18 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { hash, err = opr.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "comment hash") } case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil { - return fmt.Errorf("updating status %v: %v", bugGithubID, err) + return errors.Wrap(err, "editing status") } hash, err = opr.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "set status operation hash") } id = bugGithubID @@ -320,12 +331,12 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) if err := updateGithubIssueTitle(client, bugGithubID, opr.Title); err != nil { - return fmt.Errorf("editing comment %v: %v", bugGithubID, err) + return errors.Wrap(err, "editing title") } hash, err = opr.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "set title operation hash") } id = bugGithubID @@ -334,12 +345,12 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { case *bug.LabelChangeOperation: opr := op.(*bug.LabelChangeOperation) if err := ge.updateGithubIssueLabels(client, bugGithubID, opr.Added, opr.Removed); err != nil { - return fmt.Errorf("updating labels %v: %v", bugGithubID, err) + return errors.Wrap(err, "updating labels") } hash, err = opr.Hash() if err != nil { - return fmt.Errorf("comment hash: %v", err) + return errors.Wrap(err, "label change operation hash") } id = bugGithubID @@ -351,12 +362,12 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // mark operation as exported if err := markOperationAsExported(b, hash, id, url); err != nil { - return fmt.Errorf("marking operation as exported: %v", err) + return errors.Wrap(err, "marking operation as exported") } - } - if err := b.CommitAsNeeded(); err != nil { - return fmt.Errorf("bug commit: %v", err) + if err := b.CommitAsNeeded(); err != nil { + return errors.Wrap(err, "bug commit") + } } return nil @@ -415,49 +426,17 @@ func markOperationAsExported(b *cache.BugCache, target git.Hash, githubID, githu } // get label from github -func (ge *githubExporter) getGithubLabelID(label string) (string, error) { - url := fmt.Sprintf("%s/repos/%s/%s/labels/%s", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject], label) - - client := &http.Client{ - Timeout: defaultTimeout, - } - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", err - } - - // need the token for private repositories - req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[keyToken])) - - resp, err := client.Do(req) - if err != nil { - return "", err - } - - if resp.StatusCode != http.StatusFound { - return "", fmt.Errorf("error getting label: status code: %v", resp.StatusCode) - } - - aux := struct { - ID string `json:"id"` - NodeID string `json:"node_id"` - Color string `json:"color"` - }{} - - data, _ := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - - err = json.Unmarshal(data, &aux) - if err != nil { +func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (string, error) { + q := labelQuery{} + variables := map[string]interface{}{"name": label} + if err := gc.Query(context.TODO(), &q, variables); err != nil { return "", err } - return aux.NodeID, nil + return q.Repository.Label.ID, nil } -// create github label using api v3 -func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, error) { +func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelColor string) (string, error) { url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) client := &http.Client{ @@ -498,6 +477,22 @@ func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, e return aux.NodeID, nil } +// create github label using api v4 +func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) { + m := &createLabelMutation{} + input := createLabelInput{ + RepositoryID: ge.repositoryID, + Name: githubv4.String(label), + Color: githubv4.String(labelColor), + } + + if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + return "", err + } + + return m.CreateLabel.Label.ID, nil +} + // randomHexColor return a random hex color code func randomHexColor() string { bytes := make([]byte, 6) @@ -508,9 +503,9 @@ func randomHexColor() string { return hex.EncodeToString(bytes) } -func (ge *githubExporter) getOrCreateGithubLabelID(repositoryID, label string) (string, error) { +func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, repositoryID, label string) (string, error) { // try to get label id - labelID, err := ge.getGithubLabelID(label) + labelID, err := ge.getGithubLabelID(gc, label) if err == nil { return labelID, nil } @@ -520,7 +515,7 @@ func (ge *githubExporter) getOrCreateGithubLabelID(repositoryID, label string) ( color := randomHexColor() // create label and return id - labelID, err = ge.createGithubLabel(label, color) + labelID, err = ge.createGithubLabel(gc, label, color) if err != nil { return "", err } @@ -528,7 +523,7 @@ func (ge *githubExporter) getOrCreateGithubLabelID(repositoryID, label string) ( return labelID, nil } -func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) ([]githubv4.ID, error) { +func (ge *githubExporter) getLabelsIDs(gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) { ids := make([]githubv4.ID, 0, len(labels)) var err error @@ -539,9 +534,9 @@ func (ge *githubExporter) getLabelsIDs(repositoryID string, labels []bug.Label) id, ok := ge.cachedLabels[label] if !ok { // try to query label id - id, err = ge.getOrCreateGithubLabelID(repositoryID, label) + id, err = ge.getOrCreateGithubLabelID(gc, repositoryID, label) if err != nil { - return nil, fmt.Errorf("get or create github label: %v", err) + return nil, errors.Wrap(err, "get or create github label") } // cache label id @@ -653,9 +648,9 @@ func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { // update github issue labels func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { - addedIDs, err := ge.getLabelsIDs(labelableID, added) + addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) if err != nil { - return fmt.Errorf("getting added labels ids: %v", err) + return errors.Wrap(err, "getting added labels ids") } m := &addLabelsToLabelableMutation{} @@ -669,9 +664,9 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable return err } - removedIDs, err := ge.getLabelsIDs(labelableID, added) + removedIDs, err := ge.getLabelsIDs(gc, labelableID, added) if err != nil { - return fmt.Errorf("getting added labels ids: %v", err) + return errors.Wrap(err, "getting added labels ids") } m2 := &removeLabelsFromLabelableMutation{} -- cgit From 4ad3d336c69922a7e34c782e995db9f1adcbc403 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 19:46:42 +0200 Subject: [bridge/github] use context.WithTimeout on all graphql queries --- bridge/github/export.go | 62 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 10 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index a41efc99..e76e95f0 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -429,7 +429,12 @@ func markOperationAsExported(b *cache.BugCache, target git.Hash, githubID, githu func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (string, error) { q := labelQuery{} variables := map[string]interface{}{"name": label} - if err := gc.Query(context.TODO(), &q, variables); err != nil { + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Query(ctx, &q, variables); err != nil { return "", err } @@ -486,7 +491,11 @@ func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelC Color: githubv4.String(labelColor), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return "", err } @@ -558,7 +567,11 @@ func createGithubIssue(gc *githubv4.Client, repositoryID, title, body string) (s Body: (*githubv4.String)(&body), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return "", "", err } @@ -574,7 +587,11 @@ func addCommentGithubIssue(gc *githubv4.Client, subjectID string, body string) ( Body: githubv4.String(body), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return "", "", err } @@ -589,7 +606,11 @@ func editCommentGithubIssue(gc *githubv4.Client, commentID, body string) (string Body: githubv4.String(body), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return "", "", err } @@ -611,7 +632,11 @@ func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) State: &state, } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return err } @@ -625,7 +650,11 @@ func updateGithubIssueBody(gc *githubv4.Client, id string, body string) error { Body: (*githubv4.String)(&body), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return err } @@ -639,7 +668,11 @@ func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { Title: (*githubv4.String)(&title), } - if err := gc.Mutate(context.TODO(), m, input, nil); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { return err } @@ -648,6 +681,7 @@ func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { // update github issue labels func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { + addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) if err != nil { return errors.Wrap(err, "getting added labels ids") @@ -659,11 +693,16 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable LabelIDs: addedIDs, } + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + // add labels - if err := gc.Mutate(context.TODO(), m, inputAdd, nil); err != nil { + if err := gc.Mutate(ctx, m, inputAdd, nil); err != nil { + cancel() return err } + cancel() removedIDs, err := ge.getLabelsIDs(gc, labelableID, added) if err != nil { return errors.Wrap(err, "getting added labels ids") @@ -675,8 +714,11 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable LabelIDs: removedIDs, } + ctx2, cancel2 := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel2() + // remove label labels - if err := gc.Mutate(context.TODO(), m2, inputRemove, nil); err != nil { + if err := gc.Mutate(ctx2, m2, inputRemove, nil); err != nil { return err } -- cgit From 23f2c58e920ea239842e0303af0b5bba5e1d3665 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 20:08:51 +0200 Subject: [bridge/github] create label: Generate hexColor from label rgba --- bridge/github/export.go | 58 +++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index e76e95f0..df8400da 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -1,12 +1,11 @@ package github import ( + "bytes" "context" - "encoding/hex" "encoding/json" "fmt" "io/ioutil" - "math/rand" "net/http" "time" @@ -448,7 +447,21 @@ func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelCol Timeout: defaultTimeout, } - req, err := http.NewRequest("POST", url, nil) + params := struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + }{ + Name: label, + Color: labelColor, + } + + data, err := json.Marshal(params) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) if err != nil { return "", err } @@ -462,16 +475,18 @@ func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelCol } if resp.StatusCode != http.StatusCreated { + //d, _ := ioutil.ReadAll(resp.Body) + //fmt.Println(string(d)) return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode) } aux := struct { - ID string `json:"id"` + ID int `json:"id"` NodeID string `json:"node_id"` Color string `json:"color"` }{} - data, _ := ioutil.ReadAll(resp.Body) + data, _ = ioutil.ReadAll(resp.Body) defer resp.Body.Close() err = json.Unmarshal(data, &aux) @@ -479,6 +494,7 @@ func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelCol return "", err } + fmt.Println("ezzz", aux.NodeID) return aux.NodeID, nil } @@ -502,29 +518,21 @@ func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelC return m.CreateLabel.Label.ID, nil } -// randomHexColor return a random hex color code -func randomHexColor() string { - bytes := make([]byte, 6) - if _, err := rand.Read(bytes); err != nil { - return "fffff" - } - - return hex.EncodeToString(bytes) -} - -func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, repositoryID, label string) (string, error) { +func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) { // try to get label id - labelID, err := ge.getGithubLabelID(gc, label) + labelID, err := ge.getGithubLabelID(gc, string(label)) if err == nil { return labelID, nil } - // random color - //TODO: no random - color := randomHexColor() + // hex color + rgba := label.RGBA() + hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + + fmt.Println("creating color", label, hexColor) // create label and return id - labelID, err = ge.createGithubLabel(gc, label, color) + labelID, err = ge.createGithubLabel(gc, string(label), hexColor) if err != nil { return "", err } @@ -537,10 +545,8 @@ func (ge *githubExporter) getLabelsIDs(gc *githubv4.Client, repositoryID string, var err error // check labels ids - for _, l := range labels { - label := string(l) - - id, ok := ge.cachedLabels[label] + for _, label := range labels { + id, ok := ge.cachedLabels[string(label)] if !ok { // try to query label id id, err = ge.getOrCreateGithubLabelID(gc, repositoryID, label) @@ -549,7 +555,7 @@ func (ge *githubExporter) getLabelsIDs(gc *githubv4.Client, repositoryID string, } // cache label id - ge.cachedLabels[label] = id + ge.cachedLabels[string(label)] = id } ids = append(ids, githubv4.ID(id)) -- cgit From 12cc74e22b35d08d2d342a1b445d967e3e09c1f6 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 22 Jun 2019 22:28:04 +0200 Subject: [bridge/github] correct label & comment edit mutations bugs --- bridge/github/export.go | 60 ++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index df8400da..393146d6 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -426,17 +426,25 @@ func markOperationAsExported(b *cache.BugCache, target git.Hash, githubID, githu // get label from github func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (string, error) { - q := labelQuery{} - variables := map[string]interface{}{"name": label} + q := &labelQuery{} + variables := map[string]interface{}{ + "label": githubv4.String(label), + "owner": githubv4.String(ge.conf[keyOwner]), + "name": githubv4.String(ge.conf[keyProject]), + } parentCtx := context.Background() ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() - if err := gc.Query(ctx, &q, variables); err != nil { + if err := gc.Query(ctx, q, variables); err != nil { return "", err } + if q.Repository.Label.ID == "" { + return "", fmt.Errorf("label not found") + } + return q.Repository.Label.ID, nil } @@ -475,8 +483,6 @@ func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelCol } if resp.StatusCode != http.StatusCreated { - //d, _ := ioutil.ReadAll(resp.Body) - //fmt.Println(string(d)) return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode) } @@ -494,13 +500,12 @@ func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelCol return "", err } - fmt.Println("ezzz", aux.NodeID) return aux.NodeID, nil } // create github label using api v4 func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) { - m := &createLabelMutation{} + m := createLabelMutation{} input := createLabelInput{ RepositoryID: ge.repositoryID, Name: githubv4.String(label), @@ -511,7 +516,7 @@ func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelC ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) defer cancel() - if err := gc.Mutate(ctx, m, input, nil); err != nil { + if err := gc.Mutate(ctx, &m, input, nil); err != nil { return "", err } @@ -529,8 +534,6 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito rgba := label.RGBA() hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) - fmt.Println("creating color", label, hexColor) - // create label and return id labelID, err = ge.createGithubLabel(gc, string(label), hexColor) if err != nil { @@ -620,8 +623,7 @@ func editCommentGithubIssue(gc *githubv4.Client, commentID, body string) (string return "", "", err } - comment := m.IssueComment - return commentID, comment.URL, nil + return commentID, m.UpdateIssueComment.IssueComment.URL, nil } func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) error { @@ -687,7 +689,6 @@ func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { // update github issue labels func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { - addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) if err != nil { return errors.Wrap(err, "getting added labels ids") @@ -707,25 +708,28 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable cancel() return err } - cancel() - removedIDs, err := ge.getLabelsIDs(gc, labelableID, added) - if err != nil { - return errors.Wrap(err, "getting added labels ids") - } - m2 := &removeLabelsFromLabelableMutation{} - inputRemove := githubv4.RemoveLabelsFromLabelableInput{ - LabelableID: labelableID, - LabelIDs: removedIDs, - } + if len(removed) > 0 { + removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) + if err != nil { + return errors.Wrap(err, "getting added labels ids") + } + + m2 := &removeLabelsFromLabelableMutation{} + inputRemove := githubv4.RemoveLabelsFromLabelableInput{ + LabelableID: labelableID, + LabelIDs: removedIDs, + } - ctx2, cancel2 := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel2() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + // remove label labels + if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { + return err + } - // remove label labels - if err := gc.Mutate(ctx2, m2, inputRemove, nil); err != nil { - return err } return nil -- cgit From fc09f2a492348f3b6013308697cbc86e15e8ab47 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 23 Jun 2019 00:52:30 +0200 Subject: [bridge/github] exporter: Add test cases correct delete repo improve tests set user identity --- bridge/github/export.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 393146d6..947d5aca 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -35,7 +35,7 @@ type githubExporter struct { // cache identities clients identityClient map[string]*githubv4.Client - // map identity with their tokens + // map identities with their tokens identityToken map[string]string // github repository ID @@ -201,6 +201,9 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { return errors.Wrap(err, "exporting github issue") } + // incr exported bugs + ge.exportedBugs++ + hash, err := createOp.Hash() if err != nil { return errors.Wrap(err, "comment hash") -- cgit From 0dea0f6a7673865b16e71f1998c3b0db33a54514 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 23 Jun 2019 13:50:45 +0200 Subject: [bridge/github] simplify export operation hashs [bridge/github] exporter tests: add more test cases and global tests [bridge/github] rename export_query to export_mutation [bridge/github] exporter: Log number of exported issues and labels [bridge/github] Improve comments --- bridge/github/export.go | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 947d5aca..1ab6ebca 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -29,6 +29,9 @@ type githubExporter struct { // number of exported bugs exportedBugs int + // number of exported labels + exportedLabels int + // export only bugs taged with one of these origins onlyOrigins []string @@ -146,6 +149,7 @@ bugLoop: } } + fmt.Printf("Successfully exported %d issues and %d labels to Github\n", ge.exportedBugs, ge.exportedLabels) return nil } @@ -272,10 +276,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { return errors.Wrap(err, "adding comment") } - hash, err = opr.Hash() - if err != nil { - return errors.Wrap(err, "comment hash") - } + // cache comment id + ge.cachedIDs[hash.String()] = id case *bug.EditCommentOperation: @@ -311,22 +313,12 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { url = eurl } - hash, err = opr.Hash() - if err != nil { - return errors.Wrap(err, "comment hash") - } - case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil { return errors.Wrap(err, "editing status") } - hash, err = opr.Hash() - if err != nil { - return errors.Wrap(err, "set status operation hash") - } - id = bugGithubID url = bugGithubURL @@ -336,11 +328,6 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { return errors.Wrap(err, "editing title") } - hash, err = opr.Hash() - if err != nil { - return errors.Wrap(err, "set title operation hash") - } - id = bugGithubID url = bugGithubURL @@ -350,11 +337,6 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { return errors.Wrap(err, "updating labels") } - hash, err = opr.Hash() - if err != nil { - return errors.Wrap(err, "label change operation hash") - } - id = bugGithubID url = bugGithubURL @@ -367,6 +349,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { return errors.Wrap(err, "marking operation as exported") } + // commit at each operation export to avoid exporting same events multiple times if err := b.CommitAsNeeded(); err != nil { return errors.Wrap(err, "bug commit") } @@ -451,7 +434,7 @@ func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (s return q.Repository.Label.ID, nil } -func (ge *githubExporter) createGithubLabel(gc *githubv4.Client, label, labelColor string) (string, error) { +func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, error) { url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) client := &http.Client{ @@ -538,11 +521,14 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) // create label and return id - labelID, err = ge.createGithubLabel(gc, string(label), hexColor) + // NOTE: since createLabel mutation is still in preview mode we use github api v4 to create labels + // see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview + labelID, err = ge.createGithubLabel(string(label), hexColor) if err != nil { return "", err } + ge.exportedLabels++ return labelID, nil } -- cgit From 31eebdf9da8cd0f6afd7999175fb53cc135a1833 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 23 Jun 2019 17:54:38 +0200 Subject: [bridge/github] Correcte some types and add comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit General improvements Co-Authored-By: Michael Muré empty array check an empty array is not nil Co-Authored-By: Michael Muré --- bridge/github/export.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 1ab6ebca..9e394334 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -22,7 +22,7 @@ var ( ErrMissingIdentityToken = errors.New("missing identity token") ) -// githubImporter implement the Importer interface +// githubExporter implement the Exporter interface type githubExporter struct { conf core.Configuration @@ -63,8 +63,11 @@ func (ge *githubExporter) Init(conf core.Configuration) error { return nil } +// allowOrigin verify that origin is allowed to get exported. +// if the exporter was initialized with no specified origins, it will return +// true for all origins func (ge *githubExporter) allowOrigin(origin string) bool { - if ge.onlyOrigins == nil { + if len(ge.onlyOrigins) == 0 { return true } @@ -78,7 +81,7 @@ func (ge *githubExporter) allowOrigin(origin string) bool { } // getIdentityClient return an identity github api v4 client -// if no client were found it will initilize it from the known tokens map and cache it for next it use +// if no client were found it will initilize it from the known tokens map and cache it for next use func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { client, ok := ge.identityClient[id] if ok { @@ -143,6 +146,7 @@ bugLoop: return err } + // avoid calling exportBug multiple times for the same bug continue bugLoop } } @@ -163,7 +167,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // Special case: // if a user try to export a bug that is not already exported to Github (or imported - // from Github) and he is not the author of the bug. There is nothing we can do. + // from Github) and we do not have the token of the bug author, there is nothing we can do. // first operation is always createOp createOp := snapshot.Operations[0].(*bug.CreateOperation) @@ -184,6 +188,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // if we find github ID, github URL must be found too panic("expected to find github issue URL") } + // will be used to mark operation related to a bug as exported bugGithubID = githubID bugGithubURL = githubURL @@ -427,6 +432,7 @@ func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (s return "", err } + // if label id is empty, it means there is no such label in this Github repository if q.Repository.Label.ID == "" { return "", fmt.Errorf("label not found") } @@ -434,7 +440,7 @@ func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (s return q.Repository.Label.ID, nil } -func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, error) { +func (ge *githubExporter) createGithubLabel(label, color string) (string, error) { url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) client := &http.Client{ @@ -447,7 +453,7 @@ func (ge *githubExporter) createGithubLabel(label, labelColor string) (string, e Description string `json:"description"` }{ Name: label, - Color: labelColor, + Color: color, } data, err := json.Marshal(params) @@ -516,7 +522,7 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito return labelID, nil } - // hex color + // RGBA to hex color rgba := label.RGBA() hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) -- cgit From 570ae5f75e034523a1d7d94a04febe944d36399b Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 23 Jun 2019 19:20:10 +0200 Subject: [bug] add snapshot.GetCreateMetadata method [bug] add snapshot.HasParticipant(id string) [bug] add snapshot.HasAnyParticipant(ids ...string) [bug] add snapshot.HasActor(id string) [bug] add snapshot.HasAnyActor(ids ...string) [bridge/github] improve comments [bridge/github] exporter tests: register deleteRepository in cleaner [bridge/github] tests rebase --- bridge/github/export.go | 67 +++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 33 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 9e394334..00ba1585 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -122,8 +122,13 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) erro return err } + allIdentitiesIds := []string{} + for id := range ge.identityToken { + allIdentitiesIds = append(allIdentitiesIds, id) + } + allBugsIds := repo.AllBugsIds() -bugLoop: + for _, id := range allBugsIds { b, err := repo.ResolveBug(id) if err != nil { @@ -137,20 +142,13 @@ bugLoop: continue } - for _, p := range snapshot.Participants { - // if we have a token for one of the participants - for userId := range ge.identityToken { - if p.Id() == userId { - // try to export the bug and it associated events - if err := ge.exportBug(b, since); err != nil { - return err - } - - // avoid calling exportBug multiple times for the same bug - continue bugLoop - } + if snapshot.HasAnyParticipant(allIdentitiesIds...) { + // try to export the bug and it associated events + if err := ge.exportBug(b, since); err != nil { + return err } } + } fmt.Printf("Successfully exported %d issues and %d labels to Github\n", ge.exportedBugs, ge.exportedLabels) @@ -171,7 +169,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // first operation is always createOp createOp := snapshot.Operations[0].(*bug.CreateOperation) - author := createOp.GetAuthor() + author := snapshot.Author // skip bug if origin is not allowed origin, ok := createOp.GetMetadata(keyOrigin) @@ -440,6 +438,9 @@ func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (s return q.Repository.Label.ID, nil } +// create a new label and return it github id +// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels +// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview func (ge *githubExporter) createGithubLabel(label, color string) (string, error) { url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) @@ -495,6 +496,7 @@ func (ge *githubExporter) createGithubLabel(label, color string) (string, error) return aux.NodeID, nil } +/** // create github label using api v4 func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) { m := createLabelMutation{} @@ -514,6 +516,7 @@ func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelC return m.CreateLabel.Label.ID, nil } +*/ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) { // try to get label id @@ -526,9 +529,6 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito rgba := label.RGBA() hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) - // create label and return id - // NOTE: since createLabel mutation is still in preview mode we use github api v4 to create labels - // see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview labelID, err = ge.createGithubLabel(string(label), hexColor) if err != nil { return "", err @@ -705,26 +705,27 @@ func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelable } cancel() - if len(removed) > 0 { - removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) - if err != nil { - return errors.Wrap(err, "getting added labels ids") - } + if len(removed) == 0 { + return nil + } - m2 := &removeLabelsFromLabelableMutation{} - inputRemove := githubv4.RemoveLabelsFromLabelableInput{ - LabelableID: labelableID, - LabelIDs: removedIDs, - } + removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) + if err != nil { + return errors.Wrap(err, "getting added labels ids") + } - ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() + m2 := &removeLabelsFromLabelableMutation{} + inputRemove := githubv4.RemoveLabelsFromLabelableInput{ + LabelableID: labelableID, + LabelIDs: removedIDs, + } - // remove label labels - if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { - return err - } + ctx, cancel = context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + // remove label labels + if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { + return err } return nil -- cgit From 1f365b236900d429cbfdaa1807163a110e6e4624 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 25 Jun 2019 01:33:48 +0200 Subject: [core] Implement ExportResults Use ExportResult chan to send export events Remove exportedBugs and exportedLabels --- bridge/github/export.go | 185 +++++++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 71 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 00ba1585..40044ca1 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -26,13 +26,7 @@ var ( type githubExporter struct { conf core.Configuration - // number of exported bugs - exportedBugs int - - // number of exported labels - exportedLabels int - - // export only bugs taged with one of these origins + // export only bugs tagged with one of these origins onlyOrigins []string // cache identities clients @@ -81,7 +75,7 @@ func (ge *githubExporter) allowOrigin(origin string) bool { } // getIdentityClient return an identity github api v4 client -// if no client were found it will initilize it from the known tokens map and cache it for next use +// if no client were found it will initialize it from the known tokens map and cache it for next use func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { client, ok := ge.identityClient[id] if ok { @@ -103,60 +97,68 @@ func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) } // ExportAll export all event made by the current user to Github -func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) error { - user, err := repo.GetUserIdentity() - if err != nil { - return err - } +func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) <-chan core.ExportResult { + out := make(chan core.ExportResult) - ge.identityToken[user.Id()] = ge.conf[keyToken] + go func(out chan<- core.ExportResult) { + defer close(out) - // get repository node id - ge.repositoryID, err = getRepositoryNodeID( - ge.conf[keyOwner], - ge.conf[keyProject], - ge.conf[keyToken], - ) - - if err != nil { - return err - } + user, err := repo.GetUserIdentity() + if err != nil { + out <- core.NewExportError(err, "") + return + } - allIdentitiesIds := []string{} - for id := range ge.identityToken { - allIdentitiesIds = append(allIdentitiesIds, id) - } + ge.identityToken[user.Id()] = ge.conf[keyToken] - allBugsIds := repo.AllBugsIds() + // get repository node id + ge.repositoryID, err = getRepositoryNodeID( + ge.conf[keyOwner], + ge.conf[keyProject], + ge.conf[keyToken], + ) - for _, id := range allBugsIds { - b, err := repo.ResolveBug(id) if err != nil { - return err + out <- core.NewExportError(err, ge.repositoryID) + return } - snapshot := b.Snapshot() - - // ignore issues created before since date - if snapshot.CreatedAt.Before(since) { - continue + var allIdentitiesIds []string + for id := range ge.identityToken { + allIdentitiesIds = append(allIdentitiesIds, id) } - if snapshot.HasAnyParticipant(allIdentitiesIds...) { - // try to export the bug and it associated events - if err := ge.exportBug(b, since); err != nil { - return err + allBugsIds := repo.AllBugsIds() + + for _, id := range allBugsIds { + b, err := repo.ResolveBug(id) + if err != nil { + out <- core.NewExportError(err, id) + return } - } - } + snapshot := b.Snapshot() - fmt.Printf("Successfully exported %d issues and %d labels to Github\n", ge.exportedBugs, ge.exportedLabels) - return nil + // ignore issues created before since date + if snapshot.CreatedAt.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } + + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + ge.exportBug(b, since, out) + } else { + out <- core.NewExportNothing(id, "not an actor") + } + } + }(out) + + return out } // exportBug publish bugs and related events -func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { +func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { snapshot := b.Snapshot() var bugGithubID string @@ -174,8 +176,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // skip bug if origin is not allowed origin, ok := createOp.GetMetadata(keyOrigin) if ok && !ge.allowOrigin(origin) { - // TODO print a warn ? - return nil + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue taged with origin: %s", origin)) + return } // get github bug ID @@ -187,6 +189,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { panic("expected to find github issue URL") } + out <- core.NewExportNothing(b.Id(), "bug already exported") // will be used to mark operation related to a bug as exported bugGithubID = githubID bugGithubURL = githubURL @@ -197,33 +200,41 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { if err != nil { // if bug is still not exported and we do not have the author stop the execution - fmt.Println("warning: skipping issue due to missing token for bug creator") + // fmt.Println("warning: skipping issue due to missing token for bug creator") // this is not an error, don't export bug - return nil + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) + return } // create bug id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message) if err != nil { - return errors.Wrap(err, "exporting github issue") + err := errors.Wrap(err, "exporting github issue") + out <- core.NewExportError(err, b.Id()) + return } - // incr exported bugs - ge.exportedBugs++ + out <- core.NewExportBug(b.Id()) hash, err := createOp.Hash() if err != nil { - return errors.Wrap(err, "comment hash") + err := errors.Wrap(err, "comment hash") + out <- core.NewExportError(err, b.Id()) + return } // mark bug creation operation as exported if err := markOperationAsExported(b, hash, id, url); err != nil { - return errors.Wrap(err, "marking operation as exported") + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return } // commit operation to avoid creating multiple issues with multiple pushes if err := b.CommitAsNeeded(); err != nil { - return errors.Wrap(err, "bug commit") + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return } // cache bug github ID and URL @@ -234,7 +245,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // get createOp hash hash, err := createOp.Hash() if err != nil { - return err + out <- core.NewExportError(err, b.Id()) + return } bugCreationHash = hash.String() @@ -251,20 +263,23 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // get operation hash hash, err := op.Hash() if err != nil { - return errors.Wrap(err, "operation hash") + err := errors.Wrap(err, "operation hash") + out <- core.NewExportError(err, b.Id()) + return } // ignore imported (or exported) operations from github // cache the ID of already exported or imported issues and events from Github if id, ok := op.GetMetadata(keyGithubId); ok { ge.cachedIDs[hash.String()] = id + out <- core.NewExportNothing(hash.String(), "already exported operation") continue } opAuthor := op.GetAuthor() client, err := ge.getIdentityClient(opAuthor.Id()) if err != nil { - // don't export operation + out <- core.NewExportNothing(hash.String(), "missing operation author token") continue } @@ -276,9 +291,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // send operation to github id, url, err = addCommentGithubIssue(client, bugGithubID, opr.Message) if err != nil { - return errors.Wrap(err, "adding comment") + err := errors.Wrap(err, "adding comment") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportComment(hash.String()) + // cache comment id ge.cachedIDs[hash.String()] = id @@ -292,9 +311,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // case bug creation operation: we need to edit the Github issue if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil { - return errors.Wrap(err, "editing issue") + err := errors.Wrap(err, "editing issue") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportCommentEdition(hash.String()) + id = bugGithubID url = bugGithubURL @@ -308,9 +331,13 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { eid, eurl, err := editCommentGithubIssue(client, commentID, opr.Message) if err != nil { - return errors.Wrap(err, "editing comment") + err := errors.Wrap(err, "editing comment") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportCommentEdition(hash.String()) + // use comment id/url instead of issue id/url id = eid url = eurl @@ -319,27 +346,39 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil { - return errors.Wrap(err, "editing status") + err := errors.Wrap(err, "editing status") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportStatusChange(hash.String()) + id = bugGithubID url = bugGithubURL case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) if err := updateGithubIssueTitle(client, bugGithubID, opr.Title); err != nil { - return errors.Wrap(err, "editing title") + err := errors.Wrap(err, "editing title") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportTitleEdition(hash.String()) + id = bugGithubID url = bugGithubURL case *bug.LabelChangeOperation: opr := op.(*bug.LabelChangeOperation) if err := ge.updateGithubIssueLabels(client, bugGithubID, opr.Added, opr.Removed); err != nil { - return errors.Wrap(err, "updating labels") + err := errors.Wrap(err, "updating labels") + out <- core.NewExportError(err, b.Id()) + return } + out <- core.NewExportLabelChange(hash.String()) + id = bugGithubID url = bugGithubURL @@ -349,16 +388,18 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time) error { // mark operation as exported if err := markOperationAsExported(b, hash, id, url); err != nil { - return errors.Wrap(err, "marking operation as exported") + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return } // commit at each operation export to avoid exporting same events multiple times if err := b.CommitAsNeeded(); err != nil { - return errors.Wrap(err, "bug commit") + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return } } - - return nil } // getRepositoryNodeID request github api v3 to get repository node id @@ -391,7 +432,10 @@ func getRepositoryNodeID(owner, project, token string) (string, error) { }{} data, _ := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() + err = resp.Body.Close() + if err != nil { + return "", err + } err = json.Unmarshal(data, &aux) if err != nil { @@ -534,7 +578,6 @@ func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, reposito return "", err } - ge.exportedLabels++ return labelID, nil } -- cgit From 93e731fd0d365c0cf49dbb7aea371e48f46e1f11 Mon Sep 17 00:00:00 2001 From: Amine Date: Fri, 5 Jul 2019 18:32:51 +0200 Subject: [bridge/github] improve comments and documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [bridge/github] improve error handling and tests Co-Authored-By: Michael Muré --- bridge/github/export.go | 77 +++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 40044ca1..5e5b1a21 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -40,7 +40,7 @@ type githubExporter struct { // cache identifiers used to speed up exporting operations // cleared for each bug - cachedIDs map[string]string + cachedOperationIDs map[string]string // cache labels used to speed up exporting labels events cachedLabels map[string]string @@ -52,7 +52,7 @@ func (ge *githubExporter) Init(conf core.Configuration) error { //TODO: initialize with multiple tokens ge.identityToken = make(map[string]string) ge.identityClient = make(map[string]*githubv4.Client) - ge.cachedIDs = make(map[string]string) + ge.cachedOperationIDs = make(map[string]string) ge.cachedLabels = make(map[string]string) return nil } @@ -74,7 +74,7 @@ func (ge *githubExporter) allowOrigin(origin string) bool { return false } -// getIdentityClient return an identity github api v4 client +// getIdentityClient return a githubv4 API client configured with the access token of the given identity. // if no client were found it will initialize it from the known tokens map and cache it for next use func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { client, ok := ge.identityClient[id] @@ -97,31 +97,29 @@ func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) } // ExportAll export all event made by the current user to Github -func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) <-chan core.ExportResult { +func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { out := make(chan core.ExportResult) - go func(out chan<- core.ExportResult) { - defer close(out) + user, err := repo.GetUserIdentity() + if err != nil { + return nil, err + } - user, err := repo.GetUserIdentity() - if err != nil { - out <- core.NewExportError(err, "") - return - } + ge.identityToken[user.Id()] = ge.conf[keyToken] - ge.identityToken[user.Id()] = ge.conf[keyToken] + // get repository node id + ge.repositoryID, err = getRepositoryNodeID( + ge.conf[keyOwner], + ge.conf[keyProject], + ge.conf[keyToken], + ) - // get repository node id - ge.repositoryID, err = getRepositoryNodeID( - ge.conf[keyOwner], - ge.conf[keyProject], - ge.conf[keyToken], - ) + if err != nil { + return nil, err + } - if err != nil { - out <- core.NewExportError(err, ge.repositoryID) - return - } + go func() { + defer close(out) var allIdentitiesIds []string for id := range ge.identityToken { @@ -140,6 +138,7 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) <-ch snapshot := b.Snapshot() // ignore issues created before since date + // TODO: compare the Lamport time instead of using the unix time if snapshot.CreatedAt.Before(since) { out <- core.NewExportNothing(b.Id(), "bug created before the since date") continue @@ -152,9 +151,9 @@ func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) <-ch out <- core.NewExportNothing(id, "not an actor") } } - }(out) + }() - return out + return out, nil } // exportBug publish bugs and related events @@ -176,7 +175,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan // skip bug if origin is not allowed origin, ok := createOp.GetMetadata(keyOrigin) if ok && !ge.allowOrigin(origin) { - out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue taged with origin: %s", origin)) + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) return } @@ -186,7 +185,8 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan githubURL, ok := createOp.GetMetadata(keyGithubUrl) if !ok { // if we find github ID, github URL must be found too - panic("expected to find github issue URL") + err := fmt.Errorf("expected to find github issue URL") + out <- core.NewExportError(err, b.Id()) } out <- core.NewExportNothing(b.Id(), "bug already exported") @@ -199,9 +199,6 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan client, err := ge.getIdentityClient(author.Id()) if err != nil { // if bug is still not exported and we do not have the author stop the execution - - // fmt.Println("warning: skipping issue due to missing token for bug creator") - // this is not an error, don't export bug out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) return } @@ -252,7 +249,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan bugCreationHash = hash.String() // cache operation github id - ge.cachedIDs[bugCreationHash] = bugGithubID + ge.cachedOperationIDs[bugCreationHash] = bugGithubID for _, op := range snapshot.Operations[1:] { // ignore SetMetadata operations @@ -268,10 +265,10 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan return } - // ignore imported (or exported) operations from github + // ignore operations already existing in github (due to import or export) // cache the ID of already exported or imported issues and events from Github if id, ok := op.GetMetadata(keyGithubId); ok { - ge.cachedIDs[hash.String()] = id + ge.cachedOperationIDs[hash.String()] = id out <- core.NewExportNothing(hash.String(), "already exported operation") continue } @@ -299,7 +296,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan out <- core.NewExportComment(hash.String()) // cache comment id - ge.cachedIDs[hash.String()] = id + ge.cachedOperationIDs[hash.String()] = id case *bug.EditCommentOperation: @@ -324,7 +321,7 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan } else { // case comment edition operation: we need to edit the Github comment - commentID, ok := ge.cachedIDs[targetHash] + commentID, ok := ge.cachedOperationIDs[targetHash] if !ok { panic("unexpected error: comment id not found") } @@ -424,7 +421,7 @@ func getRepositoryNodeID(owner, project, token string) (string, error) { } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("error retrieving repository node id %v", resp.StatusCode) + return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode) } aux := struct { @@ -668,9 +665,15 @@ func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) m := &updateIssueMutation{} // set state - state := githubv4.IssueStateClosed - if status == bug.OpenStatus { + var state githubv4.IssueState + + switch status { + case bug.OpenStatus: + state = githubv4.IssueStateOpen + case bug.ClosedStatus: state = githubv4.IssueStateOpen + default: + panic("unknown bug state") } input := githubv4.UpdateIssueInput{ -- cgit From 3e181168eadb1d6a13e67583c16219200d052fcc Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 5 Jul 2019 20:55:24 +0200 Subject: [bridge/github] improve export label change function [bridge/core] rename `EventStatus` to `ExportEvent` --- bridge/github/export.go | 91 +++++++++++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 34 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index 5e5b1a21..c4ca42da 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -7,6 +7,8 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" + "sync" "time" "github.com/pkg/errors" @@ -671,7 +673,7 @@ func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) case bug.OpenStatus: state = githubv4.IssueStateOpen case bug.ClosedStatus: - state = githubv4.IssueStateOpen + state = githubv4.IssueStateClosed default: panic("unknown bug state") } @@ -730,49 +732,70 @@ func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { // update github issue labels func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { - addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) - if err != nil { - return errors.Wrap(err, "getting added labels ids") - } - - m := &addLabelsToLabelableMutation{} - inputAdd := githubv4.AddLabelsToLabelableInput{ - LabelableID: labelableID, - LabelIDs: addedIDs, - } + var errs []string + var wg sync.WaitGroup parentCtx := context.Background() - ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) - // add labels - if err := gc.Mutate(ctx, m, inputAdd, nil); err != nil { - cancel() - return err - } - cancel() + if len(added) > 0 { + wg.Add(1) + go func() { + defer wg.Done() - if len(removed) == 0 { - return nil - } + addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) + if err != nil { + errs = append(errs, errors.Wrap(err, "getting added labels ids").Error()) + return + } - removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) - if err != nil { - return errors.Wrap(err, "getting added labels ids") + m := &addLabelsToLabelableMutation{} + inputAdd := githubv4.AddLabelsToLabelableInput{ + LabelableID: labelableID, + LabelIDs: addedIDs, + } + + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + // add labels + if err := gc.Mutate(ctx, m, inputAdd, nil); err != nil { + errs = append(errs, err.Error()) + } + }() } - m2 := &removeLabelsFromLabelableMutation{} - inputRemove := githubv4.RemoveLabelsFromLabelableInput{ - LabelableID: labelableID, - LabelIDs: removedIDs, + if len(removed) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + + removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) + if err != nil { + errs = append(errs, errors.Wrap(err, "getting added labels ids").Error()) + return + } + + m2 := &removeLabelsFromLabelableMutation{} + inputRemove := githubv4.RemoveLabelsFromLabelableInput{ + LabelableID: labelableID, + LabelIDs: removedIDs, + } + + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + // remove label labels + if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { + errs = append(errs, err.Error()) + } + }() } - ctx, cancel = context.WithTimeout(parentCtx, defaultTimeout) - defer cancel() + wg.Wait() - // remove label labels - if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { - return err + if len(errs) == 0 { + return nil } - return nil + return fmt.Errorf("label change error: %v", strings.Join(errs, "\n")) } -- cgit From 6323c7b10663024f03d6c8c641852c702a5d8a04 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sat, 6 Jul 2019 00:23:19 +0200 Subject: [bridge/github] ignore imported bugs from other repositories --- bridge/github/export.go | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) (limited to 'bridge/github/export.go') diff --git a/bridge/github/export.go b/bridge/github/export.go index c4ca42da..b4351bdb 100644 --- a/bridge/github/export.go +++ b/bridge/github/export.go @@ -28,9 +28,6 @@ var ( type githubExporter struct { conf core.Configuration - // export only bugs tagged with one of these origins - onlyOrigins []string - // cache identities clients identityClient map[string]*githubv4.Client @@ -59,23 +56,6 @@ func (ge *githubExporter) Init(conf core.Configuration) error { return nil } -// allowOrigin verify that origin is allowed to get exported. -// if the exporter was initialized with no specified origins, it will return -// true for all origins -func (ge *githubExporter) allowOrigin(origin string) bool { - if len(ge.onlyOrigins) == 0 { - return true - } - - for _, o := range ge.onlyOrigins { - if origin == o { - return true - } - } - - return false -} - // getIdentityClient return a githubv4 API client configured with the access token of the given identity. // if no client were found it will initialize it from the known tokens map and cache it for next use func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { @@ -175,22 +155,36 @@ func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan author := snapshot.Author // skip bug if origin is not allowed - origin, ok := createOp.GetMetadata(keyOrigin) - if ok && !ge.allowOrigin(origin) { + origin, ok := snapshot.GetCreateMetadata(keyOrigin) + if ok && origin != target { out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) return } // get github bug ID - githubID, ok := createOp.GetMetadata(keyGithubId) + githubID, ok := snapshot.GetCreateMetadata(keyGithubId) if ok { - githubURL, ok := createOp.GetMetadata(keyGithubUrl) + githubURL, ok := snapshot.GetCreateMetadata(keyGithubUrl) if !ok { // if we find github ID, github URL must be found too err := fmt.Errorf("expected to find github issue URL") out <- core.NewExportError(err, b.Id()) } + // extract owner and project + owner, project, err := splitURL(githubURL) + if err != nil { + err := fmt.Errorf("bad project url: %v", err) + out <- core.NewExportError(err, b.Id()) + return + } + + // ignore issue comming from other repositories + if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] { + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL)) + return + } + out <- core.NewExportNothing(b.Id(), "bug already exported") // will be used to mark operation related to a bug as exported bugGithubID = githubID -- cgit