diff options
Diffstat (limited to 'bridge')
-rw-r--r-- | bridge/core/bridge.go | 8 | ||||
-rw-r--r-- | bridge/core/export.go | 101 | ||||
-rw-r--r-- | bridge/core/interfaces.go | 2 | ||||
-rw-r--r-- | bridge/github/export.go | 795 | ||||
-rw-r--r-- | bridge/github/export_mutation.go | 74 | ||||
-rw-r--r-- | bridge/github/export_test.go | 340 | ||||
-rw-r--r-- | bridge/github/github.go | 4 | ||||
-rw-r--r-- | bridge/github/import.go | 13 | ||||
-rw-r--r-- | bridge/github/import_query.go | 11 | ||||
-rw-r--r-- | bridge/github/import_test.go | 2 | ||||
-rw-r--r-- | bridge/github/iterator.go | 27 |
11 files changed, 1359 insertions, 18 deletions
diff --git a/bridge/core/bridge.go b/bridge/core/bridge.go index 1b960e0e..d04e88e4 100644 --- a/bridge/core/bridge.go +++ b/bridge/core/bridge.go @@ -297,20 +297,20 @@ func (b *Bridge) ImportAll(since time.Time) error { return importer.ImportAll(b.repo, since) } -func (b *Bridge) ExportAll(since time.Time) error { +func (b *Bridge) ExportAll(since time.Time) (<-chan ExportResult, error) { exporter := b.getExporter() if exporter == nil { - return ErrExportNotSupported + return nil, ErrExportNotSupported } err := b.ensureConfig() if err != nil { - return err + return nil, err } err = b.ensureInit() if err != nil { - return err + return nil, err } return exporter.ExportAll(b.repo, since) diff --git a/bridge/core/export.go b/bridge/core/export.go new file mode 100644 index 00000000..080ced80 --- /dev/null +++ b/bridge/core/export.go @@ -0,0 +1,101 @@ +package core + +import "fmt" + +type ExportEvent int + +const ( + _ ExportEvent = iota + ExportEventBug + ExportEventComment + ExportEventCommentEdition + ExportEventStatusChange + ExportEventTitleEdition + ExportEventLabelChange + ExportEventNothing +) + +type ExportResult struct { + Err error + Event ExportEvent + ID string + Reason string +} + +func (er ExportResult) String() string { + switch er.Event { + case ExportEventBug: + return "new issue" + case ExportEventComment: + return "new comment" + case ExportEventCommentEdition: + return "updated comment" + case ExportEventStatusChange: + return "changed status" + case ExportEventTitleEdition: + return "changed title" + case ExportEventLabelChange: + return "changed label" + case ExportEventNothing: + return fmt.Sprintf("no event: %v", er.Reason) + default: + panic("unknown export result") + } +} + +func NewExportError(err error, reason string) ExportResult { + return ExportResult{ + Err: err, + Reason: reason, + } +} + +func NewExportNothing(id string, reason string) ExportResult { + return ExportResult{ + ID: id, + Reason: reason, + Event: ExportEventNothing, + } +} + +func NewExportBug(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventBug, + } +} + +func NewExportComment(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventComment, + } +} + +func NewExportCommentEdition(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventCommentEdition, + } +} + +func NewExportStatusChange(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventStatusChange, + } +} + +func NewExportLabelChange(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventLabelChange, + } +} + +func NewExportTitleEdition(id string) ExportResult { + return ExportResult{ + ID: id, + Event: ExportEventTitleEdition, + } +} diff --git a/bridge/core/interfaces.go b/bridge/core/interfaces.go index 37fdb3d7..76d66fb4 100644 --- a/bridge/core/interfaces.go +++ b/bridge/core/interfaces.go @@ -34,5 +34,5 @@ type Importer interface { type Exporter interface { Init(conf Configuration) error - ExportAll(repo *cache.RepoCache, since time.Time) error + ExportAll(repo *cache.RepoCache, since time.Time) (<-chan ExportResult, error) } diff --git a/bridge/github/export.go b/bridge/github/export.go new file mode 100644 index 00000000..b4351bdb --- /dev/null +++ b/bridge/github/export.go @@ -0,0 +1,795 @@ +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/shurcooL/githubv4" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/util/git" +) + +var ( + ErrMissingIdentityToken = errors.New("missing identity token") +) + +// githubExporter implement the Exporter interface +type githubExporter struct { + conf core.Configuration + + // cache identities clients + identityClient map[string]*githubv4.Client + + // map identities with their tokens + identityToken map[string]string + + // github repository ID + repositoryID string + + // cache identifiers used to speed up exporting operations + // cleared for each bug + cachedOperationIDs map[string]string + + // cache labels used to speed up exporting labels events + cachedLabels map[string]string +} + +// Init . +func (ge *githubExporter) Init(conf core.Configuration) error { + ge.conf = conf + //TODO: initialize with multiple tokens + ge.identityToken = make(map[string]string) + ge.identityClient = make(map[string]*githubv4.Client) + ge.cachedOperationIDs = make(map[string]string) + ge.cachedLabels = make(map[string]string) + return nil +} + +// getIdentityClient return a githubv4 API client configured with the access token of the given identity. +// if no client were found it will initialize it from the known tokens map and cache it for next use +func (ge *githubExporter) getIdentityClient(id string) (*githubv4.Client, error) { + client, ok := ge.identityClient[id] + if ok { + return client, nil + } + + // get token + token, ok := ge.identityToken[id] + if !ok { + return nil, ErrMissingIdentityToken + } + + // create client + client = buildClient(token) + // cache client + ge.identityClient[id] = client + + return client, nil +} + +// ExportAll export all event made by the current user to Github +func (ge *githubExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { + out := make(chan core.ExportResult) + + user, err := repo.GetUserIdentity() + if err != nil { + return nil, err + } + + ge.identityToken[user.Id()] = ge.conf[keyToken] + + // get repository node id + ge.repositoryID, err = getRepositoryNodeID( + ge.conf[keyOwner], + ge.conf[keyProject], + ge.conf[keyToken], + ) + + if err != nil { + return nil, err + } + + go func() { + defer close(out) + + var allIdentitiesIds []string + for id := range ge.identityToken { + allIdentitiesIds = append(allIdentitiesIds, id) + } + + allBugsIds := repo.AllBugsIds() + + for _, id := range allBugsIds { + b, err := repo.ResolveBug(id) + if err != nil { + out <- core.NewExportError(err, id) + return + } + + snapshot := b.Snapshot() + + // ignore issues created before since date + // TODO: compare the Lamport time instead of using the unix time + if snapshot.CreatedAt.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } + + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + ge.exportBug(b, since, out) + } else { + out <- core.NewExportNothing(id, "not an actor") + } + } + }() + + return out, nil +} + +// exportBug publish bugs and related events +func (ge *githubExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { + snapshot := b.Snapshot() + + var bugGithubID string + var bugGithubURL string + var bugCreationHash string + + // Special case: + // if a user try to export a bug that is not already exported to Github (or imported + // from Github) and we do not have the token of the bug author, there is nothing we can do. + + // first operation is always createOp + createOp := snapshot.Operations[0].(*bug.CreateOperation) + author := snapshot.Author + + // skip bug if origin is not allowed + origin, ok := snapshot.GetCreateMetadata(keyOrigin) + if ok && origin != target { + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) + return + } + + // get github bug ID + githubID, ok := snapshot.GetCreateMetadata(keyGithubId) + if ok { + githubURL, ok := snapshot.GetCreateMetadata(keyGithubUrl) + if !ok { + // if we find github ID, github URL must be found too + err := fmt.Errorf("expected to find github issue URL") + out <- core.NewExportError(err, b.Id()) + } + + // extract owner and project + owner, project, err := splitURL(githubURL) + if err != nil { + err := fmt.Errorf("bad project url: %v", err) + out <- core.NewExportError(err, b.Id()) + return + } + + // ignore issue comming from other repositories + if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] { + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL)) + return + } + + out <- core.NewExportNothing(b.Id(), "bug already exported") + // will be used to mark operation related to a bug as exported + bugGithubID = githubID + bugGithubURL = githubURL + + } else { + // check that we have a token for operation author + client, err := ge.getIdentityClient(author.Id()) + if err != nil { + // if bug is still not exported and we do not have the author stop the execution + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) + return + } + + // create bug + id, url, err := createGithubIssue(client, ge.repositoryID, createOp.Title, createOp.Message) + if err != nil { + err := errors.Wrap(err, "exporting github issue") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportBug(b.Id()) + + hash, err := createOp.Hash() + if err != nil { + err := errors.Wrap(err, "comment hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // mark bug creation operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit operation to avoid creating multiple issues with multiple pushes + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + + // cache bug github ID and URL + bugGithubID = id + bugGithubURL = url + } + + // get createOp hash + hash, err := createOp.Hash() + if err != nil { + out <- core.NewExportError(err, b.Id()) + return + } + + bugCreationHash = hash.String() + + // cache operation github id + ge.cachedOperationIDs[bugCreationHash] = bugGithubID + + for _, op := range snapshot.Operations[1:] { + // ignore SetMetadata operations + if _, ok := op.(*bug.SetMetadataOperation); ok { + continue + } + + // get operation hash + hash, err := op.Hash() + if err != nil { + err := errors.Wrap(err, "operation hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // ignore operations already existing in github (due to import or export) + // cache the ID of already exported or imported issues and events from Github + if id, ok := op.GetMetadata(keyGithubId); ok { + ge.cachedOperationIDs[hash.String()] = id + out <- core.NewExportNothing(hash.String(), "already exported operation") + continue + } + + opAuthor := op.GetAuthor() + client, err := ge.getIdentityClient(opAuthor.Id()) + if err != nil { + out <- core.NewExportNothing(hash.String(), "missing operation author token") + continue + } + + var id, url string + switch op.(type) { + case *bug.AddCommentOperation: + opr := op.(*bug.AddCommentOperation) + + // send operation to github + id, url, err = addCommentGithubIssue(client, bugGithubID, opr.Message) + if err != nil { + err := errors.Wrap(err, "adding comment") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportComment(hash.String()) + + // cache comment id + ge.cachedOperationIDs[hash.String()] = id + + case *bug.EditCommentOperation: + + opr := op.(*bug.EditCommentOperation) + targetHash := opr.Target.String() + + // Since github doesn't consider the issue body as a comment + if targetHash == bugCreationHash { + + // case bug creation operation: we need to edit the Github issue + if err := updateGithubIssueBody(client, bugGithubID, opr.Message); err != nil { + err := errors.Wrap(err, "editing issue") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + id = bugGithubID + url = bugGithubURL + + } else { + + // case comment edition operation: we need to edit the Github comment + commentID, ok := ge.cachedOperationIDs[targetHash] + if !ok { + panic("unexpected error: comment id not found") + } + + eid, eurl, err := editCommentGithubIssue(client, commentID, opr.Message) + if err != nil { + err := errors.Wrap(err, "editing comment") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + // use comment id/url instead of issue id/url + id = eid + url = eurl + } + + case *bug.SetStatusOperation: + opr := op.(*bug.SetStatusOperation) + if err := updateGithubIssueStatus(client, bugGithubID, opr.Status); err != nil { + err := errors.Wrap(err, "editing status") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportStatusChange(hash.String()) + + id = bugGithubID + url = bugGithubURL + + case *bug.SetTitleOperation: + opr := op.(*bug.SetTitleOperation) + if err := updateGithubIssueTitle(client, bugGithubID, opr.Title); err != nil { + err := errors.Wrap(err, "editing title") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportTitleEdition(hash.String()) + + id = bugGithubID + url = bugGithubURL + + case *bug.LabelChangeOperation: + opr := op.(*bug.LabelChangeOperation) + if err := ge.updateGithubIssueLabels(client, bugGithubID, opr.Added, opr.Removed); err != nil { + err := errors.Wrap(err, "updating labels") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportLabelChange(hash.String()) + + id = bugGithubID + url = bugGithubURL + + default: + panic("unhandled operation type case") + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit at each operation export to avoid exporting same events multiple times + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + } +} + +// getRepositoryNodeID request github api v3 to get repository node id +func getRepositoryNodeID(owner, project, token string) (string, error) { + url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode) + } + + aux := struct { + NodeID string `json:"node_id"` + }{} + + data, _ := ioutil.ReadAll(resp.Body) + err = resp.Body.Close() + if err != nil { + return "", err + } + + err = json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + return aux.NodeID, nil +} + +func markOperationAsExported(b *cache.BugCache, target git.Hash, githubID, githubURL string) error { + _, err := b.SetMetadata( + target, + map[string]string{ + keyGithubId: githubID, + keyGithubUrl: githubURL, + }, + ) + + return err +} + +// get label from github +func (ge *githubExporter) getGithubLabelID(gc *githubv4.Client, label string) (string, error) { + q := &labelQuery{} + variables := map[string]interface{}{ + "label": githubv4.String(label), + "owner": githubv4.String(ge.conf[keyOwner]), + "name": githubv4.String(ge.conf[keyProject]), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Query(ctx, q, variables); err != nil { + return "", err + } + + // if label id is empty, it means there is no such label in this Github repository + if q.Repository.Label.ID == "" { + return "", fmt.Errorf("label not found") + } + + return q.Repository.Label.ID, nil +} + +// create a new label and return it github id +// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels +// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview +func (ge *githubExporter) createGithubLabel(label, color string) (string, error) { + url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject]) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + params := struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + }{ + Name: label, + Color: color, + } + + data, err := json.Marshal(params) + if err != nil { + return "", err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return "", err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[keyToken])) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode) + } + + aux := struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Color string `json:"color"` + }{} + + data, _ = ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + + err = json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + return aux.NodeID, nil +} + +/** +// create github label using api v4 +func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) { + m := createLabelMutation{} + input := createLabelInput{ + RepositoryID: ge.repositoryID, + Name: githubv4.String(label), + Color: githubv4.String(labelColor), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, &m, input, nil); err != nil { + return "", err + } + + return m.CreateLabel.Label.ID, nil +} +*/ + +func (ge *githubExporter) getOrCreateGithubLabelID(gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) { + // try to get label id + labelID, err := ge.getGithubLabelID(gc, string(label)) + if err == nil { + return labelID, nil + } + + // RGBA to hex color + rgba := label.RGBA() + hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + + labelID, err = ge.createGithubLabel(string(label), hexColor) + if err != nil { + return "", err + } + + return labelID, nil +} + +func (ge *githubExporter) getLabelsIDs(gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) { + ids := make([]githubv4.ID, 0, len(labels)) + var err error + + // check labels ids + for _, label := range labels { + id, ok := ge.cachedLabels[string(label)] + if !ok { + // try to query label id + id, err = ge.getOrCreateGithubLabelID(gc, repositoryID, label) + if err != nil { + return nil, errors.Wrap(err, "get or create github label") + } + + // cache label id + ge.cachedLabels[string(label)] = id + } + + ids = append(ids, githubv4.ID(id)) + } + + return ids, nil +} + +// create a github issue and return it ID +func createGithubIssue(gc *githubv4.Client, repositoryID, title, body string) (string, string, error) { + m := &createIssueMutation{} + input := githubv4.CreateIssueInput{ + RepositoryID: repositoryID, + Title: githubv4.String(title), + Body: (*githubv4.String)(&body), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return "", "", err + } + + issue := m.CreateIssue.Issue + return issue.ID, issue.URL, nil +} + +// add a comment to an issue and return it ID +func addCommentGithubIssue(gc *githubv4.Client, subjectID string, body string) (string, string, error) { + m := &addCommentToIssueMutation{} + input := githubv4.AddCommentInput{ + SubjectID: subjectID, + Body: githubv4.String(body), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return "", "", err + } + + node := m.AddComment.CommentEdge.Node + return node.ID, node.URL, nil +} + +func editCommentGithubIssue(gc *githubv4.Client, commentID, body string) (string, string, error) { + m := &updateIssueCommentMutation{} + input := githubv4.UpdateIssueCommentInput{ + ID: commentID, + Body: githubv4.String(body), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return "", "", err + } + + return commentID, m.UpdateIssueComment.IssueComment.URL, nil +} + +func updateGithubIssueStatus(gc *githubv4.Client, id string, status bug.Status) error { + m := &updateIssueMutation{} + + // set state + var state githubv4.IssueState + + switch status { + case bug.OpenStatus: + state = githubv4.IssueStateOpen + case bug.ClosedStatus: + state = githubv4.IssueStateClosed + default: + panic("unknown bug state") + } + + input := githubv4.UpdateIssueInput{ + ID: id, + State: &state, + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return err + } + + return nil +} + +func updateGithubIssueBody(gc *githubv4.Client, id string, body string) error { + m := &updateIssueMutation{} + input := githubv4.UpdateIssueInput{ + ID: id, + Body: (*githubv4.String)(&body), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return err + } + + return nil +} + +func updateGithubIssueTitle(gc *githubv4.Client, id, title string) error { + m := &updateIssueMutation{} + input := githubv4.UpdateIssueInput{ + ID: id, + Title: (*githubv4.String)(&title), + } + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := gc.Mutate(ctx, m, input, nil); err != nil { + return err + } + + return nil +} + +// update github issue labels +func (ge *githubExporter) updateGithubIssueLabels(gc *githubv4.Client, labelableID string, added, removed []bug.Label) error { + var errs []string + var wg sync.WaitGroup + + parentCtx := context.Background() + + if len(added) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + + addedIDs, err := ge.getLabelsIDs(gc, labelableID, added) + if err != nil { + errs = append(errs, errors.Wrap(err, "getting added labels ids").Error()) + return + } + + m := &addLabelsToLabelableMutation{} + inputAdd := githubv4.AddLabelsToLabelableInput{ + LabelableID: labelableID, + LabelIDs: addedIDs, + } + + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + // add labels + if err := gc.Mutate(ctx, m, inputAdd, nil); err != nil { + errs = append(errs, err.Error()) + } + }() + } + + if len(removed) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + + removedIDs, err := ge.getLabelsIDs(gc, labelableID, removed) + if err != nil { + errs = append(errs, errors.Wrap(err, "getting added labels ids").Error()) + return + } + + m2 := &removeLabelsFromLabelableMutation{} + inputRemove := githubv4.RemoveLabelsFromLabelableInput{ + LabelableID: labelableID, + LabelIDs: removedIDs, + } + + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + // remove label labels + if err := gc.Mutate(ctx, m2, inputRemove, nil); err != nil { + errs = append(errs, err.Error()) + } + }() + } + + wg.Wait() + + if len(errs) == 0 { + return nil + } + + return fmt.Errorf("label change error: %v", strings.Join(errs, "\n")) +} diff --git a/bridge/github/export_mutation.go b/bridge/github/export_mutation.go new file mode 100644 index 00000000..cf77f344 --- /dev/null +++ b/bridge/github/export_mutation.go @@ -0,0 +1,74 @@ +package github + +type createIssueMutation struct { + CreateIssue struct { + Issue struct { + ID string `graphql:"id"` + URL string `graphql:"url"` + } + } `graphql:"createIssue(input:$input)"` +} + +type updateIssueMutation struct { + UpdateIssue struct { + Issue struct { + ID string `graphql:"id"` + URL string `graphql:"url"` + } + } `graphql:"updateIssue(input:$input)"` +} + +type addCommentToIssueMutation struct { + AddComment struct { + CommentEdge struct { + Node struct { + ID string `graphql:"id"` + URL string `graphql:"url"` + } + } + } `graphql:"addComment(input:$input)"` +} + +type updateIssueCommentMutation struct { + UpdateIssueComment struct { + IssueComment struct { + ID string `graphql:"id"` + URL string `graphql:"url"` + } `graphql:"issueComment"` + } `graphql:"updateIssueComment(input:$input)"` +} + +type removeLabelsFromLabelableMutation struct { + AddLabels struct { + Labelable struct { + Typename string `graphql:"__typename"` + } + } `graphql:"removeLabelsFromLabelable(input:$input)"` +} + +type addLabelsToLabelableMutation struct { + RemoveLabels struct { + Labelable struct { + Typename string `graphql:"__typename"` + } + } `graphql:"addLabelsToLabelable(input:$input)"` +} + +/** +type createLabelMutation struct { + CreateLabel struct { + Label struct { + ID string `graphql:"id"` + } `graphql:"label"` + } `graphql:"createLabel(input: $input)"` +} + +type createLabelInput struct { + Color githubv4.String `json:"color"` + Description *githubv4.String `json:"description,omitempty"` + Name githubv4.String `json:"name"` + RepositoryID githubv4.ID `json:"repositoryId"` + + ClientMutationID *githubv4.String `json:"clientMutationId,omitempty"` +} +*/ diff --git a/bridge/github/export_test.go b/bridge/github/export_test.go new file mode 100644 index 00000000..80660e77 --- /dev/null +++ b/bridge/github/export_test.go @@ -0,0 +1,340 @@ +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/interrupt" +) + +const ( + testRepoBaseName = "git-bug-test-github-exporter" +) + +type testCase struct { + name string + bug *cache.BugCache + numOrOp int // number of original operations +} + +func testCases(t *testing.T, repo *cache.RepoCache, identity *cache.IdentityCache) []*testCase { + // simple bug + simpleBug, _, err := repo.NewBug("simple bug", "new bug") + require.NoError(t, err) + + // bug with comments + bugWithComments, _, err := repo.NewBug("bug with comments", "new bug") + require.NoError(t, err) + + _, err = bugWithComments.AddComment("new comment") + require.NoError(t, err) + + // bug with label changes + bugLabelChange, _, err := repo.NewBug("bug label change", "new bug") + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"bug"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"core"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels(nil, []string{"bug"}) + require.NoError(t, err) + + // bug with comments editions + bugWithCommentEditions, createOp, err := repo.NewBug("bug with comments editions", "new bug") + require.NoError(t, err) + + createOpHash, err := createOp.Hash() + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(createOpHash, "first comment edited") + require.NoError(t, err) + + commentOp, err := bugWithCommentEditions.AddComment("first comment") + require.NoError(t, err) + + commentOpHash, err := commentOp.Hash() + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(commentOpHash, "first comment edited") + require.NoError(t, err) + + // bug status changed + bugStatusChanged, _, err := repo.NewBug("bug status changed", "new bug") + require.NoError(t, err) + + _, err = bugStatusChanged.Close() + require.NoError(t, err) + + _, err = bugStatusChanged.Open() + require.NoError(t, err) + + // bug title changed + bugTitleEdited, _, err := repo.NewBug("bug title edited", "new bug") + require.NoError(t, err) + + _, err = bugTitleEdited.SetTitle("bug title edited again") + require.NoError(t, err) + + return []*testCase{ + &testCase{ + name: "simple bug", + bug: simpleBug, + numOrOp: 1, + }, + &testCase{ + name: "bug with comments", + bug: bugWithComments, + numOrOp: 2, + }, + &testCase{ + name: "bug label change", + bug: bugLabelChange, + numOrOp: 4, + }, + &testCase{ + name: "bug with comment editions", + bug: bugWithCommentEditions, + numOrOp: 4, + }, + &testCase{ + name: "bug changed status", + bug: bugStatusChanged, + numOrOp: 3, + }, + &testCase{ + name: "bug title edited", + bug: bugTitleEdited, + numOrOp: 2, + }, + } +} + +func TestPushPull(t *testing.T) { + // repo owner + user := os.Getenv("GITHUB_TEST_USER") + + // token must have 'repo' and 'delete_repo' scopes + token := os.Getenv("GITHUB_TOKEN_ADMIN") + if token == "" { + t.Skip("Env var GITHUB_TOKEN_ADMIN missing") + } + + // create repo backend + repo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repo) + + backend, err := cache.NewRepoCache(repo) + require.NoError(t, err) + + // set author identity + author, err := backend.NewIdentity("test identity", "test@test.org") + require.NoError(t, err) + + err = backend.SetUserIdentity(author) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + tests := testCases(t, backend, author) + + // generate project name + projectName := generateRepoName() + + // create target Github repository + err = createRepository(projectName, token) + require.NoError(t, err) + + fmt.Println("created repository", projectName) + + // Make sure to remove the Github repository when the test end + defer func(t *testing.T) { + if err := deleteRepository(projectName, user, token); err != nil { + t.Fatal(err) + } + fmt.Println("deleted repository:", projectName) + }(t) + + interrupt.RegisterCleaner(func() error { + return deleteRepository(projectName, user, token) + }) + + // initialize exporter + exporter := &githubExporter{} + err = exporter.Init(core.Configuration{ + keyOwner: user, + keyProject: projectName, + keyToken: token, + }) + require.NoError(t, err) + + start := time.Now() + + // export all bugs + events, err := exporter.ExportAll(backend, time.Time{}) + require.NoError(t, err) + + for result := range events { + require.NoError(t, result.Err) + } + require.NoError(t, err) + + fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds()) + + repoTwo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repoTwo) + + // create a second backend + backendTwo, err := cache.NewRepoCache(repoTwo) + require.NoError(t, err) + + importer := &githubImporter{} + err = importer.Init(core.Configuration{ + keyOwner: user, + keyProject: projectName, + keyToken: token, + }) + require.NoError(t, err) + + // import all exported bugs to the second backend + err = importer.ImportAll(backendTwo, time.Time{}) + require.NoError(t, err) + + require.Len(t, backendTwo.AllBugsIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // for each operation a SetMetadataOperation will be added + // so number of operations should double + require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2) + + // verify operation have correct metadata + for _, op := range tt.bug.Snapshot().Operations { + // Check if the originals operations (*not* SetMetadata) are tagged properly + if _, ok := op.(*bug.SetMetadataOperation); !ok { + _, haveIDMetadata := op.GetMetadata(keyGithubId) + require.True(t, haveIDMetadata) + + _, haveURLMetada := op.GetMetadata(keyGithubUrl) + require.True(t, haveURLMetada) + } + } + + // get bug github ID + bugGithubID, ok := tt.bug.Snapshot().GetCreateMetadata(keyGithubId) + require.True(t, ok) + + // retrieve bug from backendTwo + importedBug, err := backendTwo.ResolveBugCreateMetadata(keyGithubId, bugGithubID) + require.NoError(t, err) + + // verify bug have same number of original operations + require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp) + + // verify bugs are taged with origin=github + issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(keyOrigin) + require.True(t, ok) + require.Equal(t, issueOrigin, target) + + //TODO: maybe more tests to ensure bug final state + }) + } +} + +func generateRepoName() string { + rand.Seed(time.Now().UnixNano()) + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, 8) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return fmt.Sprintf("%s-%s", testRepoBaseName, string(b)) +} + +// create repository need a token with scope 'repo' +func createRepository(project, token string) error { + // This function use the V3 Github API because repository creation is not supported yet on the V4 API. + url := fmt.Sprintf("%s/user/repos", githubV3Url) + + params := struct { + Name string `json:"name"` + Description string `json:"description"` + Private bool `json:"private"` + HasIssues bool `json:"has_issues"` + }{ + Name: project, + Description: "git-bug exporter temporary test repository", + Private: true, + HasIssues: true, + } + + data, err := json.Marshal(params) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + + return resp.Body.Close() +} + +// delete repository need a token with scope 'delete_repo' +func deleteRepository(project, owner, token string) error { + // This function use the V3 Github API because repository removal is not supported yet on the V4 API. + url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("error deleting repository") + } + + return nil +} diff --git a/bridge/github/github.go b/bridge/github/github.go index 3e717ee9..176bdd84 100644 --- a/bridge/github/github.go +++ b/bridge/github/github.go @@ -17,7 +17,7 @@ func init() { type Github struct{} func (*Github) Target() string { - return "github" + return target } func (*Github) NewImporter() core.Importer { @@ -25,7 +25,7 @@ func (*Github) NewImporter() core.Importer { } func (*Github) NewExporter() core.Exporter { - return nil + return &githubExporter{} } func buildClient(token string) *githubv4.Client { diff --git a/bridge/github/import.go b/bridge/github/import.go index 6cfd022e..c162d70e 100644 --- a/bridge/github/import.go +++ b/bridge/github/import.go @@ -16,6 +16,7 @@ import ( ) const ( + keyOrigin = "origin" keyGithubId = "github-id" keyGithubUrl = "github-url" keyGithubLogin = "github-login" @@ -113,6 +114,7 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline cleanText, nil, map[string]string{ + keyOrigin: target, keyGithubId: parseId(issue.Id), keyGithubUrl: issue.Url.String(), }) @@ -147,6 +149,7 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline cleanText, nil, map[string]string{ + keyOrigin: target, keyGithubId: parseId(issue.Id), keyGithubUrl: issue.Url.String(), }, @@ -502,7 +505,7 @@ func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, return nil, err } - var q userQuery + var q ghostQuery variables := map[string]interface{}{ "login": githubv4.String("ghost"), @@ -510,7 +513,11 @@ func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, gc := buildClient(gi.conf[keyToken]) - err = gc.Query(context.TODO(), &q, variables) + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + err = gc.Query(ctx, &q, variables) if err != nil { return nil, err } @@ -522,7 +529,7 @@ func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, return repo.NewIdentityRaw( name, - string(q.User.Email), + "", string(q.User.Login), string(q.User.AvatarUrl), map[string]string{ diff --git a/bridge/github/import_query.go b/bridge/github/import_query.go index 4d5886f6..89d7859d 100644 --- a/bridge/github/import_query.go +++ b/bridge/github/import_query.go @@ -160,11 +160,18 @@ type commentEditQuery struct { } `graphql:"repository(owner: $owner, name: $name)"` } -type userQuery struct { +type ghostQuery struct { User struct { Login githubv4.String AvatarUrl githubv4.String Name *githubv4.String - Email githubv4.String } `graphql:"user(login: $login)"` } + +type labelQuery struct { + Repository struct { + Label struct { + ID string `graphql:"id"` + } `graphql:"label(name: $label)"` + } `graphql:"repository(owner: $owner, name: $name)"` +} diff --git a/bridge/github/import_test.go b/bridge/github/import_test.go index 24356f34..5ba87993 100644 --- a/bridge/github/import_test.go +++ b/bridge/github/import_test.go @@ -190,7 +190,7 @@ func Test_Importer(t *testing.T) { assert.Equal(t, op.(*bug.EditCommentOperation).Author.Name(), ops[i].(*bug.EditCommentOperation).Author.Name()) default: - panic("Unknown operation type") + panic("unknown operation type") } } }) diff --git a/bridge/github/iterator.go b/bridge/github/iterator.go index fcf72b8f..6d63cf42 100644 --- a/bridge/github/iterator.go +++ b/bridge/github/iterator.go @@ -59,7 +59,7 @@ type iterator struct { commentEdit commentEditIterator } -// NewIterator create and initalize a new iterator +// NewIterator create and initialize a new iterator func NewIterator(owner, project, token string, since time.Time) *iterator { i := &iterator{ gc: buildClient(token), @@ -147,7 +147,11 @@ func (i *iterator) Error() error { } func (i *iterator) queryIssue() bool { - if err := i.gc.Query(context.TODO(), &i.timeline.query, i.timeline.variables); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := i.gc.Query(ctx, &i.timeline.query, i.timeline.variables); err != nil { i.err = err return false } @@ -220,7 +224,12 @@ func (i *iterator) NextTimelineItem() bool { // more timelines, query them i.timeline.variables["timelineAfter"] = i.timeline.query.Repository.Issues.Nodes[0].Timeline.PageInfo.EndCursor - if err := i.gc.Query(context.TODO(), &i.timeline.query, i.timeline.variables); err != nil { + + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := i.gc.Query(ctx, &i.timeline.query, i.timeline.variables); err != nil { i.err = err return false } @@ -236,7 +245,11 @@ func (i *iterator) TimelineItemValue() timelineItem { } func (i *iterator) queryIssueEdit() bool { - if err := i.gc.Query(context.TODO(), &i.issueEdit.query, i.issueEdit.variables); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := i.gc.Query(ctx, &i.issueEdit.query, i.issueEdit.variables); err != nil { i.err = err //i.timeline.issueEdit.index = -1 return false @@ -334,7 +347,11 @@ func (i *iterator) IssueEditValue() userContentEdit { } func (i *iterator) queryCommentEdit() bool { - if err := i.gc.Query(context.TODO(), &i.commentEdit.query, i.commentEdit.variables); err != nil { + parentCtx := context.Background() + ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout) + defer cancel() + + if err := i.gc.Query(ctx, &i.commentEdit.query, i.commentEdit.variables); err != nil { i.err = err return false } |