From 01c0f644b2b94589f4f597a90d6245d5e2c0ad17 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 9 Jul 2019 22:56:38 +0200 Subject: bridge/gitlab: init new bridge --- bridge/gitlab/gitlab.go | 28 ++++++++++++++++++++++++++++ bridge/gitlab/import.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 bridge/gitlab/gitlab.go create mode 100644 bridge/gitlab/import.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go new file mode 100644 index 00000000..538ae715 --- /dev/null +++ b/bridge/gitlab/gitlab.go @@ -0,0 +1,28 @@ +package gitlab + +import ( + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/xanzy/go-gitlab" +) + +func init() { + core.Register(&Gitlab{}) +} + +type Gitlab struct{} + +func (*Gitlab) Target() string { + return target +} + +func (*Gitlab) NewImporter() core.Importer { + return &gitlabImporter{} +} + +func (*Gitlab) NewExporter() core.Exporter { + return &gitlabExporter{} +} + +func buildClient(token string) *gitlab.Client { + return gitlab.NewClient(nil, token) +} diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go new file mode 100644 index 00000000..dec90a6c --- /dev/null +++ b/bridge/gitlab/import.go @@ -0,0 +1,30 @@ +package gitlab + +import ( + "time" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/cache" +) + +const ( + keyGitlabLogin = "gitlab-login" +) + +type gitlabImporter struct { + conf core.Configuration + + // number of imported issues + importedIssues int + + // number of imported identities + importedIdentities int +} + +func (*gitlabImporter) Init(conf core.Configuration) error { + return nil +} + +func (*gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { + return nil +} -- cgit From cfd565350873f7ee9cd2c6b4935382d1037ff34c Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 9 Jul 2019 22:58:15 +0200 Subject: bridge/gitlab: init exporter --- bridge/gitlab/export.go | 466 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 bridge/gitlab/export.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go new file mode 100644 index 00000000..6f234940 --- /dev/null +++ b/bridge/gitlab/export.go @@ -0,0 +1,466 @@ +package gitlab + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/xanzy/go-gitlab" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/util/git" +) + +var ( + ErrMissingIdentityToken = errors.New("missing identity token") +) + +const ( + keyGitlabId = "gitlab-id" + keyGitlabUrl = "gitlab-url" + keyOrigin = "origin" +) + +// gitlabExporter implement the Exporter interface +type gitlabExporter struct { + conf core.Configuration + + // cache identities clients + identityClient map[string]*gitlab.Client + + // map identities with their tokens + identityToken map[string]string + + // gitlab. repository ID + repositoryID string + + // cache identifiers used to speed up exporting operations + // cleared for each bug + cachedOperationIDs map[string]string + + // cache labels used to speed up exporting labels events + cachedLabels map[string]string +} + +// Init . +func (ge *gitlabExporter) Init(conf core.Configuration) error { + ge.conf = conf + //TODO: initialize with multiple tokens + ge.identityToken = make(map[string]string) + ge.identityClient = make(map[string]*gitlab.Client) + ge.cachedOperationIDs = make(map[string]string) + ge.cachedLabels = make(map[string]string) + return nil +} + +// getIdentityClient return a gitlab v4 API client configured with the access token of the given identity. +// if no client were found it will initialize it from the known tokens map and cache it for next use +func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { + client, ok := ge.identityClient[id] + if ok { + return client, nil + } + + // get token + token, ok := ge.identityToken[id] + if !ok { + return nil, ErrMissingIdentityToken + } + + // create client + client = buildClient(token) + // cache client + ge.identityClient[id] = client + + //client.Labels.CreateLabel() + + return client, nil +} + +// ExportAll export all event made by the current user to Gitlab +func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { + out := make(chan core.ExportResult) + + user, err := repo.GetUserIdentity() + if err != nil { + return nil, err + } + + ge.identityToken[user.Id()] = ge.conf[keyToken] + + // get repository node id + ge.repositoryID, err = getRepositoryNodeID( + "", "", + ge.conf[keyToken], + ) + + if err != nil { + return nil, err + } + + go func() { + defer close(out) + + var allIdentitiesIds []string + for id := range ge.identityToken { + allIdentitiesIds = append(allIdentitiesIds, id) + } + + allBugsIds := repo.AllBugsIds() + + for _, id := range allBugsIds { + b, err := repo.ResolveBug(id) + if err != nil { + out <- core.NewExportError(err, id) + return + } + + snapshot := b.Snapshot() + + // ignore issues created before since date + // TODO: compare the Lamport time instead of using the unix time + if snapshot.CreatedAt.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } + + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + ge.exportBug(b, since, out) + } else { + out <- core.NewExportNothing(id, "not an actor") + } + } + }() + + return out, nil +} + +// exportBug publish bugs and related events +func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { + snapshot := b.Snapshot() + + var bugGitlabID string + var bugGitlabURL string + var bugCreationHash string + + // Special case: + // if a user try to export a bug that is not already exported to Gitlab (or imported + // from Gitlab) and we do not have the token of the bug author, there is nothing we can do. + + // first operation is always createOp + createOp := snapshot.Operations[0].(*bug.CreateOperation) + author := snapshot.Author + + // skip bug if origin is not allowed + origin, ok := snapshot.GetCreateMetadata(keyOrigin) + if ok && origin != target { + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) + return + } + + // get gitlab bug ID + gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) + if ok { + gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl) + if !ok { + // if we find gitlab ID, gitlab URL must be found too + err := fmt.Errorf("expected to find gitlab issue URL") + out <- core.NewExportError(err, b.Id()) + } + + //FIXME: + // ignore issue comming from other repositories + + out <- core.NewExportNothing(b.Id(), "bug already exported") + // will be used to mark operation related to a bug as exported + bugGitlabID = gitlabID + bugGitlabURL = gitlabURL + + } else { + // check that we have a token for operation author + client, err := ge.getIdentityClient(author.Id()) + if err != nil { + // if bug is still not exported and we do not have the author stop the execution + out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) + return + } + + // create bug + id, url, err := createGitlabIssue(client, ge.repositoryID, createOp.Title, createOp.Message) + if err != nil { + err := errors.Wrap(err, "exporting gitlab issue") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportBug(b.Id()) + + hash, err := createOp.Hash() + if err != nil { + err := errors.Wrap(err, "comment hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // mark bug creation operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit operation to avoid creating multiple issues with multiple pushes + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + + // cache bug gitlab ID and URL + bugGitlabID = id + bugGitlabURL = url + } + + // get createOp hash + hash, err := createOp.Hash() + if err != nil { + out <- core.NewExportError(err, b.Id()) + return + } + + bugCreationHash = hash.String() + + // cache operation gitlab id + ge.cachedOperationIDs[bugCreationHash] = bugGitlabID + + for _, op := range snapshot.Operations[1:] { + // ignore SetMetadata operations + if _, ok := op.(*bug.SetMetadataOperation); ok { + continue + } + + // get operation hash + hash, err := op.Hash() + if err != nil { + err := errors.Wrap(err, "operation hash") + out <- core.NewExportError(err, b.Id()) + return + } + + // ignore operations already existing in gitlab (due to import or export) + // cache the ID of already exported or imported issues and events from Gitlab + if id, ok := op.GetMetadata(keyGitlabId); ok { + ge.cachedOperationIDs[hash.String()] = id + out <- core.NewExportNothing(hash.String(), "already exported operation") + continue + } + + opAuthor := op.GetAuthor() + client, err := ge.getIdentityClient(opAuthor.Id()) + if err != nil { + out <- core.NewExportNothing(hash.String(), "missing operation author token") + continue + } + + var id, url string + switch op.(type) { + case *bug.AddCommentOperation: + opr := op.(*bug.AddCommentOperation) + + // send operation to gitlab + id, url, err = addCommentGitlabIssue(client, bugGitlabID, opr.Message) + if err != nil { + err := errors.Wrap(err, "adding comment") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportComment(hash.String()) + + // cache comment id + ge.cachedOperationIDs[hash.String()] = id + + case *bug.EditCommentOperation: + + opr := op.(*bug.EditCommentOperation) + targetHash := opr.Target.String() + + // Since gitlab doesn't consider the issue body as a comment + if targetHash == bugCreationHash { + + // case bug creation operation: we need to edit the Gitlab issue + if err := updateGitlabIssueBody(client, bugGitlabID, opr.Message); err != nil { + err := errors.Wrap(err, "editing issue") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + } else { + + // case comment edition operation: we need to edit the Gitlab comment + commentID, ok := ge.cachedOperationIDs[targetHash] + if !ok { + panic("unexpected error: comment id not found") + } + + eid, eurl, err := editCommentGitlabIssue(client, commentID, opr.Message) + if err != nil { + err := errors.Wrap(err, "editing comment") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportCommentEdition(hash.String()) + + // use comment id/url instead of issue id/url + id = eid + url = eurl + } + + case *bug.SetStatusOperation: + opr := op.(*bug.SetStatusOperation) + if err := updateGitlabIssueStatus(client, bugGitlabID, opr.Status); err != nil { + err := errors.Wrap(err, "editing status") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportStatusChange(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + case *bug.SetTitleOperation: + opr := op.(*bug.SetTitleOperation) + if err := updateGitlabIssueTitle(client, bugGitlabID, opr.Title); err != nil { + err := errors.Wrap(err, "editing title") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportTitleEdition(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + case *bug.LabelChangeOperation: + opr := op.(*bug.LabelChangeOperation) + if err := ge.updateGitlabIssueLabels(client, bugGitlabID, opr.Added, opr.Removed); err != nil { + err := errors.Wrap(err, "updating labels") + out <- core.NewExportError(err, b.Id()) + return + } + + out <- core.NewExportLabelChange(hash.String()) + + id = bugGitlabID + url = bugGitlabURL + + default: + panic("unhandled operation type case") + } + + // mark operation as exported + if err := markOperationAsExported(b, hash, id, url); err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return + } + + // commit at each operation export to avoid exporting same events multiple times + if err := b.CommitAsNeeded(); err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return + } + } +} + +// getRepositoryNodeID request gitlab api v3 to get repository node id +func getRepositoryNodeID(owner, project, token string) (string, error) { + return "", nil +} + +func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { + _, err := b.SetMetadata( + target, + map[string]string{ + keyGitlabId: gitlabID, + keyGitlabUrl: gitlabURL, + }, + ) + + return err +} + +// get label from gitlab +func (ge *gitlabExporter) getGitlabLabelID(gc *gitlab.Client, label string) (string, error) { + return "", nil +} + +func (ge *gitlabExporter) createGitlabLabel(label, color string) (string, error) { + return "", nil +} + +func (ge *gitlabExporter) getOrCreateGitlabLabelID(gc *gitlab.Client, repositoryID string, label bug.Label) (string, error) { + // try to get label id + labelID, err := ge.getGitlabLabelID(gc, string(label)) + if err == nil { + return labelID, nil + } + + // RGBA to hex color + rgba := label.RGBA() + hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + + labelID, err = ge.createGitlabLabel(string(label), hexColor) + if err != nil { + return "", err + } + + return labelID, nil +} + +func (ge *gitlabExporter) getLabelsIDs(gc *gitlab.Client, repositoryID string, labels []bug.Label) ([]string, error) { + return []string{}, nil +} + +// create a gitlab. issue and return it ID +func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (string, string, error) { + return "", "", nil + +} + +// add a comment to an issue and return it ID +func addCommentGitlabIssue(gc *gitlab.Client, subjectID string, body string) (string, string, error) { + return "", "", nil +} + +func editCommentGitlabIssue(gc *gitlab.Client, commentID, body string) (string, string, error) { + return "", "", nil +} + +func updateGitlabIssueStatus(gc *gitlab.Client, id string, status bug.Status) error { + return nil +} + +func updateGitlabIssueBody(gc *gitlab.Client, id string, body string) error { + return nil +} + +func updateGitlabIssueTitle(gc *gitlab.Client, id, title string) error { + return nil +} + +// update gitlab. issue labels +func (ge *gitlabExporter) updateGitlabIssueLabels(gc *gitlab.Client, labelableID string, added, removed []bug.Label) error { + return nil +} -- cgit From a1a1d4868b7a32e90343b3c3b085fb523b20b8e2 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 9 Jul 2019 22:58:47 +0200 Subject: bridge/gitlab: add bridge configure --- bridge/gitlab/config.go | 518 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 bridge/gitlab/config.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go new file mode 100644 index 00000000..997494dd --- /dev/null +++ b/bridge/gitlab/config.go @@ -0,0 +1,518 @@ +package gitlab + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + neturl "net/url" + "os" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/pkg/errors" + "github.com/xanzy/go-gitlab" + "golang.org/x/crypto/ssh/terminal" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/repository" +) + +const ( + target = "gitlab" + gitlabV4Url = "https://gitlab.com/api/v4" + keyID = "id" + keyTarget = "target" + keyToken = "token" + + defaultTimeout = 60 * time.Second +) + +//note to my self: bridge configure --target=gitlab --url=$URL + +var ( + ErrBadProjectURL = errors.New("bad project url") +) + +func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) (core.Configuration, error) { + if params.Project != "" { + fmt.Println("warning: --project is ineffective for a gitlab bridge") + } + if params.Owner != "" { + fmt.Println("warning: --owner is ineffective for a gitlab bridge") + } + + conf := make(core.Configuration) + var err error + var url string + var token string + var projectID string + + // get project url + if params.URL != "" { + url = params.URL + + } else { + // remote suggestions + remotes, err := repo.GetRemotes() + if err != nil { + return nil, err + } + + // terminal prompt + url, err = promptURL(remotes) + if err != nil { + return nil, err + } + } + + // get user token + if params.Token != "" { + token = params.Token + } else { + token, err = promptTokenOptions(url) + if err != nil { + return nil, err + } + } + + var ok bool + // validate project url and get it ID + ok, projectID, err = validateProjectURL(url, token) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("invalid project id or wrong token scope") + } + + conf[keyID] = projectID + conf[keyToken] = token + conf[keyTarget] = target + + return conf, nil +} + +func (*Gitlab) ValidateConfig(conf core.Configuration) error { + if v, ok := conf[keyTarget]; !ok { + return fmt.Errorf("missing %s key", keyTarget) + } else if v != target { + return fmt.Errorf("unexpected target name: %v", v) + } + + if _, ok := conf[keyToken]; !ok { + return fmt.Errorf("missing %s key", keyToken) + } + + if _, ok := conf[keyID]; !ok { + return fmt.Errorf("missing %s key", keyID) + } + + return nil +} + +func requestToken(note, username, password string, scope string) (*http.Response, error) { + return requestTokenWith2FA(note, username, password, "", scope) +} + +//TODO: FIX THIS ONE +func requestTokenWith2FA(note, username, password, otpCode string, scope string) (*http.Response, error) { + url := fmt.Sprintf("%s/authorizations", gitlabV4Url) + params := struct { + Scopes []string `json:"scopes"` + Note string `json:"note"` + Fingerprint string `json:"fingerprint"` + }{ + Scopes: []string{scope}, + Note: note, + Fingerprint: randomFingerprint(), + } + + data, err := json.Marshal(params) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + req.SetBasicAuth(username, password) + req.Header.Set("Content-Type", "application/json") + + if otpCode != "" { + req.Header.Set("X-GitHub-OTP", otpCode) + } + + client := &http.Client{ + Timeout: defaultTimeout, + } + + return client.Do(req) +} + +func decodeBody(body io.ReadCloser) (string, error) { + data, _ := ioutil.ReadAll(body) + + aux := struct { + Token string `json:"token"` + }{} + + err := json.Unmarshal(data, &aux) + if err != nil { + return "", err + } + + if aux.Token == "" { + return "", fmt.Errorf("no token found in response: %s", string(data)) + } + + return aux.Token, nil +} + +func randomFingerprint() string { + // Doesn't have to be crypto secure, it's just to avoid token collision + rand.Seed(time.Now().UnixNano()) + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, 32) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func promptTokenOptions(url string) (string, error) { + for { + fmt.Println() + fmt.Println("[1]: user provided token") + fmt.Println("[2]: interactive token creation") + fmt.Print("Select option: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + fmt.Println() + if err != nil { + return "", err + } + + line = strings.TrimRight(line, "\n") + + index, err := strconv.Atoi(line) + if err != nil || (index != 1 && index != 2) { + fmt.Println("invalid input") + continue + } + + if index == 1 { + return promptToken() + } + + return loginAndRequestToken(url) + } +} + +func promptToken() (string, error) { + fmt.Println("You can generate a new token by visiting https://gitlab.com/settings/tokens.") + fmt.Println("Choose 'Generate new token' and set the necessary access scope for your repository.") + fmt.Println() + fmt.Println("The access scope depend on the type of repository.") + fmt.Println("Public:") + fmt.Println(" - 'public_repo': to be able to read public repositories") + fmt.Println("Private:") + fmt.Println(" - 'repo' : to be able to read private repositories") + fmt.Println() + + re, err := regexp.Compile(`^[a-zA-Z0-9\-]{20}`) + if err != nil { + panic("regexp compile:" + err.Error()) + } + + for { + fmt.Print("Enter token: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return "", err + } + + token := strings.TrimRight(line, "\n") + if re.MatchString(token) { + return token, nil + } + + fmt.Println("token is invalid") + } +} + +// TODO: FIX THIS ONE TOO +func loginAndRequestToken(url string) (string, error) { + + // prompt project visibility to know the token scope needed for the repository + isPublic, err := promptProjectVisibility() + if err != nil { + return "", err + } + + username, err := promptUsername() + if err != nil { + return "", err + } + + password, err := promptPassword() + if err != nil { + return "", err + } + + var scope string + //TODO: Gitlab scopes + if isPublic { + // public_repo is requested to be able to read public repositories + scope = "public_repo" + } else { + // 'repo' is request to be able to read private repositories + // /!\ token will have read/write rights on every private repository you have access to + scope = "repo" + } + + // Attempt to authenticate and create a token + + note := fmt.Sprintf("git-bug - %s/%s", url) + + resp, err := requestToken(note, username, password, scope) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + // Handle 2FA is needed + OTPHeader := resp.Header.Get("X-GitHub-OTP") + if resp.StatusCode == http.StatusUnauthorized && OTPHeader != "" { + otpCode, err := prompt2FA() + if err != nil { + return "", err + } + + resp, err = requestTokenWith2FA(note, username, password, otpCode, scope) + if err != nil { + return "", err + } + + defer resp.Body.Close() + } + + if resp.StatusCode == http.StatusCreated { + return decodeBody(resp.Body) + } + + b, _ := ioutil.ReadAll(resp.Body) + return "", fmt.Errorf("error creating token %v: %v", resp.StatusCode, string(b)) +} + +func promptUsername() (string, error) { + for { + fmt.Print("username: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return "", err + } + + line = strings.TrimRight(line, "\n") + + ok, err := validateUsername(line) + if err != nil { + return "", err + } + if ok { + return line, nil + } + + fmt.Println("invalid username") + } +} + +func promptURL(remotes map[string]string) (string, error) { + validRemotes := getValidGitlabRemoteURLs(remotes) + if len(validRemotes) > 0 { + for { + fmt.Println("\nDetected projects:") + + // print valid remote gitlab urls + for i, remote := range validRemotes { + fmt.Printf("[%d]: %v\n", i+1, remote) + } + + fmt.Printf("\n[0]: Another project\n\n") + fmt.Printf("Select option: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return "", err + } + + line = strings.TrimRight(line, "\n") + + index, err := strconv.Atoi(line) + if err != nil || (index < 0 && index >= len(validRemotes)) { + fmt.Println("invalid input") + continue + } + + // if user want to enter another project url break this loop + if index == 0 { + break + } + + return validRemotes[index-1], nil + } + } + + // manually enter gitlab url + for { + fmt.Print("Gitlab project URL: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + if err != nil { + return "", err + } + + url := strings.TrimRight(line, "\n") + if line == "" { + fmt.Println("URL is empty") + continue + } + + return url, nil + } +} + +func splitURL(url string) (string, string, error) { + cleanUrl := strings.TrimSuffix(url, ".git") + objectUrl, err := neturl.Parse(cleanUrl) + if err != nil { + return "", "", nil + } + + return fmt.Sprintf("%s%s", objectUrl.Host, objectUrl.Path), objectUrl.Path, nil +} + +func getValidGitlabRemoteURLs(remotes map[string]string) []string { + urls := make([]string, 0, len(remotes)) + for _, u := range remotes { + url, _, err := splitURL(u) + if err != nil { + continue + } + + urls = append(urls, url) + } + + return urls +} + +func validateUsername(username string) (bool, error) { + // no need for a token for this action + client := buildClient("") + + users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &username}) + if err != nil { + return false, err + } + + if len(users) == 0 { + return false, fmt.Errorf("username not found") + } else if len(users) > 1 { + return false, fmt.Errorf("found multiple matches") + } + + return users[0].Username == username, nil +} + +func validateProjectURL(url, token string) (bool, string, error) { + client := buildClient(token) + + _, projectPath, err := splitURL(url) + if err != nil { + return false, "", err + } + + project, _, err := client.Projects.GetProject(projectPath[1:], &gitlab.GetProjectOptions{}) + if err != nil { + return false, "", err + } + projectID := strconv.Itoa(project.ID) + + return true, projectID, nil +} + +func promptPassword() (string, error) { + for { + fmt.Print("password: ") + + bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) + // new line for coherent formatting, ReadPassword clip the normal new line + // entered by the user + fmt.Println() + + if err != nil { + return "", err + } + + if len(bytePassword) > 0 { + return string(bytePassword), nil + } + + fmt.Println("password is empty") + } +} + +func prompt2FA() (string, error) { + for { + fmt.Print("two-factor authentication code: ") + + byte2fa, err := terminal.ReadPassword(int(syscall.Stdin)) + fmt.Println() + if err != nil { + return "", err + } + + if len(byte2fa) > 0 { + return string(byte2fa), nil + } + + fmt.Println("code is empty") + } +} + +func promptProjectVisibility() (bool, error) { + for { + fmt.Println("[1]: public") + fmt.Println("[2]: private") + fmt.Print("repository visibility: ") + + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + fmt.Println() + if err != nil { + return false, err + } + + line = strings.TrimRight(line, "\n") + + index, err := strconv.Atoi(line) + if err != nil || (index != 0 && index != 1) { + fmt.Println("invalid input") + continue + } + + // return true for public repositories, false for private + return index == 0, nil + } +} -- cgit From 35a033c0f1f17fadc4525927aace4f4043038c7e Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 10 Jul 2019 00:41:43 +0200 Subject: bridge/gitlab: bridge project validation bridge/gitlab: token generation --- bridge/gitlab/config.go | 230 ++++++++++-------------------------------------- bridge/gitlab/export.go | 11 +++ bridge/gitlab/gitlab.go | 5 ++ 3 files changed, 61 insertions(+), 185 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index 997494dd..a6fd6622 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -2,13 +2,7 @@ package gitlab import ( "bufio" - "bytes" - "encoding/json" "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" neturl "net/url" "os" "regexp" @@ -28,15 +22,13 @@ import ( const ( target = "gitlab" gitlabV4Url = "https://gitlab.com/api/v4" - keyID = "id" + keyID = "project-id" keyTarget = "target" keyToken = "token" defaultTimeout = 60 * time.Second ) -//note to my self: bridge configure --target=gitlab --url=$URL - var ( ErrBadProjectURL = errors.New("bad project url") ) @@ -53,7 +45,6 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( var err error var url string var token string - var projectID string // get project url if params.URL != "" { @@ -85,7 +76,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( var ok bool // validate project url and get it ID - ok, projectID, err = validateProjectURL(url, token) + ok, id, err := validateProjectURL(url, token) if err != nil { return nil, err } @@ -93,7 +84,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( return nil, fmt.Errorf("invalid project id or wrong token scope") } - conf[keyID] = projectID + conf[keyID] = strconv.Itoa(id) conf[keyToken] = token conf[keyTarget] = target @@ -118,75 +109,19 @@ func (*Gitlab) ValidateConfig(conf core.Configuration) error { return nil } -func requestToken(note, username, password string, scope string) (*http.Response, error) { - return requestTokenWith2FA(note, username, password, "", scope) -} - -//TODO: FIX THIS ONE -func requestTokenWith2FA(note, username, password, otpCode string, scope string) (*http.Response, error) { - url := fmt.Sprintf("%s/authorizations", gitlabV4Url) - params := struct { - Scopes []string `json:"scopes"` - Note string `json:"note"` - Fingerprint string `json:"fingerprint"` - }{ - Scopes: []string{scope}, - Note: note, - Fingerprint: randomFingerprint(), - } - - data, err := json.Marshal(params) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) - if err != nil { - return nil, err - } - - req.SetBasicAuth(username, password) - req.Header.Set("Content-Type", "application/json") - - if otpCode != "" { - req.Header.Set("X-GitHub-OTP", otpCode) - } - - client := &http.Client{ - Timeout: defaultTimeout, - } - - return client.Do(req) -} - -func decodeBody(body io.ReadCloser) (string, error) { - data, _ := ioutil.ReadAll(body) - - aux := struct { - Token string `json:"token"` - }{} - - err := json.Unmarshal(data, &aux) +func requestToken(client *gitlab.Client, userID int, name string, scopes ...string) (string, error) { + impToken, _, err := client.Users.CreateImpersonationToken( + userID, + &gitlab.CreateImpersonationTokenOptions{ + Name: &name, + Scopes: &scopes, + }, + ) if err != nil { return "", err } - if aux.Token == "" { - return "", fmt.Errorf("no token found in response: %s", string(data)) - } - - return aux.Token, nil -} - -func randomFingerprint() string { - // Doesn't have to be crypto secure, it's just to avoid token collision - rand.Seed(time.Now().UnixNano()) - var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - b := make([]rune, 32) - for i := range b { - b[i] = letterRunes[rand.Intn(len(letterRunes))] - } - return string(b) + return impToken.Token, nil } func promptTokenOptions(url string) (string, error) { @@ -222,11 +157,7 @@ func promptToken() (string, error) { fmt.Println("You can generate a new token by visiting https://gitlab.com/settings/tokens.") fmt.Println("Choose 'Generate new token' and set the necessary access scope for your repository.") fmt.Println() - fmt.Println("The access scope depend on the type of repository.") - fmt.Println("Public:") - fmt.Println(" - 'public_repo': to be able to read public repositories") - fmt.Println("Private:") - fmt.Println(" - 'repo' : to be able to read private repositories") + fmt.Println("'api' scope access : access scope: to be able to make api calls") fmt.Println() re, err := regexp.Compile(`^[a-zA-Z0-9\-]{20}`) @@ -251,15 +182,7 @@ func promptToken() (string, error) { } } -// TODO: FIX THIS ONE TOO func loginAndRequestToken(url string) (string, error) { - - // prompt project visibility to know the token scope needed for the repository - isPublic, err := promptProjectVisibility() - if err != nil { - return "", err - } - username, err := promptUsername() if err != nil { return "", err @@ -270,50 +193,26 @@ func loginAndRequestToken(url string) (string, error) { return "", err } - var scope string - //TODO: Gitlab scopes - if isPublic { - // public_repo is requested to be able to read public repositories - scope = "public_repo" - } else { - // 'repo' is request to be able to read private repositories - // /!\ token will have read/write rights on every private repository you have access to - scope = "repo" - } - // Attempt to authenticate and create a token - note := fmt.Sprintf("git-bug - %s/%s", url) + note := fmt.Sprintf("git-bug - %s", url) - resp, err := requestToken(note, username, password, scope) + ok, id, err := validateUsername(username) if err != nil { return "", err } - - defer resp.Body.Close() - - // Handle 2FA is needed - OTPHeader := resp.Header.Get("X-GitHub-OTP") - if resp.StatusCode == http.StatusUnauthorized && OTPHeader != "" { - otpCode, err := prompt2FA() - if err != nil { - return "", err - } - - resp, err = requestTokenWith2FA(note, username, password, otpCode, scope) - if err != nil { - return "", err - } - - defer resp.Body.Close() + if !ok { + return "", fmt.Errorf("invalid username") } - if resp.StatusCode == http.StatusCreated { - return decodeBody(resp.Body) + client, err := buildClientFromUsernameAndPassword(username, password) + if err != nil { + return "", err } - b, _ := ioutil.ReadAll(resp.Body) - return "", fmt.Errorf("error creating token %v: %v", resp.StatusCode, string(b)) + fmt.Println(username, password) + + return requestToken(client, id, note, "api") } func promptUsername() (string, error) { @@ -327,7 +226,7 @@ func promptUsername() (string, error) { line = strings.TrimRight(line, "\n") - ok, err := validateUsername(line) + ok, _, err := validateUsername(line) if err != nil { return "", err } @@ -394,63 +293,67 @@ func promptURL(remotes map[string]string) (string, error) { } } -func splitURL(url string) (string, string, error) { +func getProjectPath(url string) (string, error) { + cleanUrl := strings.TrimSuffix(url, ".git") objectUrl, err := neturl.Parse(cleanUrl) if err != nil { - return "", "", nil + return "", nil } - return fmt.Sprintf("%s%s", objectUrl.Host, objectUrl.Path), objectUrl.Path, nil + return objectUrl.Path[1:], nil } func getValidGitlabRemoteURLs(remotes map[string]string) []string { urls := make([]string, 0, len(remotes)) for _, u := range remotes { - url, _, err := splitURL(u) + path, err := getProjectPath(u) if err != nil { continue } - urls = append(urls, url) + urls = append(urls, fmt.Sprintf("%s%s", "gitlab.com", path)) } return urls } -func validateUsername(username string) (bool, error) { +func validateUsername(username string) (bool, int, error) { // no need for a token for this action client := buildClient("") users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &username}) if err != nil { - return false, err + return false, 0, err } if len(users) == 0 { - return false, fmt.Errorf("username not found") + return false, 0, fmt.Errorf("username not found") } else if len(users) > 1 { - return false, fmt.Errorf("found multiple matches") + return false, 0, fmt.Errorf("found multiple matches") + } + + if users[0].Username == username { + return true, users[0].ID, nil } - return users[0].Username == username, nil + return false, 0, nil } -func validateProjectURL(url, token string) (bool, string, error) { +func validateProjectURL(url, token string) (bool, int, error) { client := buildClient(token) - _, projectPath, err := splitURL(url) + projectPath, err := getProjectPath(url) if err != nil { - return false, "", err + return false, 0, err } - project, _, err := client.Projects.GetProject(projectPath[1:], &gitlab.GetProjectOptions{}) + project, _, err := client.Projects.GetProject(projectPath, &gitlab.GetProjectOptions{}) if err != nil { - return false, "", err + return false, 0, err } - projectID := strconv.Itoa(project.ID) - return true, projectID, nil + return true, project.ID, nil } func promptPassword() (string, error) { @@ -473,46 +376,3 @@ func promptPassword() (string, error) { fmt.Println("password is empty") } } - -func prompt2FA() (string, error) { - for { - fmt.Print("two-factor authentication code: ") - - byte2fa, err := terminal.ReadPassword(int(syscall.Stdin)) - fmt.Println() - if err != nil { - return "", err - } - - if len(byte2fa) > 0 { - return string(byte2fa), nil - } - - fmt.Println("code is empty") - } -} - -func promptProjectVisibility() (bool, error) { - for { - fmt.Println("[1]: public") - fmt.Println("[2]: private") - fmt.Print("repository visibility: ") - - line, err := bufio.NewReader(os.Stdin).ReadString('\n') - fmt.Println() - if err != nil { - return false, err - } - - line = strings.TrimRight(line, "\n") - - index, err := strconv.Atoi(line) - if err != nil || (index != 0 && index != 1) { - fmt.Println("invalid input") - continue - } - - // return true for public repositories, false for private - return index == 0, nil - } -} diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 6f234940..4e3b6040 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -403,10 +403,21 @@ func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitla // get label from gitlab func (ge *gitlabExporter) getGitlabLabelID(gc *gitlab.Client, label string) (string, error) { + return "", nil } func (ge *gitlabExporter) createGitlabLabel(label, color string) (string, error) { + client := buildClient(ge.conf[keyToken]) + _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ + Name: &label, + Color: &color, + }) + + if err != nil { + return "", err + } + return "", nil } diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index 538ae715..7a375d6e 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -26,3 +26,8 @@ func (*Gitlab) NewExporter() core.Exporter { func buildClient(token string) *gitlab.Client { return gitlab.NewClient(nil, token) } + +func buildClientFromUsernameAndPassword(username, password string) (*gitlab.Client, error) { + return gitlab.NewBasicAuthClient(nil, "https://gitlab.com", username, password) + +} -- cgit From 8ee136e9fc9012ba3ef37b03f7c17f3dfad51e91 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 12 Jul 2019 17:30:14 +0200 Subject: bridge/gitlab: add issue iterator --- bridge/gitlab/iterator.go | 212 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 bridge/gitlab/iterator.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go new file mode 100644 index 00000000..724d1c73 --- /dev/null +++ b/bridge/gitlab/iterator.go @@ -0,0 +1,212 @@ +package gitlab + +import ( + "time" + + "github.com/xanzy/go-gitlab" +) + +type issueIterator struct { + page int + index int + cache []*gitlab.Issue +} + +type commentIterator struct { + page int + index int + cache []*gitlab.Note +} + +type iterator struct { + // gitlab api v4 client + gc *gitlab.Client + + // if since is given the iterator will query only the updated + // issues after this date + since time.Time + + // project id + project string + + // number of issues and notes to query at once + capacity int + + // sticky error + err error + + // issues iterator + issue *issueIterator + + // comments iterator + comment *commentIterator +} + +// NewIterator create a new iterator +func NewIterator(projectID, token string, capacity int, since time.Time) *iterator { + return &iterator{ + gc: buildClient(token), + project: projectID, + since: since, + capacity: capacity, + issue: &issueIterator{ + index: -1, + page: 1, + }, + comment: &commentIterator{ + index: -1, + page: 1, + }, + } +} + +// Error return last encountered error +func (i *iterator) Error() error { + return i.err +} + +func (i *iterator) getIssues() ([]*gitlab.Issue, error) { + scope := "all" + issues, _, err := i.gc.Issues.ListProjectIssues( + i.project, + &gitlab.ListProjectIssuesOptions{ + ListOptions: gitlab.ListOptions{ + Page: i.issue.page, + PerPage: i.capacity, + }, + Scope: &scope, + UpdatedAfter: &i.since, + }, + ) + + return issues, err +} + +func (i *iterator) NextIssue() bool { + // first query + if i.issue.cache == nil { + issues, err := i.getIssues() + if err != nil { + i.err = err + return false + } + + // if repository doesn't have any issues + if len(issues) == 0 { + return false + } + + i.issue.cache = issues + i.issue.index++ + return true + } + + if i.err != nil { + return false + } + + // move cursor index + if i.issue.index < min(i.capacity, len(i.issue.cache)) { + i.issue.index++ + return true + } + + // query next issues + issues, err := i.getIssues() + if err != nil { + i.err = err + return false + } + + // no more issues to query + if len(issues) == 0 { + return false + } + + i.issue.page++ + i.issue.index = 0 + i.comment.index = 0 + + return true +} + +func (i *iterator) IssueValue() *gitlab.Issue { + return i.issue.cache[i.issue.index] +} + +func (i *iterator) getComments() ([]*gitlab.Note, error) { + notes, _, err := i.gc.Notes.ListIssueNotes( + i.project, + i.IssueValue().IID, + &gitlab.ListIssueNotesOptions{ + ListOptions: gitlab.ListOptions{ + Page: i.issue.page, + PerPage: i.capacity, + }, + }, + ) + + return notes, err +} + +func (i *iterator) NextComment() bool { + if i.err != nil { + return false + } + + if len(i.comment.cache) == 0 { + // query next issues + comments, err := i.getComments() + if err != nil { + i.err = err + return false + } + + if len(comments) == 0 { + i.comment.index = 0 + i.comment.page = 1 + return false + } + + i.comment.page++ + i.comment.index = 0 + + return true + } + + // move cursor index + if i.comment.index < min(i.capacity, len(i.comment.cache)) { + i.comment.index++ + return true + } + + // query next issues + comments, err := i.getComments() + if err != nil { + i.err = err + return false + } + + if len(comments) == 0 { + i.comment.index = 0 + i.comment.page = 1 + return false + } + + i.comment.page++ + i.comment.index = 0 + + return false +} + +func (i *iterator) CommentValue() *gitlab.Note { + return i.comment.cache[i.comment.index] +} + +func min(a, b int) int { + if a > b { + return b + } + + return a +} -- cgit From 51445256496f2eb629f62c638faa6199356eb8e6 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 12 Jul 2019 17:31:01 +0200 Subject: bridge/gitlab: remove request token methodes --- bridge/gitlab/config.go | 98 +++++-------------------------------------------- 1 file changed, 9 insertions(+), 89 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index a6fd6622..392452c6 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -8,23 +8,21 @@ import ( "regexp" "strconv" "strings" - "syscall" "time" "github.com/pkg/errors" "github.com/xanzy/go-gitlab" - "golang.org/x/crypto/ssh/terminal" "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/repository" ) const ( - target = "gitlab" - gitlabV4Url = "https://gitlab.com/api/v4" - keyID = "project-id" - keyTarget = "target" - keyToken = "token" + target = "gitlab" + gitlabV4Url = "https://gitlab.com/api/v4" + keyProjectID = "project-id" + keyTarget = "target" + keyToken = "token" defaultTimeout = 60 * time.Second ) @@ -84,7 +82,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( return nil, fmt.Errorf("invalid project id or wrong token scope") } - conf[keyID] = strconv.Itoa(id) + conf[keyProjectID] = strconv.Itoa(id) conf[keyToken] = token conf[keyTarget] = target @@ -102,8 +100,8 @@ func (*Gitlab) ValidateConfig(conf core.Configuration) error { return fmt.Errorf("missing %s key", keyToken) } - if _, ok := conf[keyID]; !ok { - return fmt.Errorf("missing %s key", keyID) + if _, ok := conf[keyProjectID]; !ok { + return fmt.Errorf("missing %s key", keyProjectID) } return nil @@ -124,6 +122,7 @@ func requestToken(client *gitlab.Client, userID int, name string, scopes ...stri return impToken.Token, nil } +//TODO fix this func promptTokenOptions(url string) (string, error) { for { fmt.Println() @@ -148,8 +147,6 @@ func promptTokenOptions(url string) (string, error) { if index == 1 { return promptToken() } - - return loginAndRequestToken(url) } } @@ -182,62 +179,6 @@ func promptToken() (string, error) { } } -func loginAndRequestToken(url string) (string, error) { - username, err := promptUsername() - if err != nil { - return "", err - } - - password, err := promptPassword() - if err != nil { - return "", err - } - - // Attempt to authenticate and create a token - - note := fmt.Sprintf("git-bug - %s", url) - - ok, id, err := validateUsername(username) - if err != nil { - return "", err - } - if !ok { - return "", fmt.Errorf("invalid username") - } - - client, err := buildClientFromUsernameAndPassword(username, password) - if err != nil { - return "", err - } - - fmt.Println(username, password) - - return requestToken(client, id, note, "api") -} - -func promptUsername() (string, error) { - for { - fmt.Print("username: ") - - line, err := bufio.NewReader(os.Stdin).ReadString('\n') - if err != nil { - return "", err - } - - line = strings.TrimRight(line, "\n") - - ok, _, err := validateUsername(line) - if err != nil { - return "", err - } - if ok { - return line, nil - } - - fmt.Println("invalid username") - } -} - func promptURL(remotes map[string]string) (string, error) { validRemotes := getValidGitlabRemoteURLs(remotes) if len(validRemotes) > 0 { @@ -355,24 +296,3 @@ func validateProjectURL(url, token string) (bool, int, error) { return true, project.ID, nil } - -func promptPassword() (string, error) { - for { - fmt.Print("password: ") - - bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) - // new line for coherent formatting, ReadPassword clip the normal new line - // entered by the user - fmt.Println() - - if err != nil { - return "", err - } - - if len(bytePassword) > 0 { - return string(bytePassword), nil - } - - fmt.Println("password is empty") - } -} -- cgit From aea88180d3ccd6594d52f1489fd0a7ffc65ec178 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 12 Jul 2019 17:31:23 +0200 Subject: bridge/gitlab: add method to query all project labels --- bridge/gitlab/export.go | 82 ++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 31 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 4e3b6040..2f70e01a 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -2,6 +2,7 @@ package gitlab import ( "fmt" + "strconv" "time" "github.com/pkg/errors" @@ -33,7 +34,7 @@ type gitlabExporter struct { // map identities with their tokens identityToken map[string]string - // gitlab. repository ID + // gitlab repository ID repositoryID string // cache identifiers used to speed up exporting operations @@ -74,8 +75,6 @@ func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { // cache client ge.identityClient[id] = client - //client.Labels.CreateLabel() - return client, nil } @@ -91,10 +90,7 @@ func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-c ge.identityToken[user.Id()] = ge.conf[keyToken] // get repository node id - ge.repositoryID, err = getRepositoryNodeID( - "", "", - ge.conf[keyToken], - ) + ge.repositoryID = ge.conf[keyProjectID] if err != nil { return nil, err @@ -384,11 +380,6 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan } } -// getRepositoryNodeID request gitlab api v3 to get repository node id -func getRepositoryNodeID(owner, project, token string) (string, error) { - return "", nil -} - func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { _, err := b.SetMetadata( target, @@ -401,43 +392,72 @@ func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitla return err } -// get label from gitlab -func (ge *gitlabExporter) getGitlabLabelID(gc *gitlab.Client, label string) (string, error) { +func (ge *gitlabExporter) getGitlabLabelID(label string) (string, error) { + id, ok := ge.cachedLabels[label] + if !ok { + return "", fmt.Errorf("non cached label") + } - return "", nil + return id, nil } -func (ge *gitlabExporter) createGitlabLabel(label, color string) (string, error) { - client := buildClient(ge.conf[keyToken]) - _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ - Name: &label, - Color: &color, - }) +// get label from gitlab +func (ge *gitlabExporter) loadLabelsFromGitlab(client *gitlab.Client) error { + + labels, _, err := client.Labels.ListLabels( + ge.repositoryID, + &gitlab.ListLabelsOptions{ + Page: 0, + }, + ) if err != nil { - return "", err + return err } - return "", nil -} + for _, label := range labels { + ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) + } + + for page := 2; len(labels) != 0; page++ { -func (ge *gitlabExporter) getOrCreateGitlabLabelID(gc *gitlab.Client, repositoryID string, label bug.Label) (string, error) { - // try to get label id - labelID, err := ge.getGitlabLabelID(gc, string(label)) - if err == nil { - return labelID, nil + labels, _, err = client.Labels.ListLabels( + ge.repositoryID, + &gitlab.ListLabelsOptions{ + Page: page, + }, + ) + + if err != nil { + return err + } + + for _, label := range labels { + ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) + } } + return nil +} + +func (ge *gitlabExporter) createGitlabLabel(gc *gitlab.Client, label bug.Label) (string, error) { + client := buildClient(ge.conf[keyToken]) + // RGBA to hex color rgba := label.RGBA() hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) + name := label.String() + + _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ + Name: &name, + Color: &hexColor, + }) - labelID, err = ge.createGitlabLabel(string(label), hexColor) if err != nil { return "", err } - return labelID, nil + return "", nil } func (ge *gitlabExporter) getLabelsIDs(gc *gitlab.Client, repositoryID string, labels []bug.Label) ([]string, error) { -- cgit From 6c02f0951da7962ccdcc50628887ef4eda2eb3b7 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 12 Jul 2019 17:37:49 +0200 Subject: bridge/gitlab: prompt only for user provided token --- bridge/gitlab/config.go | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index 392452c6..7faddd6d 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -66,7 +66,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( if params.Token != "" { token = params.Token } else { - token, err = promptTokenOptions(url) + token, err = promptToken() if err != nil { return nil, err } @@ -122,34 +122,6 @@ func requestToken(client *gitlab.Client, userID int, name string, scopes ...stri return impToken.Token, nil } -//TODO fix this -func promptTokenOptions(url string) (string, error) { - for { - fmt.Println() - fmt.Println("[1]: user provided token") - fmt.Println("[2]: interactive token creation") - fmt.Print("Select option: ") - - line, err := bufio.NewReader(os.Stdin).ReadString('\n') - fmt.Println() - if err != nil { - return "", err - } - - line = strings.TrimRight(line, "\n") - - index, err := strconv.Atoi(line) - if err != nil || (index != 1 && index != 2) { - fmt.Println("invalid input") - continue - } - - if index == 1 { - return promptToken() - } - } -} - func promptToken() (string, error) { fmt.Println("You can generate a new token by visiting https://gitlab.com/settings/tokens.") fmt.Println("Choose 'Generate new token' and set the necessary access scope for your repository.") -- cgit From 612264a00f352a12620bab3c72cd0b11b304f5e2 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 12 Jul 2019 17:48:57 +0200 Subject: bridge/gitlab: fix iterator out of index bug --- bridge/gitlab/iterator.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 724d1c73..a0f2907b 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -106,7 +106,7 @@ func (i *iterator) NextIssue() bool { } // move cursor index - if i.issue.index < min(i.capacity, len(i.issue.cache)) { + if i.issue.index < min(i.capacity, len(i.issue.cache))-1 { i.issue.index++ return true } @@ -126,6 +126,7 @@ func (i *iterator) NextIssue() bool { i.issue.page++ i.issue.index = 0 i.comment.index = 0 + i.issue.cache = issues return true } @@ -149,6 +150,10 @@ func (i *iterator) getComments() ([]*gitlab.Note, error) { return notes, err } +func (i *iterator) getNextComments() bool { + return false +} + func (i *iterator) NextComment() bool { if i.err != nil { return false @@ -168,6 +173,7 @@ func (i *iterator) NextComment() bool { return false } + i.comment.cache = comments i.comment.page++ i.comment.index = 0 @@ -175,7 +181,7 @@ func (i *iterator) NextComment() bool { } // move cursor index - if i.comment.index < min(i.capacity, len(i.comment.cache)) { + if i.comment.index < min(i.capacity, len(i.comment.cache))-1 { i.comment.index++ return true } @@ -193,6 +199,7 @@ func (i *iterator) NextComment() bool { return false } + i.comment.cache = comments i.comment.page++ i.comment.index = 0 -- cgit From b512108ac76724f150184fff8c0499539e42dd9a Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 14 Jul 2019 15:54:09 +0200 Subject: bridge/gitlab: fix iterator bugs and enhacements --- bridge/gitlab/iterator.go | 153 ++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 86 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index a0f2907b..bab35d95 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -12,12 +12,18 @@ type issueIterator struct { cache []*gitlab.Issue } -type commentIterator struct { +type noteIterator struct { page int index int cache []*gitlab.Note } +type labelEventIterator struct { + page int + index int + cache []*gitlab.LabelEvent +} + type iterator struct { // gitlab api v4 client gc *gitlab.Client @@ -38,8 +44,10 @@ type iterator struct { // issues iterator issue *issueIterator - // comments iterator - comment *commentIterator + // notes iterator + note *noteIterator + + labelEvent *labelEventIterator } // NewIterator create a new iterator @@ -53,7 +61,11 @@ func NewIterator(projectID, token string, capacity int, since time.Time) *iterat index: -1, page: 1, }, - comment: &commentIterator{ + note: ¬eIterator{ + index: -1, + page: 1, + }, + labelEvent: &labelEventIterator{ index: -1, page: 1, }, @@ -65,7 +77,8 @@ func (i *iterator) Error() error { return i.err } -func (i *iterator) getIssues() ([]*gitlab.Issue, error) { +func (i *iterator) getNextIssues() bool { + sort := "asc" scope := "all" issues, _, err := i.gc.Issues.ListProjectIssues( i.project, @@ -76,29 +89,33 @@ func (i *iterator) getIssues() ([]*gitlab.Issue, error) { }, Scope: &scope, UpdatedAfter: &i.since, + Sort: &sort, }, ) - return issues, err + if err != nil { + i.err = err + return false + } + + // if repository doesn't have any issues + if len(issues) == 0 { + return false + } + + i.issue.cache = issues + i.issue.index = 0 + i.issue.page++ + i.note.index = -1 + i.note.cache = nil + + return true } func (i *iterator) NextIssue() bool { // first query if i.issue.cache == nil { - issues, err := i.getIssues() - if err != nil { - i.err = err - return false - } - - // if repository doesn't have any issues - if len(issues) == 0 { - return false - } - - i.issue.cache = issues - i.issue.index++ - return true + return i.getNextIssues() } if i.err != nil { @@ -111,103 +128,67 @@ func (i *iterator) NextIssue() bool { return true } - // query next issues - issues, err := i.getIssues() - if err != nil { - i.err = err - return false - } - - // no more issues to query - if len(issues) == 0 { - return false - } - - i.issue.page++ - i.issue.index = 0 - i.comment.index = 0 - i.issue.cache = issues - - return true + return i.getNextIssues() } func (i *iterator) IssueValue() *gitlab.Issue { return i.issue.cache[i.issue.index] } -func (i *iterator) getComments() ([]*gitlab.Note, error) { +func (i *iterator) getNextNotes() bool { + sort := "asc" + order := "created_at" notes, _, err := i.gc.Notes.ListIssueNotes( i.project, i.IssueValue().IID, &gitlab.ListIssueNotesOptions{ ListOptions: gitlab.ListOptions{ - Page: i.issue.page, + Page: i.note.page, PerPage: i.capacity, }, + Sort: &sort, + OrderBy: &order, }, ) - return notes, err -} + if err != nil { + i.err = err + return false + } + + if len(notes) == 0 { + i.note.index = -1 + i.note.page = 1 + i.note.cache = nil + return false + } -func (i *iterator) getNextComments() bool { - return false + i.note.cache = notes + i.note.page++ + i.note.index = 0 + return true } -func (i *iterator) NextComment() bool { +func (i *iterator) NextNote() bool { if i.err != nil { return false } - if len(i.comment.cache) == 0 { - // query next issues - comments, err := i.getComments() - if err != nil { - i.err = err - return false - } - - if len(comments) == 0 { - i.comment.index = 0 - i.comment.page = 1 - return false - } - - i.comment.cache = comments - i.comment.page++ - i.comment.index = 0 - - return true + if len(i.note.cache) == 0 { + return i.getNextNotes() } // move cursor index - if i.comment.index < min(i.capacity, len(i.comment.cache))-1 { - i.comment.index++ + if i.note.index < min(i.capacity, len(i.note.cache))-1 { + i.note.index++ return true } - // query next issues - comments, err := i.getComments() - if err != nil { - i.err = err - return false - } - - if len(comments) == 0 { - i.comment.index = 0 - i.comment.page = 1 - return false - } - - i.comment.cache = comments - i.comment.page++ - i.comment.index = 0 - - return false + return i.getNextNotes() } -func (i *iterator) CommentValue() *gitlab.Note { - return i.comment.cache[i.comment.index] +func (i *iterator) NoteValue() *gitlab.Note { + return i.note.cache[i.note.index] } func min(a, b int) int { -- cgit From 89227f92b5e900fb2d77680ca09966534ebd2bc1 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 14 Jul 2019 15:54:48 +0200 Subject: bridge/gitlab: add iterator LabelEvents --- bridge/gitlab/iterator.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index bab35d95..f2fb0c2b 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -191,6 +191,58 @@ func (i *iterator) NoteValue() *gitlab.Note { return i.note.cache[i.note.index] } +func (i *iterator) getNextLabelEvents() bool { + labelEvents, _, err := i.gc.ResourceLabelEvents.ListIssueLabelEvents( + i.project, + i.IssueValue().IID, + &gitlab.ListLabelEventsOptions{ + ListOptions: gitlab.ListOptions{ + Page: i.labelEvent.page, + PerPage: i.capacity, + }, + }, + ) + + if err != nil { + i.err = err + return false + } + + if len(labelEvents) == 0 { + i.labelEvent.page = 1 + i.labelEvent.index = -1 + i.labelEvent.cache = nil + return false + } + + i.labelEvent.cache = labelEvents + i.labelEvent.page++ + i.labelEvent.index = 0 + return true +} + +func (i *iterator) NextLabelEvent() bool { + if i.err != nil { + return false + } + + if len(i.labelEvent.cache) == 0 { + return i.getNextLabelEvents() + } + + // move cursor index + if i.labelEvent.index < min(i.capacity, len(i.labelEvent.cache))-1 { + i.labelEvent.index++ + return true + } + + return i.getNextLabelEvents() +} + +func (i *iterator) LabelEventValue() *gitlab.LabelEvent { + return i.labelEvent.cache[i.labelEvent.index] +} + func min(a, b int) int { if a > b { return b -- cgit From 53f99d3b8549ea64afa84608fd2f15f732a96a68 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 14 Jul 2019 19:09:05 +0200 Subject: bridge/gitlab: add import note utilities bridge/gitlab: set default capacity to 20 --- bridge/gitlab/import_notes.go | 94 +++++++++++++++++++++++++++++++++++++++++++ bridge/gitlab/iterator.go | 4 +- 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 bridge/gitlab/import_notes.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import_notes.go b/bridge/gitlab/import_notes.go new file mode 100644 index 00000000..438967de --- /dev/null +++ b/bridge/gitlab/import_notes.go @@ -0,0 +1,94 @@ +package gitlab + +import ( + "strings" + + "github.com/xanzy/go-gitlab" +) + +type NoteType int + +const ( + _ NoteType = iota + NOTE_COMMENT + NOTE_TITLE_CHANGED + NOTE_DESCRIPTION_CHANGED + NOTE_CLOSED + NOTE_REOPENED + NOTE_LOCKED + NOTE_UNLOCKED + NOTE_CHANGED_DUEDATE + NOTE_REMOVED_DUEDATE + NOTE_ASSIGNED + NOTE_UNASSIGNED + NOTE_CHANGED_MILESTONE + NOTE_REMOVED_MILESTONE + NOTE_UNKNOWN +) + +// GetNoteType parses note body a give it type +// Since gitlab api return all these NoteType event as the same object +// and doesn't provide a field to specify the note type. We must parse the +// note body to detect it type. +func GetNoteType(n *gitlab.Note) (NoteType, string) { + if n.Body == "closed" { + return NOTE_CLOSED, "" + } + + if n.Body == "reopened" { + return NOTE_REOPENED, "" + } + + if n.Body == "changed the description" { + return NOTE_DESCRIPTION_CHANGED, "" + } + + if n.Body == "locked this issue" { + return NOTE_LOCKED, "" + } + + if n.Body == "unlocked this issue" { + return NOTE_UNLOCKED, "" + } + + if strings.HasPrefix(n.Body, "changed title from") { + return NOTE_TITLE_CHANGED, getNewTitle(n.Body) + } + + if strings.HasPrefix(n.Body, "changed due date to") { + return NOTE_CHANGED_DUEDATE, "" + } + + if n.Body == "removed due date" { + return NOTE_REMOVED_DUEDATE, "" + } + + if strings.HasPrefix(n.Body, "assigned to @") { + return NOTE_ASSIGNED, "" + } + + if strings.HasPrefix(n.Body, "unassigned @") { + return NOTE_UNASSIGNED, "" + } + + if strings.HasPrefix(n.Body, "changed milestone to %") { + return NOTE_CHANGED_MILESTONE, "" + } + + if strings.HasPrefix(n.Body, "removed milestone") { + return NOTE_REMOVED_MILESTONE, "" + } + + // comment don't have a specific format + return NOTE_COMMENT, n.Body +} + +// getNewTitle parses body diff given by gitlab api and return it final form +// examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**" +// "changed title from **fourth issue{- changed-}** to **fourth issue**" +func getNewTitle(diff string) string { + newTitle := strings.Split(diff, "** to **")[1] + newTitle = strings.Replace(newTitle, "{+", "", -1) + newTitle = strings.Replace(newTitle, "+}", "", -1) + return strings.TrimSuffix(newTitle, "**") +} diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index f2fb0c2b..8502504d 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -51,12 +51,12 @@ type iterator struct { } // NewIterator create a new iterator -func NewIterator(projectID, token string, capacity int, since time.Time) *iterator { +func NewIterator(projectID, token string, since time.Time) *iterator { return &iterator{ gc: buildClient(token), project: projectID, since: since, - capacity: capacity, + capacity: 20, issue: &issueIterator{ index: -1, page: 1, -- cgit From 8b6c896369bc48599bc97181a3f3a85a9425af87 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 00:06:42 +0200 Subject: bridge/gitlab: complete importer --- bridge/gitlab/import.go | 311 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 309 insertions(+), 2 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index dec90a6c..1ac8eaf3 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -1,10 +1,15 @@ package gitlab import ( + "fmt" "time" + "github.com/xanzy/go-gitlab" + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/util/text" ) const ( @@ -14,6 +19,9 @@ const ( type gitlabImporter struct { conf core.Configuration + // iterator + iterator *iterator + // number of imported issues importedIssues int @@ -21,10 +29,309 @@ type gitlabImporter struct { importedIdentities int } -func (*gitlabImporter) Init(conf core.Configuration) error { +func (gi *gitlabImporter) Init(conf core.Configuration) error { + gi.conf = conf + return nil +} + +func (gi *gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { + gi.iterator = NewIterator(gi.conf[keyProjectID], gi.conf[keyToken], since) + + // Loop over all matching issues + for gi.iterator.NextIssue() { + issue := gi.iterator.IssueValue() + fmt.Printf("importing issue: %v\n", issue.Title) + + // create issue + b, err := gi.ensureIssue(repo, issue) + if err != nil { + return fmt.Errorf("issue creation: %v", err) + } + + // Loop over all notes + for gi.iterator.NextNote() { + note := gi.iterator.NoteValue() + if err := gi.ensureNote(repo, b, note); err != nil { + return fmt.Errorf("note creation: %v", err) + } + } + + // Loop over all label events + for gi.iterator.NextLabelEvent() { + labelEvent := gi.iterator.LabelEventValue() + if err := gi.ensureLabelEvent(repo, b, labelEvent); err != nil { + return fmt.Errorf("label event creation: %v", err) + } + + } + + // commit bug state + if err := b.CommitAsNeeded(); err != nil { + return fmt.Errorf("bug commit: %v", err) + } + } + + if err := gi.iterator.Error(); err != nil { + fmt.Printf("import error: %v\n", err) + return err + } + + fmt.Printf("Successfully imported %d issues and %d identities from Gitlab\n", gi.importedIssues, gi.importedIdentities) return nil } -func (*gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { +func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue) (*cache.BugCache, error) { + // ensure issue author + author, err := gi.ensurePerson(repo, issue.Author.ID) + if err != nil { + return nil, err + } + + // resolve bug + b, err := repo.ResolveBugCreateMetadata(keyGitlabUrl, issue.WebURL) + if err != nil && err != bug.ErrBugNotExist { + return nil, err + } + + if err == bug.ErrBugNotExist { + cleanText, err := text.Cleanup(string(issue.Description)) + if err != nil { + return nil, err + } + + // create bug + b, _, err = repo.NewBugRaw( + author, + issue.CreatedAt.Unix(), + issue.Title, + cleanText, + nil, + map[string]string{ + keyOrigin: target, + keyGitlabId: parseID(issue.ID), + keyGitlabUrl: issue.WebURL, + }, + ) + + if err != nil { + return nil, err + } + + // importing a new bug + gi.importedIssues++ + + return b, nil + } + + return nil, nil +} + +func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, note *gitlab.Note) error { + id := parseID(note.ID) + + hash, err := b.ResolveOperationWithMetadata(keyGitlabId, id) + if err != cache.ErrNoMatchingOp { + return err + } + + // ensure issue author + author, err := gi.ensurePerson(repo, note.Author.ID) + if err != nil { + return err + } + + noteType, body := GetNoteType(note) + switch noteType { + case NOTE_CLOSED: + _, err = b.CloseRaw( + author, + note.CreatedAt.Unix(), + map[string]string{ + keyGitlabId: id, + }, + ) + return err + + case NOTE_REOPENED: + _, err = b.OpenRaw( + author, + note.CreatedAt.Unix(), + map[string]string{ + keyGitlabId: id, + }, + ) + return err + + case NOTE_DESCRIPTION_CHANGED: + issue := gi.iterator.IssueValue() + + // since gitlab doesn't provide the issue history + // we should check for "changed the description" notes and compare issue texts + + if issue.Description != b.Snapshot().Comments[0].Message { + // comment edition + _, err = b.EditCommentRaw( + author, + note.UpdatedAt.Unix(), + target, + issue.Description, + map[string]string{ + keyGitlabId: id, + keyGitlabUrl: "", + }, + ) + + return err + + } + + case NOTE_COMMENT: + + cleanText, err := text.Cleanup(body) + if err != nil { + return err + } + + // if we didn't import the comment + if err == cache.ErrNoMatchingOp { + + // add comment operation + _, err = b.AddCommentRaw( + author, + note.CreatedAt.Unix(), + cleanText, + nil, + map[string]string{ + keyGitlabId: id, + keyGitlabUrl: "", + }, + ) + + return err + } + + // if comment was already exported + + // if note wasn't updated + if note.UpdatedAt.Equal(*note.CreatedAt) { + return nil + } + + // search for last comment update + timeline, err := b.Snapshot().SearchTimelineItem(hash) + if err != nil { + return err + } + + item, ok := timeline.(*bug.AddCommentTimelineItem) + if !ok { + return fmt.Errorf("expected add comment time line") + } + + // compare local bug comment with the new note body + if item.Message != cleanText { + // comment edition + _, err = b.EditCommentRaw( + author, + note.UpdatedAt.Unix(), + target, + cleanText, + map[string]string{ + // no metadata unique metadata to store + keyGitlabId: "", + keyGitlabUrl: "", + }, + ) + + return err + } + + return nil + + case NOTE_TITLE_CHANGED: + + _, err = b.SetTitleRaw( + author, + note.CreatedAt.Unix(), + body, + map[string]string{ + keyGitlabId: id, + keyGitlabUrl: "", + }, + ) + + return err + + default: + // non handled note types + + return nil + } + return nil } + +func (gi *gitlabImporter) ensureLabelEvent(repo *cache.RepoCache, b *cache.BugCache, labelEvent *gitlab.LabelEvent) error { + _, err := b.ResolveOperationWithMetadata(keyGitlabId, parseID(labelEvent.ID)) + if err != cache.ErrNoMatchingOp { + return err + } + + // ensure issue author + author, err := gi.ensurePerson(repo, labelEvent.User.ID) + if err != nil { + return err + } + + switch labelEvent.Action { + case "add": + _, err = b.ForceChangeLabelsRaw( + author, + labelEvent.CreatedAt.Unix(), + []string{labelEvent.Label.Name}, + nil, + map[string]string{ + keyGitlabId: parseID(labelEvent.ID), + }, + ) + + case "remove": + _, err = b.ForceChangeLabelsRaw( + author, + labelEvent.CreatedAt.Unix(), + nil, + []string{labelEvent.Label.Name}, + map[string]string{ + keyGitlabId: parseID(labelEvent.ID), + }, + ) + + default: + panic("unexpected label event action") + } + + return err +} + +func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.IdentityCache, error) { + client := buildClient(gi.conf["token"]) + + user, _, err := client.Users.GetUser(id) + if err != nil { + return nil, err + } + + return repo.NewIdentityRaw( + user.Name, + user.PublicEmail, + user.Username, + user.AvatarURL, + map[string]string{ + keyGitlabLogin: user.Username, + }, + ) +} + +func parseID(id int) string { + return fmt.Sprintf("%d", id) +} -- cgit From ffb8d34e4f04b678b3f4785a8247762ca7c06c4a Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 00:09:02 +0200 Subject: bridge/gitlab: check identity cache in ensurePerson --- bridge/gitlab/import.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 1ac8eaf3..4eeb3813 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -2,6 +2,7 @@ package gitlab import ( "fmt" + "strconv" "time" "github.com/xanzy/go-gitlab" @@ -9,6 +10,7 @@ import ( "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/util/text" ) @@ -314,6 +316,18 @@ func (gi *gitlabImporter) ensureLabelEvent(repo *cache.RepoCache, b *cache.BugCa } func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.IdentityCache, error) { + // Look first in the cache + i, err := repo.ResolveIdentityImmutableMetadata(keyGitlabId, strconv.Itoa(id)) + if err == nil { + return i, nil + } + if _, ok := err.(identity.ErrMultipleMatch); ok { + return nil, err + } + + // importing a new identity + gi.importedIdentities++ + client := buildClient(gi.conf["token"]) user, _, err := client.Users.GetUser(id) @@ -327,6 +341,7 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id user.Username, user.AvatarURL, map[string]string{ + keyGitlabId: strconv.Itoa(id), keyGitlabLogin: user.Username, }, ) -- cgit From e012b6c6226a0f4121e036b8829f28f238d87e46 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 18:46:46 +0200 Subject: bridge/gitlab: check notes system field --- bridge/gitlab/import_notes.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import_notes.go b/bridge/gitlab/import_notes.go index 438967de..37bf2834 100644 --- a/bridge/gitlab/import_notes.go +++ b/bridge/gitlab/import_notes.go @@ -31,6 +31,10 @@ const ( // and doesn't provide a field to specify the note type. We must parse the // note body to detect it type. func GetNoteType(n *gitlab.Note) (NoteType, string) { + if !n.System { + return NOTE_COMMENT, n.Body + } + if n.Body == "closed" { return NOTE_CLOSED, "" } @@ -79,8 +83,7 @@ func GetNoteType(n *gitlab.Note) (NoteType, string) { return NOTE_REMOVED_MILESTONE, "" } - // comment don't have a specific format - return NOTE_COMMENT, n.Body + return NOTE_UNKNOWN, "" } // getNewTitle parses body diff given by gitlab api and return it final form -- cgit From 76a389c93da284c4178789d378b4a2bbd8214934 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 18:54:19 +0200 Subject: bridge/gitlab: make resolve error unique within the importer --- bridge/gitlab/import.go | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 4eeb3813..d94dbe2e 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -64,7 +64,6 @@ func (gi *gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) erro if err := gi.ensureLabelEvent(repo, b, labelEvent); err != nil { return fmt.Errorf("label event creation: %v", err) } - } // commit bug state @@ -131,17 +130,17 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, note *gitlab.Note) error { id := parseID(note.ID) - hash, err := b.ResolveOperationWithMetadata(keyGitlabId, id) - if err != cache.ErrNoMatchingOp { - return err - } - // ensure issue author author, err := gi.ensurePerson(repo, note.Author.ID) if err != nil { return err } + hash, errResolve := b.ResolveOperationWithMetadata(keyGitlabId, id) + if err != nil && err != cache.ErrNoMatchingOp { + return err + } + noteType, body := GetNoteType(note) switch noteType { case NOTE_CLOSED: @@ -149,7 +148,8 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n author, note.CreatedAt.Unix(), map[string]string{ - keyGitlabId: id, + keyGitlabId: id, + keyGitlabUrl: "", }, ) return err @@ -159,7 +159,8 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n author, note.CreatedAt.Unix(), map[string]string{ - keyGitlabId: id, + keyGitlabId: id, + keyGitlabUrl: "", }, ) return err @@ -171,6 +172,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n // we should check for "changed the description" notes and compare issue texts if issue.Description != b.Snapshot().Comments[0].Message { + // comment edition _, err = b.EditCommentRaw( author, @@ -195,7 +197,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n } // if we didn't import the comment - if err == cache.ErrNoMatchingOp { + if errResolve == cache.ErrNoMatchingOp { // add comment operation _, err = b.AddCommentRaw( @@ -220,18 +222,13 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n } // search for last comment update - timeline, err := b.Snapshot().SearchTimelineItem(hash) + comment, err := b.Snapshot().SearchComment(hash) if err != nil { return err } - item, ok := timeline.(*bug.AddCommentTimelineItem) - if !ok { - return fmt.Errorf("expected add comment time line") - } - // compare local bug comment with the new note body - if item.Message != cleanText { + if comment.Message != cleanText { // comment edition _, err = b.EditCommentRaw( author, @@ -251,7 +248,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n return nil case NOTE_TITLE_CHANGED: - + // title change events are given new notes _, err = b.SetTitleRaw( author, note.CreatedAt.Unix(), @@ -265,8 +262,8 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n return err default: - // non handled note types - + // non handled note types, this is not an error + //TODO: send warning via channel return nil } -- cgit From 05a3aec1a89f1b8c56a464558bbe39f96a42b267 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 18:54:32 +0200 Subject: bridge/gitlab: add import unit tests --- bridge/gitlab/import_test.go | 151 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 bridge/gitlab/import_test.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import_test.go b/bridge/gitlab/import_test.go new file mode 100644 index 00000000..ddf0553e --- /dev/null +++ b/bridge/gitlab/import_test.go @@ -0,0 +1,151 @@ +package gitlab + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/MichaelMure/git-bug/bridge/core" + "github.com/MichaelMure/git-bug/bug" + "github.com/MichaelMure/git-bug/cache" + "github.com/MichaelMure/git-bug/identity" + "github.com/MichaelMure/git-bug/repository" + "github.com/MichaelMure/git-bug/util/interrupt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + //_ "github.com/motemen/go-loghttp/global" +) + +func TestImport(t *testing.T) { + author := identity.NewIdentity("Amine hilaly", "hilalyamine@gmail.com") + tests := []struct { + name string + url string + bug *bug.Snapshot + }{ + { + name: "simple issue", + url: "https://gitlab.com/a-hilaly/git-bug-test/issues/1", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewAddCommentOp(author, 0, "second comment", nil), + }, + }, + }, + { + name: "empty issue", + url: "https://gitlab.com/a-hilaly/git-bug-test/issues/2", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "empty issue", "", nil), + }, + }, + }, + { + name: "complex issue", + url: "https://gitlab.com/a-hilaly/git-bug-test/issues/3", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "complex issue", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "### header\n\n**bold**\n\n_italic_\n\n> with quote\n\n`inline code`\n\n```\nmultiline code\n```\n\n- bulleted\n- list\n\n1. numbered\n1. list\n\n- [ ] task\n- [x] list\n\n@MichaelMure mention\n\n#2 reference issue\n#3 auto-reference issue", nil), + bug.NewSetTitleOp(author, 0, "complex issue edited", "complex issue"), + bug.NewSetTitleOp(author, 0, "complex issue", "complex issue edited"), + bug.NewSetStatusOp(author, 0, bug.ClosedStatus), + bug.NewSetStatusOp(author, 0, bug.OpenStatus), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"bug"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{"critical"}, []bug.Label{}), + bug.NewLabelChangeOperation(author, 0, []bug.Label{}, []bug.Label{"critical"}), + }, + }, + }, + { + name: "editions", + url: "https://gitlab.com/a-hilaly/git-bug-test/issues/4", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "editions", "initial comment edited", nil), + bug.NewAddCommentOp(author, 0, "first comment edited", nil), + }, + }, + }, + } + + repo := repository.CreateTestRepo(false) + defer repository.CleanupTestRepos(t, repo) + + backend, err := cache.NewRepoCache(repo) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + token := os.Getenv("GITLAB_API_TOKEN") + if token == "" { + t.Skip("Env var GITLAB_API_TOKEN missing") + } + + projectID := os.Getenv("GITLAB_PROJECT_ID") + if projectID == "" { + t.Skip("Env var GITLAB_PROJECT_ID missing") + } + + importer := &gitlabImporter{} + err = importer.Init(core.Configuration{ + keyProjectID: projectID, + keyToken: token, + }) + require.NoError(t, err) + + start := time.Now() + err = importer.ImportAll(backend, time.Time{}) + require.NoError(t, err) + + fmt.Printf("test repository imported in %f seconds\n", time.Since(start).Seconds()) + + require.Len(t, backend.AllBugsIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := backend.ResolveBugCreateMetadata(keyGitlabUrl, tt.url) + require.NoError(t, err) + + ops := b.Snapshot().Operations + assert.Len(t, tt.bug.Operations, len(ops)) + + for i, op := range tt.bug.Operations { + + require.IsType(t, ops[i], op) + + switch op.(type) { + case *bug.CreateOperation: + assert.Equal(t, op.(*bug.CreateOperation).Title, ops[i].(*bug.CreateOperation).Title) + assert.Equal(t, op.(*bug.CreateOperation).Message, ops[i].(*bug.CreateOperation).Message) + assert.Equal(t, op.(*bug.CreateOperation).Author.Name(), ops[i].(*bug.CreateOperation).Author.Name()) + case *bug.SetStatusOperation: + assert.Equal(t, op.(*bug.SetStatusOperation).Status, ops[i].(*bug.SetStatusOperation).Status) + assert.Equal(t, op.(*bug.SetStatusOperation).Author.Name(), ops[i].(*bug.SetStatusOperation).Author.Name()) + case *bug.SetTitleOperation: + assert.Equal(t, op.(*bug.SetTitleOperation).Was, ops[i].(*bug.SetTitleOperation).Was) + assert.Equal(t, op.(*bug.SetTitleOperation).Title, ops[i].(*bug.SetTitleOperation).Title) + assert.Equal(t, op.(*bug.SetTitleOperation).Author.Name(), ops[i].(*bug.SetTitleOperation).Author.Name()) + case *bug.LabelChangeOperation: + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Added, ops[i].(*bug.LabelChangeOperation).Added) + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Removed, ops[i].(*bug.LabelChangeOperation).Removed) + assert.Equal(t, op.(*bug.LabelChangeOperation).Author.Name(), ops[i].(*bug.LabelChangeOperation).Author.Name()) + case *bug.AddCommentOperation: + assert.Equal(t, op.(*bug.AddCommentOperation).Message, ops[i].(*bug.AddCommentOperation).Message) + assert.Equal(t, op.(*bug.AddCommentOperation).Author.Name(), ops[i].(*bug.AddCommentOperation).Author.Name()) + case *bug.EditCommentOperation: + assert.Equal(t, op.(*bug.EditCommentOperation).Message, ops[i].(*bug.EditCommentOperation).Message) + assert.Equal(t, op.(*bug.EditCommentOperation).Author.Name(), ops[i].(*bug.EditCommentOperation).Author.Name()) + + default: + panic("unknown operation type") + } + } + }) + } +} -- cgit From ce3a2788ab3bc9f205bcc27b03355155d641c2f4 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Wed, 17 Jul 2019 22:41:42 +0200 Subject: bridge/gitlab: fix note error handling bug bridge/gitlab: remove unused functions --- bridge/gitlab/config.go | 37 ------------------------------------- bridge/gitlab/gitlab.go | 3 ++- bridge/gitlab/import.go | 2 +- bridge/gitlab/import_test.go | 6 +++--- 4 files changed, 6 insertions(+), 42 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index 7faddd6d..cac2d91e 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -107,21 +107,6 @@ func (*Gitlab) ValidateConfig(conf core.Configuration) error { return nil } -func requestToken(client *gitlab.Client, userID int, name string, scopes ...string) (string, error) { - impToken, _, err := client.Users.CreateImpersonationToken( - userID, - &gitlab.CreateImpersonationTokenOptions{ - Name: &name, - Scopes: &scopes, - }, - ) - if err != nil { - return "", err - } - - return impToken.Token, nil -} - func promptToken() (string, error) { fmt.Println("You can generate a new token by visiting https://gitlab.com/settings/tokens.") fmt.Println("Choose 'Generate new token' and set the necessary access scope for your repository.") @@ -231,28 +216,6 @@ func getValidGitlabRemoteURLs(remotes map[string]string) []string { return urls } -func validateUsername(username string) (bool, int, error) { - // no need for a token for this action - client := buildClient("") - - users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &username}) - if err != nil { - return false, 0, err - } - - if len(users) == 0 { - return false, 0, fmt.Errorf("username not found") - } else if len(users) > 1 { - return false, 0, fmt.Errorf("found multiple matches") - } - - if users[0].Username == username { - return true, users[0].ID, nil - } - - return false, 0, nil -} - func validateProjectURL(url, token string) (bool, int, error) { client := buildClient(token) diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index 7a375d6e..9f5807cd 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -1,8 +1,9 @@ package gitlab import ( - "github.com/MichaelMure/git-bug/bridge/core" "github.com/xanzy/go-gitlab" + + "github.com/MichaelMure/git-bug/bridge/core" ) func init() { diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index d94dbe2e..93ad8570 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -137,7 +137,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n } hash, errResolve := b.ResolveOperationWithMetadata(keyGitlabId, id) - if err != nil && err != cache.ErrNoMatchingOp { + if errResolve != cache.ErrNoMatchingOp { return err } diff --git a/bridge/gitlab/import_test.go b/bridge/gitlab/import_test.go index ddf0553e..4a49cfe9 100644 --- a/bridge/gitlab/import_test.go +++ b/bridge/gitlab/import_test.go @@ -6,15 +6,15 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/repository" "github.com/MichaelMure/git-bug/util/interrupt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - //_ "github.com/motemen/go-loghttp/global" ) func TestImport(t *testing.T) { -- cgit From 7726bbdbcf8aa49548ecbcfc323b47b0ec034f54 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 19 Jul 2019 18:49:28 +0200 Subject: bridge/gitlab: add bridge config tests --- bridge/gitlab/config.go | 22 ++++-------- bridge/gitlab/config_test.go | 81 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 bridge/gitlab/config_test.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index cac2d91e..efef5993 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -8,7 +8,6 @@ import ( "regexp" "strconv" "strings" - "time" "github.com/pkg/errors" "github.com/xanzy/go-gitlab" @@ -17,16 +16,6 @@ import ( "github.com/MichaelMure/git-bug/repository" ) -const ( - target = "gitlab" - gitlabV4Url = "https://gitlab.com/api/v4" - keyProjectID = "project-id" - keyTarget = "target" - keyToken = "token" - - defaultTimeout = 60 * time.Second -) - var ( ErrBadProjectURL = errors.New("bad project url") ) @@ -108,10 +97,10 @@ func (*Gitlab) ValidateConfig(conf core.Configuration) error { } func promptToken() (string, error) { - fmt.Println("You can generate a new token by visiting https://gitlab.com/settings/tokens.") - fmt.Println("Choose 'Generate new token' and set the necessary access scope for your repository.") + fmt.Println("You can generate a new token by visiting https://gitlab.com/profile/personal_access_tokens.") + fmt.Println("Choose 'Create personal access token' and set the necessary access scope for your repository.") fmt.Println() - fmt.Println("'api' scope access : access scope: to be able to make api calls") + fmt.Println("'api' access scope: to be able to make api calls") fmt.Println() re, err := regexp.Compile(`^[a-zA-Z0-9\-]{20}`) @@ -192,13 +181,14 @@ func promptURL(remotes map[string]string) (string, error) { } func getProjectPath(url string) (string, error) { - cleanUrl := strings.TrimSuffix(url, ".git") + cleanUrl = strings.Replace(cleanUrl, "git@", "https://", 1) objectUrl, err := neturl.Parse(cleanUrl) if err != nil { - return "", nil + return "", err } + fmt.Println(objectUrl.Path) return objectUrl.Path[1:], nil } diff --git a/bridge/gitlab/config_test.go b/bridge/gitlab/config_test.go new file mode 100644 index 00000000..248cdb66 --- /dev/null +++ b/bridge/gitlab/config_test.go @@ -0,0 +1,81 @@ +package gitlab + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProjectPath(t *testing.T) { + type args struct { + url string + } + type want struct { + path string + err error + } + tests := []struct { + name string + args args + want want + }{ + { + name: "default url", + args: args{ + url: "https://gitlab.com/MichaelMure/git-bug", + }, + want: want{ + path: "MichaelMure/git-bug", + err: nil, + }, + }, + { + name: "multiple sub groups", + args: args{ + url: "https://gitlab.com/MichaelMure/group/subgroup/git-bug", + }, + want: want{ + path: "MichaelMure/group/subgroup/git-bug", + err: nil, + }, + }, + { + name: "default url with git extension", + args: args{ + url: "https://gitlab.com/MichaelMure/git-bug.git", + }, + want: want{ + path: "MichaelMure/git-bug", + err: nil, + }, + }, + { + name: "url with git protocol", + args: args{ + url: "git://gitlab.com/MichaelMure/git-bug.git", + }, + want: want{ + path: "MichaelMure/git-bug", + err: nil, + }, + }, + { + name: "ssh url", + args: args{ + url: "git@gitlab.com/MichaelMure/git-bug.git", + }, + want: want{ + path: "MichaelMure/git-bug", + err: nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := getProjectPath(tt.args.url) + assert.Equal(t, tt.want.path, path) + assert.Equal(t, tt.want.err, err) + }) + } +} -- cgit From b9a533804940989617890b84d5008e8bb7c8e15b Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 19 Jul 2019 18:56:58 +0200 Subject: bridge/gitlab: move constants to gitlab.go --- bridge/gitlab/export.go | 57 +++++++++++++++++++++++++++++++++++-------------- bridge/gitlab/gitlab.go | 17 +++++++++++++++ bridge/gitlab/import.go | 6 +----- 3 files changed, 59 insertions(+), 21 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 2f70e01a..bf48e31a 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -18,12 +18,6 @@ var ( ErrMissingIdentityToken = errors.New("missing identity token") ) -const ( - keyGitlabId = "gitlab-id" - keyGitlabUrl = "gitlab-url" - keyOrigin = "origin" -) - // gitlabExporter implement the Exporter interface type gitlabExporter struct { conf core.Configuration @@ -288,7 +282,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan if targetHash == bugCreationHash { // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(client, bugGitlabID, opr.Message); err != nil { + if err := updateGitlabIssueBody(client, 0, opr.Message); err != nil { err := errors.Wrap(err, "editing issue") out <- core.NewExportError(err, b.Id()) return @@ -323,7 +317,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan case *bug.SetStatusOperation: opr := op.(*bug.SetStatusOperation) - if err := updateGitlabIssueStatus(client, bugGitlabID, opr.Status); err != nil { + if err := updateGitlabIssueStatus(client, 0, opr.Status); err != nil { err := errors.Wrap(err, "editing status") out <- core.NewExportError(err, b.Id()) return @@ -336,7 +330,7 @@ func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan case *bug.SetTitleOperation: opr := op.(*bug.SetTitleOperation) - if err := updateGitlabIssueTitle(client, bugGitlabID, opr.Title); err != nil { + if err := updateGitlabIssueTitle(client, 0, opr.Title); err != nil { err := errors.Wrap(err, "editing title") out <- core.NewExportError(err, b.Id()) return @@ -466,8 +460,16 @@ func (ge *gitlabExporter) getLabelsIDs(gc *gitlab.Client, repositoryID string, l // create a gitlab. issue and return it ID func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (string, string, error) { - return "", "", nil + issue, _, err := gc.Issues.CreateIssue(repositoryID, &gitlab.CreateIssueOptions{ + Title: &title, + Description: &body, + }) + if err != nil { + return "", "", err + } + + return strconv.Itoa(issue.IID), issue.WebURL, nil } // add a comment to an issue and return it ID @@ -479,16 +481,39 @@ func editCommentGitlabIssue(gc *gitlab.Client, commentID, body string) (string, return "", "", nil } -func updateGitlabIssueStatus(gc *gitlab.Client, id string, status bug.Status) error { - return nil +func updateGitlabIssueStatus(gc *gitlab.Client, id int, status bug.Status) error { + var state string + + switch status { + case bug.OpenStatus: + state = "reopen" + case bug.ClosedStatus: + state = "close" + default: + panic("unknown bug state") + } + + _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ + StateEvent: &state, + }) + + return err } -func updateGitlabIssueBody(gc *gitlab.Client, id string, body string) error { - return nil +func updateGitlabIssueBody(gc *gitlab.Client, id int, body string) error { + _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ + Description: &body, + }) + + return err } -func updateGitlabIssueTitle(gc *gitlab.Client, id, title string) error { - return nil +func updateGitlabIssueTitle(gc *gitlab.Client, id int, title string) error { + _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ + Title: &title, + }) + + return err } // update gitlab. issue labels diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index 9f5807cd..d0faf1ef 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -1,11 +1,28 @@ package gitlab import ( + "time" + "github.com/xanzy/go-gitlab" "github.com/MichaelMure/git-bug/bridge/core" ) +const ( + target = "gitlab" + gitlabV4Url = "https://gitlab.com/api/v4" + + keyProjectID = "project-id" + keyGitlabId = "gitlab-id" + keyGitlabUrl = "gitlab-url" + keyGitlabLogin = "gitlab-login" + keyToken = "token" + keyTarget = "target" + keyOrigin = "origin" + + defaultTimeout = 60 * time.Second +) + func init() { core.Register(&Gitlab{}) } diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 93ad8570..6227767b 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -14,10 +14,6 @@ import ( "github.com/MichaelMure/git-bug/util/text" ) -const ( - keyGitlabLogin = "gitlab-login" -) - type gitlabImporter struct { conf core.Configuration @@ -170,7 +166,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n // since gitlab doesn't provide the issue history // we should check for "changed the description" notes and compare issue texts - + // TODO: Check only one time and ignore next 'description change' within one issue if issue.Description != b.Snapshot().Comments[0].Message { // comment edition -- cgit From 5e2eb5000bea9ef1268c3ff0dd24c8126f850d85 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 19 Jul 2019 19:35:27 +0200 Subject: bridge/gitlab: remove exporter --- bridge/gitlab/export.go | 496 +----------------------------------------------- 1 file changed, 1 insertion(+), 495 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index bf48e31a..0aafeef9 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -1,17 +1,12 @@ package gitlab import ( - "fmt" - "strconv" "time" "github.com/pkg/errors" - "github.com/xanzy/go-gitlab" "github.com/MichaelMure/git-bug/bridge/core" - "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" - "github.com/MichaelMure/git-bug/util/git" ) var ( @@ -20,503 +15,14 @@ var ( // gitlabExporter implement the Exporter interface type gitlabExporter struct { - conf core.Configuration - - // cache identities clients - identityClient map[string]*gitlab.Client - - // map identities with their tokens - identityToken map[string]string - - // gitlab repository ID - repositoryID string - - // cache identifiers used to speed up exporting operations - // cleared for each bug - cachedOperationIDs map[string]string - - // cache labels used to speed up exporting labels events - cachedLabels map[string]string } // Init . func (ge *gitlabExporter) Init(conf core.Configuration) error { - ge.conf = conf - //TODO: initialize with multiple tokens - ge.identityToken = make(map[string]string) - ge.identityClient = make(map[string]*gitlab.Client) - ge.cachedOperationIDs = make(map[string]string) - ge.cachedLabels = make(map[string]string) return nil } -// getIdentityClient return a gitlab v4 API client configured with the access token of the given identity. -// if no client were found it will initialize it from the known tokens map and cache it for next use -func (ge *gitlabExporter) getIdentityClient(id string) (*gitlab.Client, error) { - client, ok := ge.identityClient[id] - if ok { - return client, nil - } - - // get token - token, ok := ge.identityToken[id] - if !ok { - return nil, ErrMissingIdentityToken - } - - // create client - client = buildClient(token) - // cache client - ge.identityClient[id] = client - - return client, nil -} - // ExportAll export all event made by the current user to Gitlab func (ge *gitlabExporter) ExportAll(repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { - out := make(chan core.ExportResult) - - user, err := repo.GetUserIdentity() - if err != nil { - return nil, err - } - - ge.identityToken[user.Id()] = ge.conf[keyToken] - - // get repository node id - ge.repositoryID = ge.conf[keyProjectID] - - if err != nil { - return nil, err - } - - go func() { - defer close(out) - - var allIdentitiesIds []string - for id := range ge.identityToken { - allIdentitiesIds = append(allIdentitiesIds, id) - } - - allBugsIds := repo.AllBugsIds() - - for _, id := range allBugsIds { - b, err := repo.ResolveBug(id) - if err != nil { - out <- core.NewExportError(err, id) - return - } - - snapshot := b.Snapshot() - - // ignore issues created before since date - // TODO: compare the Lamport time instead of using the unix time - if snapshot.CreatedAt.Before(since) { - out <- core.NewExportNothing(b.Id(), "bug created before the since date") - continue - } - - if snapshot.HasAnyActor(allIdentitiesIds...) { - // try to export the bug and it associated events - ge.exportBug(b, since, out) - } else { - out <- core.NewExportNothing(id, "not an actor") - } - } - }() - - return out, nil -} - -// exportBug publish bugs and related events -func (ge *gitlabExporter) exportBug(b *cache.BugCache, since time.Time, out chan<- core.ExportResult) { - snapshot := b.Snapshot() - - var bugGitlabID string - var bugGitlabURL string - var bugCreationHash string - - // Special case: - // if a user try to export a bug that is not already exported to Gitlab (or imported - // from Gitlab) and we do not have the token of the bug author, there is nothing we can do. - - // first operation is always createOp - createOp := snapshot.Operations[0].(*bug.CreateOperation) - author := snapshot.Author - - // skip bug if origin is not allowed - origin, ok := snapshot.GetCreateMetadata(keyOrigin) - if ok && origin != target { - out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) - return - } - - // get gitlab bug ID - gitlabID, ok := snapshot.GetCreateMetadata(keyGitlabId) - if ok { - gitlabURL, ok := snapshot.GetCreateMetadata(keyGitlabUrl) - if !ok { - // if we find gitlab ID, gitlab URL must be found too - err := fmt.Errorf("expected to find gitlab issue URL") - out <- core.NewExportError(err, b.Id()) - } - - //FIXME: - // ignore issue comming from other repositories - - out <- core.NewExportNothing(b.Id(), "bug already exported") - // will be used to mark operation related to a bug as exported - bugGitlabID = gitlabID - bugGitlabURL = gitlabURL - - } else { - // check that we have a token for operation author - client, err := ge.getIdentityClient(author.Id()) - if err != nil { - // if bug is still not exported and we do not have the author stop the execution - out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token")) - return - } - - // create bug - id, url, err := createGitlabIssue(client, ge.repositoryID, createOp.Title, createOp.Message) - if err != nil { - err := errors.Wrap(err, "exporting gitlab issue") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportBug(b.Id()) - - hash, err := createOp.Hash() - if err != nil { - err := errors.Wrap(err, "comment hash") - out <- core.NewExportError(err, b.Id()) - return - } - - // mark bug creation operation as exported - if err := markOperationAsExported(b, hash, id, url); err != nil { - err := errors.Wrap(err, "marking operation as exported") - out <- core.NewExportError(err, b.Id()) - return - } - - // commit operation to avoid creating multiple issues with multiple pushes - if err := b.CommitAsNeeded(); err != nil { - err := errors.Wrap(err, "bug commit") - out <- core.NewExportError(err, b.Id()) - return - } - - // cache bug gitlab ID and URL - bugGitlabID = id - bugGitlabURL = url - } - - // get createOp hash - hash, err := createOp.Hash() - if err != nil { - out <- core.NewExportError(err, b.Id()) - return - } - - bugCreationHash = hash.String() - - // cache operation gitlab id - ge.cachedOperationIDs[bugCreationHash] = bugGitlabID - - for _, op := range snapshot.Operations[1:] { - // ignore SetMetadata operations - if _, ok := op.(*bug.SetMetadataOperation); ok { - continue - } - - // get operation hash - hash, err := op.Hash() - if err != nil { - err := errors.Wrap(err, "operation hash") - out <- core.NewExportError(err, b.Id()) - return - } - - // ignore operations already existing in gitlab (due to import or export) - // cache the ID of already exported or imported issues and events from Gitlab - if id, ok := op.GetMetadata(keyGitlabId); ok { - ge.cachedOperationIDs[hash.String()] = id - out <- core.NewExportNothing(hash.String(), "already exported operation") - continue - } - - opAuthor := op.GetAuthor() - client, err := ge.getIdentityClient(opAuthor.Id()) - if err != nil { - out <- core.NewExportNothing(hash.String(), "missing operation author token") - continue - } - - var id, url string - switch op.(type) { - case *bug.AddCommentOperation: - opr := op.(*bug.AddCommentOperation) - - // send operation to gitlab - id, url, err = addCommentGitlabIssue(client, bugGitlabID, opr.Message) - if err != nil { - err := errors.Wrap(err, "adding comment") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportComment(hash.String()) - - // cache comment id - ge.cachedOperationIDs[hash.String()] = id - - case *bug.EditCommentOperation: - - opr := op.(*bug.EditCommentOperation) - targetHash := opr.Target.String() - - // Since gitlab doesn't consider the issue body as a comment - if targetHash == bugCreationHash { - - // case bug creation operation: we need to edit the Gitlab issue - if err := updateGitlabIssueBody(client, 0, opr.Message); err != nil { - err := errors.Wrap(err, "editing issue") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportCommentEdition(hash.String()) - - id = bugGitlabID - url = bugGitlabURL - - } else { - - // case comment edition operation: we need to edit the Gitlab comment - commentID, ok := ge.cachedOperationIDs[targetHash] - if !ok { - panic("unexpected error: comment id not found") - } - - eid, eurl, err := editCommentGitlabIssue(client, commentID, opr.Message) - if err != nil { - err := errors.Wrap(err, "editing comment") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportCommentEdition(hash.String()) - - // use comment id/url instead of issue id/url - id = eid - url = eurl - } - - case *bug.SetStatusOperation: - opr := op.(*bug.SetStatusOperation) - if err := updateGitlabIssueStatus(client, 0, opr.Status); err != nil { - err := errors.Wrap(err, "editing status") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportStatusChange(hash.String()) - - id = bugGitlabID - url = bugGitlabURL - - case *bug.SetTitleOperation: - opr := op.(*bug.SetTitleOperation) - if err := updateGitlabIssueTitle(client, 0, opr.Title); err != nil { - err := errors.Wrap(err, "editing title") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportTitleEdition(hash.String()) - - id = bugGitlabID - url = bugGitlabURL - - case *bug.LabelChangeOperation: - opr := op.(*bug.LabelChangeOperation) - if err := ge.updateGitlabIssueLabels(client, bugGitlabID, opr.Added, opr.Removed); err != nil { - err := errors.Wrap(err, "updating labels") - out <- core.NewExportError(err, b.Id()) - return - } - - out <- core.NewExportLabelChange(hash.String()) - - id = bugGitlabID - url = bugGitlabURL - - default: - panic("unhandled operation type case") - } - - // mark operation as exported - if err := markOperationAsExported(b, hash, id, url); err != nil { - err := errors.Wrap(err, "marking operation as exported") - out <- core.NewExportError(err, b.Id()) - return - } - - // commit at each operation export to avoid exporting same events multiple times - if err := b.CommitAsNeeded(); err != nil { - err := errors.Wrap(err, "bug commit") - out <- core.NewExportError(err, b.Id()) - return - } - } -} - -func markOperationAsExported(b *cache.BugCache, target git.Hash, gitlabID, gitlabURL string) error { - _, err := b.SetMetadata( - target, - map[string]string{ - keyGitlabId: gitlabID, - keyGitlabUrl: gitlabURL, - }, - ) - - return err -} - -func (ge *gitlabExporter) getGitlabLabelID(label string) (string, error) { - id, ok := ge.cachedLabels[label] - if !ok { - return "", fmt.Errorf("non cached label") - } - - return id, nil -} - -// get label from gitlab -func (ge *gitlabExporter) loadLabelsFromGitlab(client *gitlab.Client) error { - - labels, _, err := client.Labels.ListLabels( - ge.repositoryID, - &gitlab.ListLabelsOptions{ - Page: 0, - }, - ) - - if err != nil { - return err - } - - for _, label := range labels { - ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) - } - - for page := 2; len(labels) != 0; page++ { - - labels, _, err = client.Labels.ListLabels( - ge.repositoryID, - &gitlab.ListLabelsOptions{ - Page: page, - }, - ) - - if err != nil { - return err - } - - for _, label := range labels { - ge.cachedLabels[label.Name] = strconv.Itoa(label.ID) - } - } - - return nil -} - -func (ge *gitlabExporter) createGitlabLabel(gc *gitlab.Client, label bug.Label) (string, error) { - client := buildClient(ge.conf[keyToken]) - - // RGBA to hex color - rgba := label.RGBA() - hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B) - name := label.String() - - _, _, err := client.Labels.CreateLabel(ge.repositoryID, &gitlab.CreateLabelOptions{ - Name: &name, - Color: &hexColor, - }) - - if err != nil { - return "", err - } - - return "", nil -} - -func (ge *gitlabExporter) getLabelsIDs(gc *gitlab.Client, repositoryID string, labels []bug.Label) ([]string, error) { - return []string{}, nil -} - -// create a gitlab. issue and return it ID -func createGitlabIssue(gc *gitlab.Client, repositoryID, title, body string) (string, string, error) { - issue, _, err := gc.Issues.CreateIssue(repositoryID, &gitlab.CreateIssueOptions{ - Title: &title, - Description: &body, - }) - - if err != nil { - return "", "", err - } - - return strconv.Itoa(issue.IID), issue.WebURL, nil -} - -// add a comment to an issue and return it ID -func addCommentGitlabIssue(gc *gitlab.Client, subjectID string, body string) (string, string, error) { - return "", "", nil -} - -func editCommentGitlabIssue(gc *gitlab.Client, commentID, body string) (string, string, error) { - return "", "", nil -} - -func updateGitlabIssueStatus(gc *gitlab.Client, id int, status bug.Status) error { - var state string - - switch status { - case bug.OpenStatus: - state = "reopen" - case bug.ClosedStatus: - state = "close" - default: - panic("unknown bug state") - } - - _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ - StateEvent: &state, - }) - - return err -} - -func updateGitlabIssueBody(gc *gitlab.Client, id int, body string) error { - _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ - Description: &body, - }) - - return err -} - -func updateGitlabIssueTitle(gc *gitlab.Client, id int, title string) error { - _, _, err := gc.Issues.UpdateIssue("", id, &gitlab.UpdateIssueOptions{ - Title: &title, - }) - - return err -} - -// update gitlab. issue labels -func (ge *gitlabExporter) updateGitlabIssueLabels(gc *gitlab.Client, labelableID string, added, removed []bug.Label) error { - return nil + return nil, nil } -- cgit From b18507836cd1716ba842e82eb8d42c82b8d62d71 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Fri, 19 Jul 2019 19:39:15 +0200 Subject: bridge/gitlab: add gitlab client default timeout bridge/gitlab: fix import bug --- bridge/gitlab/gitlab.go | 14 ++++++-------- bridge/gitlab/import.go | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index d0faf1ef..743ab172 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -1,6 +1,7 @@ package gitlab import ( + "net/http" "time" "github.com/xanzy/go-gitlab" @@ -9,9 +10,7 @@ import ( ) const ( - target = "gitlab" - gitlabV4Url = "https://gitlab.com/api/v4" - + target = "gitlab" keyProjectID = "project-id" keyGitlabId = "gitlab-id" keyGitlabUrl = "gitlab-url" @@ -42,10 +41,9 @@ func (*Gitlab) NewExporter() core.Exporter { } func buildClient(token string) *gitlab.Client { - return gitlab.NewClient(nil, token) -} - -func buildClientFromUsernameAndPassword(username, password string) (*gitlab.Client, error) { - return gitlab.NewBasicAuthClient(nil, "https://gitlab.com", username, password) + client := &http.Client{ + Timeout: defaultTimeout, + } + return gitlab.NewClient(client, token) } diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 6227767b..6869a103 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -120,7 +120,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue return b, nil } - return nil, nil + return b, nil } func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, note *gitlab.Note) error { -- cgit From b27647c7a0dd95fdbbe4d22962129615fc5c9325 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Mon, 22 Jul 2019 00:13:47 +0200 Subject: bridge/gitlab: Fix test project path bridge/gitlab: update comments --- bridge/gitlab/import.go | 4 ++++ bridge/gitlab/import_notes.go | 5 +---- bridge/gitlab/import_test.go | 10 +++++----- bridge/gitlab/iterator.go | 1 + 4 files changed, 11 insertions(+), 9 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 6869a103..b2db13d0 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -14,6 +14,7 @@ import ( "github.com/MichaelMure/git-bug/util/text" ) +// gitlabImporter implement the Importer interface type gitlabImporter struct { conf core.Configuration @@ -32,6 +33,8 @@ func (gi *gitlabImporter) Init(conf core.Configuration) error { return nil } +// ImportAll iterate over all the configured repository issues (notes) and ensure the creation +// of the missing issues / comments / label events / title changes ... func (gi *gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { gi.iterator = NewIterator(gi.conf[keyProjectID], gi.conf[keyToken], since) @@ -90,6 +93,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue return nil, err } + // if bug was never imported if err == bug.ErrBugNotExist { cleanText, err := text.Cleanup(string(issue.Description)) if err != nil { diff --git a/bridge/gitlab/import_notes.go b/bridge/gitlab/import_notes.go index 37bf2834..a096b14e 100644 --- a/bridge/gitlab/import_notes.go +++ b/bridge/gitlab/import_notes.go @@ -26,10 +26,7 @@ const ( NOTE_UNKNOWN ) -// GetNoteType parses note body a give it type -// Since gitlab api return all these NoteType event as the same object -// and doesn't provide a field to specify the note type. We must parse the -// note body to detect it type. +// GetNoteType parse a note system and body and return the note type and it content func GetNoteType(n *gitlab.Note) (NoteType, string) { if !n.System { return NOTE_COMMENT, n.Body diff --git a/bridge/gitlab/import_test.go b/bridge/gitlab/import_test.go index 4a49cfe9..c38d3ce3 100644 --- a/bridge/gitlab/import_test.go +++ b/bridge/gitlab/import_test.go @@ -26,7 +26,7 @@ func TestImport(t *testing.T) { }{ { name: "simple issue", - url: "https://gitlab.com/a-hilaly/git-bug-test/issues/1", + url: "https://gitlab.com/git-bug/test/issues/1", bug: &bug.Snapshot{ Operations: []bug.Operation{ bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), @@ -37,7 +37,7 @@ func TestImport(t *testing.T) { }, { name: "empty issue", - url: "https://gitlab.com/a-hilaly/git-bug-test/issues/2", + url: "https://gitlab.com/git-bug/test/issues/2", bug: &bug.Snapshot{ Operations: []bug.Operation{ bug.NewCreateOp(author, 0, "empty issue", "", nil), @@ -46,7 +46,7 @@ func TestImport(t *testing.T) { }, { name: "complex issue", - url: "https://gitlab.com/a-hilaly/git-bug-test/issues/3", + url: "https://gitlab.com/git-bug/test/issues/3", bug: &bug.Snapshot{ Operations: []bug.Operation{ bug.NewCreateOp(author, 0, "complex issue", "initial comment", nil), @@ -63,7 +63,7 @@ func TestImport(t *testing.T) { }, { name: "editions", - url: "https://gitlab.com/a-hilaly/git-bug-test/issues/4", + url: "https://gitlab.com/git-bug/test/issues/4", bug: &bug.Snapshot{ Operations: []bug.Operation{ bug.NewCreateOp(author, 0, "editions", "initial comment edited", nil), @@ -113,7 +113,7 @@ func TestImport(t *testing.T) { require.NoError(t, err) ops := b.Snapshot().Operations - assert.Len(t, tt.bug.Operations, len(ops)) + require.Len(t, tt.bug.Operations, len(ops)) for i, op := range tt.bug.Operations { diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 8502504d..5a627ade 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -47,6 +47,7 @@ type iterator struct { // notes iterator note *noteIterator + // labelEvent iterator labelEvent *labelEventIterator } -- cgit From ece2cb126293361212d7673fea976876af7b811b Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Mon, 22 Jul 2019 18:56:14 +0200 Subject: bridge/gitlab: improve tests and errors bridge/gitlab: global fixes --- bridge/gitlab/config.go | 19 +++++++++---------- bridge/gitlab/config_test.go | 9 +++++++++ bridge/gitlab/import.go | 14 ++++++++------ 3 files changed, 26 insertions(+), 16 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index efef5993..a375bab2 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -3,7 +3,7 @@ package gitlab import ( "bufio" "fmt" - neturl "net/url" + "net/url" "os" "regexp" "strconv" @@ -41,13 +41,13 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( // remote suggestions remotes, err := repo.GetRemotes() if err != nil { - return nil, err + return nil, errors.Wrap(err, "getting remotes") } // terminal prompt url, err = promptURL(remotes) if err != nil { - return nil, err + return nil, errors.Wrap(err, "url prompt") } } @@ -57,7 +57,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( } else { token, err = promptToken() if err != nil { - return nil, err + return nil, errors.Wrap(err, "token prompt") } } @@ -65,7 +65,7 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( // validate project url and get it ID ok, id, err := validateProjectURL(url, token) if err != nil { - return nil, err + return nil, errors.Wrap(err, "project validation") } if !ok { return nil, fmt.Errorf("invalid project id or wrong token scope") @@ -180,15 +180,14 @@ func promptURL(remotes map[string]string) (string, error) { } } -func getProjectPath(url string) (string, error) { - cleanUrl := strings.TrimSuffix(url, ".git") +func getProjectPath(projectUrl string) (string, error) { + cleanUrl := strings.TrimSuffix(projectUrl, ".git") cleanUrl = strings.Replace(cleanUrl, "git@", "https://", 1) - objectUrl, err := neturl.Parse(cleanUrl) + objectUrl, err := url.Parse(cleanUrl) if err != nil { - return "", err + return "", ErrBadProjectURL } - fmt.Println(objectUrl.Path) return objectUrl.Path[1:], nil } diff --git a/bridge/gitlab/config_test.go b/bridge/gitlab/config_test.go index 248cdb66..87469796 100644 --- a/bridge/gitlab/config_test.go +++ b/bridge/gitlab/config_test.go @@ -69,6 +69,15 @@ func TestProjectPath(t *testing.T) { err: nil, }, }, + { + name: "bad url", + args: args{ + url: "---,%gitlab.com/MichaelMure/git-bug.git", + }, + want: want{ + err: ErrBadProjectURL, + }, + }, } for _, tt := range tests { diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index b2db13d0..67d9aa25 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -95,7 +95,7 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue // if bug was never imported if err == bug.ErrBugNotExist { - cleanText, err := text.Cleanup(string(issue.Description)) + cleanText, err := text.Cleanup(issue.Description) if err != nil { return nil, err } @@ -261,10 +261,12 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n return err - default: - // non handled note types, this is not an error + case NOTE_UNKNOWN: //TODO: send warning via channel return nil + + default: + panic("unhandled note type") } return nil @@ -322,9 +324,6 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id return nil, err } - // importing a new identity - gi.importedIdentities++ - client := buildClient(gi.conf["token"]) user, _, err := client.Users.GetUser(id) @@ -332,6 +331,9 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id return nil, err } + // importing a new identity + gi.importedIdentities++ + return repo.NewIdentityRaw( user.Name, user.PublicEmail, -- cgit From d098a96407c55281c28bfdea9925df587b4d4400 Mon Sep 17 00:00:00 2001 From: Amine Date: Tue, 23 Jul 2019 17:10:07 +0200 Subject: bridge/gitlab: global code and comment updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Michael MurĂ© --- bridge/gitlab/config.go | 8 ++++---- bridge/gitlab/iterator.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index a375bab2..dbbd1bd9 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -62,13 +62,13 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( } var ok bool - // validate project url and get it ID + // validate project url and get its ID ok, id, err := validateProjectURL(url, token) if err != nil { return nil, errors.Wrap(err, "project validation") } if !ok { - return nil, fmt.Errorf("invalid project id or wrong token scope") + return nil, fmt.Errorf("invalid project id or incorrect token scope") } conf[keyProjectID] = strconv.Itoa(id) @@ -121,7 +121,7 @@ func promptToken() (string, error) { return token, nil } - fmt.Println("token is invalid") + fmt.Println("token format is invalid") } } @@ -147,7 +147,7 @@ func promptURL(remotes map[string]string) (string, error) { line = strings.TrimRight(line, "\n") index, err := strconv.Atoi(line) - if err != nil || (index < 0 && index >= len(validRemotes)) { + if err != nil || (index < 0 && index > len(validRemotes)) { fmt.Println("invalid input") continue } diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 5a627ade..8b7177f6 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -124,7 +124,7 @@ func (i *iterator) NextIssue() bool { } // move cursor index - if i.issue.index < min(i.capacity, len(i.issue.cache))-1 { + if i.issue.index < len(i.issue.cache)-1 { i.issue.index++ return true } @@ -180,7 +180,7 @@ func (i *iterator) NextNote() bool { } // move cursor index - if i.note.index < min(i.capacity, len(i.note.cache))-1 { + if i.note.index < len(i.note.cache)-1 { i.note.index++ return true } @@ -232,7 +232,7 @@ func (i *iterator) NextLabelEvent() bool { } // move cursor index - if i.labelEvent.index < min(i.capacity, len(i.labelEvent.cache))-1 { + if i.labelEvent.index < len(i.labelEvent.cache)-1 { i.labelEvent.index++ return true } -- cgit From 0329bfdf440ec48c5c5c5c6dbe2ca8519d99b706 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 23 Jul 2019 17:29:53 +0200 Subject: bridge/gitlab: change validateProjectURL signature bridge/gitlab: code cleanup --- bridge/gitlab/config.go | 26 +++++++++++--------------- bridge/gitlab/export.go | 3 +-- bridge/gitlab/gitlab.go | 12 ++++++------ bridge/gitlab/import.go | 6 +++--- bridge/gitlab/iterator.go | 8 -------- 5 files changed, 21 insertions(+), 34 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/config.go b/bridge/gitlab/config.go index dbbd1bd9..15172871 100644 --- a/bridge/gitlab/config.go +++ b/bridge/gitlab/config.go @@ -61,26 +61,22 @@ func (*Gitlab) Configure(repo repository.RepoCommon, params core.BridgeParams) ( } } - var ok bool // validate project url and get its ID - ok, id, err := validateProjectURL(url, token) + id, err := validateProjectURL(url, token) if err != nil { return nil, errors.Wrap(err, "project validation") } - if !ok { - return nil, fmt.Errorf("invalid project id or incorrect token scope") - } conf[keyProjectID] = strconv.Itoa(id) conf[keyToken] = token - conf[keyTarget] = target + conf[core.KeyTarget] = target return conf, nil } func (*Gitlab) ValidateConfig(conf core.Configuration) error { - if v, ok := conf[keyTarget]; !ok { - return fmt.Errorf("missing %s key", keyTarget) + if v, ok := conf[core.KeyTarget]; !ok { + return fmt.Errorf("missing %s key", core.KeyTarget) } else if v != target { return fmt.Errorf("unexpected target name: %v", v) } @@ -147,7 +143,7 @@ func promptURL(remotes map[string]string) (string, error) { line = strings.TrimRight(line, "\n") index, err := strconv.Atoi(line) - if err != nil || (index < 0 && index > len(validRemotes)) { + if err != nil || index < 0 || index > len(validRemotes) { fmt.Println("invalid input") continue } @@ -205,18 +201,18 @@ func getValidGitlabRemoteURLs(remotes map[string]string) []string { return urls } -func validateProjectURL(url, token string) (bool, int, error) { - client := buildClient(token) - +func validateProjectURL(url, token string) (int, error) { projectPath, err := getProjectPath(url) if err != nil { - return false, 0, err + return 0, err } + client := buildClient(token) + project, _, err := client.Projects.GetProject(projectPath, &gitlab.GetProjectOptions{}) if err != nil { - return false, 0, err + return 0, err } - return true, project.ID, nil + return project.ID, nil } diff --git a/bridge/gitlab/export.go b/bridge/gitlab/export.go index 0aafeef9..85b5ee2e 100644 --- a/bridge/gitlab/export.go +++ b/bridge/gitlab/export.go @@ -14,8 +14,7 @@ var ( ) // gitlabExporter implement the Exporter interface -type gitlabExporter struct { -} +type gitlabExporter struct{} // Init . func (ge *gitlabExporter) Init(conf core.Configuration) error { diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index 743ab172..a52ea2c1 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -10,14 +10,14 @@ import ( ) const ( - target = "gitlab" - keyProjectID = "project-id" + target = "gitlab" + keyGitlabId = "gitlab-id" keyGitlabUrl = "gitlab-url" keyGitlabLogin = "gitlab-login" - keyToken = "token" - keyTarget = "target" - keyOrigin = "origin" + + keyProjectID = "project-id" + keyToken = "token" defaultTimeout = 60 * time.Second ) @@ -37,7 +37,7 @@ func (*Gitlab) NewImporter() core.Importer { } func (*Gitlab) NewExporter() core.Exporter { - return &gitlabExporter{} + return nil } func buildClient(token string) *gitlab.Client { diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 67d9aa25..b19587a9 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -108,9 +108,9 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue cleanText, nil, map[string]string{ - keyOrigin: target, - keyGitlabId: parseID(issue.ID), - keyGitlabUrl: issue.WebURL, + core.KeyOrigin: target, + keyGitlabId: parseID(issue.ID), + keyGitlabUrl: issue.WebURL, }, ) diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 8b7177f6..320f2a50 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -243,11 +243,3 @@ func (i *iterator) NextLabelEvent() bool { func (i *iterator) LabelEventValue() *gitlab.LabelEvent { return i.labelEvent.cache[i.labelEvent.index] } - -func min(a, b int) int { - if a > b { - return b - } - - return a -} -- cgit From 0c8f1c3a58c4707284cc6368af26715520a220c0 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 23 Jul 2019 18:40:10 +0200 Subject: bridge/gitlab: fix comment edition target hash in the import bridge/gitlab: global changes, typo fixes, comments addition --- bridge/gitlab/gitlab.go | 7 +-- bridge/gitlab/import.go | 103 ++++++++++++++++++------------------------ bridge/gitlab/import_notes.go | 2 + bridge/gitlab/iterator.go | 24 ++++------ 4 files changed, 61 insertions(+), 75 deletions(-) (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/gitlab.go b/bridge/gitlab/gitlab.go index a52ea2c1..63c212ed 100644 --- a/bridge/gitlab/gitlab.go +++ b/bridge/gitlab/gitlab.go @@ -12,9 +12,10 @@ import ( const ( target = "gitlab" - keyGitlabId = "gitlab-id" - keyGitlabUrl = "gitlab-url" - keyGitlabLogin = "gitlab-login" + keyGitlabId = "gitlab-id" + keyGitlabUrl = "gitlab-url" + keyGitlabLogin = "gitlab-login" + keyGitlabProject = "gitlab-project-id" keyProjectID = "project-id" keyToken = "token" diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index b19587a9..20c586a7 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -11,6 +11,7 @@ import ( "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/identity" + "github.com/MichaelMure/git-bug/util/git" "github.com/MichaelMure/git-bug/util/text" ) @@ -65,17 +66,17 @@ func (gi *gitlabImporter) ImportAll(repo *cache.RepoCache, since time.Time) erro } } + if err := gi.iterator.Error(); err != nil { + fmt.Printf("import error: %v\n", err) + return err + } + // commit bug state if err := b.CommitAsNeeded(); err != nil { return fmt.Errorf("bug commit: %v", err) } } - if err := gi.iterator.Error(); err != nil { - fmt.Printf("import error: %v\n", err) - return err - } - fmt.Printf("Successfully imported %d issues and %d identities from Gitlab\n", gi.importedIssues, gi.importedIdentities) return nil } @@ -93,37 +94,38 @@ func (gi *gitlabImporter) ensureIssue(repo *cache.RepoCache, issue *gitlab.Issue return nil, err } - // if bug was never imported - if err == bug.ErrBugNotExist { - cleanText, err := text.Cleanup(issue.Description) - if err != nil { - return nil, err - } - - // create bug - b, _, err = repo.NewBugRaw( - author, - issue.CreatedAt.Unix(), - issue.Title, - cleanText, - nil, - map[string]string{ - core.KeyOrigin: target, - keyGitlabId: parseID(issue.ID), - keyGitlabUrl: issue.WebURL, - }, - ) + if err == nil { + return b, nil + } - if err != nil { - return nil, err - } + // if bug was never imported + cleanText, err := text.Cleanup(issue.Description) + if err != nil { + return nil, err + } - // importing a new bug - gi.importedIssues++ + // create bug + b, _, err = repo.NewBugRaw( + author, + issue.CreatedAt.Unix(), + issue.Title, + cleanText, + nil, + map[string]string{ + core.KeyOrigin: target, + keyGitlabId: parseID(issue.ID), + keyGitlabUrl: issue.WebURL, + keyGitlabProject: gi.conf[keyProjectID], + }, + ) - return b, nil + if err != nil { + return nil, err } + // importing a new bug + gi.importedIssues++ + return b, nil } @@ -138,7 +140,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n hash, errResolve := b.ResolveOperationWithMetadata(keyGitlabId, id) if errResolve != cache.ErrNoMatchingOp { - return err + return errResolve } noteType, body := GetNoteType(note) @@ -148,8 +150,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n author, note.CreatedAt.Unix(), map[string]string{ - keyGitlabId: id, - keyGitlabUrl: "", + keyGitlabId: id, }, ) return err @@ -159,8 +160,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n author, note.CreatedAt.Unix(), map[string]string{ - keyGitlabId: id, - keyGitlabUrl: "", + keyGitlabId: id, }, ) return err @@ -168,25 +168,24 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n case NOTE_DESCRIPTION_CHANGED: issue := gi.iterator.IssueValue() + firstComment := b.Snapshot().Comments[0] // since gitlab doesn't provide the issue history // we should check for "changed the description" notes and compare issue texts // TODO: Check only one time and ignore next 'description change' within one issue - if issue.Description != b.Snapshot().Comments[0].Message { + if issue.Description != firstComment.Message { // comment edition _, err = b.EditCommentRaw( author, note.UpdatedAt.Unix(), - target, + git.Hash(firstComment.Id()), issue.Description, map[string]string{ - keyGitlabId: id, - keyGitlabUrl: "", + keyGitlabId: id, }, ) return err - } case NOTE_COMMENT: @@ -206,8 +205,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n cleanText, nil, map[string]string{ - keyGitlabId: id, - keyGitlabUrl: "", + keyGitlabId: id, }, ) @@ -216,11 +214,6 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n // if comment was already exported - // if note wasn't updated - if note.UpdatedAt.Equal(*note.CreatedAt) { - return nil - } - // search for last comment update comment, err := b.Snapshot().SearchComment(hash) if err != nil { @@ -233,13 +226,9 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n _, err = b.EditCommentRaw( author, note.UpdatedAt.Unix(), - target, + hash, cleanText, - map[string]string{ - // no metadata unique metadata to store - keyGitlabId: "", - keyGitlabUrl: "", - }, + nil, ) return err @@ -254,15 +243,13 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n note.CreatedAt.Unix(), body, map[string]string{ - keyGitlabId: id, - keyGitlabUrl: "", + keyGitlabId: id, }, ) return err case NOTE_UNKNOWN: - //TODO: send warning via channel return nil default: @@ -308,7 +295,7 @@ func (gi *gitlabImporter) ensureLabelEvent(repo *cache.RepoCache, b *cache.BugCa ) default: - panic("unexpected label event action") + err = fmt.Errorf("unexpected label event action") } return err diff --git a/bridge/gitlab/import_notes.go b/bridge/gitlab/import_notes.go index a096b14e..06c0a242 100644 --- a/bridge/gitlab/import_notes.go +++ b/bridge/gitlab/import_notes.go @@ -28,6 +28,8 @@ const ( // GetNoteType parse a note system and body and return the note type and it content func GetNoteType(n *gitlab.Note) (NoteType, string) { + // when a note is a comment system is set to false + // when a note is a different event system is set to true if !n.System { return NOTE_COMMENT, n.Body } diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 320f2a50..73f1614e 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -28,8 +28,8 @@ type iterator struct { // gitlab api v4 client gc *gitlab.Client - // if since is given the iterator will query only the updated - // issues after this date + // if since is given the iterator will query only the issues + // updated after this date since time.Time // project id @@ -79,8 +79,6 @@ func (i *iterator) Error() error { } func (i *iterator) getNextIssues() bool { - sort := "asc" - scope := "all" issues, _, err := i.gc.Issues.ListProjectIssues( i.project, &gitlab.ListProjectIssuesOptions{ @@ -88,9 +86,9 @@ func (i *iterator) getNextIssues() bool { Page: i.issue.page, PerPage: i.capacity, }, - Scope: &scope, + Scope: gitlab.String("all"), UpdatedAfter: &i.since, - Sort: &sort, + Sort: gitlab.String("asc"), }, ) @@ -114,15 +112,15 @@ func (i *iterator) getNextIssues() bool { } func (i *iterator) NextIssue() bool { + if i.err != nil { + return false + } + // first query if i.issue.cache == nil { return i.getNextIssues() } - if i.err != nil { - return false - } - // move cursor index if i.issue.index < len(i.issue.cache)-1 { i.issue.index++ @@ -137,8 +135,6 @@ func (i *iterator) IssueValue() *gitlab.Issue { } func (i *iterator) getNextNotes() bool { - sort := "asc" - order := "created_at" notes, _, err := i.gc.Notes.ListIssueNotes( i.project, i.IssueValue().IID, @@ -147,8 +143,8 @@ func (i *iterator) getNextNotes() bool { Page: i.note.page, PerPage: i.capacity, }, - Sort: &sort, - OrderBy: &order, + Sort: gitlab.String("asc"), + OrderBy: gitlab.String("created_at"), }, ) -- cgit From 29fdd37ce69b48aa9fc3c1b829ff67818041068f Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 23 Jul 2019 19:16:52 +0200 Subject: bridge/github: add getNewTitle tests --- bridge/gitlab/export_test.go | 1 + bridge/gitlab/import.go | 12 ++++---- bridge/gitlab/import_notes.go | 2 ++ bridge/gitlab/import_notes_test.go | 56 ++++++++++++++++++++++++++++++++++++++ bridge/gitlab/iterator.go | 1 + 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 bridge/gitlab/export_test.go create mode 100644 bridge/gitlab/import_notes_test.go (limited to 'bridge/gitlab') diff --git a/bridge/gitlab/export_test.go b/bridge/gitlab/export_test.go new file mode 100644 index 00000000..4d4b35af --- /dev/null +++ b/bridge/gitlab/export_test.go @@ -0,0 +1 @@ +package gitlab diff --git a/bridge/gitlab/import.go b/bridge/gitlab/import.go index 20c586a7..8f4ceec9 100644 --- a/bridge/gitlab/import.go +++ b/bridge/gitlab/import.go @@ -138,11 +138,6 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n return err } - hash, errResolve := b.ResolveOperationWithMetadata(keyGitlabId, id) - if errResolve != cache.ErrNoMatchingOp { - return errResolve - } - noteType, body := GetNoteType(note) switch noteType { case NOTE_CLOSED: @@ -189,6 +184,10 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n } case NOTE_COMMENT: + hash, errResolve := b.ResolveOperationWithMetadata(keyGitlabId, id) + if errResolve != cache.ErrNoMatchingOp { + return errResolve + } cleanText, err := text.Cleanup(body) if err != nil { @@ -226,7 +225,7 @@ func (gi *gitlabImporter) ensureNote(repo *cache.RepoCache, b *cache.BugCache, n _, err = b.EditCommentRaw( author, note.UpdatedAt.Unix(), - hash, + git.Hash(comment.Id()), cleanText, nil, ) @@ -327,6 +326,7 @@ func (gi *gitlabImporter) ensurePerson(repo *cache.RepoCache, id int) (*cache.Id user.Username, user.AvatarURL, map[string]string{ + // because Gitlab keyGitlabId: strconv.Itoa(id), keyGitlabLogin: user.Username, }, diff --git a/bridge/gitlab/import_notes.go b/bridge/gitlab/import_notes.go index 06c0a242..85da3158 100644 --- a/bridge/gitlab/import_notes.go +++ b/bridge/gitlab/import_notes.go @@ -30,6 +30,7 @@ const ( func GetNoteType(n *gitlab.Note) (NoteType, string) { // when a note is a comment system is set to false // when a note is a different event system is set to true + // because Gitlab if !n.System { return NOTE_COMMENT, n.Body } @@ -88,6 +89,7 @@ func GetNoteType(n *gitlab.Note) (NoteType, string) { // getNewTitle parses body diff given by gitlab api and return it final form // examples: "changed title from **fourth issue** to **fourth issue{+ changed+}**" // "changed title from **fourth issue{- changed-}** to **fourth issue**" +// because Gitlab func getNewTitle(diff string) string { newTitle := strings.Split(diff, "** to **")[1] newTitle = strings.Replace(newTitle, "{+", "", -1) diff --git a/bridge/gitlab/import_notes_test.go b/bridge/gitlab/import_notes_test.go new file mode 100644 index 00000000..c7b5ab56 --- /dev/null +++ b/bridge/gitlab/import_notes_test.go @@ -0,0 +1,56 @@ +package gitlab + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetNewTitle(t *testing.T) { + type args struct { + diff string + } + type want struct { + title string + } + tests := []struct { + name string + args args + want want + }{ + { + name: "addition diff", + args: args{ + diff: "**first issue** to **first issue{+ edited+}**", + }, + want: want{ + title: "first issue edited", + }, + }, + { + name: "deletion diff", + args: args{ + diff: "**first issue{- edited-}** to **first issue**", + }, + want: want{ + title: "first issue", + }, + }, + { + name: "mixed diff", + args: args{ + diff: "**first {-issue-}** to **first {+bug+}**", + }, + want: want{ + title: "first bug", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + title := getNewTitle(tt.args.diff) + assert.Equal(t, tt.want.title, title) + }) + } +} diff --git a/bridge/gitlab/iterator.go b/bridge/gitlab/iterator.go index 73f1614e..883fea9c 100644 --- a/bridge/gitlab/iterator.go +++ b/bridge/gitlab/iterator.go @@ -218,6 +218,7 @@ func (i *iterator) getNextLabelEvents() bool { return true } +// because Gitlab func (i *iterator) NextLabelEvent() bool { if i.err != nil { return false -- cgit