From 5ca326af83b90531d4d0c502bb1beabbe1b48c55 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Tue, 13 Aug 2019 19:51:14 +0200 Subject: bridge/core: add context.Context to ImportAll and ExportAll signatures bridge/core: add ImportResult objects to stream import events bridge/core: launchpad support asynchronous import bridge/github: cancellable export and import functions bridge/gitlab: cancellable export and import functions commands: bridge pull/push gracefull kill bridge/github: fix github import bridge/github: use simple context for imports bridge/core: name parameters in interfaces github/core: Add EventError to export and import events types bridge/gitlab: add context support in gitlab requests functions bridge/gitlab: remove imported events count from importer logic bridge/github: remove imported events count from importer logic bridge/github: add context support in query and muration requets bridge/github: fix bug duplicate editions after multiple calls bridge/core: import import and export events String methods bridge/gitlab: fix error handling in note import events commands/bridge: Add statistics about imports and exports bridge/gitlab: properly handle context cancellation bridge/github: improve error handling bridge: break iterators on context cancel or timeout bridge: add context timeout support bridge: improve event formating and error handling commands: handle interrupt and switch cases bridge/github: add export mutation timeouts bridge: fix race condition bug in the github and gitlab importers bridge/github: improve context error handling --- bridge/launchpad/import.go | 185 +++++++++++++++++++++++++-------------------- 1 file changed, 101 insertions(+), 84 deletions(-) (limited to 'bridge/launchpad/import.go') diff --git a/bridge/launchpad/import.go b/bridge/launchpad/import.go index 7ef11416..7f50d898 100644 --- a/bridge/launchpad/import.go +++ b/bridge/launchpad/import.go @@ -1,11 +1,10 @@ package launchpad import ( + "context" "fmt" "time" - "github.com/pkg/errors" - "github.com/MichaelMure/git-bug/bridge/core" "github.com/MichaelMure/git-bug/bug" "github.com/MichaelMure/git-bug/cache" @@ -45,98 +44,116 @@ func (li *launchpadImporter) ensurePerson(repo *cache.RepoCache, owner LPPerson) ) } -func (li *launchpadImporter) ImportAll(repo *cache.RepoCache, since time.Time) error { +func (li *launchpadImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) { + out := make(chan core.ImportResult) lpAPI := new(launchpadAPI) err := lpAPI.Init() if err != nil { - return err + return nil, err } - lpBugs, err := lpAPI.SearchTasks(li.conf["project"]) + lpBugs, err := lpAPI.SearchTasks(ctx, li.conf["project"]) if err != nil { - return err + return nil, err } - for _, lpBug := range lpBugs { - var b *cache.BugCache - var err error - - lpBugID := fmt.Sprintf("%d", lpBug.ID) - b, err = repo.ResolveBugCreateMetadata(keyLaunchpadID, lpBugID) - if err != nil && err != bug.ErrBugNotExist { - return err - } - - owner, err := li.ensurePerson(repo, lpBug.Owner) - if err != nil { - return err - } - - if err == bug.ErrBugNotExist { - createdAt, _ := time.Parse(time.RFC3339, lpBug.CreatedAt) - b, _, err = repo.NewBugRaw( - owner, - createdAt.Unix(), - lpBug.Title, - lpBug.Description, - nil, - map[string]string{ - keyLaunchpadID: lpBugID, - }, - ) - if err != nil { - return errors.Wrapf(err, "failed to add bug id #%s", lpBugID) + go func() { + for _, lpBug := range lpBugs { + select { + case <-ctx.Done(): + return + default: + lpBugID := fmt.Sprintf("%d", lpBug.ID) + b, err := repo.ResolveBugCreateMetadata(keyLaunchpadID, lpBugID) + if err != nil && err != bug.ErrBugNotExist { + out <- core.NewImportError(err, entity.Id(lpBugID)) + return + } + + owner, err := li.ensurePerson(repo, lpBug.Owner) + if err != nil { + out <- core.NewImportError(err, entity.Id(lpBugID)) + return + } + + if err == bug.ErrBugNotExist { + createdAt, _ := time.Parse(time.RFC3339, lpBug.CreatedAt) + b, _, err = repo.NewBugRaw( + owner, + createdAt.Unix(), + lpBug.Title, + lpBug.Description, + nil, + map[string]string{ + keyLaunchpadID: lpBugID, + }, + ) + if err != nil { + out <- core.NewImportError(err, entity.Id(lpBugID)) + return + } + + out <- core.NewImportBug(b.Id()) + + } + + /* Handle messages */ + if len(lpBug.Messages) == 0 { + err := fmt.Sprintf("bug doesn't have any comments") + out <- core.NewImportNothing(entity.Id(lpBugID), err) + return + } + + // The Launchpad API returns the bug description as the first + // comment, so skip it. + for _, lpMessage := range lpBug.Messages[1:] { + _, err := b.ResolveOperationWithMetadata(keyLaunchpadID, lpMessage.ID) + if err != nil && err != cache.ErrNoMatchingOp { + out <- core.NewImportError(err, entity.Id(lpMessage.ID)) + return + } + + // If this comment already exists, we are probably + // updating an existing bug. We do not want to duplicate + // the comments, so let us just skip this one. + // TODO: Can Launchpad comments be edited? + if err == nil { + continue + } + + owner, err := li.ensurePerson(repo, lpMessage.Owner) + if err != nil { + out <- core.NewImportError(err, "") + return + } + + // This is a new comment, we can add it. + createdAt, _ := time.Parse(time.RFC3339, lpMessage.CreatedAt) + op, err := b.AddCommentRaw( + owner, + createdAt.Unix(), + lpMessage.Content, + nil, + map[string]string{ + keyLaunchpadID: lpMessage.ID, + }) + if err != nil { + out <- core.NewImportError(err, op.Id()) + return + } + + out <- core.NewImportComment(op.Id()) + } + + err = b.CommitAsNeeded() + if err != nil { + out <- core.NewImportError(err, "") + return + } } - } else { - /* TODO: Update bug */ - fmt.Println("TODO: Update bug") } + }() - /* Handle messages */ - if len(lpBug.Messages) == 0 { - return errors.Wrapf(err, "failed to fetch comments for bug #%s", lpBugID) - } - - // The Launchpad API returns the bug description as the first - // comment, so skip it. - for _, lpMessage := range lpBug.Messages[1:] { - _, err := b.ResolveOperationWithMetadata(keyLaunchpadID, lpMessage.ID) - if err != nil && err != cache.ErrNoMatchingOp { - return errors.Wrapf(err, "failed to fetch comments for bug #%s", lpBugID) - } - - // If this comment already exists, we are probably - // updating an existing bug. We do not want to duplicate - // the comments, so let us just skip this one. - // TODO: Can Launchpad comments be edited? - if err == nil { - continue - } - - owner, err := li.ensurePerson(repo, lpMessage.Owner) - if err != nil { - return err - } - - // This is a new comment, we can add it. - createdAt, _ := time.Parse(time.RFC3339, lpMessage.CreatedAt) - _, err = b.AddCommentRaw( - owner, - createdAt.Unix(), - lpMessage.Content, - nil, - map[string]string{ - keyLaunchpadID: lpMessage.ID, - }) - if err != nil { - return errors.Wrapf(err, "failed to add comment to bug #%s", lpBugID) - } - } - err = b.CommitAsNeeded() - if err != nil { - return err - } - } - return nil + return out, nil } -- cgit