From 614bc5a2c53409a8e9c85ab31a104428069aa7b3 Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 25 Oct 2019 16:42:48 +0200 Subject: commands: support bridge imports after a given date and resumable imports --- bridge/core/bridge.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 7 deletions(-) (limited to 'bridge/core') diff --git a/bridge/core/bridge.go b/bridge/core/bridge.go index 47a89389..0165994a 100644 --- a/bridge/core/bridge.go +++ b/bridge/core/bridge.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "time" @@ -291,7 +292,7 @@ func (b *Bridge) ensureInit() error { return nil } -func (b *Bridge) ImportAll(ctx context.Context, since time.Time) (<-chan ImportResult, error) { +func (b *Bridge) startImportSince(ctx context.Context, since time.Time) (<-chan ImportResult, error) { importer := b.getImporter() if importer == nil { return nil, ErrImportNotSupported @@ -307,24 +308,88 @@ func (b *Bridge) ImportAll(ctx context.Context, since time.Time) (<-chan ImportR return nil, err } - return importer.ImportAll(ctx, b.repo, since) + events, err := importer.ImportAll(ctx, b.repo, since) + if err != nil { + return nil, err + } + + return events, nil +} + +func (b *Bridge) ImportAllSince(ctx context.Context, since time.Time) (<-chan ImportResult, error) { + if since.Equal(time.Time{}) { + lastImportTimeStr, err := b.repo.LocalConfig().ReadString(fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name)) + if err == nil { + lastImportTime, err := strconv.Atoi(lastImportTimeStr) + if err != nil { + return nil, err + } + since = time.Unix(int64(lastImportTime), 0) + } + } + + importStartTime := time.Now().Unix() + + events, err := b.startImportSince(ctx, since) + if err != nil { + return nil, err + } + + out := make(chan ImportResult) + go func() { + defer close(out) + + for event := range events { + out <- event + } + + // do not store last import time if context was cancelled + if ctx.Err() == nil { + err = b.repo.LocalConfig().StoreString(fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name), strconv.Itoa(int(importStartTime))) + } + + }() + + return out, nil } -func (b *Bridge) ExportAll(ctx context.Context, since time.Time) (<-chan ExportResult, error) { +func (b *Bridge) ImportAll(ctx context.Context) (<-chan ImportResult, error) { + return b.startImportSince(ctx, time.Time{}) +} + +func (b *Bridge) ExportAll(ctx context.Context, since time.Time) error { exporter := b.getExporter() if exporter == nil { - return nil, ErrExportNotSupported + return ErrExportNotSupported } err := b.ensureConfig() if err != nil { - return nil, err + return err } err = b.ensureInit() if err != nil { - return nil, err + return err + } + + events, err := exporter.ExportAll(ctx, b.repo, since) + if err != nil { + return err } - return exporter.ExportAll(ctx, b.repo, since) + exportedIssues := 0 + for result := range events { + if result.Event != ExportEventNothing { + fmt.Println(result.String()) + } + + switch result.Event { + case ExportEventBug: + exportedIssues++ + } + } + + fmt.Printf("exported %d issues with %s bridge\n", exportedIssues, b.Name) + return nil } -- cgit From 57e23c8ada0a9d921a6b68187a76eb5c8b8a407d Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Sun, 3 Nov 2019 16:46:51 +0100 Subject: bridge: improvement on the import resume feature --- bridge/core/bridge.go | 49 ++++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) (limited to 'bridge/core') diff --git a/bridge/core/bridge.go b/bridge/core/bridge.go index 0165994a..ddf041f1 100644 --- a/bridge/core/bridge.go +++ b/bridge/core/bridge.go @@ -7,7 +7,6 @@ import ( "reflect" "regexp" "sort" - "strconv" "strings" "time" @@ -292,7 +291,10 @@ func (b *Bridge) ensureInit() error { return nil } -func (b *Bridge) startImportSince(ctx context.Context, since time.Time) (<-chan ImportResult, error) { +func (b *Bridge) ImportAllSince(ctx context.Context, since time.Time) (<-chan ImportResult, error) { + // 5 seconds before the actual start just to be sure. + importStartTime := time.Now().Add(-5 * time.Second) + importer := b.getImporter() if importer == nil { return nil, ErrImportNotSupported @@ -313,48 +315,37 @@ func (b *Bridge) startImportSince(ctx context.Context, since time.Time) (<-chan return nil, err } - return events, nil -} - -func (b *Bridge) ImportAllSince(ctx context.Context, since time.Time) (<-chan ImportResult, error) { - if since.Equal(time.Time{}) { - lastImportTimeStr, err := b.repo.LocalConfig().ReadString(fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name)) - if err == nil { - lastImportTime, err := strconv.Atoi(lastImportTimeStr) - if err != nil { - return nil, err - } - since = time.Unix(int64(lastImportTime), 0) - } - } - - importStartTime := time.Now().Unix() - - events, err := b.startImportSince(ctx, since) - if err != nil { - return nil, err - } - out := make(chan ImportResult) go func() { defer close(out) + noError := true + // relay all events while checking that everything went well for event := range events { + if event.Err != nil { + noError = false + } out <- event } - // do not store last import time if context was cancelled - if ctx.Err() == nil { - err = b.repo.LocalConfig().StoreString(fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name), strconv.Itoa(int(importStartTime))) + // store the last import time ONLY if no error happened + if noError { + key := fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name) + err = b.repo.LocalConfig().StoreTimestamp(key, importStartTime) } - }() return out, nil } func (b *Bridge) ImportAll(ctx context.Context) (<-chan ImportResult, error) { - return b.startImportSince(ctx, time.Time{}) + // If possible, restart from the last import time + lastImport, err := b.repo.LocalConfig().ReadTimestamp(fmt.Sprintf("git-bug.bridge.%s.lastImportTime", b.Name)) + if err == nil { + return b.ImportAllSince(ctx, lastImport) + } + + return b.ImportAllSince(ctx, time.Time{}) } func (b *Bridge) ExportAll(ctx context.Context, since time.Time) error { -- cgit