From 7df34aa7a40be0d7b759ea3ef42cedf9f25f32b2 Mon Sep 17 00:00:00 2001 From: Michael Muré Date: Tue, 10 Jan 2023 21:36:03 +0100 Subject: commands: add a nice terminal progress bar when building the cache One issue remaining is that writing the cache takes significant time, but I don't know how to reflect that nicely to the user. --- cache/repo_cache.go | 27 ++++------ cache/subcache.go | 113 ++++++++++++++++++++++++++++------------ commands/execenv/env.go | 63 +++++++++++++++++----- commands/execenv/env_testing.go | 7 +++ commands/webui.go | 17 ++---- entities/identity/identity.go | 10 +++- entity/dag/entity.go | 10 +++- entity/streamed.go | 7 ++- go.mod | 7 ++- go.sum | 12 ++++- 10 files changed, 192 insertions(+), 81 deletions(-) diff --git a/cache/repo_cache.go b/cache/repo_cache.go index 99e9abbd..9c7e945a 100644 --- a/cache/repo_cache.go +++ b/cache/repo_cache.go @@ -30,7 +30,7 @@ var _ repository.RepoKeyring = &RepoCache{} type cacheMgmt interface { Typename() string Load() error - Build() error + Build() <-chan BuildEvent SetCacheSize(size int) MergeAll(remote string) <-chan entity.MergeResult GetNamespace() string @@ -212,6 +212,7 @@ const ( BuildEventCacheIsBuilt BuildEventRemoveLock BuildEventStarted + BuildEventProgress BuildEventFinished ) @@ -223,6 +224,10 @@ type BuildEvent struct { Typename string // Event is the type of the event. Event BuildEventType + // Total is the total number of element being built. Set if Event is BuildEventStarted. + Total int64 + // Progress is the current count of processed element. Set if Event is BuildEventProgress. + Progress int64 } func (c *RepoCache) buildCache(events chan BuildEvent) { @@ -233,23 +238,13 @@ func (c *RepoCache) buildCache(events chan BuildEvent) { wg.Add(1) go func(subcache cacheMgmt) { defer wg.Done() - events <- BuildEvent{ - Typename: subcache.Typename(), - Event: BuildEventStarted, - } - err := subcache.Build() - if err != nil { - events <- BuildEvent{ - Typename: subcache.Typename(), - Err: err, + buildEvents := subcache.Build() + for buildEvent := range buildEvents { + events <- buildEvent + if buildEvent.Err != nil { + return } - return - } - - events <- BuildEvent{ - Typename: subcache.Typename(), - Event: BuildEventFinished, } }(subcache) } diff --git a/cache/subcache.go b/cache/subcache.go index 7b599b10..29155e82 100644 --- a/cache/subcache.go +++ b/cache/subcache.go @@ -185,51 +185,98 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) write() error { return f.Close() } -func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error { - sc.excerpts = make(map[entity.Id]ExcerptT) +func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() <-chan BuildEvent { + out := make(chan BuildEvent) - allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers()) + go func() { + defer close(out) - index, err := sc.repo.GetIndex(sc.namespace) - if err != nil { - return err - } + out <- BuildEvent{ + Typename: sc.typename, + Event: BuildEventStarted, + } - // wipe the index just to be sure - err = index.Clear() - if err != nil { - return err - } + sc.excerpts = make(map[entity.Id]ExcerptT) + + allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers()) - indexer, indexEnd := index.IndexBatch() + index, err := sc.repo.GetIndex(sc.namespace) + if err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: err, + } + return + } - for e := range allEntities { - if e.Err != nil { - return e.Err + // wipe the index just to be sure + err = index.Clear() + if err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: err, + } + return } - cached := sc.makeCached(e.Entity, sc.entityUpdated) - sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached) - // might as well keep them in memory - sc.cached[e.Entity.Id()] = cached + indexer, indexEnd := index.IndexBatch() + + for e := range allEntities { + if e.Err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: e.Err, + } + return + } + + cached := sc.makeCached(e.Entity, sc.entityUpdated) + sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached) + // might as well keep them in memory + sc.cached[e.Entity.Id()] = cached + + indexData := sc.makeIndexData(cached) + if err := indexer(e.Entity.Id().String(), indexData); err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: err, + } + return + } + + out <- BuildEvent{ + Typename: sc.typename, + Event: BuildEventProgress, + Progress: e.CurrentEntity, + Total: e.TotalEntities, + } + } - indexData := sc.makeIndexData(cached) - if err := indexer(e.Entity.Id().String(), indexData); err != nil { - return err + err = indexEnd() + if err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: err, + } + return } - } - err = indexEnd() - if err != nil { - return err - } + err = sc.write() + if err != nil { + out <- BuildEvent{ + Typename: sc.typename, + Err: err, + } + return + } - err = sc.write() - if err != nil { - return err - } + out <- BuildEvent{ + Typename: sc.typename, + Event: BuildEventFinished, + } + }() - return nil + return out } func (sc *SubCache[EntityT, ExcerptT, CacheT]) SetCacheSize(size int) { diff --git a/commands/execenv/env.go b/commands/execenv/env.go index 4be7c247..d2d1c301 100644 --- a/commands/execenv/env.go +++ b/commands/execenv/env.go @@ -7,6 +7,8 @@ import ( "os" "github.com/spf13/cobra" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/entities/identity" @@ -50,6 +52,10 @@ type Out interface { // Reset clear what has been recorded as written in the output before. // This only works in test scenario. Reset() + + // Raw return the underlying io.Writer, or itself if not. + // This is useful if something need to access the raw file descriptor. + Raw() io.Writer } type out struct { @@ -89,6 +95,10 @@ func (o out) Reset() { panic("only work with a test env") } +func (o out) Raw() io.Writer { + return o.Writer +} + // LoadRepo is a pre-run function that load the repository for use in a command func LoadRepo(env *Env) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { @@ -143,18 +153,9 @@ func LoadBackend(env *Env) func(*cobra.Command, []string) error { var events chan cache.BuildEvent env.Backend, events = cache.NewRepoCache(env.Repo) - for event := range events { - if event.Err != nil { - return event.Err - } - switch event.Event { - case cache.BuildEventCacheIsBuilt: - env.Err.Println("Building cache... ") - case cache.BuildEventStarted: - env.Err.Printf("[%s] started\n", event.Typename) - case cache.BuildEventFinished: - env.Err.Printf("[%s] done\n", event.Typename) - } + err = CacheBuildProgressBar(env, events) + if err != nil { + return err } cleaner := func(env *Env) interrupt.CleanerFunc { @@ -213,3 +214,41 @@ func CloseBackend(env *Env, runE func(cmd *cobra.Command, args []string) error) return err } } + +func CacheBuildProgressBar(env *Env, events chan cache.BuildEvent) error { + var progress *mpb.Progress + var bars = make(map[string]*mpb.Bar) + + for event := range events { + if event.Err != nil { + return event.Err + } + + if progress == nil { + progress = mpb.New(mpb.WithOutput(env.Err.Raw())) + } + + switch event.Event { + case cache.BuildEventCacheIsBuilt: + env.Err.Println("Building cache... ") + case cache.BuildEventStarted: + bars[event.Typename] = progress.AddBar(-1, + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators( + decor.Name(event.Typename, decor.WCSyncSpace), + decor.CountersNoUnit("%d / %d", decor.WCSyncSpace), + ), + mpb.AppendDecorators(decor.Percentage(decor.WCSyncSpace)), + ) + case cache.BuildEventProgress: + bars[event.Typename].SetTotal(event.Total, false) + bars[event.Typename].SetCurrent(event.Progress) + } + } + + if progress != nil { + progress.Shutdown() + } + + return nil +} diff --git a/commands/execenv/env_testing.go b/commands/execenv/env_testing.go index 34eafc9c..6eb9c69d 100644 --- a/commands/execenv/env_testing.go +++ b/commands/execenv/env_testing.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "testing" "github.com/stretchr/testify/require" @@ -12,6 +13,8 @@ import ( "github.com/MichaelMure/git-bug/repository" ) +var _ Out = &TestOut{} + type TestOut struct { *bytes.Buffer } @@ -37,6 +40,10 @@ func (te *TestOut) PrintJSON(v interface{}) error { return nil } +func (te *TestOut) Raw() io.Writer { + return te.Buffer +} + func NewTestEnv(t *testing.T) *Env { t.Helper() diff --git a/commands/webui.go b/commands/webui.go index 0b3b24a6..31313146 100644 --- a/commands/webui.go +++ b/commands/webui.go @@ -108,19 +108,10 @@ func runWebUI(env *execenv.Env, opts webUIOptions) error { mrc := cache.NewMultiRepoCache() _, events := mrc.RegisterDefaultRepository(env.Repo) - for event := range events { - if event.Err != nil { - env.Err.Printf("Cache building error [%s]: %v\n", event.Typename, event.Err) - continue - } - switch event.Event { - case cache.BuildEventCacheIsBuilt: - env.Err.Println("Building cache... ") - case cache.BuildEventStarted: - env.Err.Printf("[%s] started\n", event.Typename) - case cache.BuildEventFinished: - env.Err.Printf("[%s] done\n", event.Typename) - } + + err := execenv.CacheBuildProgressBar(env, events) + if err != nil { + return err } var errOut io.Writer diff --git a/entities/identity/identity.go b/entities/identity/identity.go index 22ce652c..e654cb2b 100644 --- a/entities/identity/identity.go +++ b/entities/identity/identity.go @@ -238,6 +238,9 @@ func readAll(repo repository.ClockedRepo, refPrefix string) <-chan entity.Stream return } + total := int64(len(refs)) + current := int64(1) + for _, ref := range refs { i, err := read(repo, ref) @@ -246,7 +249,12 @@ func readAll(repo repository.ClockedRepo, refPrefix string) <-chan entity.Stream return } - out <- entity.StreamedEntity[*Identity]{Entity: i} + out <- entity.StreamedEntity[*Identity]{ + Entity: i, + CurrentEntity: current, + TotalEntities: total, + } + current++ } }() diff --git a/entity/dag/entity.go b/entity/dag/entity.go index 2028e1b4..f8dbd53d 100644 --- a/entity/dag/entity.go +++ b/entity/dag/entity.go @@ -314,6 +314,9 @@ func ReadAll[EntityT entity.Interface](def Definition, wrapper func(e *Entity) E return } + total := int64(len(refs)) + current := int64(1) + for _, ref := range refs { e, err := read[EntityT](def, wrapper, repo, resolvers, ref) @@ -322,7 +325,12 @@ func ReadAll[EntityT entity.Interface](def Definition, wrapper func(e *Entity) E return } - out <- entity.StreamedEntity[EntityT]{Entity: e} + out <- entity.StreamedEntity[EntityT]{ + Entity: e, + CurrentEntity: current, + TotalEntities: total, + } + current++ } }() diff --git a/entity/streamed.go b/entity/streamed.go index 789224a3..33124ef0 100644 --- a/entity/streamed.go +++ b/entity/streamed.go @@ -1,6 +1,11 @@ package entity type StreamedEntity[EntityT Interface] struct { - Entity EntityT Err error + Entity EntityT + + // CurrentEntity is the index of the current entity being streamed, to express progress. + CurrentEntity int64 + // TotalEntities is the total count of expected entities, if known. + TotalEntities int64 } diff --git a/go.mod b/go.mod index 57896eb3..a5384ab2 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 + github.com/vbauerster/mpb/v8 v8.1.4 github.com/vektah/gqlparser/v2 v2.5.1 github.com/xanzy/go-gitlab v0.77.0 golang.org/x/crypto v0.5.0 @@ -36,6 +37,8 @@ require ( ) require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/cloudflare/circl v1.3.1 // indirect github.com/lithammer/dedent v1.1.0 // indirect github.com/owenrumney/go-sarif v1.0.11 // indirect @@ -86,14 +89,14 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.0.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-runewidth v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/philhofer/fwd v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/praetorian-inc/gokart v0.5.1 - github.com/rivo/uniseg v0.1.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect diff --git a/go.sum b/go.sum index 980af4bb..3ca2ebdf 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,10 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0A github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -209,8 +213,9 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -243,8 +248,9 @@ github.com/praetorian-inc/gokart v0.5.1 h1:GYUM69qskrRibZUAEwKEm/pd/j/SFzlFnQnhx github.com/praetorian-inc/gokart v0.5.1/go.mod h1:GuA97YgdXwqOVsnHY6PCvV1t9t0Jsk3Zcd6sbTXj4uI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -298,6 +304,8 @@ github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4= github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY= +github.com/vbauerster/mpb/v8 v8.1.4 h1:MOcLTIbbAA892wVjRiuFHa1nRlNvifQMDVh12Bq/xIs= +github.com/vbauerster/mpb/v8 v8.1.4/go.mod h1:2fRME8lCLU9gwJwghZb1bO9A3Plc8KPeQ/ayGj+Ek4I= github.com/vektah/gqlparser/v2 v2.5.1 h1:ZGu+bquAY23jsxDRcYpWjttRZrUz07LbiY77gUOHcr4= github.com/vektah/gqlparser/v2 v2.5.1/go.mod h1:mPgqFBu/woKTVYWyNk8cO3kh4S/f4aRFZrvOnp3hmCs= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -- cgit