From a9a384972649182c188396c5f996ae617d35188a Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Mon, 24 Aug 2026 12:07:30 +0200 Subject: Add progress bars to the long-running CLI processes (backtest, backfill, extract-osm) --- README.md | 1 + cli/cli.go | 48 ++++++++++++++++++++++++++++++++++++++----- mountain_pass/extract.go | 32 +++++++++++++++++++++++++++-- mountain_pass/extract_test.go | 21 +++++++++++++++++++ ride/store.go | 12 +++++++++-- ride/store_test.go | 14 +++++++++++++ 6 files changed, 119 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index be3a756..9b24b65 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A Go toolkit for analyzing cycling rides from GPX exports: parse rides, detect c - **Ride table sorting** — sorts every ride-library column through clickable server-side headers and query parameters - **Ride library filters** — filters rides by distance, moving duration, start date, Cotacol score, and route availability through shareable query URLs - **Computed moving time** — computes moving time from the GPX track with a speed-threshold algorithm, persists it as the canonical value (keeping Strava's as `external_moving_time_s`), and benchmarks against that reference via `-backtest` +- **CLI progress bars** — shows a live progress bar for the long-running CLI processes (`-fetch-osm`, `-extract-osm`, `-backfill`, `-backtest`) - **Ride detail maps** — opens a stored ride with its recorded route on an interactive OpenStreetMap map - **Strava import progress** — streams live import progress with an aggregate progress bar and prevents overlapping syncs - **Analytics second chart** — plots normalized difficulty (Cotacol per 100 km) against moving duration alongside speed vs. Cotacol diff --git a/cli/cli.go b/cli/cli.go index e2138cd..1fdb866 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -18,6 +18,7 @@ import ( "github.com/martinlehoux/biking_home/ride" "github.com/martinlehoux/biking_home/web" "github.com/martinlehoux/kagamigo/kcore" + "github.com/schollz/progressbar/v3" ) var ( @@ -34,10 +35,34 @@ func Run(db *sql.DB, configPath string, appConfig config.Config) { err := mountain_pass.FetchFrancePBF("france-latest.osm.pbf") kcore.Expect(err, "failed to download France OSM PBF") case *extractOSM != "": - _, err := mountain_pass.ExtractMountainPasses(context.Background(), *extractOSM, db) + enabled := progressBarEnabled() + var bar *progressbar.ProgressBar + _, err := mountain_pass.ExtractMountainPasses(context.Background(), *extractOSM, db, func(current, total int64) { + if bar == nil && enabled { + bar = progressbar.DefaultBytes(total, "Extracting mountain passes") + } + if bar != nil { + bar.Set64(current) + } + }) + if bar != nil { + bar.Finish() + } kcore.Expect(err, "failed to extract mountain passes from OSM") case *backfill: - count, err := ride.Backfill(db) + enabled := progressBarEnabled() + var bar *progressbar.ProgressBar + count, err := ride.Backfill(db, func(current, total int) { + if bar == nil && enabled { + bar = progressbar.Default(int64(total), "Backfilling ride values") + } + if bar != nil { + bar.Set(current) + } + }) + if bar != nil { + bar.Finish() + } kcore.Expect(err, "failed to backfill ride values") slog.Info("Backfilled ride values", "rides", count) case *backtest: @@ -47,6 +72,14 @@ func Run(db *sql.DB, configPath string, appConfig config.Config) { } } +// progressBarEnabled reports whether stderr is attached to a terminal, so the +// progress bar only renders for interactive runs and stays out of piped or +// redirected output. +func progressBarEnabled() bool { + fi, err := os.Stderr.Stat() + return err == nil && fi.Mode()&os.ModeCharDevice != 0 +} + func runServer(db *sql.DB, configPath string, appConfig config.Config) { server := web.NewServer(db, configPath).WithPprof(*enablePprof) if *enablePprof { @@ -70,9 +103,14 @@ func runBacktest(db *sql.DB) error { return err } var samples []backtestSample - for i, item := range items { - if i > 0 && i%50 == 0 { - slog.Info("Backtest progress", "processed", i, "total", len(items)) + var bar *progressbar.ProgressBar + if progressBarEnabled() { + bar = progressbar.Default(int64(len(items)), "Backtesting rides") + defer bar.Finish() + } + for _, item := range items { + if bar != nil { + bar.Add(1) } if !isBacktestEligible(item) { continue diff --git a/mountain_pass/extract.go b/mountain_pass/extract.go index 705dbd3..9643603 100644 --- a/mountain_pass/extract.go +++ b/mountain_pass/extract.go @@ -24,14 +24,28 @@ const insertStatement = ` longitude = excluded.longitude ` -func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int, error) { +// ExtractMountainPasses scans pbfPath for mountain pass nodes and upserts them +// into the database. An optional reporter is invoked as scanning progresses +// with bytes scanned and the total file size, so callers (e.g. the CLI) can +// render progress. +func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB, reporters ...func(current, total int64)) (int, error) { file, err := os.Open(pbfPath) if err != nil { return 0, fmt.Errorf("failed to open pbf file: %w", err) } defer file.Close() - scanner := osmpbf.New(ctx, file, 4) + info, err := file.Stat() + if err != nil { + return 0, fmt.Errorf("failed to stat pbf file: %w", err) + } + total := info.Size() + report := func(int64, int64) {} + if len(reporters) > 0 { + report = reporters[0] + } + reader := &countingReader{r: file} + scanner := osmpbf.New(ctx, reader, 4) scanner.SkipWays = true scanner.SkipRelations = true scanner.FilterNode = isMountainPass @@ -50,6 +64,7 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int count := 0 for scanner.Scan() { + report(reader.n, total) node, ok := scanner.Object().(*osm.Node) if !ok { continue @@ -74,6 +89,19 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int return count, nil } +// countingReader tracks the number of bytes read from an underlying reader, +// used to report scan progress against the PBF file size. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + func isMountainPass(node *osm.Node) bool { if node.Tags.Find("mountain_pass") == "yes" { return true diff --git a/mountain_pass/extract_test.go b/mountain_pass/extract_test.go index 636372c..7eae33b 100644 --- a/mountain_pass/extract_test.go +++ b/mountain_pass/extract_test.go @@ -1,10 +1,13 @@ package mountain_pass import ( + "io" + "strings" "testing" "github.com/paulmach/osm" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestIsMountainPass(t *testing.T) { @@ -47,3 +50,21 @@ func TestParseElevation(t *testing.T) { assert.Equal(t, c.ok, ok) } } + +func TestCountingReaderTracksBytes(t *testing.T) { + const content = "0123456789abcdef" + reader := &countingReader{r: strings.NewReader(content)} + + buf := make([]byte, 4) + var total int64 + for { + n, err := reader.Read(buf) + total += int64(n) + require.Equal(t, total, reader.n) + if err == io.EOF { + break + } + require.NoError(t, err) + } + assert.Equal(t, int64(len(content)), reader.n) +} diff --git a/ride/store.go b/ride/store.go index 162b912..56cd087 100644 --- a/ride/store.go +++ b/ride/store.go @@ -91,7 +91,10 @@ func Save(db *sql.DB, r StoredRide) error { return err } -func Backfill(db *sql.DB) (int, error) { +// Backfill recomputes and persists stored ride values for every ride. An +// optional reporter is invoked with the number of rides processed and the +// total, so callers (e.g. the CLI) can render progress. +func Backfill(db *sql.DB, reporters ...func(current, total int)) (int, error) { rows, err := db.Query("SELECT " + columns + " FROM rides ORDER BY id") if err != nil { return 0, err @@ -113,7 +116,12 @@ func Backfill(db *sql.DB) (int, error) { return 0, err } backfilled := 0 - for _, item := range pending { + report := func(int, int) {} + if len(reporters) > 0 { + report = reporters[0] + } + for i, item := range pending { + report(i+1, len(pending)) if err := Save(db, item); err != nil { slog.Warn("Failed to backfill ride values", "ride", item.ExternalID, "file", item.GPXPath, "error", err) continue diff --git a/ride/store_test.go b/ride/store_test.go index 877f4e5..89c4830 100644 --- a/ride/store_test.go +++ b/ride/store_test.go @@ -197,6 +197,20 @@ func TestBackfill(t *testing.T) { assert.Equal(t, 1, count) } +func TestBackfillReportsProgress(t *testing.T) { + db := newTestDB(t) + require.NoError(t, Save(db, sampleRide(t))) + + var calls [][2]int + count, err := Backfill(db, func(current, total int) { + calls = append(calls, [2]int{current, total}) + }) + require.NoError(t, err) + assert.Equal(t, 1, count) + require.Len(t, calls, 1) + assert.Equal(t, [2]int{1, 1}, calls[0]) +} + func TestBackfillSkipsInvalidRide(t *testing.T) { db := newTestDB(t) _, err := db.Exec(` -- cgit v1.3