Add progress bars to the long-running CLI processes (backtest, backfill, extract-osm)

6 files changed, 119 insertions, 9 deletions

diff --git a/README.md b/README.md
index be3a756..9b24b65 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,7 @@ A Go toolkit for analyzing cycling rides from GPX exports: parse rides, detect c
 - **Ride table sorting** — sorts every ride-library column through clickable server-side headers and query parameters
 - **Ride library filters** — filters rides by distance, moving duration, start date, Cotacol score, and route availability through shareable query URLs
 - **Computed moving time** — computes moving time from the GPX track with a speed-threshold algorithm, persists it as the canonical value (keeping Strava's as `external_moving_time_s`), and benchmarks against that reference via `-backtest`
+- **CLI progress bars** — shows a live progress bar for the long-running CLI processes (`-fetch-osm`, `-extract-osm`, `-backfill`, `-backtest`)
 - **Ride detail maps** — opens a stored ride with its recorded route on an interactive OpenStreetMap map
 - **Strava import progress** — streams live import progress with an aggregate progress bar and prevents overlapping syncs
 - **Analytics second chart** — plots normalized difficulty (Cotacol per 100 km) against moving duration alongside speed vs. Cotacol
diff --git a/cli/cli.go b/cli/cli.go
index e2138cd..1fdb866 100644
--- a/cli/cli.go
+++ b/cli/cli.go
@@ -18,6 +18,7 @@ import (
 	"github.com/martinlehoux/biking_home/ride"
 	"github.com/martinlehoux/biking_home/web"
 	"github.com/martinlehoux/kagamigo/kcore"
+	"github.com/schollz/progressbar/v3"
 )
 
 var (
@@ -34,10 +35,34 @@ func Run(db *sql.DB, configPath string, appConfig config.Config) {
 		err := mountain_pass.FetchFrancePBF("france-latest.osm.pbf")
 		kcore.Expect(err, "failed to download France OSM PBF")
 	case *extractOSM != "":
-		_, err := mountain_pass.ExtractMountainPasses(context.Background(), *extractOSM, db)
+		enabled := progressBarEnabled()
+		var bar *progressbar.ProgressBar
+		_, err := mountain_pass.ExtractMountainPasses(context.Background(), *extractOSM, db, func(current, total int64) {
+			if bar == nil && enabled {
+				bar = progressbar.DefaultBytes(total, "Extracting mountain passes")
+			}
+			if bar != nil {
+				bar.Set64(current)
+			}
+		})
+		if bar != nil {
+			bar.Finish()
+		}
 		kcore.Expect(err, "failed to extract mountain passes from OSM")
 	case *backfill:
-		count, err := ride.Backfill(db)
+		enabled := progressBarEnabled()
+		var bar *progressbar.ProgressBar
+		count, err := ride.Backfill(db, func(current, total int) {
+			if bar == nil && enabled {
+				bar = progressbar.Default(int64(total), "Backfilling ride values")
+			}
+			if bar != nil {
+				bar.Set(current)
+			}
+		})
+		if bar != nil {
+			bar.Finish()
+		}
 		kcore.Expect(err, "failed to backfill ride values")
 		slog.Info("Backfilled ride values", "rides", count)
 	case *backtest:
@@ -47,6 +72,14 @@ func Run(db *sql.DB, configPath string, appConfig config.Config) {
 	}
 }
 
+// progressBarEnabled reports whether stderr is attached to a terminal, so the
+// progress bar only renders for interactive runs and stays out of piped or
+// redirected output.
+func progressBarEnabled() bool {
+	fi, err := os.Stderr.Stat()
+	return err == nil && fi.Mode()&os.ModeCharDevice != 0
+}
+
 func runServer(db *sql.DB, configPath string, appConfig config.Config) {
 	server := web.NewServer(db, configPath).WithPprof(*enablePprof)
 	if *enablePprof {
@@ -70,9 +103,14 @@ func runBacktest(db *sql.DB) error {
 		return err
 	}
 	var samples []backtestSample
-	for i, item := range items {
-		if i > 0 && i%50 == 0 {
-			slog.Info("Backtest progress", "processed", i, "total", len(items))
+	var bar *progressbar.ProgressBar
+	if progressBarEnabled() {
+		bar = progressbar.Default(int64(len(items)), "Backtesting rides")
+		defer bar.Finish()
+	}
+	for _, item := range items {
+		if bar != nil {
+			bar.Add(1)
 		}
 		if !isBacktestEligible(item) {
 			continue
diff --git a/mountain_pass/extract.go b/mountain_pass/extract.go
index 705dbd3..9643603 100644
--- a/mountain_pass/extract.go
+++ b/mountain_pass/extract.go
@@ -24,14 +24,28 @@ const insertStatement = `
 		longitude = excluded.longitude
 `
 
-func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int, error) {
+// ExtractMountainPasses scans pbfPath for mountain pass nodes and upserts them
+// into the database. An optional reporter is invoked as scanning progresses
+// with bytes scanned and the total file size, so callers (e.g. the CLI) can
+// render progress.
+func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB, reporters ...func(current, total int64)) (int, error) {
 	file, err := os.Open(pbfPath)
 	if err != nil {
 		return 0, fmt.Errorf("failed to open pbf file: %w", err)
 	}
 	defer file.Close()
 
-	scanner := osmpbf.New(ctx, file, 4)
+	info, err := file.Stat()
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat pbf file: %w", err)
+	}
+	total := info.Size()
+	report := func(int64, int64) {}
+	if len(reporters) > 0 {
+		report = reporters[0]
+	}
+	reader := &countingReader{r: file}
+	scanner := osmpbf.New(ctx, reader, 4)
 	scanner.SkipWays = true
 	scanner.SkipRelations = true
 	scanner.FilterNode = isMountainPass
@@ -50,6 +64,7 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int
 
 	count := 0
 	for scanner.Scan() {
+		report(reader.n, total)
 		node, ok := scanner.Object().(*osm.Node)
 		if !ok {
 			continue
@@ -74,6 +89,19 @@ func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int
 	return count, nil
 }
 
+// countingReader tracks the number of bytes read from an underlying reader,
+// used to report scan progress against the PBF file size.
+type countingReader struct {
+	r io.Reader
+	n int64
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+	n, err := c.r.Read(p)
+	c.n += int64(n)
+	return n, err
+}
+
 func isMountainPass(node *osm.Node) bool {
 	if node.Tags.Find("mountain_pass") == "yes" {
 		return true
diff --git a/mountain_pass/extract_test.go b/mountain_pass/extract_test.go
index 636372c..7eae33b 100644
--- a/mountain_pass/extract_test.go
+++ b/mountain_pass/extract_test.go
@@ -1,10 +1,13 @@
 package mountain_pass
 
 import (
+	"io"
+	"strings"
 	"testing"
 
 	"github.com/paulmach/osm"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIsMountainPass(t *testing.T) {
@@ -47,3 +50,21 @@ func TestParseElevation(t *testing.T) {
 		assert.Equal(t, c.ok, ok)
 	}
 }
+
+func TestCountingReaderTracksBytes(t *testing.T) {
+	const content = "0123456789abcdef"
+	reader := &countingReader{r: strings.NewReader(content)}
+
+	buf := make([]byte, 4)
+	var total int64
+	for {
+		n, err := reader.Read(buf)
+		total += int64(n)
+		require.Equal(t, total, reader.n)
+		if err == io.EOF {
+			break
+		}
+		require.NoError(t, err)
+	}
+	assert.Equal(t, int64(len(content)), reader.n)
+}
diff --git a/ride/store.go b/ride/store.go
index 162b912..56cd087 100644
--- a/ride/store.go
+++ b/ride/store.go
@@ -91,7 +91,10 @@ func Save(db *sql.DB, r StoredRide) error {
 	return err
 }
 
-func Backfill(db *sql.DB) (int, error) {
+// Backfill recomputes and persists stored ride values for every ride. An
+// optional reporter is invoked with the number of rides processed and the
+// total, so callers (e.g. the CLI) can render progress.
+func Backfill(db *sql.DB, reporters ...func(current, total int)) (int, error) {
 	rows, err := db.Query("SELECT " + columns + " FROM rides ORDER BY id")
 	if err != nil {
 		return 0, err
@@ -113,7 +116,12 @@ func Backfill(db *sql.DB) (int, error) {
 		return 0, err
 	}
 	backfilled := 0
-	for _, item := range pending {
+	report := func(int, int) {}
+	if len(reporters) > 0 {
+		report = reporters[0]
+	}
+	for i, item := range pending {
+		report(i+1, len(pending))
 		if err := Save(db, item); err != nil {
 			slog.Warn("Failed to backfill ride values", "ride", item.ExternalID, "file", item.GPXPath, "error", err)
 			continue
diff --git a/ride/store_test.go b/ride/store_test.go
index 877f4e5..89c4830 100644
--- a/ride/store_test.go
+++ b/ride/store_test.go
@@ -197,6 +197,20 @@ func TestBackfill(t *testing.T) {
 	assert.Equal(t, 1, count)
 }
 
+func TestBackfillReportsProgress(t *testing.T) {
+	db := newTestDB(t)
+	require.NoError(t, Save(db, sampleRide(t)))
+
+	var calls [][2]int
+	count, err := Backfill(db, func(current, total int) {
+		calls = append(calls, [2]int{current, total})
+	})
+	require.NoError(t, err)
+	assert.Equal(t, 1, count)
+	require.Len(t, calls, 1)
+	assert.Equal(t, [2]int{1, 1}, calls[0])
+}
+
 func TestBackfillSkipsInvalidRide(t *testing.T) {
 	db := newTestDB(t)
 	_, err := db.Exec(`