Verify downloaded France OSM PBF against geofabrik's published MD5

3 files changed, 198 insertions, 7 deletions

diff --git a/README.md b/README.md
index 40e9346..6175e44 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@ A Go toolkit for analyzing cycling rides from GPX exports: parse rides, detect c
 - **Ride library filters** — filters rides by distance, moving duration, start date, Cotacol score, and route availability through shareable query URLs
 - **Computed moving time** — computes moving time from the GPX track with a speed-threshold algorithm, persists it as the canonical value (keeping Strava's as `external_moving_time_s`), and benchmarks against that reference via `-backtest`
 - **CLI progress bars** — shows a live progress bar for the long-running CLI processes (`-fetch-osm`, `-extract-osm`, `-backfill`, `-backtest`)
+- **Verified OSM download** — verifies the France PBF against geofabrik's published MD5 after downloading (and on already-complete files), keeping a corrupt file for inspection instead of silently trusting its size
 - **Ride detail maps** — opens a stored ride with its recorded route on an interactive OpenStreetMap map
 - **Strava import progress** — streams live import progress with an aggregate progress bar and prevents overlapping syncs
 - **Analytics second chart** — plots normalized difficulty (Cotacol per 100 km) against moving duration alongside speed vs. Cotacol
diff --git a/mountain_pass/fetch.go b/mountain_pass/fetch.go
index 4be2d87..94b0b5e 100644
--- a/mountain_pass/fetch.go
+++ b/mountain_pass/fetch.go
@@ -1,12 +1,15 @@
 package mountain_pass
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
 	"os"
+	"strings"
 
 	"github.com/schollz/progressbar/v3"
 )
@@ -14,12 +17,19 @@ import (
 const francePBFRUL = "https://download.geofabrik.de/europe/france-latest.osm.pbf"
 
 // FetchFrancePBF downloads the full France OSM PBF to destPath, resuming a
-// partial download when the file already exists and skipping the download
-// entirely when it is already complete.
+// partial download when the file already exists, skipping the download
+// entirely when it is already complete, and verifying the result against
+// geofabrik's published MD5 checksum.
 func FetchFrancePBF(destPath string) error {
-	client := &http.Client{}
+	return fetchFrancePBF(&http.Client{}, francePBFRUL, destPath)
+}
+
+// fetchFrancePBF is the testable core of FetchFrancePBF. baseURL is the
+// server URL whose .md5 sibling carries the published checksum.
+func fetchFrancePBF(client *http.Client, baseURL, destPath string) error {
+	checksumURL := baseURL + ".md5"
 
-	head, err := client.Head(francePBFRUL)
+	head, err := client.Head(baseURL)
 	if err != nil {
 		return fmt.Errorf("failed to check remote size: %w", err)
 	}
@@ -35,7 +45,7 @@ func FetchFrancePBF(destPath string) error {
 		offset = info.Size()
 		if offset >= total {
 			slog.Info("Already downloaded", "file", destPath, "bytes", offset)
-			return nil
+			return verifyPBF(client, checksumURL, destPath)
 		}
 		if offset > 0 {
 			slog.Info("Resuming download", "file", destPath, "bytes", offset, "remaining", total-offset)
@@ -44,7 +54,7 @@ func FetchFrancePBF(destPath string) error {
 		return fmt.Errorf("failed to stat %s: %w", destPath, statErr)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, francePBFRUL, nil)
+	req, err := http.NewRequest(http.MethodGet, baseURL, nil)
 	if err != nil {
 		return err
 	}
@@ -89,10 +99,69 @@ func FetchFrancePBF(destPath string) error {
 	}
 	_, err = io.Copy(io.MultiWriter(file, bar), res.Body)
 	if err != nil {
-		return fmt.Errorf("failed to download %s: %w", francePBFRUL, err)
+		return fmt.Errorf("failed to download %s: %w", baseURL, err)
 	}
 	if size, err := file.Stat(); err == nil && size.Size() != total {
 		return fmt.Errorf("download incomplete: got %d bytes, want %d", size.Size(), total)
 	}
+	return verifyPBF(client, checksumURL, destPath)
+}
+
+// verifyPBF verifies destPath against the published checksum served at
+// checksumURL. If the checksum endpoint is unreachable it logs a warning and
+// proceeds unverified rather than blocking the download; on mismatch it
+// returns an error without deleting the corrupt file so the user can inspect
+// it and remove it manually before re-fetching.
+func verifyPBF(client *http.Client, checksumURL, destPath string) error {
+	want, err := expectedPBFMD5(client, checksumURL)
+	if err != nil {
+		slog.Warn("Could not fetch expected checksum, skipping verification", "url", checksumURL, "err", err)
+		return nil
+	}
+	got, err := pbfMD5(destPath)
+	if err != nil {
+		return fmt.Errorf("failed to compute MD5 of %s: %w", destPath, err)
+	}
+	if !strings.EqualFold(got, want) {
+		return fmt.Errorf("checksum mismatch on %s: local md5 %s != expected %s; keeping the corrupt file for inspection, remove it manually before re-fetching",
+			destPath, got, want)
+	}
+	slog.Info("Checksum verified", "file", destPath, "md5", got)
 	return nil
 }
+
+// expectedPBFMD5 fetches checksumURL and returns its first whitespace-separated
+// token. Geofabrik's .md5 files follow the "<hex> <filename>" format.
+func expectedPBFMD5(client *http.Client, checksumURL string) (string, error) {
+	res, err := client.Get(checksumURL)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+	body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return "", err
+	}
+	fields := strings.Fields(string(body))
+	if len(fields) == 0 {
+		return "", errors.New("empty checksum file")
+	}
+	return fields[0], nil
+}
+
+// pbfMD5 returns the lowercase hex MD5 digest of the file at path.
+func pbfMD5(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := md5.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
diff --git a/mountain_pass/fetch_test.go b/mountain_pass/fetch_test.go
new file mode 100644
index 0000000..0e6304a
--- /dev/null
+++ b/mountain_pass/fetch_test.go
@@ -0,0 +1,121 @@
+package mountain_pass
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// osmd5Hex mirrors the production helper in the test so the expected checksum
+// served by the fake geofabrik server is derived independently of pbfMD5.
+func osmd5Hex(data []byte) string {
+	sum := md5.Sum(data)
+	return hex.EncodeToString(sum[:])
+}
+
+// fakeGeofabrik serves a PBF payload and an optional .md5 sibling.
+func fakeGeofabrik(t *testing.T, pbf []byte, md5Body string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/france-latest.osm.pbf", func(w http.ResponseWriter, r *http.Request) {
+		w.Write(pbf)
+	})
+	mux.HandleFunc("/france-latest.osm.pbf.md5", func(w http.ResponseWriter, r *http.Request) {
+		if md5Body == "" {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		w.Write([]byte(md5Body))
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestFetchFrancePBFDownloadsAndVerifies(t *testing.T) {
+	pbf := []byte("some-valid-pbf-content-1")
+	srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+"  france-latest.osm.pbf\n")
+	dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf")
+
+	err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest)
+	require.NoError(t, err)
+
+	got, err := os.ReadFile(dest)
+	require.NoError(t, err)
+	assert.Equal(t, pbf, got)
+}
+
+func TestFetchFrancePBFMismatchRetainsFile(t *testing.T) {
+	pbf := []byte("some-valid-pbf-content-2")
+	srv := fakeGeofabrik(t, pbf, strings.Repeat("0", 32)+"  france-latest.osm.pbf\n")
+	dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf")
+
+	err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "checksum mismatch")
+
+	// The corrupt file is kept for inspection, not deleted.
+	got, readErr := os.ReadFile(dest)
+	require.NoError(t, readErr)
+	assert.Equal(t, pbf, got)
+}
+
+func TestFetchFrancePBFAlreadyCompleteVerified(t *testing.T) {
+	pbf := []byte("some-valid-pbf-content-3")
+	srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+"  france-latest.osm.pbf\n")
+	dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf")
+	require.NoError(t, os.WriteFile(dest, pbf, 0o644))
+
+	err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest)
+	require.NoError(t, err)
+
+	got, _ := os.ReadFile(dest)
+	assert.Equal(t, pbf, got)
+}
+
+func TestFetchFrancePBFAlreadyCompleteCorrupt(t *testing.T) {
+	pbf := []byte("the-real-content")
+	corrupt := []byte(strings.Repeat("x", len(pbf)))
+	require.Equal(t, len(pbf), len(corrupt), "same length to prove size-only check is not enough")
+	srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+"  france-latest.osm.pbf\n")
+	dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf")
+	require.NoError(t, os.WriteFile(dest, corrupt, 0o644))
+
+	err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "checksum mismatch")
+
+	got, _ := os.ReadFile(dest)
+	assert.Equal(t, corrupt, got, "corrupt file retained for inspection")
+}
+
+func TestFetchFrancePBFSkipsVerificationWhenChecksumUnreachable(t *testing.T) {
+	pbf := []byte("some-valid-pbf-content-4")
+	srv := fakeGeofabrik(t, pbf, "") // md5 endpoint returns 404
+	dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf")
+
+	err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest)
+	require.NoError(t, err)
+
+	got, _ := os.ReadFile(dest)
+	assert.Equal(t, pbf, got)
+}
+
+func TestPBFMD5(t *testing.T) {
+	data := []byte("known-content-for-md5-test")
+	expected := osmd5Hex(data)
+	path := filepath.Join(t.TempDir(), "sample.bin")
+	require.NoError(t, os.WriteFile(path, data, 0o644))
+
+	got, err := pbfMD5(path)
+	require.NoError(t, err)
+	assert.Equal(t, expected, got)
+}