diff options
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | mountain_pass/fetch.go | 83 | ||||
| -rw-r--r-- | mountain_pass/fetch_test.go | 121 |
3 files changed, 198 insertions, 7 deletions
@@ -21,6 +21,7 @@ A Go toolkit for analyzing cycling rides from GPX exports: parse rides, detect c - **Ride library filters** — filters rides by distance, moving duration, start date, Cotacol score, and route availability through shareable query URLs - **Computed moving time** — computes moving time from the GPX track with a speed-threshold algorithm, persists it as the canonical value (keeping Strava's as `external_moving_time_s`), and benchmarks against that reference via `-backtest` - **CLI progress bars** — shows a live progress bar for the long-running CLI processes (`-fetch-osm`, `-extract-osm`, `-backfill`, `-backtest`) +- **Verified OSM download** — verifies the France PBF against geofabrik's published MD5 after downloading (and on already-complete files), keeping a corrupt file for inspection instead of silently trusting its size - **Ride detail maps** — opens a stored ride with its recorded route on an interactive OpenStreetMap map - **Strava import progress** — streams live import progress with an aggregate progress bar and prevents overlapping syncs - **Analytics second chart** — plots normalized difficulty (Cotacol per 100 km) against moving duration alongside speed vs. Cotacol diff --git a/mountain_pass/fetch.go b/mountain_pass/fetch.go index 4be2d87..94b0b5e 100644 --- a/mountain_pass/fetch.go +++ b/mountain_pass/fetch.go @@ -1,12 +1,15 @@ package mountain_pass import ( + "crypto/md5" + "encoding/hex" "errors" "fmt" "io" "log/slog" "net/http" "os" + "strings" "github.com/schollz/progressbar/v3" ) @@ -14,12 +17,19 @@ import ( const francePBFRUL = "https://download.geofabrik.de/europe/france-latest.osm.pbf" // FetchFrancePBF downloads the full France OSM PBF to destPath, resuming a -// partial download when the file already exists and skipping the download -// entirely when it is already complete. +// partial download when the file already exists, skipping the download +// entirely when it is already complete, and verifying the result against +// geofabrik's published MD5 checksum. func FetchFrancePBF(destPath string) error { - client := &http.Client{} + return fetchFrancePBF(&http.Client{}, francePBFRUL, destPath) +} + +// fetchFrancePBF is the testable core of FetchFrancePBF. baseURL is the +// server URL whose .md5 sibling carries the published checksum. +func fetchFrancePBF(client *http.Client, baseURL, destPath string) error { + checksumURL := baseURL + ".md5" - head, err := client.Head(francePBFRUL) + head, err := client.Head(baseURL) if err != nil { return fmt.Errorf("failed to check remote size: %w", err) } @@ -35,7 +45,7 @@ func FetchFrancePBF(destPath string) error { offset = info.Size() if offset >= total { slog.Info("Already downloaded", "file", destPath, "bytes", offset) - return nil + return verifyPBF(client, checksumURL, destPath) } if offset > 0 { slog.Info("Resuming download", "file", destPath, "bytes", offset, "remaining", total-offset) @@ -44,7 +54,7 @@ func FetchFrancePBF(destPath string) error { return fmt.Errorf("failed to stat %s: %w", destPath, statErr) } - req, err := http.NewRequest(http.MethodGet, francePBFRUL, nil) + req, err := http.NewRequest(http.MethodGet, baseURL, nil) if err != nil { return err } @@ -89,10 +99,69 @@ func FetchFrancePBF(destPath string) error { } _, err = io.Copy(io.MultiWriter(file, bar), res.Body) if err != nil { - return fmt.Errorf("failed to download %s: %w", francePBFRUL, err) + return fmt.Errorf("failed to download %s: %w", baseURL, err) } if size, err := file.Stat(); err == nil && size.Size() != total { return fmt.Errorf("download incomplete: got %d bytes, want %d", size.Size(), total) } + return verifyPBF(client, checksumURL, destPath) +} + +// verifyPBF verifies destPath against the published checksum served at +// checksumURL. If the checksum endpoint is unreachable it logs a warning and +// proceeds unverified rather than blocking the download; on mismatch it +// returns an error without deleting the corrupt file so the user can inspect +// it and remove it manually before re-fetching. +func verifyPBF(client *http.Client, checksumURL, destPath string) error { + want, err := expectedPBFMD5(client, checksumURL) + if err != nil { + slog.Warn("Could not fetch expected checksum, skipping verification", "url", checksumURL, "err", err) + return nil + } + got, err := pbfMD5(destPath) + if err != nil { + return fmt.Errorf("failed to compute MD5 of %s: %w", destPath, err) + } + if !strings.EqualFold(got, want) { + return fmt.Errorf("checksum mismatch on %s: local md5 %s != expected %s; keeping the corrupt file for inspection, remove it manually before re-fetching", + destPath, got, want) + } + slog.Info("Checksum verified", "file", destPath, "md5", got) return nil } + +// expectedPBFMD5 fetches checksumURL and returns its first whitespace-separated +// token. Geofabrik's .md5 files follow the "<hex> <filename>" format. +func expectedPBFMD5(client *http.Client, checksumURL string) (string, error) { + res, err := client.Get(checksumURL) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if err != nil { + return "", err + } + fields := strings.Fields(string(body)) + if len(fields) == 0 { + return "", errors.New("empty checksum file") + } + return fields[0], nil +} + +// pbfMD5 returns the lowercase hex MD5 digest of the file at path. +func pbfMD5(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/mountain_pass/fetch_test.go b/mountain_pass/fetch_test.go new file mode 100644 index 0000000..0e6304a --- /dev/null +++ b/mountain_pass/fetch_test.go @@ -0,0 +1,121 @@ +package mountain_pass + +import ( + "crypto/md5" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// osmd5Hex mirrors the production helper in the test so the expected checksum +// served by the fake geofabrik server is derived independently of pbfMD5. +func osmd5Hex(data []byte) string { + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]) +} + +// fakeGeofabrik serves a PBF payload and an optional .md5 sibling. +func fakeGeofabrik(t *testing.T, pbf []byte, md5Body string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/france-latest.osm.pbf", func(w http.ResponseWriter, r *http.Request) { + w.Write(pbf) + }) + mux.HandleFunc("/france-latest.osm.pbf.md5", func(w http.ResponseWriter, r *http.Request) { + if md5Body == "" { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Write([]byte(md5Body)) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func TestFetchFrancePBFDownloadsAndVerifies(t *testing.T) { + pbf := []byte("some-valid-pbf-content-1") + srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+" france-latest.osm.pbf\n") + dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf") + + err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest) + require.NoError(t, err) + + got, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, pbf, got) +} + +func TestFetchFrancePBFMismatchRetainsFile(t *testing.T) { + pbf := []byte("some-valid-pbf-content-2") + srv := fakeGeofabrik(t, pbf, strings.Repeat("0", 32)+" france-latest.osm.pbf\n") + dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf") + + err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "checksum mismatch") + + // The corrupt file is kept for inspection, not deleted. + got, readErr := os.ReadFile(dest) + require.NoError(t, readErr) + assert.Equal(t, pbf, got) +} + +func TestFetchFrancePBFAlreadyCompleteVerified(t *testing.T) { + pbf := []byte("some-valid-pbf-content-3") + srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+" france-latest.osm.pbf\n") + dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf") + require.NoError(t, os.WriteFile(dest, pbf, 0o644)) + + err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest) + require.NoError(t, err) + + got, _ := os.ReadFile(dest) + assert.Equal(t, pbf, got) +} + +func TestFetchFrancePBFAlreadyCompleteCorrupt(t *testing.T) { + pbf := []byte("the-real-content") + corrupt := []byte(strings.Repeat("x", len(pbf))) + require.Equal(t, len(pbf), len(corrupt), "same length to prove size-only check is not enough") + srv := fakeGeofabrik(t, pbf, osmd5Hex(pbf)+" france-latest.osm.pbf\n") + dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf") + require.NoError(t, os.WriteFile(dest, corrupt, 0o644)) + + err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "checksum mismatch") + + got, _ := os.ReadFile(dest) + assert.Equal(t, corrupt, got, "corrupt file retained for inspection") +} + +func TestFetchFrancePBFSkipsVerificationWhenChecksumUnreachable(t *testing.T) { + pbf := []byte("some-valid-pbf-content-4") + srv := fakeGeofabrik(t, pbf, "") // md5 endpoint returns 404 + dest := filepath.Join(t.TempDir(), "france-latest.osm.pbf") + + err := fetchFrancePBF(srv.Client(), srv.URL+"/france-latest.osm.pbf", dest) + require.NoError(t, err) + + got, _ := os.ReadFile(dest) + assert.Equal(t, pbf, got) +} + +func TestPBFMD5(t *testing.T) { + data := []byte("known-content-for-md5-test") + expected := osmd5Hex(data) + path := filepath.Join(t.TempDir(), "sample.bin") + require.NoError(t, os.WriteFile(path, data, 0o644)) + + got, err := pbfMD5(path) + require.NoError(t, err) + assert.Equal(t, expected, got) +} |