aboutsummaryrefslogtreecommitdiff
path: root/mountain_pass
diff options
context:
space:
mode:
authorMartin Kagamino Lehoux <martin@lehoux.net>2026-08-21 10:30:27 +0200
committerMartin Kagamino Lehoux <martin@lehoux.net>2026-08-21 10:30:27 +0200
commit44f7333d9deef81d3468e20b378c63692a9406ec (patch)
treeaeacb99247dba4e48ee61e2792f5476a3770795a /mountain_pass
parent6667ee479c06c14bb182f36a0a9ee704fca6218d (diff)
Use OSM as the single source of mountain passes
Replace the centcols CSV download/enrichment pipeline with OSM PBF extraction as the primary (only) source: merge osmpass into mountain_pass, drop the centcols table and model fields, filter to named passes with elevation, and remove the centcols CLI flags.
Diffstat (limited to 'mountain_pass')
-rw-r--r--mountain_pass/detection.go24
-rw-r--r--mountain_pass/detection_test.go36
-rw-r--r--mountain_pass/downloader.go222
-rw-r--r--mountain_pass/downloader_test.go75
-rw-r--r--mountain_pass/extract.go98
-rw-r--r--mountain_pass/extract_test.go49
-rw-r--r--mountain_pass/fetch.go98
7 files changed, 290 insertions, 312 deletions
diff --git a/mountain_pass/detection.go b/mountain_pass/detection.go
index 4c54493..b71837b 100644
--- a/mountain_pass/detection.go
+++ b/mountain_pass/detection.go
@@ -9,6 +9,15 @@ import (
"github.com/martinlehoux/biking_home/ride"
)
+// MountainPass is a pass extracted from OpenStreetMap: a node tagged
+// mountain_pass=yes (or legacy natural=mountain_pass) with a name and an
+// elevation, as filtered by the loaders.
+type MountainPass struct {
+ Name string
+ Elevation int
+ Coord *geodist.Coord
+}
+
type Crossing struct {
Pass MountainPass
DistanceToM float64
@@ -55,12 +64,13 @@ func rideBounds(route ride.Ride, marginM float64) *coordinateBounds {
func loadMountainPasses(db *sql.DB, bounds *coordinateBounds) ([]MountainPass, error) {
query := `
- SELECT external_id, name, country_code, department_code, elevation, latitude, longitude
- FROM mountain_passes`
+ SELECT name, elevation, latitude, longitude
+ FROM mountain_passes
+ WHERE elevation IS NOT NULL
+ AND name IS NOT NULL AND name != ''`
args := make([]any, 0, 4)
if bounds != nil {
query += `
- WHERE latitude IS NOT NULL AND longitude IS NOT NULL
AND latitude BETWEEN ? AND ?
AND longitude BETWEEN ? AND ?`
args = append(args, bounds.minLatitude, bounds.maxLatitude, bounds.minLongitude, bounds.maxLongitude)
@@ -76,13 +86,11 @@ func loadMountainPasses(db *sql.DB, bounds *coordinateBounds) ([]MountainPass, e
mountainPasses := make([]MountainPass, 0)
for rows.Next() {
var mountainPass MountainPass
- var latitude, longitude sql.NullFloat64
- if err := rows.Scan(&mountainPass.ExternalID, &mountainPass.Name, &mountainPass.CountryCode, &mountainPass.DepartmentCode, &mountainPass.Elevation, &latitude, &longitude); err != nil {
+ var latitude, longitude float64
+ if err := rows.Scan(&mountainPass.Name, &mountainPass.Elevation, &latitude, &longitude); err != nil {
return nil, err
}
- if latitude.Valid && longitude.Valid {
- mountainPass.Coord = &geodist.Coord{Lat: latitude.Float64, Lon: longitude.Float64}
- }
+ mountainPass.Coord = &geodist.Coord{Lat: latitude, Lon: longitude}
mountainPasses = append(mountainPasses, mountainPass)
}
return mountainPasses, rows.Err()
diff --git a/mountain_pass/detection_test.go b/mountain_pass/detection_test.go
index d86e67b..f55532b 100644
--- a/mountain_pass/detection_test.go
+++ b/mountain_pass/detection_test.go
@@ -28,14 +28,15 @@ func TestDetectCrossingsFindsPass(t *testing.T) {
func TestLoadMountainPassesAroundRideUsesExpandedBounds(t *testing.T) {
db := dbtest.New(t)
_, err := db.Exec(`
- INSERT INTO mountain_passes (external_id, name, country_code, department_code, elevation, latitude, longitude)
- VALUES (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)
+ INSERT INTO mountain_passes (osm_id, name, elevation, latitude, longitude)
+ VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)
`,
- "550e8400-e29b-41d4-a716-446655440000", "On route", "FR", "13", 400, 43.05, 5.05,
- "6ba7b810-9dad-41d1-80b4-00c04fd430c8", "Within margin", "FR", "13", 500, 43.18, 5.20,
- "6ba7b811-9dad-41d1-80b4-00c04fd430c8", "Outside margin", "FR", "13", 600, 43.21, 5.20,
- "6ba7b812-9dad-41d1-80b4-00c04fd430c8", "Missing coordinates", "FR", "13", 700, nil, nil,
- "6ba7b813-9dad-41d1-80b4-00c04fd430c8", "On route edge", "FR", "13", 300, 43.10, 5.10,
+ 1, "On route", 400, 43.05, 5.05,
+ 2, "Within margin", 500, 43.18, 5.20,
+ 3, "Outside margin", 600, 43.21, 5.20,
+ 4, "No elevation", nil, 43.19, 5.20,
+ 5, "", 700, 43.17, 5.20,
+ 6, "On route edge", 300, 43.10, 5.10,
)
require.NoError(t, err)
@@ -53,6 +54,27 @@ func TestLoadMountainPassesAroundRideUsesExpandedBounds(t *testing.T) {
assert.Contains(t, []string{passes[1].Name, passes[2].Name}, "Within margin")
}
+func TestLoadMountainPassesFiltersElevationAndName(t *testing.T) {
+ db := dbtest.New(t)
+ _, err := db.Exec(`
+ INSERT INTO mountain_passes (osm_id, name, elevation, latitude, longitude)
+ VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)
+ `,
+ 1, "Named pass", 400, 43.05, 5.05,
+ 2, "No elevation", nil, 43.06, 5.06,
+ 3, "", 300, 43.07, 5.07,
+ 4, "No name or elevation", nil, 43.08, 5.08,
+ )
+ require.NoError(t, err)
+
+ passes, err := mountain_pass.LoadMountainPasses(db)
+ require.NoError(t, err)
+ require.Len(t, passes, 1)
+ assert.Equal(t, "Named pass", passes[0].Name)
+ assert.Equal(t, 400, passes[0].Elevation)
+ assert.NotNil(t, passes[0].Coord)
+}
+
func TestDetectCrossingsIgnoresDistantPass(t *testing.T) {
r := rideFromPoint(t, 43.62, 5.43, 447)
far := mountain_pass.MountainPass{
diff --git a/mountain_pass/downloader.go b/mountain_pass/downloader.go
deleted file mode 100644
index 00ff586..0000000
--- a/mountain_pass/downloader.go
+++ /dev/null
@@ -1,222 +0,0 @@
-package mountain_pass
-
-import (
- "bytes"
- "database/sql"
- "encoding/csv"
- "errors"
- "fmt"
- "io"
- "log/slog"
- "net/http"
- "os"
- "strconv"
- "strings"
- "time"
-
- "github.com/jftuga/geodist"
- "github.com/martinlehoux/kagamigo/kcore"
- "github.com/schollz/progressbar/v3"
-)
-
-type MountainPass struct {
- Name string
- ExternalID string
- CountryCode string
- DepartmentCode string
- Elevation int
- Coord *geodist.Coord
-}
-
-func parseMountainPasses(reader io.Reader) ([]MountainPass, error) {
- mountainPasses := make([]MountainPass, 0)
- csvReader := csv.NewReader(reader)
- csvReader.Comma = '\t'
- csvReader.LazyQuotes = true
- rows, err := csvReader.ReadAll()
- if err != nil {
- return nil, kcore.Wrap(err, "Failed to read mountain pass CSV data")
- }
- rows = rows[1:]
- for _, row := range rows {
- if len(row) < 4 {
- return nil, fmt.Errorf("unexpected row with %d fields: %v", len(row), row)
- }
- countryCode, departmentCode, err := splitCountryDepartment(row[0])
- if err != nil {
- return nil, err
- }
- codes := strings.Split(row[1], " | ")
- names := strings.Split(row[2], " | ")
- altitudes := strings.Split(row[3], " | ")
- if len(codes) != len(names) || len(codes) != len(altitudes) {
- return nil, fmt.Errorf("inconsistent multi-pass row: %v", row)
- }
- for i := range codes {
- elevation, err := strconv.Atoi(altitudes[i])
- if err != nil {
- return nil, fmt.Errorf("failed to parse elevation %q: %w", altitudes[i], err)
- }
- mountainPasses = append(mountainPasses, MountainPass{
- ExternalID: "centcols/" + codes[i],
- CountryCode: countryCode,
- Name: names[i],
- DepartmentCode: departmentCode,
- Elevation: elevation,
- })
- }
- }
-
- return mountainPasses, nil
-}
-
-func splitCountryDepartment(brevet string) (string, string, error) {
- parts := strings.Split(brevet, "-")
- if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
- return "", "", fmt.Errorf("unexpected brevet code: %q", brevet)
- }
- return parts[0], parts[1], nil
-}
-
-func getCSVData(departmentCode string) ([]byte, error) {
- url := "https://www.centcols.org/membres/cols/csvbrevet.php?id=FR" + departmentCode + "&enc=UTF-8"
- res, err := http.Get(url)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, statusError{statusCode: res.StatusCode}
- }
- return io.ReadAll(res.Body)
-}
-
-type statusError struct {
- statusCode int
-}
-
-func (e statusError) Error() string {
- return fmt.Sprintf("unexpected status code: %d", e.statusCode)
-}
-
-const downloadRetries = 3
-
-func departmentCacheFile(departmentCode string) string {
- return fmt.Sprintf("debug_department_%s.csv", departmentCode)
-}
-
-func loadCachedDepartment(filename string) ([]MountainPass, bool, error) {
- data, err := os.ReadFile(filename)
- if err != nil {
- if errors.Is(err, os.ErrNotExist) {
- return nil, false, nil
- }
- return nil, false, err
- }
- mountainPasses, err := parseMountainPasses(bytes.NewReader(data))
- if err != nil {
- return nil, false, nil
- }
- return mountainPasses, true, nil
-}
-
-func downloadDepartment(departmentCode string, resume bool) ([]MountainPass, error) {
- filename := departmentCacheFile(departmentCode)
- if resume {
- mountainPasses, found, err := loadCachedDepartment(filename)
- if err != nil {
- return nil, err
- }
- if found {
- return mountainPasses, nil
- }
- }
- var lastErr error
- backoff := time.Second
- for attempt := 1; attempt <= downloadRetries; attempt++ {
- data, err := getCSVData(departmentCode)
- if err != nil {
- lastErr = err
- var statusErr statusError
- if errors.As(err, &statusErr) && statusErr.statusCode == http.StatusTooManyRequests {
- backoff = time.Minute
- }
- slog.Warn("Retrying department download", "department", departmentCode, "attempt", attempt, "error", err)
- time.Sleep(backoff)
- backoff *= 2
- continue
- }
- mountainPasses, err := parseMountainPasses(bytes.NewReader(data))
- if err != nil {
- if writeErr := os.WriteFile(filename, data, 0o644); writeErr != nil {
- return nil, kcore.Wrap(writeErr, "Failed to write dump file")
- }
- return nil, err
- }
- if writeErr := os.WriteFile(filename, data, 0o644); writeErr != nil {
- return nil, kcore.Wrap(writeErr, "Failed to write dump file")
- }
- return mountainPasses, nil
- }
- return nil, lastErr
-}
-
-func upsertMountainPasses(db *sql.DB, mountainPasses []MountainPass) error {
- tx, err := db.Begin()
- kcore.Expect(err, "Failed to begin transaction")
- defer tx.Rollback()
- stmt, err := tx.Prepare(`
- INSERT INTO mountain_passes (external_id, name, country_code, department_code, elevation)
- VALUES (?, ?, ?, ?, ?)
- ON CONFLICT(external_id) DO UPDATE SET
- name = excluded.name,
- country_code = excluded.country_code,
- department_code = excluded.department_code,
- elevation = excluded.elevation
- `)
- kcore.Expect(err, "Failed to prepare statement")
- defer stmt.Close()
-
- for _, mountainPass := range mountainPasses {
- _, err := stmt.Exec(mountainPass.ExternalID, mountainPass.Name, mountainPass.CountryCode, mountainPass.DepartmentCode, mountainPass.Elevation)
- kcore.Expect(err, "Failed to execute statement")
- }
-
- return tx.Commit()
-}
-
-func DownloadMountainPasses(db *sql.DB, delay time.Duration, resume bool) error {
- mountainPasses := make([]MountainPass, 0)
- bar := progressbar.Default(90)
- ticker := time.NewTicker(delay)
- for i := 1; i <= 90; i++ {
- <-ticker.C
- bar.Add(1)
- departmentCode := fmt.Sprintf("%02d", i)
- bar.Describe(fmt.Sprintf("Downloading mountain passes for department_code=%s", departmentCode))
- departmentMountainPasses, err := downloadDepartment(departmentCode, resume)
- if err != nil {
- return kcore.Wrap(err, "Failed to download mountain passes for department "+departmentCode)
- }
- mountainPasses = append(mountainPasses, departmentMountainPasses...)
- }
-
- return upsertMountainPasses(db, mountainPasses)
-}
-
-func ImportCachedDepartments(db *sql.DB, departmentCodes []string) (int, error) {
- mountainPasses := make([]MountainPass, 0)
- for _, departmentCode := range departmentCodes {
- departmentMountainPasses, found, err := loadCachedDepartment(departmentCacheFile(departmentCode))
- if err != nil {
- return 0, err
- }
- if !found {
- slog.Warn("No cached department file", "department", departmentCode)
- continue
- }
- slog.Info("Loaded cached department", "department", departmentCode, "passes", len(departmentMountainPasses))
- mountainPasses = append(mountainPasses, departmentMountainPasses...)
- }
- return len(mountainPasses), upsertMountainPasses(db, mountainPasses)
-}
diff --git a/mountain_pass/downloader_test.go b/mountain_pass/downloader_test.go
deleted file mode 100644
index f7f5dda..0000000
--- a/mountain_pass/downloader_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package mountain_pass
-
-import (
- "os"
- "path/filepath"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestParseMountainPasses(t *testing.T) {
- csv := "Brevet\tcode\tnom\taltitude\n" +
- "FR-01\tFR-01-1500\tCol du Colombier\t1498\n" +
- "FR-06\tFR-06-0730c | FR-06-0706\tCol de Castillon | Col de Castillon (Tunnel)\t730 | 706\n" +
- "FR-13\tFR-13-0135\tCol du Télégraphe \"Chappe\"\t149\n" +
- "FR-90\tFR-88-1171\tCol du Ballon d'Alsace\t1171\n"
- mountainPasses, err := parseMountainPasses(strings.NewReader(csv))
- require.NoError(t, err)
-
- expected := []MountainPass{
- {ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498},
- {ExternalID: "centcols/FR-06-0730c", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon", Elevation: 730},
- {ExternalID: "centcols/FR-06-0706", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon (Tunnel)", Elevation: 706},
- {ExternalID: "centcols/FR-13-0135", CountryCode: "FR", DepartmentCode: "13", Name: "Col du Télégraphe \"Chappe\"", Elevation: 149},
- {ExternalID: "centcols/FR-88-1171", CountryCode: "FR", DepartmentCode: "90", Name: "Col du Ballon d'Alsace", Elevation: 1171},
- }
- assert.Equal(t, expected, mountainPasses)
-}
-
-func TestParseMountainPassesHeaderOnly(t *testing.T) {
- mountainPasses, err := parseMountainPasses(strings.NewReader("Brevet\tcode\tnom\taltitude\n"))
- require.NoError(t, err)
- assert.Empty(t, mountainPasses)
-}
-
-func TestSplitCountryDepartment(t *testing.T) {
- country, department, err := splitCountryDepartment("FR-01")
- require.NoError(t, err)
- assert.Equal(t, "FR", country)
- assert.Equal(t, "01", department)
-
- _, _, err = splitCountryDepartment("F-01-2")
- assert.Error(t, err)
-}
-
-func TestLoadCachedDepartment(t *testing.T) {
- cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv")
- err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\tFR-01-1500\tCol du Colombier\t1498\n"), 0o644)
- require.NoError(t, err)
-
- mountainPasses, found, err := loadCachedDepartment(cacheFile)
- require.NoError(t, err)
- assert.True(t, found)
- assert.Equal(t, []MountainPass{{ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498}}, mountainPasses)
-}
-
-func TestLoadCachedDepartmentMissing(t *testing.T) {
- mountainPasses, found, err := loadCachedDepartment(filepath.Join(t.TempDir(), "does-not-exist.csv"))
- require.NoError(t, err)
- assert.False(t, found)
- assert.Nil(t, mountainPasses)
-}
-
-func TestLoadCachedDepartmentCorrupt(t *testing.T) {
- cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv")
- err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\n"), 0o644)
- require.NoError(t, err)
-
- mountainPasses, found, err := loadCachedDepartment(cacheFile)
- require.NoError(t, err)
- assert.False(t, found)
- assert.Nil(t, mountainPasses)
-}
diff --git a/mountain_pass/extract.go b/mountain_pass/extract.go
new file mode 100644
index 0000000..81bed68
--- /dev/null
+++ b/mountain_pass/extract.go
@@ -0,0 +1,98 @@
+package mountain_pass
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "io"
+ "log/slog"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/paulmach/osm"
+ "github.com/paulmach/osm/osmpbf"
+)
+
+const insertStatement = `
+ INSERT INTO mountain_passes (osm_id, name, elevation, latitude, longitude)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(osm_id) DO UPDATE SET
+ name = excluded.name,
+ elevation = excluded.elevation,
+ latitude = excluded.latitude,
+ longitude = excluded.longitude
+`
+
+func ExtractMountainPasses(ctx context.Context, pbfPath string, db *sql.DB) (int, error) {
+ file, err := os.Open(pbfPath)
+ if err != nil {
+ return 0, fmt.Errorf("failed to open pbf file: %w", err)
+ }
+ defer file.Close()
+
+ scanner := osmpbf.New(ctx, file, 4)
+ scanner.SkipWays = true
+ scanner.SkipRelations = true
+ scanner.FilterNode = isMountainPass
+
+ tx, err := db.BeginTx(ctx, nil)
+ if err != nil {
+ return 0, fmt.Errorf("failed to begin transaction: %w", err)
+ }
+ defer tx.Rollback()
+
+ stmt, err := tx.PrepareContext(ctx, insertStatement)
+ if err != nil {
+ return 0, fmt.Errorf("failed to prepare insert: %w", err)
+ }
+ defer stmt.Close()
+
+ count := 0
+ for scanner.Scan() {
+ node, ok := scanner.Object().(*osm.Node)
+ if !ok {
+ continue
+ }
+ elevation, hasElevation := parseElevation(node.Tags.Find("ele"))
+ var elevationValue any
+ if hasElevation {
+ elevationValue = elevation
+ }
+ if _, err := stmt.ExecContext(ctx, node.ID, node.Tags.Find("name"), elevationValue, node.Lat, node.Lon); err != nil {
+ return count, fmt.Errorf("failed to insert osm pass %d: %w", node.ID, err)
+ }
+ count++
+ }
+ if err := scanner.Err(); err != nil && err != io.EOF {
+ return count, fmt.Errorf("failed to scan pbf: %w", err)
+ }
+ if err := tx.Commit(); err != nil {
+ return count, fmt.Errorf("failed to commit transaction: %w", err)
+ }
+ slog.Info("Extracted mountain passes", "count", count, "source", pbfPath)
+ return count, nil
+}
+
+func isMountainPass(node *osm.Node) bool {
+ if node.Tags.Find("mountain_pass") == "yes" {
+ return true
+ }
+ return node.Tags.Find("natural") == "mountain_pass"
+}
+
+func parseElevation(ele string) (int, bool) {
+ ele = strings.TrimSpace(ele)
+ if ele == "" {
+ return 0, false
+ }
+ for _, candidate := range strings.Fields(ele) {
+ if value, err := strconv.Atoi(candidate); err == nil {
+ return value, true
+ }
+ if value, err := strconv.ParseFloat(candidate, 64); err == nil {
+ return int(value), true
+ }
+ }
+ return 0, false
+}
diff --git a/mountain_pass/extract_test.go b/mountain_pass/extract_test.go
new file mode 100644
index 0000000..636372c
--- /dev/null
+++ b/mountain_pass/extract_test.go
@@ -0,0 +1,49 @@
+package mountain_pass
+
+import (
+ "testing"
+
+ "github.com/paulmach/osm"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestIsMountainPass(t *testing.T) {
+ cases := []struct {
+ name string
+ tags map[string]string
+ want bool
+ }{
+ {"tagged", map[string]string{"mountain_pass": "yes", "name": "Col de la Gineste"}, true},
+ {"legacy natural", map[string]string{"natural": "mountain_pass"}, true},
+ {"not a pass", map[string]string{"natural": "saddle", "name": "Col de la Gineste"}, false},
+ {"no tags", map[string]string{}, false},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ node := &osm.Node{Tags: osm.Tags{}}
+ for key, value := range c.tags {
+ node.Tags = append(node.Tags, osm.Tag{Key: key, Value: value})
+ }
+ assert.Equal(t, c.want, isMountainPass(node))
+ })
+ }
+}
+
+func TestParseElevation(t *testing.T) {
+ cases := []struct {
+ input string
+ want int
+ ok bool
+ }{
+ {"1320", 1320, true},
+ {"1320 m", 1320, true},
+ {" 447.5", 447, true},
+ {"", 0, false},
+ {"unknown", 0, false},
+ }
+ for _, c := range cases {
+ value, ok := parseElevation(c.input)
+ assert.Equal(t, c.want, value)
+ assert.Equal(t, c.ok, ok)
+ }
+}
diff --git a/mountain_pass/fetch.go b/mountain_pass/fetch.go
new file mode 100644
index 0000000..4be2d87
--- /dev/null
+++ b/mountain_pass/fetch.go
@@ -0,0 +1,98 @@
+package mountain_pass
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+
+ "github.com/schollz/progressbar/v3"
+)
+
+const francePBFRUL = "https://download.geofabrik.de/europe/france-latest.osm.pbf"
+
+// FetchFrancePBF downloads the full France OSM PBF to destPath, resuming a
+// partial download when the file already exists and skipping the download
+// entirely when it is already complete.
+func FetchFrancePBF(destPath string) error {
+ client := &http.Client{}
+
+ head, err := client.Head(francePBFRUL)
+ if err != nil {
+ return fmt.Errorf("failed to check remote size: %w", err)
+ }
+ head.Body.Close()
+ total := head.ContentLength
+ if total <= 0 {
+ return fmt.Errorf("unexpected remote content length: %d", total)
+ }
+
+ info, statErr := os.Stat(destPath)
+ var offset int64
+ if statErr == nil {
+ offset = info.Size()
+ if offset >= total {
+ slog.Info("Already downloaded", "file", destPath, "bytes", offset)
+ return nil
+ }
+ if offset > 0 {
+ slog.Info("Resuming download", "file", destPath, "bytes", offset, "remaining", total-offset)
+ }
+ } else if !errors.Is(statErr, os.ErrNotExist) {
+ return fmt.Errorf("failed to stat %s: %w", destPath, statErr)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, francePBFRUL, nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("User-Agent", "biking_home")
+ if offset > 0 {
+ req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
+ }
+
+ res, err := client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+
+ resuming := offset > 0 && res.StatusCode == http.StatusPartialContent
+ if !resuming {
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("unexpected status code: %d", res.StatusCode)
+ }
+ if offset > 0 {
+ slog.Warn("Server ignored Range header, restarting download", "file", destPath)
+ }
+ offset = 0
+ }
+
+ mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
+ if resuming {
+ mode = os.O_CREATE | os.O_WRONLY
+ }
+ file, err := os.OpenFile(destPath, mode, 0o644)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ bar := progressbar.DefaultBytes(res.ContentLength, "Downloading france-latest.osm.pbf")
+ if resuming {
+ // Re-append to the partial file when the server honored the range request.
+ if _, err := file.Seek(offset, io.SeekStart); err != nil {
+ return err
+ }
+ }
+ _, err = io.Copy(io.MultiWriter(file, bar), res.Body)
+ if err != nil {
+ return fmt.Errorf("failed to download %s: %w", francePBFRUL, err)
+ }
+ if size, err := file.Stat(); err == nil && size.Size() != total {
+ return fmt.Errorf("download incomplete: got %d bytes, want %d", size.Size(), total)
+ }
+ return nil
+}