summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMartin Kagamino Lehoux <martin@lehoux.net>2026-08-01 22:40:08 +0200
committerMartin Kagamino Lehoux <martin@lehoux.net>2026-08-01 22:40:08 +0200
commit80f2e72cd4818a4c0a5b5d4afb93dc23f04208e2 (patch)
treed6f0323ed7e9e29190742750d70ee8ec392573c5
parentef12ceb8105c21cbfd9cd7ac9935893086d5af51 (diff)
feat: Download mountain passes from centcols into SQLite
-rw-r--r--.gitignore4
-rw-r--r--README.md1
-rw-r--r--db/migrations/20250802140659_mountain_passes.sql12
-rw-r--r--db/schema.sql12
-rw-r--r--go.mod20
-rw-r--r--go.sum43
-rw-r--r--main.go31
-rw-r--r--mountain_pass/downloader.go199
-rw-r--r--mountain_pass/downloader_test.go75
-rw-r--r--ride/index.go8
10 files changed, 397 insertions, 8 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8a49c36
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+.env
+*.db
+*.png
+debug_department_*.csv
diff --git a/README.md b/README.md
index 55bb620..fea48d2 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@
- Storing GPX
- Export mountain pass from OSM and auto detect when pass is crossed
- https://www.jmspae.se/write-ups/kebabs-train-stations/
+- Download French mountain passes from centcols.org into a SQLite database (`go run . -download`, resume with `-resume`)
- Auto detect climbs from GPX
- Compute estimated power
- Export data from Strava, Garmin
diff --git a/db/migrations/20250802140659_mountain_passes.sql b/db/migrations/20250802140659_mountain_passes.sql
new file mode 100644
index 0000000..afe3218
--- /dev/null
+++ b/db/migrations/20250802140659_mountain_passes.sql
@@ -0,0 +1,12 @@
+-- migrate:up
+create table mountain_passes (
+ id integer primary key,
+ external_id text unique not null,
+ name text not null,
+ country_code text not null,
+ department_code text not null,
+ elevation integer not null
+);
+
+-- migrate:down
+drop table mountain_passes;
diff --git a/db/schema.sql b/db/schema.sql
new file mode 100644
index 0000000..e6796c8
--- /dev/null
+++ b/db/schema.sql
@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(255) primary key);
+CREATE TABLE mountain_passes (
+ id integer primary key,
+ external_id text unique not null,
+ name text not null,
+ country_code text not null,
+ department_code text not null,
+ elevation integer not null
+);
+-- Dbmate schema migrations
+INSERT INTO "schema_migrations" (version) VALUES
+ ('20250802140659');
diff --git a/go.mod b/go.mod
index 8eb0702..63e19a4 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ toolchain go1.23.8
require (
github.com/jftuga/geodist v1.0.0
github.com/martinlehoux/kagamigo v0.4.1
- github.com/stretchr/testify v1.8.4
+ github.com/stretchr/testify v1.9.0
github.com/tkrajina/gpxgo v1.4.0
)
@@ -17,23 +17,41 @@ require (
codeberg.org/go-pdf/fpdf v0.10.0 // indirect
git.sr.ht/~sbinet/gg v0.6.0 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
+ github.com/ClickHouse/clickhouse-go v1.5.4 // indirect
github.com/a-h/templ v0.2.747 // indirect
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
+ github.com/amacneil/dbmate v1.16.2 // indirect
github.com/bradleyjkemp/cupaloy v1.3.0 // indirect
github.com/campoy/embedmd v1.0.0 // indirect
+ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
+ github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+ github.com/joho/godotenv v1.4.0 // indirect
github.com/kataras/i18n v0.0.8 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/kr/text v0.2.0 // indirect
+ github.com/lib/pq v1.10.7 // indirect
+ github.com/mattn/go-sqlite3 v1.14.16 // indirect
+ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/rivo/uniseg v0.4.7 // indirect
+ github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/schollz/progressbar/v3 v3.18.0 // indirect
+ github.com/urfave/cli/v2 v2.24.2 // indirect
+ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/image v0.25.0 // indirect
golang.org/x/net v0.27.0 // indirect
+ golang.org/x/sys v0.34.0 // indirect
+ golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.23.0 // indirect
gonum.org/v1/plot v0.16.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+
+tool github.com/amacneil/dbmate
diff --git a/go.sum b/go.sum
index 19e7ad1..0b6280f 100644
--- a/go.sum
+++ b/go.sum
@@ -9,28 +9,43 @@ git.sr.ht/~sbinet/gg v0.6.0/go.mod h1:uucygbfC9wVPQIfrmwM2et0imr8L7KQWywX0xpFMm9
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
+github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/a-h/templ v0.2.747 h1:D0dQ2lxC3W7Dxl6fxQ/1zZHBQslSkTSvl5FxP/CfdKg=
github.com/a-h/templ v0.2.747/go.mod h1:69ObQIbrcuwPCU32ohNaWce3Cb7qM5GMiqN1K+2yop4=
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
+github.com/amacneil/dbmate v1.16.2 h1:ovhzYRR2JT5EZbISNtg7MZmLM51ZrHLKoEKMPhiFz5E=
+github.com/amacneil/dbmate v1.16.2/go.mod h1:d+2u+wE7GpLepbKxi231FXoi7thXuI1AND5CRG18RcI=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/bradleyjkemp/cupaloy v1.3.0 h1:UJ0YJuhkMXEQcaoQNSCmK8og6GEVW/eDhAZuizvcpOY=
github.com/bradleyjkemp/cupaloy v1.3.0/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk=
github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/jftuga/geodist v1.0.0 h1:WzDVdHJGyCF2lDIbTLCUJxchlvnBEQyORSB2AN9/Mns=
+github.com/jftuga/geodist v1.0.0 h1:PFPQlZtj10u8ETAYTyxE0DWMl1bwA+Xzrqb4+oLkkC0=
github.com/jftuga/geodist v1.0.0/go.mod h1:BohEDxpZ8S5ADAxW/9EKPSKWOVl0+3wHENIT40m4UO4=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/kataras/i18n v0.0.8 h1:thiDRqq4fN2sQOK5CwR5Yh1yT7swP6B3wnadILhqjhk=
github.com/kataras/i18n v0.0.8/go.mod h1:M/yRAqQ3Y7z2oSpotxG/+nPrgsJLas6t0kEJfIJk98E=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -40,16 +55,38 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/martinlehoux/kagamigo v0.4.1 h1:jI4TXyQMdm05H/Pd5RE7pfoMHq9PqdOF21oWAZOxsJ8=
github.com/martinlehoux/kagamigo v0.4.1/go.mod h1:CDkN/Kx8KQSU0kC/ISfq2gYIhVbddHD3E8r3uAlnnMI=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tkrajina/gpxgo v1.4.0 h1:cSD5uSwy3VZuNFieTEZLyRnuIwhonQEkGPkPGW4XNag=
github.com/tkrajina/gpxgo v1.4.0/go.mod h1:BXSMfUAvKiEhMEXAFM2NvNsbjsSvp394mOvdcNjettg=
+github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk=
+github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
+github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -73,7 +110,11 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
+golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
diff --git a/main.go b/main.go
index cacbe98..f072210 100644
--- a/main.go
+++ b/main.go
@@ -1,20 +1,47 @@
package main
import (
+ "database/sql"
"flag"
"log/slog"
"os"
"runtime/pprof"
+ "time"
+ mountainpass "github.com/martinlehoux/biking_home/mountain_pass"
"github.com/martinlehoux/biking_home/ride"
"github.com/martinlehoux/kagamigo/kcore"
+ _ "github.com/mattn/go-sqlite3"
)
-var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-var parser = ride.GPXRideParser{}
+var (
+ download = flag.Bool("download", false, "download mountain passes into the database")
+ resume = flag.Bool("resume", false, "skip departments already cached on disk")
+ demo = flag.Bool("demo", false, "run the climb similarity demo")
+ cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+ parser = ride.GPXRideParser{}
+)
func main() {
flag.Parse()
+ if *download {
+ runDownload()
+ return
+ }
+ if *demo {
+ runDemo()
+ }
+}
+
+func runDownload() {
+ db, err := sql.Open("sqlite3", "biking_home.db")
+ kcore.Expect(err, "failed to open database")
+ defer db.Close()
+ err = mountainpass.DownloadMountainPasses(db, 5*time.Second, *resume)
+ kcore.Expect(err, "failed to download mountain passes")
+}
+
+func runDemo() {
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
kcore.Expect(err, "failed to create CPU profile")
diff --git a/mountain_pass/downloader.go b/mountain_pass/downloader.go
new file mode 100644
index 0000000..685e0fe
--- /dev/null
+++ b/mountain_pass/downloader.go
@@ -0,0 +1,199 @@
+package mountainpass
+
+import (
+ "bytes"
+ "database/sql"
+ "encoding/csv"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/martinlehoux/kagamigo/kcore"
+ "github.com/schollz/progressbar/v3"
+)
+
+type MountainPass struct {
+ Name string
+ ExternalID string
+ CountryCode string
+ DepartmentCode string
+ Elevation int
+}
+
+func parseMountainPasses(reader io.Reader) ([]MountainPass, error) {
+ mountainPasses := make([]MountainPass, 0)
+ csvReader := csv.NewReader(reader)
+ csvReader.Comma = '\t'
+ csvReader.LazyQuotes = true
+ rows, err := csvReader.ReadAll()
+ if err != nil {
+ return nil, kcore.Wrap(err, "Failed to read mountain pass CSV data")
+ }
+ rows = rows[1:]
+ for _, row := range rows {
+ if len(row) < 4 {
+ return nil, fmt.Errorf("unexpected row with %d fields: %v", len(row), row)
+ }
+ countryCode, departmentCode, err := splitCountryDepartment(row[0])
+ if err != nil {
+ return nil, err
+ }
+ codes := strings.Split(row[1], " | ")
+ names := strings.Split(row[2], " | ")
+ altitudes := strings.Split(row[3], " | ")
+ if len(codes) != len(names) || len(codes) != len(altitudes) {
+ return nil, fmt.Errorf("inconsistent multi-pass row: %v", row)
+ }
+ for i := range codes {
+ elevation, err := strconv.Atoi(altitudes[i])
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse elevation %q: %w", altitudes[i], err)
+ }
+ mountainPasses = append(mountainPasses, MountainPass{
+ ExternalID: "centcols/" + codes[i],
+ CountryCode: countryCode,
+ Name: names[i],
+ DepartmentCode: departmentCode,
+ Elevation: elevation,
+ })
+ }
+ }
+
+ return mountainPasses, nil
+}
+
+func splitCountryDepartment(brevet string) (string, string, error) {
+ parts := strings.Split(brevet, "-")
+ if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+ return "", "", fmt.Errorf("unexpected brevet code: %q", brevet)
+ }
+ return parts[0], parts[1], nil
+}
+
+func getCSVData(departmentCode string) ([]byte, error) {
+ url := "https://www.centcols.org/membres/cols/csvbrevet.php?id=FR" + departmentCode + "&enc=UTF-8"
+ res, err := http.Get(url)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, statusError{statusCode: res.StatusCode}
+ }
+ return io.ReadAll(res.Body)
+}
+
+type statusError struct {
+ statusCode int
+}
+
+func (e statusError) Error() string {
+ return fmt.Sprintf("unexpected status code: %d", e.statusCode)
+}
+
+const downloadRetries = 3
+
+func departmentCacheFile(departmentCode string) string {
+ return fmt.Sprintf("debug_department_%s.csv", departmentCode)
+}
+
+func loadCachedDepartment(filename string) ([]MountainPass, bool, error) {
+ data, err := os.ReadFile(filename)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, false, nil
+ }
+ return nil, false, err
+ }
+ mountainPasses, err := parseMountainPasses(bytes.NewReader(data))
+ if err != nil {
+ return nil, false, nil
+ }
+ return mountainPasses, true, nil
+}
+
+func downloadDepartment(departmentCode string, resume bool) ([]MountainPass, error) {
+ filename := departmentCacheFile(departmentCode)
+ if resume {
+ mountainPasses, found, err := loadCachedDepartment(filename)
+ if err != nil {
+ return nil, err
+ }
+ if found {
+ return mountainPasses, nil
+ }
+ }
+ var lastErr error
+ backoff := time.Second
+ for attempt := 1; attempt <= downloadRetries; attempt++ {
+ data, err := getCSVData(departmentCode)
+ if err != nil {
+ lastErr = err
+ var statusErr statusError
+ if errors.As(err, &statusErr) && statusErr.statusCode == http.StatusTooManyRequests {
+ backoff = time.Minute
+ }
+ slog.Warn("Retrying department download", "department", departmentCode, "attempt", attempt, "error", err)
+ time.Sleep(backoff)
+ backoff *= 2
+ continue
+ }
+ mountainPasses, err := parseMountainPasses(bytes.NewReader(data))
+ if err != nil {
+ if writeErr := os.WriteFile(filename, data, 0644); writeErr != nil {
+ return nil, kcore.Wrap(writeErr, "Failed to write dump file")
+ }
+ return nil, err
+ }
+ if writeErr := os.WriteFile(filename, data, 0644); writeErr != nil {
+ return nil, kcore.Wrap(writeErr, "Failed to write dump file")
+ }
+ return mountainPasses, nil
+ }
+ return nil, lastErr
+}
+
+func DownloadMountainPasses(db *sql.DB, delay time.Duration, resume bool) error {
+ mountainPasses := make([]MountainPass, 0)
+ bar := progressbar.Default(90)
+ ticker := time.NewTicker(delay)
+ for i := 1; i <= 90; i++ {
+ <-ticker.C
+ bar.Add(1)
+ departmentCode := fmt.Sprintf("%02d", i)
+ bar.Describe(fmt.Sprintf("Downloading mountain passes for department_code=%s", departmentCode))
+ departmentMountainPasses, err := downloadDepartment(departmentCode, resume)
+ if err != nil {
+ return kcore.Wrap(err, "Failed to download mountain passes for department "+departmentCode)
+ }
+ mountainPasses = append(mountainPasses, departmentMountainPasses...)
+ }
+
+ tx, err := db.Begin()
+ kcore.Expect(err, "Failed to begin transaction")
+ defer tx.Rollback()
+ stmt, err := tx.Prepare(`
+ INSERT INTO mountain_passes (external_id, name, country_code, department_code, elevation)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(external_id) DO UPDATE SET
+ name = excluded.name,
+ country_code = excluded.country_code,
+ department_code = excluded.department_code,
+ elevation = excluded.elevation
+ `)
+ kcore.Expect(err, "Failed to prepare statement")
+ defer stmt.Close()
+
+ for _, mountainPass := range mountainPasses {
+ _, err := stmt.Exec(mountainPass.ExternalID, mountainPass.Name, mountainPass.CountryCode, mountainPass.DepartmentCode, mountainPass.Elevation)
+ kcore.Expect(err, "Failed to execute statement")
+ }
+
+ return tx.Commit()
+}
diff --git a/mountain_pass/downloader_test.go b/mountain_pass/downloader_test.go
new file mode 100644
index 0000000..f4ffcc8
--- /dev/null
+++ b/mountain_pass/downloader_test.go
@@ -0,0 +1,75 @@
+package mountainpass
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseMountainPasses(t *testing.T) {
+ csv := "Brevet\tcode\tnom\taltitude\n" +
+ "FR-01\tFR-01-1500\tCol du Colombier\t1498\n" +
+ "FR-06\tFR-06-0730c | FR-06-0706\tCol de Castillon | Col de Castillon (Tunnel)\t730 | 706\n" +
+ "FR-13\tFR-13-0135\tCol du Télégraphe \"Chappe\"\t149\n" +
+ "FR-90\tFR-88-1171\tCol du Ballon d'Alsace\t1171\n"
+ mountainPasses, err := parseMountainPasses(strings.NewReader(csv))
+ require.NoError(t, err)
+
+ expected := []MountainPass{
+ {ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498},
+ {ExternalID: "centcols/FR-06-0730c", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon", Elevation: 730},
+ {ExternalID: "centcols/FR-06-0706", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon (Tunnel)", Elevation: 706},
+ {ExternalID: "centcols/FR-13-0135", CountryCode: "FR", DepartmentCode: "13", Name: "Col du Télégraphe \"Chappe\"", Elevation: 149},
+ {ExternalID: "centcols/FR-88-1171", CountryCode: "FR", DepartmentCode: "90", Name: "Col du Ballon d'Alsace", Elevation: 1171},
+ }
+ assert.Equal(t, expected, mountainPasses)
+}
+
+func TestParseMountainPassesHeaderOnly(t *testing.T) {
+ mountainPasses, err := parseMountainPasses(strings.NewReader("Brevet\tcode\tnom\taltitude\n"))
+ require.NoError(t, err)
+ assert.Empty(t, mountainPasses)
+}
+
+func TestSplitCountryDepartment(t *testing.T) {
+ country, department, err := splitCountryDepartment("FR-01")
+ require.NoError(t, err)
+ assert.Equal(t, "FR", country)
+ assert.Equal(t, "01", department)
+
+ _, _, err = splitCountryDepartment("F-01-2")
+ assert.Error(t, err)
+}
+
+func TestLoadCachedDepartment(t *testing.T) {
+ cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv")
+ err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\tFR-01-1500\tCol du Colombier\t1498\n"), 0644)
+ require.NoError(t, err)
+
+ mountainPasses, found, err := loadCachedDepartment(cacheFile)
+ require.NoError(t, err)
+ assert.True(t, found)
+ assert.Equal(t, []MountainPass{{ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498}}, mountainPasses)
+}
+
+func TestLoadCachedDepartmentMissing(t *testing.T) {
+ mountainPasses, found, err := loadCachedDepartment(filepath.Join(t.TempDir(), "does-not-exist.csv"))
+ require.NoError(t, err)
+ assert.False(t, found)
+ assert.Nil(t, mountainPasses)
+}
+
+func TestLoadCachedDepartmentCorrupt(t *testing.T) {
+ cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv")
+ err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\n"), 0644)
+ require.NoError(t, err)
+
+ mountainPasses, found, err := loadCachedDepartment(cacheFile)
+ require.NoError(t, err)
+ assert.False(t, found)
+ assert.Nil(t, mountainPasses)
+}
diff --git a/ride/index.go b/ride/index.go
index 18e2ac8..79f9212 100644
--- a/ride/index.go
+++ b/ride/index.go
@@ -8,14 +8,14 @@ type RideClimbsIndex interface {
}
type FlatRideClimbsIndex struct {
- sensibility float64
+ sensitivity float64
climbs []Climb
}
-func NewFlatRideClimbsIndex(sensibility float64) *FlatRideClimbsIndex {
+func NewFlatRideClimbsIndex(sensitivity float64) *FlatRideClimbsIndex {
return &FlatRideClimbsIndex{
climbs: make([]Climb, 0),
- sensibility: sensibility,
+ sensitivity: sensitivity,
}
}
@@ -28,7 +28,7 @@ func (index *FlatRideClimbsIndex) Similar(ride Ride, climb Climb) []Climb {
for _, c := range index.climbs {
_, startDistance := geodist.HaversineDistance(c.Start().Coord, climb.Start().Coord)
_, endDistance := geodist.HaversineDistance(c.End().Coord, climb.End().Coord)
- if startDistance <= index.sensibility/1000 && endDistance <= index.sensibility/1000 {
+ if startDistance <= index.sensitivity/1000 && endDistance <= index.sensitivity/1000 {
results = append(results, c)
}
}