diff options
| author | Martin Kagamino Lehoux <martin@lehoux.net> | 2026-08-01 22:40:08 +0200 |
|---|---|---|
| committer | Martin Kagamino Lehoux <martin@lehoux.net> | 2026-08-01 22:40:08 +0200 |
| commit | 80f2e72cd4818a4c0a5b5d4afb93dc23f04208e2 (patch) | |
| tree | d6f0323ed7e9e29190742750d70ee8ec392573c5 | |
| parent | ef12ceb8105c21cbfd9cd7ac9935893086d5af51 (diff) | |
feat: Download mountain passes from centcols into SQLite
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | db/migrations/20250802140659_mountain_passes.sql | 12 | ||||
| -rw-r--r-- | db/schema.sql | 12 | ||||
| -rw-r--r-- | go.mod | 20 | ||||
| -rw-r--r-- | go.sum | 43 | ||||
| -rw-r--r-- | main.go | 31 | ||||
| -rw-r--r-- | mountain_pass/downloader.go | 199 | ||||
| -rw-r--r-- | mountain_pass/downloader_test.go | 75 | ||||
| -rw-r--r-- | ride/index.go | 8 |
10 files changed, 397 insertions, 8 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8a49c36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +*.db +*.png +debug_department_*.csv @@ -10,6 +10,7 @@ - Storing GPX - Export mountain pass from OSM and auto detect when pass is crossed - https://www.jmspae.se/write-ups/kebabs-train-stations/ +- Download French mountain passes from centcols.org into a SQLite database (`go run . -download`, resume with `-resume`) - Auto detect climbs from GPX - Compute estimated power - Export data from Strava, Garmin diff --git a/db/migrations/20250802140659_mountain_passes.sql b/db/migrations/20250802140659_mountain_passes.sql new file mode 100644 index 0000000..afe3218 --- /dev/null +++ b/db/migrations/20250802140659_mountain_passes.sql @@ -0,0 +1,12 @@ +-- migrate:up +create table mountain_passes ( + id integer primary key, + external_id text unique not null, + name text not null, + country_code text not null, + department_code text not null, + elevation integer not null +); + +-- migrate:down +drop table mountain_passes; diff --git a/db/schema.sql b/db/schema.sql new file mode 100644 index 0000000..e6796c8 --- /dev/null +++ b/db/schema.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS "schema_migrations" (version varchar(255) primary key); +CREATE TABLE mountain_passes ( + id integer primary key, + external_id text unique not null, + name text not null, + country_code text not null, + department_code text not null, + elevation integer not null +); +-- Dbmate schema migrations +INSERT INTO "schema_migrations" (version) VALUES + ('20250802140659'); @@ -7,7 +7,7 @@ toolchain go1.23.8 require ( github.com/jftuga/geodist v1.0.0 github.com/martinlehoux/kagamigo v0.4.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tkrajina/gpxgo v1.4.0 ) @@ -17,23 +17,41 @@ require ( codeberg.org/go-pdf/fpdf v0.10.0 // indirect git.sr.ht/~sbinet/gg v0.6.0 // indirect github.com/BurntSushi/toml v1.4.0 // indirect + github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/a-h/templ v0.2.747 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect + github.com/amacneil/dbmate v1.16.2 // indirect github.com/bradleyjkemp/cupaloy v1.3.0 // indirect github.com/campoy/embedmd v1.0.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/joho/godotenv v1.4.0 // indirect github.com/kataras/i18n v0.0.8 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.10.7 // indirect + github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/schollz/progressbar/v3 v3.18.0 // indirect + github.com/urfave/cli/v2 v2.24.2 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/image v0.25.0 // indirect golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect golang.org/x/text v0.23.0 // indirect gonum.org/v1/plot v0.16.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +tool github.com/amacneil/dbmate @@ -9,28 +9,43 @@ git.sr.ht/~sbinet/gg v0.6.0/go.mod h1:uucygbfC9wVPQIfrmwM2et0imr8L7KQWywX0xpFMm9 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/a-h/templ v0.2.747 h1:D0dQ2lxC3W7Dxl6fxQ/1zZHBQslSkTSvl5FxP/CfdKg= github.com/a-h/templ v0.2.747/go.mod h1:69ObQIbrcuwPCU32ohNaWce3Cb7qM5GMiqN1K+2yop4= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/amacneil/dbmate v1.16.2 h1:ovhzYRR2JT5EZbISNtg7MZmLM51ZrHLKoEKMPhiFz5E= +github.com/amacneil/dbmate v1.16.2/go.mod h1:d+2u+wE7GpLepbKxi231FXoi7thXuI1AND5CRG18RcI= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bradleyjkemp/cupaloy v1.3.0 h1:UJ0YJuhkMXEQcaoQNSCmK8og6GEVW/eDhAZuizvcpOY= github.com/bradleyjkemp/cupaloy v1.3.0/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk= github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/jftuga/geodist v1.0.0 h1:WzDVdHJGyCF2lDIbTLCUJxchlvnBEQyORSB2AN9/Mns= +github.com/jftuga/geodist v1.0.0 h1:PFPQlZtj10u8ETAYTyxE0DWMl1bwA+Xzrqb4+oLkkC0= github.com/jftuga/geodist v1.0.0/go.mod h1:BohEDxpZ8S5ADAxW/9EKPSKWOVl0+3wHENIT40m4UO4= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kataras/i18n v0.0.8 h1:thiDRqq4fN2sQOK5CwR5Yh1yT7swP6B3wnadILhqjhk= github.com/kataras/i18n v0.0.8/go.mod h1:M/yRAqQ3Y7z2oSpotxG/+nPrgsJLas6t0kEJfIJk98E= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -40,16 +55,38 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/martinlehoux/kagamigo v0.4.1 h1:jI4TXyQMdm05H/Pd5RE7pfoMHq9PqdOF21oWAZOxsJ8= github.com/martinlehoux/kagamigo v0.4.1/go.mod h1:CDkN/Kx8KQSU0kC/ISfq2gYIhVbddHD3E8r3uAlnnMI= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tkrajina/gpxgo v1.4.0 h1:cSD5uSwy3VZuNFieTEZLyRnuIwhonQEkGPkPGW4XNag= github.com/tkrajina/gpxgo v1.4.0/go.mod h1:BXSMfUAvKiEhMEXAFM2NvNsbjsSvp394mOvdcNjettg= +github.com/urfave/cli/v2 v2.24.2 h1:q1VA+ofZ8SWfEKB9xXHUD4QZaeI9e+ItEqSbfH2JBXk= +github.com/urfave/cli/v2 v2.24.2/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -73,7 +110,11 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1,20 +1,47 @@ package main import ( + "database/sql" "flag" "log/slog" "os" "runtime/pprof" + "time" + mountainpass "github.com/martinlehoux/biking_home/mountain_pass" "github.com/martinlehoux/biking_home/ride" "github.com/martinlehoux/kagamigo/kcore" + _ "github.com/mattn/go-sqlite3" ) -var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") -var parser = ride.GPXRideParser{} +var ( + download = flag.Bool("download", false, "download mountain passes into the database") + resume = flag.Bool("resume", false, "skip departments already cached on disk") + demo = flag.Bool("demo", false, "run the climb similarity demo") + cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") + parser = ride.GPXRideParser{} +) func main() { flag.Parse() + if *download { + runDownload() + return + } + if *demo { + runDemo() + } +} + +func runDownload() { + db, err := sql.Open("sqlite3", "biking_home.db") + kcore.Expect(err, "failed to open database") + defer db.Close() + err = mountainpass.DownloadMountainPasses(db, 5*time.Second, *resume) + kcore.Expect(err, "failed to download mountain passes") +} + +func runDemo() { if *cpuprofile != "" { f, err := os.Create(*cpuprofile) kcore.Expect(err, "failed to create CPU profile") diff --git a/mountain_pass/downloader.go b/mountain_pass/downloader.go new file mode 100644 index 0000000..685e0fe --- /dev/null +++ b/mountain_pass/downloader.go @@ -0,0 +1,199 @@ +package mountainpass + +import ( + "bytes" + "database/sql" + "encoding/csv" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/martinlehoux/kagamigo/kcore" + "github.com/schollz/progressbar/v3" +) + +type MountainPass struct { + Name string + ExternalID string + CountryCode string + DepartmentCode string + Elevation int +} + +func parseMountainPasses(reader io.Reader) ([]MountainPass, error) { + mountainPasses := make([]MountainPass, 0) + csvReader := csv.NewReader(reader) + csvReader.Comma = '\t' + csvReader.LazyQuotes = true + rows, err := csvReader.ReadAll() + if err != nil { + return nil, kcore.Wrap(err, "Failed to read mountain pass CSV data") + } + rows = rows[1:] + for _, row := range rows { + if len(row) < 4 { + return nil, fmt.Errorf("unexpected row with %d fields: %v", len(row), row) + } + countryCode, departmentCode, err := splitCountryDepartment(row[0]) + if err != nil { + return nil, err + } + codes := strings.Split(row[1], " | ") + names := strings.Split(row[2], " | ") + altitudes := strings.Split(row[3], " | ") + if len(codes) != len(names) || len(codes) != len(altitudes) { + return nil, fmt.Errorf("inconsistent multi-pass row: %v", row) + } + for i := range codes { + elevation, err := strconv.Atoi(altitudes[i]) + if err != nil { + return nil, fmt.Errorf("failed to parse elevation %q: %w", altitudes[i], err) + } + mountainPasses = append(mountainPasses, MountainPass{ + ExternalID: "centcols/" + codes[i], + CountryCode: countryCode, + Name: names[i], + DepartmentCode: departmentCode, + Elevation: elevation, + }) + } + } + + return mountainPasses, nil +} + +func splitCountryDepartment(brevet string) (string, string, error) { + parts := strings.Split(brevet, "-") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("unexpected brevet code: %q", brevet) + } + return parts[0], parts[1], nil +} + +func getCSVData(departmentCode string) ([]byte, error) { + url := "https://www.centcols.org/membres/cols/csvbrevet.php?id=FR" + departmentCode + "&enc=UTF-8" + res, err := http.Get(url) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, statusError{statusCode: res.StatusCode} + } + return io.ReadAll(res.Body) +} + +type statusError struct { + statusCode int +} + +func (e statusError) Error() string { + return fmt.Sprintf("unexpected status code: %d", e.statusCode) +} + +const downloadRetries = 3 + +func departmentCacheFile(departmentCode string) string { + return fmt.Sprintf("debug_department_%s.csv", departmentCode) +} + +func loadCachedDepartment(filename string) ([]MountainPass, bool, error) { + data, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + return nil, false, err + } + mountainPasses, err := parseMountainPasses(bytes.NewReader(data)) + if err != nil { + return nil, false, nil + } + return mountainPasses, true, nil +} + +func downloadDepartment(departmentCode string, resume bool) ([]MountainPass, error) { + filename := departmentCacheFile(departmentCode) + if resume { + mountainPasses, found, err := loadCachedDepartment(filename) + if err != nil { + return nil, err + } + if found { + return mountainPasses, nil + } + } + var lastErr error + backoff := time.Second + for attempt := 1; attempt <= downloadRetries; attempt++ { + data, err := getCSVData(departmentCode) + if err != nil { + lastErr = err + var statusErr statusError + if errors.As(err, &statusErr) && statusErr.statusCode == http.StatusTooManyRequests { + backoff = time.Minute + } + slog.Warn("Retrying department download", "department", departmentCode, "attempt", attempt, "error", err) + time.Sleep(backoff) + backoff *= 2 + continue + } + mountainPasses, err := parseMountainPasses(bytes.NewReader(data)) + if err != nil { + if writeErr := os.WriteFile(filename, data, 0644); writeErr != nil { + return nil, kcore.Wrap(writeErr, "Failed to write dump file") + } + return nil, err + } + if writeErr := os.WriteFile(filename, data, 0644); writeErr != nil { + return nil, kcore.Wrap(writeErr, "Failed to write dump file") + } + return mountainPasses, nil + } + return nil, lastErr +} + +func DownloadMountainPasses(db *sql.DB, delay time.Duration, resume bool) error { + mountainPasses := make([]MountainPass, 0) + bar := progressbar.Default(90) + ticker := time.NewTicker(delay) + for i := 1; i <= 90; i++ { + <-ticker.C + bar.Add(1) + departmentCode := fmt.Sprintf("%02d", i) + bar.Describe(fmt.Sprintf("Downloading mountain passes for department_code=%s", departmentCode)) + departmentMountainPasses, err := downloadDepartment(departmentCode, resume) + if err != nil { + return kcore.Wrap(err, "Failed to download mountain passes for department "+departmentCode) + } + mountainPasses = append(mountainPasses, departmentMountainPasses...) + } + + tx, err := db.Begin() + kcore.Expect(err, "Failed to begin transaction") + defer tx.Rollback() + stmt, err := tx.Prepare(` + INSERT INTO mountain_passes (external_id, name, country_code, department_code, elevation) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(external_id) DO UPDATE SET + name = excluded.name, + country_code = excluded.country_code, + department_code = excluded.department_code, + elevation = excluded.elevation + `) + kcore.Expect(err, "Failed to prepare statement") + defer stmt.Close() + + for _, mountainPass := range mountainPasses { + _, err := stmt.Exec(mountainPass.ExternalID, mountainPass.Name, mountainPass.CountryCode, mountainPass.DepartmentCode, mountainPass.Elevation) + kcore.Expect(err, "Failed to execute statement") + } + + return tx.Commit() +} diff --git a/mountain_pass/downloader_test.go b/mountain_pass/downloader_test.go new file mode 100644 index 0000000..f4ffcc8 --- /dev/null +++ b/mountain_pass/downloader_test.go @@ -0,0 +1,75 @@ +package mountainpass + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseMountainPasses(t *testing.T) { + csv := "Brevet\tcode\tnom\taltitude\n" + + "FR-01\tFR-01-1500\tCol du Colombier\t1498\n" + + "FR-06\tFR-06-0730c | FR-06-0706\tCol de Castillon | Col de Castillon (Tunnel)\t730 | 706\n" + + "FR-13\tFR-13-0135\tCol du Télégraphe \"Chappe\"\t149\n" + + "FR-90\tFR-88-1171\tCol du Ballon d'Alsace\t1171\n" + mountainPasses, err := parseMountainPasses(strings.NewReader(csv)) + require.NoError(t, err) + + expected := []MountainPass{ + {ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498}, + {ExternalID: "centcols/FR-06-0730c", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon", Elevation: 730}, + {ExternalID: "centcols/FR-06-0706", CountryCode: "FR", DepartmentCode: "06", Name: "Col de Castillon (Tunnel)", Elevation: 706}, + {ExternalID: "centcols/FR-13-0135", CountryCode: "FR", DepartmentCode: "13", Name: "Col du Télégraphe \"Chappe\"", Elevation: 149}, + {ExternalID: "centcols/FR-88-1171", CountryCode: "FR", DepartmentCode: "90", Name: "Col du Ballon d'Alsace", Elevation: 1171}, + } + assert.Equal(t, expected, mountainPasses) +} + +func TestParseMountainPassesHeaderOnly(t *testing.T) { + mountainPasses, err := parseMountainPasses(strings.NewReader("Brevet\tcode\tnom\taltitude\n")) + require.NoError(t, err) + assert.Empty(t, mountainPasses) +} + +func TestSplitCountryDepartment(t *testing.T) { + country, department, err := splitCountryDepartment("FR-01") + require.NoError(t, err) + assert.Equal(t, "FR", country) + assert.Equal(t, "01", department) + + _, _, err = splitCountryDepartment("F-01-2") + assert.Error(t, err) +} + +func TestLoadCachedDepartment(t *testing.T) { + cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv") + err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\tFR-01-1500\tCol du Colombier\t1498\n"), 0644) + require.NoError(t, err) + + mountainPasses, found, err := loadCachedDepartment(cacheFile) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, []MountainPass{{ExternalID: "centcols/FR-01-1500", CountryCode: "FR", DepartmentCode: "01", Name: "Col du Colombier", Elevation: 1498}}, mountainPasses) +} + +func TestLoadCachedDepartmentMissing(t *testing.T) { + mountainPasses, found, err := loadCachedDepartment(filepath.Join(t.TempDir(), "does-not-exist.csv")) + require.NoError(t, err) + assert.False(t, found) + assert.Nil(t, mountainPasses) +} + +func TestLoadCachedDepartmentCorrupt(t *testing.T) { + cacheFile := filepath.Join(t.TempDir(), "debug_department_01.csv") + err := os.WriteFile(cacheFile, []byte("Brevet\tcode\tnom\taltitude\nFR-01\n"), 0644) + require.NoError(t, err) + + mountainPasses, found, err := loadCachedDepartment(cacheFile) + require.NoError(t, err) + assert.False(t, found) + assert.Nil(t, mountainPasses) +} diff --git a/ride/index.go b/ride/index.go index 18e2ac8..79f9212 100644 --- a/ride/index.go +++ b/ride/index.go @@ -8,14 +8,14 @@ type RideClimbsIndex interface { } type FlatRideClimbsIndex struct { - sensibility float64 + sensitivity float64 climbs []Climb } -func NewFlatRideClimbsIndex(sensibility float64) *FlatRideClimbsIndex { +func NewFlatRideClimbsIndex(sensitivity float64) *FlatRideClimbsIndex { return &FlatRideClimbsIndex{ climbs: make([]Climb, 0), - sensibility: sensibility, + sensitivity: sensitivity, } } @@ -28,7 +28,7 @@ func (index *FlatRideClimbsIndex) Similar(ride Ride, climb Climb) []Climb { for _, c := range index.climbs { _, startDistance := geodist.HaversineDistance(c.Start().Coord, climb.Start().Coord) _, endDistance := geodist.HaversineDistance(c.End().Coord, climb.End().Coord) - if startDistance <= index.sensibility/1000 && endDistance <= index.sensibility/1000 { + if startDistance <= index.sensitivity/1000 && endDistance <= index.sensitivity/1000 { results = append(results, c) } } |