1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
package mountain_pass
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"strings"
"github.com/schollz/progressbar/v3"
)
const francePBFRUL = "https://download.geofabrik.de/europe/france-latest.osm.pbf"
// FetchFrancePBF downloads the full France OSM PBF to destPath, resuming a
// partial download when the file already exists, skipping the download
// entirely when it is already complete, and verifying the result against
// geofabrik's published MD5 checksum.
func FetchFrancePBF(destPath string) error {
return fetchFrancePBF(&http.Client{}, francePBFRUL, destPath)
}
// fetchFrancePBF is the testable core of FetchFrancePBF. baseURL is the
// server URL whose .md5 sibling carries the published checksum.
func fetchFrancePBF(client *http.Client, baseURL, destPath string) error {
checksumURL := baseURL + ".md5"
head, err := client.Head(baseURL)
if err != nil {
return fmt.Errorf("failed to check remote size: %w", err)
}
head.Body.Close()
total := head.ContentLength
if total <= 0 {
return fmt.Errorf("unexpected remote content length: %d", total)
}
info, statErr := os.Stat(destPath)
var offset int64
if statErr == nil {
offset = info.Size()
if offset >= total {
slog.Info("Already downloaded", "file", destPath, "bytes", offset)
return verifyPBF(client, checksumURL, destPath)
}
if offset > 0 {
slog.Info("Resuming download", "file", destPath, "bytes", offset, "remaining", total-offset)
}
} else if !errors.Is(statErr, os.ErrNotExist) {
return fmt.Errorf("failed to stat %s: %w", destPath, statErr)
}
req, err := http.NewRequest(http.MethodGet, baseURL, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "biking_home")
if offset > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
res, err := client.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
resuming := offset > 0 && res.StatusCode == http.StatusPartialContent
if !resuming {
if res.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
if offset > 0 {
slog.Warn("Server ignored Range header, restarting download", "file", destPath)
}
offset = 0
}
mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
if resuming {
mode = os.O_CREATE | os.O_WRONLY
}
file, err := os.OpenFile(destPath, mode, 0o644)
if err != nil {
return err
}
defer file.Close()
bar := progressbar.DefaultBytes(res.ContentLength, "Downloading france-latest.osm.pbf")
if resuming {
// Re-append to the partial file when the server honored the range request.
if _, err := file.Seek(offset, io.SeekStart); err != nil {
return err
}
}
_, err = io.Copy(io.MultiWriter(file, bar), res.Body)
if err != nil {
return fmt.Errorf("failed to download %s: %w", baseURL, err)
}
if size, err := file.Stat(); err == nil && size.Size() != total {
return fmt.Errorf("download incomplete: got %d bytes, want %d", size.Size(), total)
}
return verifyPBF(client, checksumURL, destPath)
}
// verifyPBF verifies destPath against the published checksum served at
// checksumURL. If the checksum endpoint is unreachable it logs a warning and
// proceeds unverified rather than blocking the download; on mismatch it
// returns an error without deleting the corrupt file so the user can inspect
// it and remove it manually before re-fetching.
func verifyPBF(client *http.Client, checksumURL, destPath string) error {
want, err := expectedPBFMD5(client, checksumURL)
if err != nil {
slog.Warn("Could not fetch expected checksum, skipping verification", "url", checksumURL, "err", err)
return nil
}
got, err := pbfMD5(destPath)
if err != nil {
return fmt.Errorf("failed to compute MD5 of %s: %w", destPath, err)
}
if !strings.EqualFold(got, want) {
return fmt.Errorf("checksum mismatch on %s: local md5 %s != expected %s; keeping the corrupt file for inspection, remove it manually before re-fetching",
destPath, got, want)
}
slog.Info("Checksum verified", "file", destPath, "md5", got)
return nil
}
// expectedPBFMD5 fetches checksumURL and returns its first whitespace-separated
// token. Geofabrik's .md5 files follow the "<hex> <filename>" format.
func expectedPBFMD5(client *http.Client, checksumURL string) (string, error) {
res, err := client.Get(checksumURL)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
if err != nil {
return "", err
}
fields := strings.Fields(string(body))
if len(fields) == 0 {
return "", errors.New("empty checksum file")
}
return fields[0], nil
}
// pbfMD5 returns the lowercase hex MD5 digest of the file at path.
func pbfMD5(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := md5.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
|
