package mountain_pass import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "log/slog" "net/http" "os" "strings" "github.com/schollz/progressbar/v3" ) const francePBFRUL = "https://download.geofabrik.de/europe/france-latest.osm.pbf" // FetchFrancePBF downloads the full France OSM PBF to destPath, resuming a // partial download when the file already exists, skipping the download // entirely when it is already complete, and verifying the result against // geofabrik's published MD5 checksum. func FetchFrancePBF(destPath string) error { return fetchFrancePBF(&http.Client{}, francePBFRUL, destPath) } // fetchFrancePBF is the testable core of FetchFrancePBF. baseURL is the // server URL whose .md5 sibling carries the published checksum. func fetchFrancePBF(client *http.Client, baseURL, destPath string) error { checksumURL := baseURL + ".md5" head, err := client.Head(baseURL) if err != nil { return fmt.Errorf("failed to check remote size: %w", err) } head.Body.Close() total := head.ContentLength if total <= 0 { return fmt.Errorf("unexpected remote content length: %d", total) } info, statErr := os.Stat(destPath) var offset int64 if statErr == nil { offset = info.Size() if offset >= total { slog.Info("Already downloaded", "file", destPath, "bytes", offset) return verifyPBF(client, checksumURL, destPath) } if offset > 0 { slog.Info("Resuming download", "file", destPath, "bytes", offset, "remaining", total-offset) } } else if !errors.Is(statErr, os.ErrNotExist) { return fmt.Errorf("failed to stat %s: %w", destPath, statErr) } req, err := http.NewRequest(http.MethodGet, baseURL, nil) if err != nil { return err } req.Header.Set("User-Agent", "biking_home") if offset > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } res, err := client.Do(req) if err != nil { return err } defer res.Body.Close() resuming := offset > 0 && res.StatusCode == http.StatusPartialContent if !resuming { if res.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", res.StatusCode) } if offset > 0 { slog.Warn("Server ignored Range header, restarting download", "file", destPath) } offset = 0 } mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC if resuming { mode = os.O_CREATE | os.O_WRONLY } file, err := os.OpenFile(destPath, mode, 0o644) if err != nil { return err } defer file.Close() bar := progressbar.DefaultBytes(res.ContentLength, "Downloading france-latest.osm.pbf") if resuming { // Re-append to the partial file when the server honored the range request. if _, err := file.Seek(offset, io.SeekStart); err != nil { return err } } _, err = io.Copy(io.MultiWriter(file, bar), res.Body) if err != nil { return fmt.Errorf("failed to download %s: %w", baseURL, err) } if size, err := file.Stat(); err == nil && size.Size() != total { return fmt.Errorf("download incomplete: got %d bytes, want %d", size.Size(), total) } return verifyPBF(client, checksumURL, destPath) } // verifyPBF verifies destPath against the published checksum served at // checksumURL. If the checksum endpoint is unreachable it logs a warning and // proceeds unverified rather than blocking the download; on mismatch it // returns an error without deleting the corrupt file so the user can inspect // it and remove it manually before re-fetching. func verifyPBF(client *http.Client, checksumURL, destPath string) error { want, err := expectedPBFMD5(client, checksumURL) if err != nil { slog.Warn("Could not fetch expected checksum, skipping verification", "url", checksumURL, "err", err) return nil } got, err := pbfMD5(destPath) if err != nil { return fmt.Errorf("failed to compute MD5 of %s: %w", destPath, err) } if !strings.EqualFold(got, want) { return fmt.Errorf("checksum mismatch on %s: local md5 %s != expected %s; keeping the corrupt file for inspection, remove it manually before re-fetching", destPath, got, want) } slog.Info("Checksum verified", "file", destPath, "md5", got) return nil } // expectedPBFMD5 fetches checksumURL and returns its first whitespace-separated // token. Geofabrik's .md5 files follow the " " format. func expectedPBFMD5(client *http.Client, checksumURL string) (string, error) { res, err := client.Get(checksumURL) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected status code: %d", res.StatusCode) } body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) if err != nil { return "", err } fields := strings.Fields(string(body)) if len(fields) == 0 { return "", errors.New("empty checksum file") } return fields[0], nil } // pbfMD5 returns the lowercase hex MD5 digest of the file at path. func pbfMD5(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }