diff options
| -rw-r--r-- | strava/strava.go | 171 | ||||
| -rw-r--r-- | web/frontend/sync.ts | 22 | ||||
| -rw-r--r-- | web/frontend/types.d.ts | 16 | ||||
| -rw-r--r-- | web/sync.go | 122 | ||||
| -rw-r--r-- | web/views.go | 15 |
5 files changed, 331 insertions, 15 deletions
diff --git a/strava/strava.go b/strava/strava.go index 7c555be..309d0ee 100644 --- a/strava/strava.go +++ b/strava/strava.go @@ -6,6 +6,8 @@ import ( "io" "net/http" "net/url" + "strconv" + "strings" "time" "github.com/martinlehoux/kagamigo/kcore" @@ -93,11 +95,24 @@ func requestToken(form url.Values) (Token, error) { }, nil } +type RateLimit struct { + Limit15 int `json:"limit15"` + Usage15 int `json:"usage15"` + LimitDaily int `json:"limitDaily"` + UsageDaily int `json:"usageDaily"` +} + +type RateLimitState struct { + Read RateLimit `json:"read"` + Overall RateLimit `json:"overall"` +} + type Client struct { - ClientID string - ClientSecret string - HTTP *http.Client - tokens Token + ClientID string + ClientSecret string + HTTP *http.Client + tokens Token + lastRateLimit RateLimitState } func NewClient(clientID, clientSecret string, tokens Token) *Client { @@ -108,6 +123,10 @@ func (c *Client) Tokens() Token { return c.tokens } +func (c *Client) RateLimitState() RateLimitState { + return c.lastRateLimit +} + func (c *Client) RefreshIfNeeded() (bool, error) { if time.Until(c.tokens.ExpiresAt) > 5*time.Minute { return false, nil @@ -120,6 +139,30 @@ func (c *Client) RefreshIfNeeded() (bool, error) { return true, nil } +type RateLimitError struct { + Path string + StatusCode int + Body string + RetryAfter time.Time + IsDaily bool + State RateLimitState +} + +func (e *RateLimitError) Error() string { + if !e.RetryAfter.IsZero() { + until := time.Until(e.RetryAfter).Truncate(time.Second) + if until < 0 { + until = 0 + } + scope := "15-minute" + if e.IsDaily { + scope = "daily" + } + return fmt.Sprintf("strava API %s rate limited (HTTP 429): %s; %s limit resets at %s (in %s)", e.Path, e.Body, scope, e.RetryAfter.UTC().Format(time.RFC3339), until) + } + return fmt.Sprintf("strava API %s rate limited (HTTP %d): %s", e.Path, e.StatusCode, e.Body) +} + func (c *Client) GetJSON(path string, out any) error { if _, err := c.RefreshIfNeeded(); err != nil { return err @@ -138,6 +181,20 @@ func (c *Client) GetJSON(path string, out any) error { if err != nil { return kcore.Wrap(err, "failed to read Strava API response") } + state := parseRateLimitState(resp) + if !isZeroRateLimitState(state) { + c.lastRateLimit = state + } + if resp.StatusCode == http.StatusTooManyRequests { + return &RateLimitError{ + Path: path, + StatusCode: resp.StatusCode, + Body: string(body), + RetryAfter: rateLimitRetryAfter(resp), + IsDaily: isDailyRateLimited(resp), + State: state, + } + } if resp.StatusCode != http.StatusOK { return fmt.Errorf("strava API %s failed (HTTP %d): %s", path, resp.StatusCode, body) } @@ -146,3 +203,109 @@ func (c *Client) GetJSON(path string, out any) error { } return nil } + +func rateLimitRetryAfter(resp *http.Response) time.Time { + if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" { + if secs, err := strconv.Atoi(strings.TrimSpace(retryAfter)); err == nil { + return time.Now().UTC().Add(time.Duration(secs) * time.Second) + } + if t, err := http.ParseTime(retryAfter); err == nil { + return t.UTC() + } + } + responseTime := time.Now().UTC() + if date := resp.Header.Get("Date"); date != "" { + if t, err := http.ParseTime(date); err == nil { + responseTime = t.UTC() + } + } + if isDailyRateLimited(resp) { + return nextMidnightUTC(responseTime) + } + return nextQuarterHour(responseTime) +} + +func isDailyRateLimited(resp *http.Response) bool { + for _, pair := range [][2]string{ + {resp.Header.Get("X-ReadRateLimit-Limit"), resp.Header.Get("X-ReadRateLimit-Usage")}, + {resp.Header.Get("X-RateLimit-Limit"), resp.Header.Get("X-RateLimit-Usage")}, + } { + limitPair := pair[0] + usagePair := pair[1] + if limitPair == "" || usagePair == "" { + continue + } + _, dailyLimit, ok1 := parseRateLimitPair(limitPair) + _, dailyUsage, ok2 := parseRateLimitPair(usagePair) + if !ok1 || !ok2 { + continue + } + if dailyLimit > 0 && dailyUsage >= dailyLimit { + return true + } + } + return false +} + +func parseRateLimitPair(value string) (quarter int, daily int, ok bool) { + parts := strings.Split(value, ",") + if len(parts) != 2 { + return 0, 0, false + } + q, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) + d, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) + if err1 != nil || err2 != nil { + return 0, 0, false + } + return q, d, true +} + +func nextQuarterHour(t time.Time) time.Time { + t = t.UTC().Truncate(time.Second) + minute := t.Minute() + var nextMinute int + switch { + case minute < 15: + nextMinute = 15 + case minute < 30: + nextMinute = 30 + case minute < 45: + nextMinute = 45 + default: + t = t.Add(time.Hour) + nextMinute = 0 + } + return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), nextMinute, 0, 0, time.UTC) +} + +func nextMidnightUTC(t time.Time) time.Time { + t = t.UTC() + return time.Date(t.Year(), t.Month(), t.Day()+1, 0, 0, 0, 0, time.UTC) +} + +func parseRateLimitState(resp *http.Response) RateLimitState { + var state RateLimitState + if limit, usage, ok := parseRateLimitHeaders(resp.Header.Get("X-ReadRateLimit-Limit"), resp.Header.Get("X-ReadRateLimit-Usage")); ok { + state.Read = RateLimit{Limit15: limit[0], LimitDaily: limit[1], Usage15: usage[0], UsageDaily: usage[1]} + } + if limit, usage, ok := parseRateLimitHeaders(resp.Header.Get("X-RateLimit-Limit"), resp.Header.Get("X-RateLimit-Usage")); ok { + state.Overall = RateLimit{Limit15: limit[0], LimitDaily: limit[1], Usage15: usage[0], UsageDaily: usage[1]} + } + return state +} + +func parseRateLimitHeaders(limitValue, usageValue string) ([2]int, [2]int, bool) { + if limitValue == "" || usageValue == "" { + return [2]int{}, [2]int{}, false + } + l0, l1, ok1 := parseRateLimitPair(limitValue) + u0, u1, ok2 := parseRateLimitPair(usageValue) + if !ok1 || !ok2 { + return [2]int{}, [2]int{}, false + } + return [2]int{l0, l1}, [2]int{u0, u1}, true +} + +func isZeroRateLimitState(state RateLimitState) bool { + return state.Read.Limit15 == 0 && state.Read.LimitDaily == 0 && state.Overall.Limit15 == 0 && state.Overall.LimitDaily == 0 +} diff --git a/web/frontend/sync.ts b/web/frontend/sync.ts index 000dcfc..f58d224 100644 --- a/web/frontend/sync.ts +++ b/web/frontend/sync.ts @@ -1,4 +1,4 @@ -import type { SyncErrorEvent, SyncProgress } from "./types.js"; +import type { RateLimitSnapshot, SyncErrorEvent, SyncProgress } from "./types.js"; import { requireElement } from "./dom.js"; const isSyncProgress = (data: SyncProgress | SyncErrorEvent): data is SyncProgress => "total" in data; @@ -18,12 +18,28 @@ export const mountSync = (): void => { error.textContent = message; error.hidden = false; }; + const getRateLimit = (data: SyncProgress | SyncErrorEvent): RateLimitSnapshot | undefined => { + if ("rateLimit" in data && data.rateLimit) return data.rateLimit; + if ("progress" in data && data.progress?.rateLimit) return data.progress.rateLimit; + return undefined; + }; + + const formatRateLimit = (data: SyncProgress | SyncErrorEvent): string => { + const rl = getRateLimit(data); + if (!rl) return ""; + const parts: string[] = []; + if (rl.read?.limitDaily) parts.push(`Read ${rl.read.usageDaily}/${rl.read.limitDaily} daily`); + if (rl.read?.limit15) parts.push(`${rl.read.usage15}/${rl.read.limit15} (15-min)`); + if (parts.length === 0 && rl.overall?.limitDaily) parts.push(`Overall ${rl.overall.usageDaily}/${rl.overall.limitDaily} daily`); + return parts.length ? ` · ${parts.join(" · ")}` : ""; + }; + const updateProgress = (progress: SyncProgress): void => { progressPanel.hidden = false; progressBar.max = Math.max(progress.total, 1); progressBar.value = progress.completed; progressCount.textContent = `${progress.completed} / ${progress.total}`; - progressDetails.textContent = `${progress.imported} imported · ${progress.skipped} skipped`; + progressDetails.textContent = `${progress.imported} imported · ${progress.skipped} skipped${formatRateLimit(progress)}`; }; const handleEvent = (eventName: string, data: SyncProgress | SyncErrorEvent): boolean => { if (eventName === "progress") { @@ -34,7 +50,7 @@ export const mountSync = (): void => { if (eventName === "complete") { if (!isSyncProgress(data)) throw new Error("Invalid Strava sync completion event"); updateProgress(data); - notice.textContent = `Sync complete: ${data.imported} imported, ${data.skipped} skipped.`; + notice.textContent = `Sync complete: ${data.imported} imported, ${data.skipped} skipped.${formatRateLimit(data)}`; notice.hidden = false; return true; } diff --git a/web/frontend/types.d.ts b/web/frontend/types.d.ts index 3c2a918..6ff3dcf 100644 --- a/web/frontend/types.d.ts +++ b/web/frontend/types.d.ts @@ -124,14 +124,30 @@ export interface AnalyticsPoint { movingHours: number; } +export interface RateLimit { + limit15: number; + usage15: number; + limitDaily: number; + usageDaily: number; +} + +export interface RateLimitSnapshot { + read: RateLimit; + overall: RateLimit; +} + export interface SyncProgress { total: number; completed: number; imported: number; skipped: number; + rateLimit?: RateLimitSnapshot; } export interface SyncErrorEvent { message: string; progress?: SyncProgress; + retryAfter?: string; + isRateLimited?: boolean; + rateLimit?: RateLimitSnapshot; } diff --git a/web/sync.go b/web/sync.go index b491ee0..fdb3c4e 100644 --- a/web/sync.go +++ b/web/sync.go @@ -79,19 +79,39 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Content-Type", "text/event-stream") progress, err := s.syncRides(client, appConfig.Storage.GPXDir, from, to, func(progress SyncProgress) error { + enrichProgressRateLimit(&progress, client) return writeSyncEvent(w, flusher, "progress", progress) }) if err != nil { - _ = writeSyncEvent(w, flusher, "error", syncErrorEvent{Message: err.Error(), Progress: progress}) + var rateErr *strava.RateLimitError + if errors.As(err, &rateErr) { + slog.Warn("Strava sync rate limited", "total", progress.Total, "imported", progress.Imported, "retryAfter", rateErr.RetryAfter, "isDaily", rateErr.IsDaily, "readDaily", fmt.Sprintf("%d/%d", rateErr.State.Read.UsageDaily, rateErr.State.Read.LimitDaily), "overallDaily", fmt.Sprintf("%d/%d", rateErr.State.Overall.UsageDaily, rateErr.State.Overall.LimitDaily)) + enrichProgressWithState(&progress, rateErr.State) + _ = writeSyncEvent(w, flusher, "error", syncErrorEvent{ + Message: rateLimitMessage(rateErr, progress), + Progress: progress, + RetryAfter: formatRetryAfter(rateErr.RetryAfter), + IsRateLimited: true, + RateLimit: progress.RateLimit, + }) + return + } + enrichProgressRateLimit(&progress, client) + _ = writeSyncEvent(w, flusher, "error", syncErrorEvent{Message: err.Error(), Progress: progress, RateLimit: progress.RateLimit}) return } - slog.Info("Completed Strava sync", "from", from.Format(dateFormat), "to", to.Add(-24*time.Hour).Format(dateFormat), "total", progress.Total, "imported", progress.Imported, "skipped", progress.Skipped) + enrichProgressRateLimit(&progress, client) + slog.Info("Completed Strava sync", "from", from.Format(dateFormat), "to", to.Add(-24*time.Hour).Format(dateFormat), "total", progress.Total, "imported", progress.Imported, "skipped", progress.Skipped, "readDaily", rateLimitLogValue(progress.RateLimit, true), "overallDaily", rateLimitLogValue(progress.RateLimit, false)) _ = writeSyncEvent(w, flusher, "complete", progress) } func (s *Server) syncRides(client syncClient, gpxDir string, from, to time.Time, report func(SyncProgress) error) (SyncProgress, error) { activities, err := client.List(from, to) if err != nil { + var rateErr *strava.RateLimitError + if errors.As(err, &rateErr) { + slog.Warn("Strava sync rate limited while listing activities", "retryAfter", rateErr.RetryAfter, "isDaily", rateErr.IsDaily, "error", err) + } return SyncProgress{}, err } progress := SyncProgress{Total: len(activities)} @@ -118,6 +138,11 @@ func (s *Server) syncRides(client syncClient, gpxDir string, from, to time.Time, } activity, gpxData, err := client.Get(summary.ID) if err != nil { + var rateErr *strava.RateLimitError + if errors.As(err, &rateErr) { + slog.Warn("Strava sync rate limited", "activity", summary.ID, "name", summary.Name, "retryAfter", rateErr.RetryAfter, "isDaily", rateErr.IsDaily, "completed", progress.Completed, "total", progress.Total) + return progress, err + } if errors.Is(err, strava.ErrNoTrackPoints) { _ = os.Remove(filepath.Join(gpxDir, fmt.Sprintf("activity_%d.gpx", summary.ID))) if saveErr := ride.Save(s.db, rideFromActivity(externalID, "", activity)); saveErr != nil { @@ -214,8 +239,97 @@ func (s *Server) endSync() { } type syncErrorEvent struct { - Message string `json:"message"` - Progress SyncProgress `json:"progress"` + Message string `json:"message"` + Progress SyncProgress `json:"progress"` + RetryAfter string `json:"retryAfter,omitempty"` + IsRateLimited bool `json:"isRateLimited,omitempty"` + RateLimit *RateLimitSnapshot `json:"rateLimit,omitempty"` +} + +func enrichProgressRateLimit(progress *SyncProgress, client syncClient) { + if provider, ok := client.(interface{ RateLimitState() strava.RateLimitState }); ok { + state := provider.RateLimitState() + if isZeroRateLimit(state) { + return + } + enrichProgressWithState(progress, state) + } +} + +func enrichProgressWithState(progress *SyncProgress, state strava.RateLimitState) { + progress.RateLimit = &RateLimitSnapshot{Read: state.Read, Overall: state.Overall} +} + +func isZeroRateLimit(state strava.RateLimitState) bool { + return state.Read.Limit15 == 0 && state.Read.LimitDaily == 0 && state.Overall.Limit15 == 0 && state.Overall.LimitDaily == 0 +} + +func rateLimitLogValue(snapshot *RateLimitSnapshot, read bool) string { + if snapshot == nil { + return "" + } + if read { + if snapshot.Read.LimitDaily == 0 { + return "" + } + return fmt.Sprintf("%d/%d", snapshot.Read.UsageDaily, snapshot.Read.LimitDaily) + } + if snapshot.Overall.LimitDaily == 0 { + return "" + } + return fmt.Sprintf("%d/%d", snapshot.Overall.UsageDaily, snapshot.Overall.LimitDaily) +} + +func rateLimitMessage(rateErr *strava.RateLimitError, progress SyncProgress) string { + scope := "15-minute" + if rateErr.IsDaily { + scope = "daily" + } + retryText := "" + if !rateErr.RetryAfter.IsZero() { + until := time.Until(rateErr.RetryAfter).Truncate(time.Second) + if until < 0 { + until = 0 + } + retryText = fmt.Sprintf(" Retry after %s (in %s).", rateErr.RetryAfter.UTC().Format(time.RFC3339), until) + if rateErr.IsDaily { + retryText = fmt.Sprintf(" Daily limit resets at %s (in %s).", rateErr.RetryAfter.UTC().Format(time.RFC3339), until) + } + } + usageText := formatRateLimitUsage(rateErr.State) + if usageText != "" { + usageText = " " + usageText + } + progressText := "" + if progress.Total > 0 { + remaining := progress.Total - progress.Completed + progressText = fmt.Sprintf(" Progress: %d/%d completed, %d imported, %d skipped, %d remaining.", progress.Completed, progress.Total, progress.Imported, progress.Skipped, remaining) + } + return fmt.Sprintf("Strava %s rate limit reached.%s%s%s Re-run sync after the reset time to import remaining rides.", scope, progressText, usageText, retryText) +} + +func formatRateLimitUsage(state strava.RateLimitState) string { + var parts []string + if state.Read.LimitDaily > 0 { + parts = append(parts, fmt.Sprintf("Read daily %d/%d", state.Read.UsageDaily, state.Read.LimitDaily)) + } + if state.Read.Limit15 > 0 { + parts = append(parts, fmt.Sprintf("Read 15-min %d/%d", state.Read.Usage15, state.Read.Limit15)) + } + if state.Overall.LimitDaily > 0 && (state.Overall.LimitDaily != state.Read.LimitDaily || state.Overall.UsageDaily != state.Read.UsageDaily) { + parts = append(parts, fmt.Sprintf("Overall daily %d/%d", state.Overall.UsageDaily, state.Overall.LimitDaily)) + } + if len(parts) == 0 { + return "" + } + return "Usage: " + strings.Join(parts, ", ") + "." +} + +func formatRetryAfter(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) } func writeSyncEvent(w http.ResponseWriter, flusher http.Flusher, event string, data any) error { diff --git a/web/views.go b/web/views.go index 6fd5adf..cfb31e5 100644 --- a/web/views.go +++ b/web/views.go @@ -9,6 +9,7 @@ import ( "github.com/martinlehoux/biking_home/official_climb" "github.com/martinlehoux/biking_home/ride" "github.com/martinlehoux/biking_home/rideanalysis" + "github.com/martinlehoux/biking_home/strava" ) type RideView struct { @@ -376,11 +377,17 @@ type SyncPageData struct { HasAuth bool } +type RateLimitSnapshot struct { + Read strava.RateLimit `json:"read"` + Overall strava.RateLimit `json:"overall"` +} + type SyncProgress struct { - Total int `json:"total"` - Completed int `json:"completed"` - Imported int `json:"imported"` - Skipped int `json:"skipped"` + Total int `json:"total"` + Completed int `json:"completed"` + Imported int `json:"imported"` + Skipped int `json:"skipped"` + RateLimit *RateLimitSnapshot `json:"rateLimit,omitempty"` } func formatRideDate(value time.Time) string { |