diff options
| -rw-r--r-- | .beads/issues.jsonl | 2 | ||||
| -rw-r--r-- | gallery.go | 263 | ||||
| -rw-r--r-- | gallery_test.go | 203 |
3 files changed, 345 insertions, 123 deletions
diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 8bbcec2..cc47397 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1 +1 @@ -{"id":"kagapics-77z","title":"Reduce the amount of data downloaded in a gallery","status":"open","priority":2,"issue_type":"task","owner":"martin@lehoux.net","created_at":"2026-02-13T09:00:44.379212837+01:00","created_by":"Martin Kagamino Lehoux","updated_at":"2026-02-13T09:00:44.379212837+01:00"} +{"id":"kagapics-77z","title":"Reduce the amount of data downloaded in a gallery","status":"closed","priority":2,"issue_type":"task","owner":"martin@lehoux.net","created_at":"2026-02-13T09:00:44.379212837+01:00","created_by":"Martin Kagamino Lehoux","updated_at":"2026-03-03T08:34:01.038069169+01:00","closed_at":"2026-03-03T08:34:01.038069169+01:00","close_reason":"Closed"} @@ -22,7 +22,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" "golang.org/x/crypto/bcrypt" @@ -34,9 +33,12 @@ import ( // --------------------------------------------------------------------------- const ( - thumbMaxWidth = 1600 - thumbMaxHeight = 1200 - thumbQuality = 75 + thumbMaxWidth = 400 + thumbMaxHeight = 400 + thumbQuality = 65 + thumbDirName = ".thumbs-v2" + oldThumbDirName = ".thumbs" + thumbScanInterval = 10 * time.Second maxThumbConcurrency = 4 sessionTTL = 24 * time.Hour dataFile = "galleries.json" @@ -166,10 +168,9 @@ type Server struct { // overwhelming the server (CPU/memory). thumbSemaphore chan struct{} - // peakThumbConcurrency tracks the highest number of concurrent thumbnail - // generations observed — used by tests to verify the semaphore works. - peakThumbConcurrency atomic.Int32 - currentThumbConcurrency atomic.Int32 + // thumbTrigger receives gallery names to trigger an immediate scan + // for that gallery (e.g. after an upload). Non-blocking send. + thumbTrigger chan string } // NewServer creates a fully initialised Server ready to use. @@ -181,6 +182,7 @@ func NewServer(dataDir string, maxUploadSize int64) *Server { maxUploadSize: maxUploadSize, rateLimiter: newLoginRateLimiter(), thumbSemaphore: make(chan struct{}, maxThumbConcurrency), + thumbTrigger: make(chan string, 16), } } @@ -237,19 +239,25 @@ func (s *Server) load() error { // --------------------------------------------------------------------------- func thumbDir(galleryDir string) string { - return filepath.Join(galleryDir, ".thumbs") + return filepath.Join(galleryDir, thumbDirName) } -func generateThumbGo(srcPath, dstPath string) error { +// thumbResult holds metadata about a generated thumbnail for logging. +type thumbResult struct { + srcWidth int + srcHeight int +} + +func generateThumbGo(srcPath, dstPath string) (thumbResult, error) { f, err := os.Open(srcPath) if err != nil { - return err + return thumbResult{}, err } defer f.Close() src, _, err := image.Decode(f) if err != nil { - return err + return thumbResult{}, err } bounds := src.Bounds() @@ -276,11 +284,14 @@ func generateThumbGo(srcPath, dstPath string) error { out, err := os.Create(dstPath) if err != nil { - return err + return thumbResult{}, err } defer out.Close() - return jpeg.Encode(out, dst, &jpeg.Options{Quality: thumbQuality}) + if err := jpeg.Encode(out, dst, &jpeg.Options{Quality: thumbQuality}); err != nil { + return thumbResult{}, err + } + return thumbResult{srcWidth: w, srcHeight: h}, nil } // findMagickBinary returns the ImageMagick convert command name. @@ -318,56 +329,194 @@ func generateThumbImageMagick(srcPath, dstPath string) error { return nil } -func generateThumb(srcPath, dstPath string) error { - err := generateThumbGo(srcPath, dstPath) +func generateThumb(srcPath, dstPath string) (thumbResult, error) { + res, err := generateThumbGo(srcPath, dstPath) if err == nil { - return nil + return res, nil } log.Printf("Go thumbnail failed for %s: %v — trying ImageMagick", srcPath, err) if err2 := generateThumbImageMagick(srcPath, dstPath); err2 != nil { - return fmt.Errorf("all thumbnail methods failed: go=%v, imagemagick=%v", err, err2) + return thumbResult{}, fmt.Errorf("all thumbnail methods failed: go=%v, imagemagick=%v", err, err2) } - return nil + // ImageMagick succeeded — dimensions unknown (Go couldn't decode the source). + return thumbResult{}, nil } -func (s *Server) ensureThumb(galleryDir, filename string) (string, error) { - td := thumbDir(galleryDir) - os.MkdirAll(td, 0755) - +// thumbPath returns the expected thumbnail path for a photo, or "" if it +// doesn't exist yet. +func thumbPath(galleryDir, filename string) string { thumbName := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".jpg" - thumbPath := filepath.Join(td, thumbName) + p := filepath.Join(thumbDir(galleryDir), thumbName) + if _, err := os.Stat(p); err == nil { + return p + } + return "" +} - if _, err := os.Stat(thumbPath); err == nil { - return thumbPath, nil // already exists +// --------------------------------------------------------------------------- +// Background thumbnail worker +// --------------------------------------------------------------------------- + +// triggerThumbScan sends a gallery name on the trigger channel to request +// an immediate scan. Non-blocking: if the channel is full, the periodic +// scan will pick it up. +func (s *Server) triggerThumbScan(gallery string) { + select { + case s.thumbTrigger <- gallery: + default: } +} - // Acquire semaphore to limit concurrent thumbnail generation. - s.thumbSemaphore <- struct{}{} - cur := s.currentThumbConcurrency.Add(1) - // Track peak for observability / testing. - for { - peak := s.peakThumbConcurrency.Load() - if cur <= peak || s.peakThumbConcurrency.CompareAndSwap(peak, cur) { - break +// startThumbWorker launches a background goroutine that periodically scans +// all galleries and generates missing thumbnails. +func (s *Server) startThumbWorker() { + go func() { + ticker := time.NewTicker(thumbScanInterval) + defer ticker.Stop() + + for { + select { + case gallery := <-s.thumbTrigger: + // Triggered scan for a specific gallery (e.g. after upload). + s.mu.RLock() + g, ok := s.galleries[gallery] + s.mu.RUnlock() + if ok && g.thumbsEnabled() { + s.scanGalleryThumbs(gallery) + } + case <-ticker.C: + // Periodic full scan. + s.scanAllThumbs() + } } + }() +} + +// scanAllThumbs scans every gallery and generates missing thumbnails. +func (s *Server) scanAllThumbs() { + s.mu.RLock() + names := make([]string, 0, len(s.galleries)) + for name, g := range s.galleries { + if g.thumbsEnabled() { + names = append(names, name) + } + } + s.mu.RUnlock() + + for _, name := range names { + s.scanGalleryThumbs(name) + } +} + +// formatBytes formats a byte count as a human-readable string. +func formatBytes(b int64) string { + switch { + case b >= 1<<20: + return fmt.Sprintf("%.1fMB", float64(b)/float64(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.0fKB", float64(b)/float64(1<<10)) + default: + return fmt.Sprintf("%dB", b) } - defer func() { - s.currentThumbConcurrency.Add(-1) +} + +// scanGalleryThumbs generates missing thumbnails for a single gallery. +func (s *Server) scanGalleryThumbs(name string) { + dir := s.galleryDir(name) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + + td := thumbDir(dir) + generated := 0 + galleryStart := time.Now() + + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + if !isAllowedExt(e.Name()) { + continue + } + + thumbName := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) + ".jpg" + tp := filepath.Join(td, thumbName) + + if _, err := os.Stat(tp); err == nil { + continue // already exists + } + + // Create thumbs dir if needed. + os.MkdirAll(td, 0755) + + srcPath := filepath.Join(dir, e.Name()) + + // Get source file size. + srcInfo, err := os.Stat(srcPath) + if err != nil { + continue + } + srcSize := srcInfo.Size() + + // Acquire semaphore to limit CPU usage. + s.thumbSemaphore <- struct{}{} + start := time.Now() + res, err := generateThumb(srcPath, tp) + elapsed := time.Since(start) <-s.thumbSemaphore - }() - // Double-check after acquiring the semaphore — another goroutine may have - // generated the same thumbnail while we were waiting. - if _, err := os.Stat(thumbPath); err == nil { - return thumbPath, nil + if err != nil { + log.Printf("[thumbs] failed %s/%s: %v", name, e.Name(), err) + continue + } + + // Get thumb file size. + var thumbSize int64 + if ti, err := os.Stat(tp); err == nil { + thumbSize = ti.Size() + } + + generated++ + if res.srcWidth > 0 { + log.Printf("[thumbs] generated %s/%s (%dx%d, %s → %s, %s)", + name, e.Name(), res.srcWidth, res.srcHeight, + formatBytes(srcSize), formatBytes(thumbSize), + elapsed.Round(time.Millisecond)) + } else { + log.Printf("[thumbs] generated %s/%s (%s → %s, %s)", + name, e.Name(), + formatBytes(srcSize), formatBytes(thumbSize), + elapsed.Round(time.Millisecond)) + } + } + + if generated > 0 { + log.Printf("[thumbs] gallery %q: %d new thumbnails in %s", + name, generated, time.Since(galleryStart).Round(time.Millisecond)) + } +} + +// cleanupOldThumbs removes legacy .thumbs directories from all galleries. +func (s *Server) cleanupOldThumbs() { + s.mu.RLock() + names := make([]string, 0, len(s.galleries)) + for name := range s.galleries { + names = append(names, name) } + s.mu.RUnlock() - srcPath := filepath.Join(galleryDir, filename) - if err := generateThumb(srcPath, thumbPath); err != nil { - return "", err + for _, name := range names { + oldDir := filepath.Join(s.galleryDir(name), oldThumbDirName) + if _, err := os.Stat(oldDir); err == nil { + if err := os.RemoveAll(oldDir); err != nil { + log.Printf("[thumbs] failed to remove old %s: %v", oldDir, err) + } else { + log.Printf("[thumbs] removed legacy %s", oldDir) + } + } } - return thumbPath, nil } // --------------------------------------------------------------------------- @@ -658,6 +807,10 @@ func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request, name strin uploaded++ } + if uploaded > 0 { + s.triggerThumbScan(name) + } + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]int{"uploaded": uploaded}) } @@ -677,14 +830,20 @@ func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request, name, filen filename = filepath.Base(filename) // sanitize dir := s.galleryDir(name) - thumbPath, err := s.ensureThumb(dir, filename) - if err != nil { - log.Printf("Thumbnail generation failed for %s/%s: %v", name, filename, err) - http.Error(w, "thumbnail unavailable", http.StatusInternalServerError) + // Serve pre-generated thumbnail if available, otherwise fallback to source. + if tp := thumbPath(dir, filename); tp != "" { + w.Header().Set("Cache-Control", "public, max-age=86400") + http.ServeFile(w, r, tp) + return + } + + // Thumbnail not ready yet — serve the original as fallback. + srcPath := filepath.Join(dir, filename) + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + http.NotFound(w, r) return } - w.Header().Set("Cache-Control", "public, max-age=86400") - http.ServeFile(w, r, thumbPath) + http.ServeFile(w, r, srcPath) } func (s *Server) handlePhoto(w http.ResponseWriter, r *http.Request, name, filename string) { @@ -876,6 +1035,8 @@ func main() { srv.startSessionReaper() srv.rateLimiter.startReaper() + srv.cleanupOldThumbs() + srv.startThumbWorker() log.Printf("Listening on %s (data: %s, galleries: %d)", flags["addr"], flags["data"], len(srv.galleries)) if err := http.ListenAndServe(flags["addr"], srv.Handler()); err != nil { diff --git a/gallery_test.go b/gallery_test.go index 125a19a..29539fa 100644 --- a/gallery_test.go +++ b/gallery_test.go @@ -15,8 +15,6 @@ import ( "net/url" "os" "path/filepath" - "sync" - "sync/atomic" "testing" "time" @@ -238,6 +236,92 @@ func TestThumbServing(t *testing.T) { assert.NoError(t, err, "thumbnail should be a decodable JPEG") } +func TestThumbServesSourceWhenThumbNotReady(t *testing.T) { + // When the background worker hasn't generated a thumb yet, + // handleThumb should serve the source image as fallback. + ts, srv := setupTestServer(t) + cookies := login(t, ts, "test-gallery", "secret123") + + // Write a photo directly — don't start the thumb worker, so no thumb exists. + dir := srv.galleryDir("test-gallery") + original := makeTestJPEG() + require.NoError(t, os.WriteFile(filepath.Join(dir, "nothumb.jpg"), original, 0644)) + + resp := authedGet(t, ts.URL+"/g/test-gallery/thumb/nothumb.jpg", cookies) + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, err) + + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, original, body, + "when thumbnail is not ready, should serve the source image as fallback") +} + +func TestThumbWorkerGeneratesThumbnails(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + + // Write a photo directly. + galleryDir := srv.galleryDir("test-gallery") + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, "worker-test.jpg"), makeTestJPEG(), 0644)) + + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") + + // Wait for the thumbnail to appear (up to 5s). + thumbPath := filepath.Join(galleryDir, ".thumbs-v2", "worker-test.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, "thumbnail should be generated by the background worker") + + // Verify it's a valid JPEG. + data, err := os.ReadFile(thumbPath) + require.NoError(t, err) + _, err = jpeg.Decode(bytes.NewReader(data)) + assert.NoError(t, err, "generated thumbnail should be a valid JPEG") +} + +func TestUploadTriggersThumbWorker(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + srv.startThumbWorker() + + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + + cookies := login(t, ts, "test-gallery", "secret123") + resp := uploadPhoto(t, ts, "test-gallery", cookies, "upload-trigger.jpg", makeTestJPEG()) + resp.Body.Close() + + // The upload should have triggered an immediate scan. Wait for the thumb. + thumbPath := filepath.Join(srv.galleryDir("test-gallery"), ".thumbs-v2", "upload-trigger.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, + "upload should trigger background thumbnail generation") +} + +func TestOldThumbsDirCleanedUp(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + + // Create the old .thumbs directory with some content. + oldThumbsDir := filepath.Join(srv.galleryDir("test-gallery"), ".thumbs") + require.NoError(t, os.MkdirAll(oldThumbsDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(oldThumbsDir, "old.jpg"), []byte("old"), 0644)) + + srv.cleanupOldThumbs() + + _, err := os.Stat(oldThumbsDir) + assert.True(t, os.IsNotExist(err), "old .thumbs directory should be removed") +} + func TestPhotoServing(t *testing.T) { ts, _, cookies := setupWithPhoto(t, "original.jpg") @@ -382,33 +466,38 @@ func makeUnsupportedJPEG() []byte { } func TestThumbFallbackToImageMagick(t *testing.T) { - ts, srv := setupTestServer(t) - cookies := login(t, ts, "test-gallery", "secret123") + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) // Create a JPEG that Go's stdlib can't decode but ImageMagick can. unsupportedJPEG := makeUnsupportedJPEG() - // Write directly to the gallery dir (bypass upload, test thumbnail generation). - dir := srv.galleryDir("test-gallery") - require.NoError(t, os.WriteFile(filepath.Join(dir, "tricky.jpg"), unsupportedJPEG, 0644)) + // Write directly to the gallery dir. + galleryDir := srv.galleryDir("test-gallery") + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, "tricky.jpg"), unsupportedJPEG, 0644)) - // Request the thumbnail — should use ImageMagick fallback and return a valid JPEG. - resp := authedGet(t, ts.URL+"/g/test-gallery/thumb/tricky.jpg", cookies) - defer resp.Body.Close() + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") - assert.Equal(t, 200, resp.StatusCode) - assert.Contains(t, resp.Header.Get("Content-Type"), "image/jpeg") + // Wait for the thumbnail to appear. + thumbPath := filepath.Join(galleryDir, ".thumbs-v2", "tricky.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, "thumbnail should be generated by the background worker") // The thumbnail should be a valid, fully decodable JPEG. - thumbData, err := io.ReadAll(resp.Body) + thumbData, err := os.ReadFile(thumbPath) require.NoError(t, err) _, err = jpeg.Decode(bytes.NewReader(thumbData)) assert.NoError(t, err, "thumbnail should be a fully decodable JPEG even when Go's stdlib can't decode the original") - // The thumbnail must not be the raw original served as fallback. + // The thumbnail must not be the raw original. assert.NotEqual(t, unsupportedJPEG, thumbData, - "thumbnail must not be the broken original served as fallback") + "thumbnail must not be the broken original") } // makeLargeTestJPEG creates a larger JPEG that takes more time to thumbnail. @@ -424,70 +513,42 @@ func makeLargeTestJPEG() []byte { return buf.Bytes() } -func TestConcurrentThumbnailGeneration(t *testing.T) { - ts, srv := setupTestServer(t) - cookies := login(t, ts, "test-gallery", "secret123") +func TestWorkerGeneratesAllThumbnails(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) - // Upload 40 photos directly to the gallery dir (bypass upload handler for speed). - dir := srv.galleryDir("test-gallery") - largeJPEG := makeLargeTestJPEG() - numPhotos := 40 + // Write 20 photos directly to the gallery dir. + galleryDir := srv.galleryDir("test-gallery") + numPhotos := 20 for i := 0; i < numPhotos; i++ { filename := fmt.Sprintf("photo_%03d.jpg", i) - require.NoError(t, os.WriteFile(filepath.Join(dir, filename), largeJPEG, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, filename), makeTestJPEG(), 0644)) } - // Fire all thumbnail requests concurrently, like a browser would. - var wg sync.WaitGroup - var failures atomic.Int32 - deadline := 30 * time.Second - - for i := 0; i < numPhotos; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - filename := fmt.Sprintf("photo_%03d.jpg", idx) - - client := &http.Client{Timeout: deadline} - req, err := http.NewRequest(http.MethodGet, ts.URL+"/g/test-gallery/thumb/"+filename, nil) - if err != nil { - failures.Add(1) - return - } - for _, c := range cookies { - req.AddCookie(c) - } - - resp, err := client.Do(req) - if err != nil { - failures.Add(1) - return - } - defer resp.Body.Close() + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") - if resp.StatusCode != 200 { - failures.Add(1) - return - } + // Wait for all thumbnails to appear. + thumbsDir := filepath.Join(galleryDir, ".thumbs-v2") + require.Eventually(t, func() bool { + entries, err := os.ReadDir(thumbsDir) + if err != nil { + return false + } + return len(entries) >= numPhotos + }, 10*time.Second, 100*time.Millisecond, + "all %d thumbnails should be generated by the background worker", numPhotos) - // Must be a valid JPEG - _, err = jpeg.Decode(resp.Body) - if err != nil { - failures.Add(1) - } - }(i) + // Verify each is a valid JPEG. + for i := 0; i < numPhotos; i++ { + filename := fmt.Sprintf("photo_%03d.jpg", i) + data, err := os.ReadFile(filepath.Join(thumbsDir, filename)) + require.NoError(t, err) + _, err = jpeg.Decode(bytes.NewReader(data)) + assert.NoError(t, err, "thumbnail %s should be a valid JPEG", filename) } - - wg.Wait() - assert.Equal(t, int32(0), failures.Load(), - "all %d concurrent thumbnail requests should succeed, but %d failed", numPhotos, failures.Load()) - - // Verify that thumbnail generation concurrency was bounded. - // With thumbSemaphore in place, peak concurrency must not exceed the limit. - peak := srv.peakThumbConcurrency.Load() - t.Logf("peak thumbnail generation concurrency: %d", peak) - assert.LessOrEqual(t, peak, int32(maxThumbConcurrency), - "peak concurrency (%d) should not exceed maxThumbConcurrency (%d)", peak, maxThumbConcurrency) } func setupTestServerNoThumbs(t *testing.T) (*httptest.Server, *Server) { |
