From 7a362332d02f58f3df7b4d006b99fdb421a5ecbf Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Tue, 3 Mar 2026 08:32:24 +0100 Subject: feat: background thumbnail worker with smaller thumbs Replace on-demand thumbnail generation (blocking HTTP requests) with a background goroutine that scans gallery directories every 10 seconds. Uploads trigger an immediate scan via a channel for fast availability. --- gallery_test.go | 203 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 132 insertions(+), 71 deletions(-) (limited to 'gallery_test.go') diff --git a/gallery_test.go b/gallery_test.go index 125a19a..29539fa 100644 --- a/gallery_test.go +++ b/gallery_test.go @@ -15,8 +15,6 @@ import ( "net/url" "os" "path/filepath" - "sync" - "sync/atomic" "testing" "time" @@ -238,6 +236,92 @@ func TestThumbServing(t *testing.T) { assert.NoError(t, err, "thumbnail should be a decodable JPEG") } +func TestThumbServesSourceWhenThumbNotReady(t *testing.T) { + // When the background worker hasn't generated a thumb yet, + // handleThumb should serve the source image as fallback. + ts, srv := setupTestServer(t) + cookies := login(t, ts, "test-gallery", "secret123") + + // Write a photo directly — don't start the thumb worker, so no thumb exists. + dir := srv.galleryDir("test-gallery") + original := makeTestJPEG() + require.NoError(t, os.WriteFile(filepath.Join(dir, "nothumb.jpg"), original, 0644)) + + resp := authedGet(t, ts.URL+"/g/test-gallery/thumb/nothumb.jpg", cookies) + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, err) + + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, original, body, + "when thumbnail is not ready, should serve the source image as fallback") +} + +func TestThumbWorkerGeneratesThumbnails(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + + // Write a photo directly. + galleryDir := srv.galleryDir("test-gallery") + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, "worker-test.jpg"), makeTestJPEG(), 0644)) + + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") + + // Wait for the thumbnail to appear (up to 5s). + thumbPath := filepath.Join(galleryDir, ".thumbs-v2", "worker-test.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, "thumbnail should be generated by the background worker") + + // Verify it's a valid JPEG. + data, err := os.ReadFile(thumbPath) + require.NoError(t, err) + _, err = jpeg.Decode(bytes.NewReader(data)) + assert.NoError(t, err, "generated thumbnail should be a valid JPEG") +} + +func TestUploadTriggersThumbWorker(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + srv.startThumbWorker() + + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(ts.Close) + + cookies := login(t, ts, "test-gallery", "secret123") + resp := uploadPhoto(t, ts, "test-gallery", cookies, "upload-trigger.jpg", makeTestJPEG()) + resp.Body.Close() + + // The upload should have triggered an immediate scan. Wait for the thumb. + thumbPath := filepath.Join(srv.galleryDir("test-gallery"), ".thumbs-v2", "upload-trigger.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, + "upload should trigger background thumbnail generation") +} + +func TestOldThumbsDirCleanedUp(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) + + // Create the old .thumbs directory with some content. + oldThumbsDir := filepath.Join(srv.galleryDir("test-gallery"), ".thumbs") + require.NoError(t, os.MkdirAll(oldThumbsDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(oldThumbsDir, "old.jpg"), []byte("old"), 0644)) + + srv.cleanupOldThumbs() + + _, err := os.Stat(oldThumbsDir) + assert.True(t, os.IsNotExist(err), "old .thumbs directory should be removed") +} + func TestPhotoServing(t *testing.T) { ts, _, cookies := setupWithPhoto(t, "original.jpg") @@ -382,33 +466,38 @@ func makeUnsupportedJPEG() []byte { } func TestThumbFallbackToImageMagick(t *testing.T) { - ts, srv := setupTestServer(t) - cookies := login(t, ts, "test-gallery", "secret123") + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) // Create a JPEG that Go's stdlib can't decode but ImageMagick can. unsupportedJPEG := makeUnsupportedJPEG() - // Write directly to the gallery dir (bypass upload, test thumbnail generation). - dir := srv.galleryDir("test-gallery") - require.NoError(t, os.WriteFile(filepath.Join(dir, "tricky.jpg"), unsupportedJPEG, 0644)) + // Write directly to the gallery dir. + galleryDir := srv.galleryDir("test-gallery") + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, "tricky.jpg"), unsupportedJPEG, 0644)) - // Request the thumbnail — should use ImageMagick fallback and return a valid JPEG. - resp := authedGet(t, ts.URL+"/g/test-gallery/thumb/tricky.jpg", cookies) - defer resp.Body.Close() + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") - assert.Equal(t, 200, resp.StatusCode) - assert.Contains(t, resp.Header.Get("Content-Type"), "image/jpeg") + // Wait for the thumbnail to appear. + thumbPath := filepath.Join(galleryDir, ".thumbs-v2", "tricky.jpg") + require.Eventually(t, func() bool { + _, err := os.Stat(thumbPath) + return err == nil + }, 5*time.Second, 50*time.Millisecond, "thumbnail should be generated by the background worker") // The thumbnail should be a valid, fully decodable JPEG. - thumbData, err := io.ReadAll(resp.Body) + thumbData, err := os.ReadFile(thumbPath) require.NoError(t, err) _, err = jpeg.Decode(bytes.NewReader(thumbData)) assert.NoError(t, err, "thumbnail should be a fully decodable JPEG even when Go's stdlib can't decode the original") - // The thumbnail must not be the raw original served as fallback. + // The thumbnail must not be the raw original. assert.NotEqual(t, unsupportedJPEG, thumbData, - "thumbnail must not be the broken original served as fallback") + "thumbnail must not be the broken original") } // makeLargeTestJPEG creates a larger JPEG that takes more time to thumbnail. @@ -424,70 +513,42 @@ func makeLargeTestJPEG() []byte { return buf.Bytes() } -func TestConcurrentThumbnailGeneration(t *testing.T) { - ts, srv := setupTestServer(t) - cookies := login(t, ts, "test-gallery", "secret123") +func TestWorkerGeneratesAllThumbnails(t *testing.T) { + dir := t.TempDir() + srv := NewServer(dir, 20<<20) + require.NoError(t, srv.createGallery("test-gallery", "secret123")) - // Upload 40 photos directly to the gallery dir (bypass upload handler for speed). - dir := srv.galleryDir("test-gallery") - largeJPEG := makeLargeTestJPEG() - numPhotos := 40 + // Write 20 photos directly to the gallery dir. + galleryDir := srv.galleryDir("test-gallery") + numPhotos := 20 for i := 0; i < numPhotos; i++ { filename := fmt.Sprintf("photo_%03d.jpg", i) - require.NoError(t, os.WriteFile(filepath.Join(dir, filename), largeJPEG, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(galleryDir, filename), makeTestJPEG(), 0644)) } - // Fire all thumbnail requests concurrently, like a browser would. - var wg sync.WaitGroup - var failures atomic.Int32 - deadline := 30 * time.Second + // Start the worker and trigger a scan. + srv.startThumbWorker() + srv.triggerThumbScan("test-gallery") + + // Wait for all thumbnails to appear. + thumbsDir := filepath.Join(galleryDir, ".thumbs-v2") + require.Eventually(t, func() bool { + entries, err := os.ReadDir(thumbsDir) + if err != nil { + return false + } + return len(entries) >= numPhotos + }, 10*time.Second, 100*time.Millisecond, + "all %d thumbnails should be generated by the background worker", numPhotos) + // Verify each is a valid JPEG. for i := 0; i < numPhotos; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - filename := fmt.Sprintf("photo_%03d.jpg", idx) - - client := &http.Client{Timeout: deadline} - req, err := http.NewRequest(http.MethodGet, ts.URL+"/g/test-gallery/thumb/"+filename, nil) - if err != nil { - failures.Add(1) - return - } - for _, c := range cookies { - req.AddCookie(c) - } - - resp, err := client.Do(req) - if err != nil { - failures.Add(1) - return - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - failures.Add(1) - return - } - - // Must be a valid JPEG - _, err = jpeg.Decode(resp.Body) - if err != nil { - failures.Add(1) - } - }(i) + filename := fmt.Sprintf("photo_%03d.jpg", i) + data, err := os.ReadFile(filepath.Join(thumbsDir, filename)) + require.NoError(t, err) + _, err = jpeg.Decode(bytes.NewReader(data)) + assert.NoError(t, err, "thumbnail %s should be a valid JPEG", filename) } - - wg.Wait() - assert.Equal(t, int32(0), failures.Load(), - "all %d concurrent thumbnail requests should succeed, but %d failed", numPhotos, failures.Load()) - - // Verify that thumbnail generation concurrency was bounded. - // With thumbSemaphore in place, peak concurrency must not exceed the limit. - peak := srv.peakThumbConcurrency.Load() - t.Logf("peak thumbnail generation concurrency: %d", peak) - assert.LessOrEqual(t, peak, int32(maxThumbConcurrency), - "peak concurrency (%d) should not exceed maxThumbConcurrency (%d)", peak, maxThumbConcurrency) } func setupTestServerNoThumbs(t *testing.T) (*httptest.Server, *Server) { -- cgit v1.3