From 7a362332d02f58f3df7b4d006b99fdb421a5ecbf Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Tue, 3 Mar 2026 08:32:24 +0100 Subject: feat: background thumbnail worker with smaller thumbs Replace on-demand thumbnail generation (blocking HTTP requests) with a background goroutine that scans gallery directories every 10 seconds. Uploads trigger an immediate scan via a channel for fast availability. --- gallery.go | 263 +++++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 212 insertions(+), 51 deletions(-) (limited to 'gallery.go') diff --git a/gallery.go b/gallery.go index 0a726bc..741964e 100644 --- a/gallery.go +++ b/gallery.go @@ -22,7 +22,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" "golang.org/x/crypto/bcrypt" @@ -34,9 +33,12 @@ import ( // --------------------------------------------------------------------------- const ( - thumbMaxWidth = 1600 - thumbMaxHeight = 1200 - thumbQuality = 75 + thumbMaxWidth = 400 + thumbMaxHeight = 400 + thumbQuality = 65 + thumbDirName = ".thumbs-v2" + oldThumbDirName = ".thumbs" + thumbScanInterval = 10 * time.Second maxThumbConcurrency = 4 sessionTTL = 24 * time.Hour dataFile = "galleries.json" @@ -166,10 +168,9 @@ type Server struct { // overwhelming the server (CPU/memory). thumbSemaphore chan struct{} - // peakThumbConcurrency tracks the highest number of concurrent thumbnail - // generations observed — used by tests to verify the semaphore works. - peakThumbConcurrency atomic.Int32 - currentThumbConcurrency atomic.Int32 + // thumbTrigger receives gallery names to trigger an immediate scan + // for that gallery (e.g. after an upload). Non-blocking send. + thumbTrigger chan string } // NewServer creates a fully initialised Server ready to use. @@ -181,6 +182,7 @@ func NewServer(dataDir string, maxUploadSize int64) *Server { maxUploadSize: maxUploadSize, rateLimiter: newLoginRateLimiter(), thumbSemaphore: make(chan struct{}, maxThumbConcurrency), + thumbTrigger: make(chan string, 16), } } @@ -237,19 +239,25 @@ func (s *Server) load() error { // --------------------------------------------------------------------------- func thumbDir(galleryDir string) string { - return filepath.Join(galleryDir, ".thumbs") + return filepath.Join(galleryDir, thumbDirName) } -func generateThumbGo(srcPath, dstPath string) error { +// thumbResult holds metadata about a generated thumbnail for logging. +type thumbResult struct { + srcWidth int + srcHeight int +} + +func generateThumbGo(srcPath, dstPath string) (thumbResult, error) { f, err := os.Open(srcPath) if err != nil { - return err + return thumbResult{}, err } defer f.Close() src, _, err := image.Decode(f) if err != nil { - return err + return thumbResult{}, err } bounds := src.Bounds() @@ -276,11 +284,14 @@ func generateThumbGo(srcPath, dstPath string) error { out, err := os.Create(dstPath) if err != nil { - return err + return thumbResult{}, err } defer out.Close() - return jpeg.Encode(out, dst, &jpeg.Options{Quality: thumbQuality}) + if err := jpeg.Encode(out, dst, &jpeg.Options{Quality: thumbQuality}); err != nil { + return thumbResult{}, err + } + return thumbResult{srcWidth: w, srcHeight: h}, nil } // findMagickBinary returns the ImageMagick convert command name. @@ -318,56 +329,194 @@ func generateThumbImageMagick(srcPath, dstPath string) error { return nil } -func generateThumb(srcPath, dstPath string) error { - err := generateThumbGo(srcPath, dstPath) +func generateThumb(srcPath, dstPath string) (thumbResult, error) { + res, err := generateThumbGo(srcPath, dstPath) if err == nil { - return nil + return res, nil } log.Printf("Go thumbnail failed for %s: %v — trying ImageMagick", srcPath, err) if err2 := generateThumbImageMagick(srcPath, dstPath); err2 != nil { - return fmt.Errorf("all thumbnail methods failed: go=%v, imagemagick=%v", err, err2) + return thumbResult{}, fmt.Errorf("all thumbnail methods failed: go=%v, imagemagick=%v", err, err2) } - return nil + // ImageMagick succeeded — dimensions unknown (Go couldn't decode the source). + return thumbResult{}, nil } -func (s *Server) ensureThumb(galleryDir, filename string) (string, error) { - td := thumbDir(galleryDir) - os.MkdirAll(td, 0755) - +// thumbPath returns the expected thumbnail path for a photo, or "" if it +// doesn't exist yet. +func thumbPath(galleryDir, filename string) string { thumbName := strings.TrimSuffix(filename, filepath.Ext(filename)) + ".jpg" - thumbPath := filepath.Join(td, thumbName) + p := filepath.Join(thumbDir(galleryDir), thumbName) + if _, err := os.Stat(p); err == nil { + return p + } + return "" +} - if _, err := os.Stat(thumbPath); err == nil { - return thumbPath, nil // already exists +// --------------------------------------------------------------------------- +// Background thumbnail worker +// --------------------------------------------------------------------------- + +// triggerThumbScan sends a gallery name on the trigger channel to request +// an immediate scan. Non-blocking: if the channel is full, the periodic +// scan will pick it up. +func (s *Server) triggerThumbScan(gallery string) { + select { + case s.thumbTrigger <- gallery: + default: } +} - // Acquire semaphore to limit concurrent thumbnail generation. - s.thumbSemaphore <- struct{}{} - cur := s.currentThumbConcurrency.Add(1) - // Track peak for observability / testing. - for { - peak := s.peakThumbConcurrency.Load() - if cur <= peak || s.peakThumbConcurrency.CompareAndSwap(peak, cur) { - break +// startThumbWorker launches a background goroutine that periodically scans +// all galleries and generates missing thumbnails. +func (s *Server) startThumbWorker() { + go func() { + ticker := time.NewTicker(thumbScanInterval) + defer ticker.Stop() + + for { + select { + case gallery := <-s.thumbTrigger: + // Triggered scan for a specific gallery (e.g. after upload). + s.mu.RLock() + g, ok := s.galleries[gallery] + s.mu.RUnlock() + if ok && g.thumbsEnabled() { + s.scanGalleryThumbs(gallery) + } + case <-ticker.C: + // Periodic full scan. + s.scanAllThumbs() + } } + }() +} + +// scanAllThumbs scans every gallery and generates missing thumbnails. +func (s *Server) scanAllThumbs() { + s.mu.RLock() + names := make([]string, 0, len(s.galleries)) + for name, g := range s.galleries { + if g.thumbsEnabled() { + names = append(names, name) + } + } + s.mu.RUnlock() + + for _, name := range names { + s.scanGalleryThumbs(name) + } +} + +// formatBytes formats a byte count as a human-readable string. +func formatBytes(b int64) string { + switch { + case b >= 1<<20: + return fmt.Sprintf("%.1fMB", float64(b)/float64(1<<20)) + case b >= 1<<10: + return fmt.Sprintf("%.0fKB", float64(b)/float64(1<<10)) + default: + return fmt.Sprintf("%dB", b) } - defer func() { - s.currentThumbConcurrency.Add(-1) +} + +// scanGalleryThumbs generates missing thumbnails for a single gallery. +func (s *Server) scanGalleryThumbs(name string) { + dir := s.galleryDir(name) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + + td := thumbDir(dir) + generated := 0 + galleryStart := time.Now() + + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + if !isAllowedExt(e.Name()) { + continue + } + + thumbName := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) + ".jpg" + tp := filepath.Join(td, thumbName) + + if _, err := os.Stat(tp); err == nil { + continue // already exists + } + + // Create thumbs dir if needed. + os.MkdirAll(td, 0755) + + srcPath := filepath.Join(dir, e.Name()) + + // Get source file size. + srcInfo, err := os.Stat(srcPath) + if err != nil { + continue + } + srcSize := srcInfo.Size() + + // Acquire semaphore to limit CPU usage. + s.thumbSemaphore <- struct{}{} + start := time.Now() + res, err := generateThumb(srcPath, tp) + elapsed := time.Since(start) <-s.thumbSemaphore - }() - // Double-check after acquiring the semaphore — another goroutine may have - // generated the same thumbnail while we were waiting. - if _, err := os.Stat(thumbPath); err == nil { - return thumbPath, nil + if err != nil { + log.Printf("[thumbs] failed %s/%s: %v", name, e.Name(), err) + continue + } + + // Get thumb file size. + var thumbSize int64 + if ti, err := os.Stat(tp); err == nil { + thumbSize = ti.Size() + } + + generated++ + if res.srcWidth > 0 { + log.Printf("[thumbs] generated %s/%s (%dx%d, %s → %s, %s)", + name, e.Name(), res.srcWidth, res.srcHeight, + formatBytes(srcSize), formatBytes(thumbSize), + elapsed.Round(time.Millisecond)) + } else { + log.Printf("[thumbs] generated %s/%s (%s → %s, %s)", + name, e.Name(), + formatBytes(srcSize), formatBytes(thumbSize), + elapsed.Round(time.Millisecond)) + } + } + + if generated > 0 { + log.Printf("[thumbs] gallery %q: %d new thumbnails in %s", + name, generated, time.Since(galleryStart).Round(time.Millisecond)) + } +} + +// cleanupOldThumbs removes legacy .thumbs directories from all galleries. +func (s *Server) cleanupOldThumbs() { + s.mu.RLock() + names := make([]string, 0, len(s.galleries)) + for name := range s.galleries { + names = append(names, name) } + s.mu.RUnlock() - srcPath := filepath.Join(galleryDir, filename) - if err := generateThumb(srcPath, thumbPath); err != nil { - return "", err + for _, name := range names { + oldDir := filepath.Join(s.galleryDir(name), oldThumbDirName) + if _, err := os.Stat(oldDir); err == nil { + if err := os.RemoveAll(oldDir); err != nil { + log.Printf("[thumbs] failed to remove old %s: %v", oldDir, err) + } else { + log.Printf("[thumbs] removed legacy %s", oldDir) + } + } } - return thumbPath, nil } // --------------------------------------------------------------------------- @@ -658,6 +807,10 @@ func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request, name strin uploaded++ } + if uploaded > 0 { + s.triggerThumbScan(name) + } + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]int{"uploaded": uploaded}) } @@ -677,14 +830,20 @@ func (s *Server) handleThumb(w http.ResponseWriter, r *http.Request, name, filen filename = filepath.Base(filename) // sanitize dir := s.galleryDir(name) - thumbPath, err := s.ensureThumb(dir, filename) - if err != nil { - log.Printf("Thumbnail generation failed for %s/%s: %v", name, filename, err) - http.Error(w, "thumbnail unavailable", http.StatusInternalServerError) + // Serve pre-generated thumbnail if available, otherwise fallback to source. + if tp := thumbPath(dir, filename); tp != "" { + w.Header().Set("Cache-Control", "public, max-age=86400") + http.ServeFile(w, r, tp) + return + } + + // Thumbnail not ready yet — serve the original as fallback. + srcPath := filepath.Join(dir, filename) + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + http.NotFound(w, r) return } - w.Header().Set("Cache-Control", "public, max-age=86400") - http.ServeFile(w, r, thumbPath) + http.ServeFile(w, r, srcPath) } func (s *Server) handlePhoto(w http.ResponseWriter, r *http.Request, name, filename string) { @@ -876,6 +1035,8 @@ func main() { srv.startSessionReaper() srv.rateLimiter.startReaper() + srv.cleanupOldThumbs() + srv.startThumbWorker() log.Printf("Listening on %s (data: %s, galleries: %d)", flags["addr"], flags["data"], len(srv.galleries)) if err := http.ListenAndServe(flags["addr"], srv.Handler()); err != nil { -- cgit v1.3