Improve caching

thumb.go

@@ -199,6 +199,9 @@ func pathDigest(srcFS string, info os.FileInfo) string {
 // On a cache hit (path + mtime + size unchanged) the returned data is nil,
 // so the caller knows to open the file itself. On a miss the file is read
 // once and the contents are returned for the caller to reuse.
+//
+// Hits come from memory or, after a restart, from the on-disk index (see
+// digestIndexPath), so an unchanged file is hashed only once ever.
 func sourceDigest(srcFS string, info os.FileInfo) (string, []byte, error) {
 	digestCacheMu.Lock()
 	d, ok := digestCache[srcFS]
@@ -207,18 +210,43 @@ func sourceDigest(srcFS string, info os.FileInfo) (string, []byte, error) {
 		return d.hex, nil, nil
 	}
 
+	indexFS := digestIndexPath(srcFS, info)
+	if b, err := os.ReadFile(indexFS); err == nil && len(b) == sha256.Size*2 {
+		h := string(b)
+		rememberDigest(srcFS, info, h)
+		return h, nil, nil
+	}
+
 	data, err := os.ReadFile(srcFS)
 	if err != nil {
 		return "", nil, err
 	}
 	sum := sha256.Sum256(data)
 	h := hex.EncodeToString(sum[:])
+	rememberDigest(srcFS, info, h)
 
+	// Best effort: a failed write only costs a re-hash after the next restart.
+	if err := os.MkdirAll(filepath.Dir(indexFS), 0755); err == nil {
+		if err := writeFileAtomic(indexFS, []byte(h), 0644); err != nil {
+			log.Printf("thumb digest index %s: %v", indexFS, err)
+		}
+	}
+	return h, data, nil
+}
+
+// digestIndexPath is where the content digest of srcFS is persisted, keyed on
+// path + mtime + size like pathDigest, so overwriting the file misses and
+// re-hashes. Entries for changed or deleted files are left behind; each is 64
+// bytes.
+func digestIndexPath(srcFS string, info os.FileInfo) string {
+	key := pathDigest(srcFS, info)
+	return filepath.Join(thumbCacheDir, "digests", key[:2], key)
+}
+
+func rememberDigest(srcFS string, info os.FileInfo, h string) {
 	digestCacheMu.Lock()
 	digestCache[srcFS] = digestEntry{mtime: info.ModTime(), size: info.Size(), hex: h}
 	digestCacheMu.Unlock()
-
-	return h, data, nil
 }
 
 func serveThumb(w http.ResponseWriter, r *http.Request, cacheFS string) {