Improve caching

assets/diary/calendar.html

@@ -24,4 +24,4 @@
   </table>
   {{end}}
 </div>
-<script src="/_/diary/calendar.js"></script>
+<script src="{{asset "diary/calendar.js"}}"></script>

assets/editor/main.html

@@ -78,10 +78,10 @@
     <div id="editor" class="editor-cm"></div>
     <textarea name="content" id="editor-content" hidden>{{.RawContent}}</textarea>
 </form>
-<script src="/_/editor/vendor/codemirror.bundle.js?v={{editorBundleVersion}}"></script>
+<script src="{{asset "editor/vendor/codemirror.bundle.js"}}"></script>
-<script src="/_/editor/tables.js"></script>
+<script src="{{asset "editor/tables.js"}}"></script>
-<script src="/_/editor/dates.js"></script>
+<script src="{{asset "editor/dates.js"}}"></script>
-<script src="/_/editor/movie.js"></script>
+<script src="{{asset "editor/movie.js"}}"></script>
-<script src="/_/editor/wikicomplete.js"></script>
+<script src="{{asset "editor/wikicomplete.js"}}"></script>
-<script src="/_/editor/main.js"></script>
+<script src="{{asset "editor/main.js"}}"></script>
 {{end}}

assets/fitness/main.html

@@ -47,4 +47,4 @@
     {{range .Charts}}{{template "fitnessChart" .}}{{end}}
 {{end}}
 </div>
-<script src="/_/fitness/fitness.js"></script>
+<script src="{{asset "fitness/fitness.js"}}"></script>

assets/layout.html

@@ -4,19 +4,19 @@
         <meta charset="UTF-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1, interactive-widget=resizes-content" />
         <title>{{.Title}}</title>
-        <link rel="icon" href="/_/favicon.ico" />
+        <link rel="icon" href="{{asset "favicon.ico"}}" />
         <link rel="preload" href="/_/fonts/IosevkaEtoile.woff2" as="font" type="font/woff2" crossorigin />
         <link rel="preload" href="/_/fonts/IosevkaSlab.woff2" as="font" type="font/woff2" crossorigin />
-        <link rel="stylesheet" href="/_/style.css" />
+        <link rel="stylesheet" href="{{asset "style.css"}}" />
-        <script src="/_/modal.js"></script>
+        <script src="{{asset "modal.js"}}"></script>
-        <script src="/_/global-shortcuts.js"></script>
+        <script src="{{asset "global-shortcuts.js"}}"></script>
-        <script src="/_/history-nav.js"></script>
+        <script src="{{asset "history-nav.js"}}"></script>
-        <script src="/_/scroll-sync.js" defer></script>
+        <script src="{{asset "scroll-sync.js"}}" defer></script>
-        <script src="/_/search-suggest.js" defer></script>
+        <script src="{{asset "search-suggest.js"}}" defer></script>
-        <script src="/_/tree-picker.js"></script>
+        <script src="{{asset "tree-picker.js"}}"></script>
-        <script src="/_/companion.js" defer></script>
+        <script src="{{asset "companion.js"}}" defer></script>
-        {{if not .EditMode}}<script src="/_/overlay.js" defer></script>
+        {{if not .EditMode}}<script src="{{asset "overlay.js"}}" defer></script>
-        <script src="/_/tree-sidebar.js" defer></script>{{end}}
+        <script src="{{asset "tree-sidebar.js"}}" defer></script>{{end}}
         {{block "headScripts" .}}{{end}}
     </head>
     <body>

assets/page/main.html

@@ -1,4 +1,4 @@
-{{define "headScripts"}}<script src="/_/page/actions.js"></script>{{end}}
+{{define "headScripts"}}<script src="{{asset "page/actions.js"}}"></script>{{end}}
 
 {{define "headerActions"}}{{if .CanEdit}}
 <button class="btn btn-block" onclick="newPage()" title="New page (N)"><span class="btn-icon">{{icon "new"}}</span><span class="btn-label">NEW</span></button>
@@ -44,17 +44,17 @@
     </tbody>
 </table>
 {{end}}
-{{if .CanEdit}}<script src="/_/page/view-settings.js"></script>{{end}}
+{{if .CanEdit}}<script src="{{asset "page/view-settings.js"}}"></script>{{end}}
 {{end}}
 {{if or .Content .SpecialContent}}
-<script src="/_/page/content.js"></script>
+<script src="{{asset "page/content.js"}}"></script>
-{{if not .SuppressTOC}}<script src="/_/page/toc.js"></script>{{end}}
+{{if not .SuppressTOC}}<script src="{{asset "page/toc.js"}}"></script>{{end}}
-<script src="/_/page/tasks.js"></script>
+<script src="{{asset "page/tasks.js"}}"></script>
 {{end}}
 {{if .Content}}
-<script src="/_/page/sections.js"></script>
+<script src="{{asset "page/sections.js"}}"></script>
 {{end}}
-<script src="/_/page/contents.js"></script>
+<script src="{{asset "page/contents.js"}}"></script>
 {{end}}
 
 {{define "aside"}}{{if .SidebarWidget}}<aside class="sidebar">{{.SidebarWidget}}</aside>{{end}}{{end}}

assets/quickadd.html

@@ -4,8 +4,8 @@
         <meta charset="UTF-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <title>Save link</title>
-        <link rel="icon" href="/_/favicon.ico" />
+        <link rel="icon" href="{{asset "favicon.ico"}}" />
-        <link rel="stylesheet" href="/_/style.css" />
+        <link rel="stylesheet" href="{{asset "style.css"}}" />
         <style>
             body { padding: 0.8rem; }
             .qa-form { display: flex; flex-direction: column; gap: 0.5rem; }

assets/search/main.html

@@ -1,4 +1,4 @@
-{{define "headScripts"}}<script src="/_/search/actions.js"></script>{{end}}
+{{define "headScripts"}}<script src="{{asset "search/actions.js"}}"></script>{{end}}
 
 {{define "searchQuery"}}{{.Query}}{{end}}
 

diary.go

@@ -335,7 +335,7 @@ type calendarData struct {
 	Years        []calYear
 }
 
-var diaryCalTmpl = template.Must(template.ParseFS(assets, "assets/diary/calendar.html"))
+var diaryCalTmpl = template.Must(template.New("calendar.html").Funcs(assetFuncs).ParseFS(assets, "assets/diary/calendar.html"))
 
 func computeCalendarWidget(diaryRootFS, diaryRootURL, fsPath string, depth int) template.HTML {
 	today := time.Now()

fitness.go

@@ -470,7 +470,7 @@ func alignedTicks(from, to time.Time, step xTickStep) []time.Time {
 
 // === Rendering ===
 
-var fitnessTmpl = template.Must(template.ParseFS(assets, "assets/fitness/main.html"))
+var fitnessTmpl = template.Must(template.New("main.html").Funcs(assetFuncs).ParseFS(assets, "assets/fitness/main.html"))
 
 func renderFitnessDashboard(fsPath string, weightView, weeklyView fitnessView) template.HTML {
 	data := buildFitnessDash(fsPath, weightView, weeklyView, time.Now())

main.go

@@ -2,12 +2,9 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"embed"
-	"encoding/hex"
 	"flag"
 	"html/template"
-	"io/fs"
 	"log"
 	"net/http"
 	"net/url"
@@ -23,26 +20,11 @@ import (
 //go:embed assets
 var assets embed.FS
 
-// editorBundleVersion is a short content hash of the vendored CodeMirror bundle,
+// tmplFuncs is shared by every layout-based template: asset links versioned
-// appended as ?v=… to its <script> src. The bundle is served immutable under a
+// static files (see static.go); the layout header renders the maintenance
-// stable filename, so without this query a rebuilt bundle would never reach a
+// widget.
-// client that already cached the old one (this is the editor cache-bust knob).
-var editorBundleVersion = hashAsset("assets/editor/vendor/codemirror.bundle.js")
-
-func hashAsset(name string) string {
-	b, err := assets.ReadFile(name)
-	if err != nil {
-		return ""
-	}
-	sum := sha256.Sum256(b)
-	return hex.EncodeToString(sum[:])[:12]
-}
-
-// tmplFuncs is shared by every layout-based template: the edit template appends
-// editorBundleVersion to its CodeMirror <script> src to cache-bust the bundle;
-// the layout header renders the maintenance widget.
 var tmplFuncs = template.FuncMap{
-	"editorBundleVersion": func() string { return editorBundleVersion },
+	"asset":             assetURL,
 	"fileIcon":          fileIcon,
 	"icon":              readIcon,
 	"maintenanceWidget": maintenanceWidget,
@@ -137,14 +119,7 @@ func main() {
 	}
 	h := &handler{root: root, user: *user, pass: *pass, authKey: authKey}
 
-	staticFS, _ := fs.Sub(assets, "assets")
+	http.Handle("/_/", serveStatic())
-	static := http.StripPrefix("/_/", http.FileServer(http.FS(staticFS)))
-	http.Handle("/_/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if strings.HasPrefix(r.URL.Path, "/_/fonts/") || strings.HasPrefix(r.URL.Path, "/_/editor/vendor/") {
-			w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
-		}
-		static.ServeHTTP(w, r)
-	}))
 	http.HandleFunc("/_logout", h.handleLogout)
 	http.HandleFunc("/_reindex", h.handleReindex)
 	http.HandleFunc("/_search", h.handleSearchSuggest)

quickadd.go

@@ -11,7 +11,7 @@ import (
 	"time"
 )
 
-var quickAddTmpl = template.Must(template.ParseFS(assets, "assets/quickadd.html"))
+var quickAddTmpl = template.Must(template.New("quickadd.html").Funcs(assetFuncs).ParseFS(assets, "assets/quickadd.html"))
 
 type quickAddData struct {
 	To, URL, Title string

static.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"html/template"
+	"io/fs"
+	"net/http"
+	"strings"
+)
+
+// assetVersions maps each embedded asset (path relative to assets/, e.g.
+// "page/toc.js") to a short hash of its content. Templates reference assets
+// through assetURL, which appends the hash as ?v=…, so a versioned URL names
+// one exact file content and can be cached forever; a rebuilt asset gets a
+// new URL.
+var assetVersions = hashAssets()
+
+func hashAssets() map[string]string {
+	versions := map[string]string{}
+	_ = fs.WalkDir(assets, "assets", func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return err
+		}
+		b, err := assets.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		sum := sha256.Sum256(b)
+		versions[strings.TrimPrefix(p, "assets/")] = hex.EncodeToString(sum[:])[:12]
+		return nil
+	})
+	return versions
+}
+
+// assetURL is the "asset" template func: {{asset "style.css"}} renders
+// /_/style.css?v=<hash>. An unknown name fails the template so a typo or a
+// removed file surfaces as a render error instead of a silent 404.
+func assetURL(name string) (string, error) {
+	v, ok := assetVersions[name]
+	if !ok {
+		return "", fmt.Errorf("unknown asset %q", name)
+	}
+	return "/_/" + name + "?v=" + v, nil
+}
+
+// assetFuncs is for the standalone fragment templates that do not take the
+// full tmplFuncs.
+var assetFuncs = template.FuncMap{"asset": assetURL}
+
+// serveStatic serves the embedded assets under /_/. Embedded files carry no
+// modtime, so http.FileServer alone sends no validators and browsers refetch
+// every asset on every page. Instead:
+//   - a URL carrying the current ?v= hash is immutable;
+//   - fonts are immutable too (referenced unversioned from style.css, never
+//     change);
+//   - anything else (unversioned or stale ?v=, e.g. icons from CSS) gets the
+//     content hash as ETag and revalidates, answered with 304 when unchanged.
+func serveStatic() http.Handler {
+	staticFS, _ := fs.Sub(assets, "assets")
+	static := http.StripPrefix("/_/", http.FileServer(http.FS(staticFS)))
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		name := strings.TrimPrefix(r.URL.Path, "/_/")
+		v, ok := assetVersions[name]
+		switch {
+		case ok && r.URL.Query().Get("v") == v, strings.HasPrefix(name, "fonts/"):
+			w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
+		case ok:
+			w.Header().Set("Cache-Control", "no-cache")
+			w.Header().Set("ETag", `"`+v+`"`)
+		}
+		static.ServeHTTP(w, r)
+	})
+}

static_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+)
+
+// Every {{asset "…"}} in the templates must name an embedded file, or the page
+// fails to render.
+func TestTemplateAssetsExist(t *testing.T) {
+	ref := regexp.MustCompile(`\{\{asset "([^"]+)"\}\}`)
+	_ = fs.WalkDir(assets, "assets", func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return err
+		}
+		b, _ := assets.ReadFile(p)
+		for _, m := range ref.FindAllSubmatch(b, -1) {
+			if _, err := assetURL(string(m[1])); err != nil {
+				t.Errorf("%s: %v", p, err)
+			}
+		}
+		return nil
+	})
+}
+
+func TestServeStaticCaching(t *testing.T) {
+	h := serveStatic()
+	get := func(url, ifNoneMatch string) *httptest.ResponseRecorder {
+		r := httptest.NewRequest(http.MethodGet, url, nil)
+		if ifNoneMatch != "" {
+			r.Header.Set("If-None-Match", ifNoneMatch)
+		}
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, r)
+		return w
+	}
+	v := assetVersions["style.css"]
+
+	versioned, _ := assetURL("style.css")
+	if w := get(versioned, ""); w.Code != 200 || w.Header().Get("Cache-Control") != "public, max-age=31536000, immutable" {
+		t.Errorf("versioned: %d %q", w.Code, w.Header().Get("Cache-Control"))
+	}
+	for _, u := range []string{"/_/style.css", "/_/style.css?v=stale"} {
+		w := get(u, "")
+		if w.Code != 200 || w.Header().Get("Cache-Control") != "no-cache" || w.Header().Get("ETag") != `"`+v+`"` {
+			t.Errorf("%s: %d %q %q", u, w.Code, w.Header().Get("Cache-Control"), w.Header().Get("ETag"))
+		}
+	}
+	if w := get("/_/style.css", `"`+v+`"`); w.Code != http.StatusNotModified {
+		t.Errorf("revalidate: got %d, want 304", w.Code)
+	}
+	if w := get("/_/fonts/IosevkaSlab.woff2", ""); w.Header().Get("Cache-Control") != "public, max-age=31536000, immutable" {
+		t.Errorf("font: %q", w.Header().Get("Cache-Control"))
+	}
+}

thumb.go

@@ -199,6 +199,9 @@ func pathDigest(srcFS string, info os.FileInfo) string {
 // On a cache hit (path + mtime + size unchanged) the returned data is nil,
 // so the caller knows to open the file itself. On a miss the file is read
 // once and the contents are returned for the caller to reuse.
+//
+// Hits come from memory or, after a restart, from the on-disk index (see
+// digestIndexPath), so an unchanged file is hashed only once ever.
 func sourceDigest(srcFS string, info os.FileInfo) (string, []byte, error) {
 	digestCacheMu.Lock()
 	d, ok := digestCache[srcFS]
@@ -207,18 +210,43 @@ func sourceDigest(srcFS string, info os.FileInfo) (string, []byte, error) {
 		return d.hex, nil, nil
 	}
 
+	indexFS := digestIndexPath(srcFS, info)
+	if b, err := os.ReadFile(indexFS); err == nil && len(b) == sha256.Size*2 {
+		h := string(b)
+		rememberDigest(srcFS, info, h)
+		return h, nil, nil
+	}
+
 	data, err := os.ReadFile(srcFS)
 	if err != nil {
 		return "", nil, err
 	}
 	sum := sha256.Sum256(data)
 	h := hex.EncodeToString(sum[:])
+	rememberDigest(srcFS, info, h)
 
+	// Best effort: a failed write only costs a re-hash after the next restart.
+	if err := os.MkdirAll(filepath.Dir(indexFS), 0755); err == nil {
+		if err := writeFileAtomic(indexFS, []byte(h), 0644); err != nil {
+			log.Printf("thumb digest index %s: %v", indexFS, err)
+		}
+	}
+	return h, data, nil
+}
+
+// digestIndexPath is where the content digest of srcFS is persisted, keyed on
+// path + mtime + size like pathDigest, so overwriting the file misses and
+// re-hashes. Entries for changed or deleted files are left behind; each is 64
+// bytes.
+func digestIndexPath(srcFS string, info os.FileInfo) string {
+	key := pathDigest(srcFS, info)
+	return filepath.Join(thumbCacheDir, "digests", key[:2], key)
+}
+
+func rememberDigest(srcFS string, info os.FileInfo, h string) {
 	digestCacheMu.Lock()
 	digestCache[srcFS] = digestEntry{mtime: info.ModTime(), size: info.Size(), hex: h}
 	digestCacheMu.Unlock()
-
-	return h, data, nil
 }
 
 func serveThumb(w http.ResponseWriter, r *http.Request, cacheFS string) {

thumb_image_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"image"
 	"image/jpeg"
+	"os"
+	"path/filepath"
 	"testing"
 )
 
@@ -54,3 +56,34 @@ func BenchmarkResizeBox(b *testing.B) {
 		}
 	})
 }
+
+// After a restart (empty memory cache) the digest comes from the on-disk index
+// without reading the source file.
+func TestSourceDigestPersists(t *testing.T) {
+	old := thumbCacheDir
+	thumbCacheDir = t.TempDir()
+	t.Cleanup(func() { thumbCacheDir = old })
+
+	src := filepath.Join(t.TempDir(), "a.jpg")
+	if err := os.WriteFile(src, []byte("photo"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	info, _ := os.Stat(src)
+	h1, data, err := sourceDigest(src, info)
+	if err != nil || data == nil {
+		t.Fatalf("first call: %v, data=%v", err, data)
+	}
+
+	digestCacheMu.Lock()
+	delete(digestCache, src)
+	digestCacheMu.Unlock()
+	// Removing the source proves the second call does not read it again.
+	if err := os.Remove(src); err != nil {
+		t.Fatal(err)
+	}
+
+	h2, data, err := sourceDigest(src, info)
+	if err != nil || data != nil || h2 != h1 {
+		t.Fatalf("after restart: h=%s data=%v err=%v, want %s from index", h2, data, err, h1)
+	}
+}