luxick/datascape
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add search index

Browse Source
This commit is contained in:
luxick
2026-05-07 09:41:20 +02:00
parent 2787c15d40
commit 3d3a121fa6
6 changed files with 278 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assets/layout.html
+1
View File
@@ -33,6 +33,7 @@
</main>
<footer>
<span class="muted">Request time: {{.RenderMS}} ms</span>
{{block "footerExtras" .}}{{end}}
</footer>
{{block "extras" .}}{{end}}
</body>
assets/search-actions.js
+26
View File
@@ -0,0 +1,26 @@
function rebuildIndex() {
openModal({ title: 'Rebuilding search index…', body: 'Walking the wiki tree.' });
fetch('/_reindex', { method: 'POST' })
.then(function (resp) {
if (!resp.ok) throw new Error('rebuild failed: ' + resp.status);
window.location.href = window.location.href;
})
.catch(function (err) {
closeModal();
openModal({ title: 'Rebuild failed', body: String(err), confirm: { label: 'OK' } });
});
}
document.addEventListener('DOMContentLoaded', function () {
wireDropdown(document.querySelector('[data-action="actions-drop"]'));
// Focus the search input on results pages so Tab steps directly into the
// first match — the input sits immediately before the results in DOM
// order, so the natural tab sequence is input → first result → next, …
var input = document.querySelector('.search-input');
if (input && input.value) {
input.focus();
var end = input.value.length;
try { input.setSelectionRange(end, end); } catch (e) {}
}
});
assets/search.html
+15
View File
@@ -1,3 +1,5 @@
{{define "headScripts"}}<script src="/_/search-actions.js"></script>{{end}}
{{define "searchQuery"}}{{.Query}}{{end}}
{{define "content"}}
@@ -18,3 +20,16 @@
<p class="empty">Enter a query above.</p>
{{end}}
{{end}}
{{define "footerExtras"}}
{{if not .IndexBuiltAt.IsZero}}<span class="muted">· Index: {{.IndexBuiltAt.Format "2006-01-02 15:04"}}</span>{{end}}
{{end}}
{{define "extras"}}
<div class="fab dropdown">
<button class="btn btn-fab" data-action="actions-drop" title="Actions" aria-label="Actions">≡</button>
<div class="dropdown-menu align-right open-up">
<button class="btn dropdown-item" onclick="rebuildIndex()" title="Rebuild search index">REBUILD INDEX</button>
</div>
</div>
{{end}}
main.go
+35
View File
@@ -50,6 +50,7 @@ func main() {
cacheDir := flag.String("cache", "./cache", "thumbnail cache directory")
user := flag.String("user", "", "basic auth username (empty = no auth)")
pass := flag.String("pass", "", "basic auth password")
reindexInterval := flag.Duration("reindex-interval", 30*time.Minute, "periodic search index rebuild interval (0 disables)")
flag.Parse()
root, err := filepath.Abs(*wikiDir)
@@ -85,8 +86,33 @@ func main() {
static.ServeHTTP(w, r)
}))
http.HandleFunc("/_logout", h.handleLogout)
http.HandleFunc("/_reindex", h.handleReindex)
http.Handle("/", h)
// Build the folder index off the request path so the listener can start
// accepting connections immediately. searchWiki blocks on folderIndex.ready
// so the first search after a cold start still returns correct results.
go func() {
folderIndex.buildMu.Lock()
entries := buildFolderIndex(root)
folderIndex.Lock()
folderIndex.entries = entries
folderIndex.builtAt = time.Now()
folderIndex.Unlock()
folderIndex.buildMu.Unlock()
close(folderIndex.ready)
}()
if *reindexInterval > 0 {
go func(interval time.Duration) {
t := time.NewTicker(interval)
defer t.Stop()
for range t.C {
rebuildFolderIndex(root)
}
}(*reindexInterval)
}
log.Printf("datascape listening on %s, wiki at %s", *addr, root)
log.Fatal(http.ListenAndServe(*addr, nil))
}
@@ -312,6 +338,10 @@ func (h *handler) handlePost(w http.ResponseWriter, r *http.Request, urlPath, fs
return
}
} else {
// Stat first so we know whether MkdirAll actually created the folder
// — if it did, the search index needs a new entry.
_, statErr := os.Stat(fsPath)
newlyCreated := os.IsNotExist(statErr)
if err := os.MkdirAll(fsPath, 0755); err != nil {
http.Error(w, "mkdir failed: "+err.Error(), http.StatusInternalServerError)
return
@@ -320,6 +350,11 @@ func (h *handler) handlePost(w http.ResponseWriter, r *http.Request, urlPath, fs
http.Error(w, "write failed: "+err.Error(), http.StatusInternalServerError)
return
}
if newlyCreated {
if rel, err := filepath.Rel(h.root, fsPath); err == nil {
folderIndexAdd(filepath.ToSlash(rel))
}
}
}
http.Redirect(w, r, redirectTarget, http.StatusSeeOther)
}
moves.go
+2
View File
@@ -90,6 +90,7 @@ func (h *handler) handleMove(w http.ResponseWriter, r *http.Request, srcURL, src
http.Error(w, "rename failed: "+err.Error(), http.StatusInternalServerError)
return
}
folderIndexRenameSubtree(strings.TrimPrefix(oldPath, "/"), strings.TrimPrefix(newPath, "/"))
http.Redirect(w, r, wikiTargetHref(newPath), http.StatusSeeOther)
}
@@ -109,6 +110,7 @@ func (h *handler) handleDelete(w http.ResponseWriter, r *http.Request, urlPath,
http.Error(w, "delete failed: "+err.Error(), http.StatusInternalServerError)
return
}
folderIndexRemoveSubtree(strings.TrimPrefix(normalizeMovePath(urlPath), "/"))
http.Redirect(w, r, parentURL(urlPath), http.StatusSeeOther)
}
search.go
+199 -49
View File
@@ -7,6 +7,8 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"
"unicode"
)
@@ -18,30 +20,55 @@ type searchResult struct {
}
type searchPageData struct {
Title string
Crumbs []crumb
EditMode bool
Query string
Results []searchResult
RenderMS int64
Title string
Crumbs []crumb
EditMode bool
Query string
Results []searchResult
IndexBuiltAt time.Time
RenderMS int64
}
// handleSearch walks the wiki root and renders a search results page for the
// query in r.URL.Query().Get("q"). Only invoked when path is "/" and "q" is
// present.
// folderEntry is a single indexed directory: its forward-slash relative path
// plus pre-tokenized basename so the per-query scoring loop avoids redoing
// the lowercasing and tokenization on every keystroke.
type folderEntry struct {
Path string
NameLower string
NameTokens []string
}
// folderIndex holds the in-memory directory index used by search. Writers
// always replace the entries slice wholesale so a reader that snapshots the
// header under RLock can score without holding the lock.
var folderIndex struct {
sync.RWMutex
entries []folderEntry
builtAt time.Time
buildMu sync.Mutex
ready chan struct{}
}
func init() {
folderIndex.ready = make(chan struct{})
}
// handleSearch renders the search results page for the query in
// r.URL.Query().Get("q"). Only invoked when path is "/" and "q" is present.
func (h *handler) handleSearch(w http.ResponseWriter, r *http.Request) {
query := strings.TrimSpace(r.URL.Query().Get("q"))
results := searchWiki(h.root, query)
results, builtAt := searchWiki(query)
title := "Search"
if query != "" {
title = "Search: " + query
}
data := searchPageData{
Title: title,
Crumbs: []crumb{{Name: "search", URL: "/?q=" + query}},
Query: query,
Results: results,
Title: title,
Crumbs: []crumb{{Name: "search", URL: "/?q=" + query}},
Query: query,
Results: results,
IndexBuiltAt: builtAt,
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
data.RenderMS = elapsedMS(r)
@@ -50,48 +77,39 @@ func (h *handler) handleSearch(w http.ResponseWriter, r *http.Request) {
}
}
// searchWiki walks root and scores each directory by how well the folder name
// matches the query. Page contents are not searched. Higher score = more
// relevant; exact matches rank first.
func searchWiki(root, query string) []searchResult {
// searchWiki scores the cached folder index against query. Blocks on the
// initial build so the very first request after startup serves correct
// results rather than an empty list. Returns the snapshot's builtAt so the
// UI can show how fresh the index is.
func searchWiki(query string) ([]searchResult, time.Time) {
<-folderIndex.ready
folderIndex.RLock()
entries := folderIndex.entries
builtAt := folderIndex.builtAt
folderIndex.RUnlock()
if query == "" {
return nil
return nil, builtAt
}
qLower := strings.ToLower(query)
qTokens := tokenize(qLower)
if len(qTokens) == 0 {
return nil
return nil, builtAt
}
walkRoot := resolveWalkRoot(root)
var results []searchResult
_ = filepath.WalkDir(walkRoot, func(fsPath string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if skip, walkErr := hiddenSkip(fsPath, walkRoot, d); skip {
return walkErr
}
if !d.IsDir() || fsPath == walkRoot {
return nil
}
name := d.Name()
score := scoreName(strings.ToLower(name), qLower, qTokens)
for _, e := range entries {
score := scoreName(e.NameLower, e.NameTokens, qLower, qTokens)
if score == 0 {
return nil
}
rel, relErr := filepath.Rel(walkRoot, fsPath)
if relErr != nil {
return nil
continue
}
results = append(results, searchResult{
Name: name,
URL: "/" + filepath.ToSlash(rel) + "/",
Path: filepath.ToSlash(rel),
Name: filepath.Base(e.Path),
URL: "/" + e.Path + "/",
Path: e.Path,
Score: score,
})
return nil
})
}
sort.SliceStable(results, func(i, j int) bool {
if results[i].Score != results[j].Score {
@@ -103,22 +121,20 @@ func searchWiki(root, query string) []searchResult {
}
return strings.ToLower(results[i].Name) < strings.ToLower(results[j].Name)
})
return results
return results, builtAt
}
// scoreName ranks how well nameLower matches the query. Whole-name exact
// match dominates; otherwise score is the sum of each token's best match
// against the words in the name. Position within the name does not matter —
// nesting depth is the tiebreaker, applied by the caller.
func scoreName(nameLower, qLower string, qTokens []string) int {
// against the words in the name. nameTokens is precomputed by the index.
func scoreName(nameLower string, nameTokens []string, qLower string, qTokens []string) int {
if nameLower == qLower {
return 1000
}
score := 0
nameWords := tokenize(nameLower)
for _, qt := range qTokens {
best := 0
for _, w := range nameWords {
for _, w := range nameTokens {
switch {
case w == qt:
if best < 100 {
@@ -143,6 +159,140 @@ func scoreName(nameLower, qLower string, qTokens []string) int {
return score
}
// handleReindex rebuilds the folder index synchronously and returns 204.
// The frontend reloads the page on success. Serialized via buildMu so a
// double-click waits rather than running two walks in parallel.
func (h *handler) handleReindex(w http.ResponseWriter, r *http.Request) {
if !h.checkAuth(w, r) {
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
rebuildFolderIndex(h.root)
w.WriteHeader(http.StatusNoContent)
}
// buildFolderIndex walks root and returns a fresh slice of folder entries.
// Hidden directories (`.git`, `.thumbs`, …) are pruned; the root itself is
// excluded since it cannot be a search match.
func buildFolderIndex(root string) []folderEntry {
walkRoot := resolveWalkRoot(root)
var entries []folderEntry
_ = filepath.WalkDir(walkRoot, func(fsPath string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if skip, walkErr := hiddenSkip(fsPath, walkRoot, d); skip {
return walkErr
}
if !d.IsDir() || fsPath == walkRoot {
return nil
}
rel, relErr := filepath.Rel(walkRoot, fsPath)
if relErr != nil {
return nil
}
entries = append(entries, newFolderEntry(filepath.ToSlash(rel)))
return nil
})
return entries
}
// newFolderEntry builds a folderEntry from a forward-slash relative path,
// computing the lowercased basename and its tokens once so search scoring
// doesn't have to redo it per query.
func newFolderEntry(relPath string) folderEntry {
name := relPath
if i := strings.LastIndex(relPath, "/"); i >= 0 {
name = relPath[i+1:]
}
nameLower := strings.ToLower(name)
return folderEntry{
Path: relPath,
NameLower: nameLower,
NameTokens: tokenize(nameLower),
}
}
// rebuildFolderIndex walks root and replaces the index entries atomically.
// buildMu serializes overlapping rebuilds (manual + ticker + startup) so
// the WalkDir cost is paid once even under contention.
func rebuildFolderIndex(root string) {
folderIndex.buildMu.Lock()
defer folderIndex.buildMu.Unlock()
entries := buildFolderIndex(root)
folderIndex.Lock()
folderIndex.entries = entries
folderIndex.builtAt = time.Now()
folderIndex.Unlock()
}
// folderIndexAdd appends relPath as a new entry. No-op for empty/root paths.
func folderIndexAdd(relPath string) {
relPath = strings.Trim(relPath, "/")
if relPath == "" {
return
}
folderIndex.Lock()
folderIndex.entries = append(folderIndex.entries, newFolderEntry(relPath))
folderIndex.Unlock()
}
// folderIndexRemoveSubtree drops the entry at relPath plus every descendant.
// Replaces the slice rather than mutating in place so any in-flight search
// reader keeps a valid snapshot.
func folderIndexRemoveSubtree(relPath string) {
relPath = strings.Trim(relPath, "/")
if relPath == "" {
return
}
prefix := relPath + "/"
folderIndex.Lock()
defer folderIndex.Unlock()
old := folderIndex.entries
out := make([]folderEntry, 0, len(old))
for _, e := range old {
if e.Path == relPath || strings.HasPrefix(e.Path, prefix) {
continue
}
out = append(out, e)
}
folderIndex.entries = out
}
// folderIndexRenameSubtree rewrites the path prefix for every entry under
// oldRel. The renamed root entry's basename may have changed so its
// NameLower/NameTokens are recomputed; descendants keep their basenames.
func folderIndexRenameSubtree(oldRel, newRel string) {
oldRel = strings.Trim(oldRel, "/")
newRel = strings.Trim(newRel, "/")
if oldRel == "" || newRel == "" {
return
}
oldPrefix := oldRel + "/"
folderIndex.Lock()
defer folderIndex.Unlock()
old := folderIndex.entries
out := make([]folderEntry, len(old))
for i, e := range old {
switch {
case e.Path == oldRel:
out[i] = newFolderEntry(newRel)
case strings.HasPrefix(e.Path, oldPrefix):
out[i] = folderEntry{
Path: newRel + "/" + strings.TrimPrefix(e.Path, oldPrefix),
NameLower: e.NameLower,
NameTokens: e.NameTokens,
}
default:
out[i] = e
}
}
folderIndex.entries = out
}
// resolveWalkRoot resolves symlinks so WalkDir descends into the real tree
// even when the configured wiki root is itself a symlink (as on the NAS).
func resolveWalkRoot(root string) string {