diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f7d9fd3..4da2903 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,6 +20,24 @@ jobs:
           go-version: '1.25'
           cache: true
 
+      - name: gofmt (formatting gate)
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "These files are not gofmt-clean:"; echo "$unformatted"
+            echo "Run: gofmt -w ."; exit 1
+          fi
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Cross-compile for prod target (linux/arm64)
+        # Production runs on a GH200 (linux/arm64). Pure-Go + CGO_ENABLED=0
+        # means this is a clean cross-compile on the amd64 runner — no QEMU,
+        # no self-hosted runner. Catches a compile break on the prod arch
+        # that the host-arch test build would miss.
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ./cmd/cosift
+
       - name: Run tests with coverage
         # cosift's internal/crawler hits ~25s; allow 5m total headroom.
         run: go test -race -coverprofile=coverage.out -covermode=atomic -timeout 5m ./...
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 0000000..070d714
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,148 @@
+name: deploy
+
+# Build + sign + publish a release artifact. PULL-BASED deploy:
+# this workflow never touches the production box and never holds an SSH key.
+# The box runs cosift-self-update.timer, which polls the latest GitHub Release,
+# verifies sha256 + minisign signature against a public key baked on the box,
+# snapshots, atomically swaps the binary, restarts, and health-gates with
+# auto-rollback. See deploy/scripts/README.md.
+#
+# Triggers:
+#   - push of a version tag (v*)         → cut a release for that tag
+#   - manual workflow_dispatch           → build from a chosen ref (no release
+#                                          unless a tag is also present)
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+permissions:
+  contents: write   # needed to create the GitHub Release + upload assets
+
+jobs:
+  # (a) verify — gate the release on vet + race tests + a smoke subset.
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.25'
+          cache: true
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Tests (race)
+        run: go test -race -timeout 5m ./...
+
+      - name: Smoke subset (build + serve roundtrip)
+        # make smoke builds the binary, crawls, and asserts /healthz + /search.
+        # Guarded with a timeout so a hung crawl can't wedge the release.
+        run: timeout 300 make smoke
+
+  # (b) build — reproducible static arm64 binary, sha256, minisign signature.
+  build:
+    needs: verify
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.ver.outputs.version }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0   # full history so version stamping is meaningful
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.25'
+          cache: true
+
+      - name: Resolve version stamp
+        id: ver
+        # Tag name when triggered by a tag push; otherwise the short SHA.
+        run: |
+          if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+            echo "version=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build (linux/arm64, static, trimmed)
+        env:
+          CGO_ENABLED: '0'
+          GOOS: linux
+          GOARCH: arm64
+          VERSION: ${{ steps.ver.outputs.version }}
+        # Stamp both main.version and internal/server.Version so /healthz and
+        # /metrics report the shipped version. Matches the Makefile `build`
+        # target's ldflags exactly (kept in sync deliberately).
+        run: |
+          go build -trimpath \
+            -ldflags "-s -w -X main.version=${VERSION} -X github.com/pilot-protocol/cosift/internal/server.Version=${VERSION}" \
+            -o cosift-linux-arm64 ./cmd/cosift
+
+      - name: sha256
+        run: sha256sum cosift-linux-arm64 | tee cosift-linux-arm64.sha256
+
+      - name: Install minisign
+        run: sudo apt-get update && sudo apt-get install -y minisign
+
+      - name: Sign with minisign
+        env:
+          # Repo secret. The matching PUBLIC key is baked on the box at
+          # /etc/cosift/minisign.pub. Generate with `minisign -G` (see
+          # deploy/scripts/README.md). NEVER commit the secret key.
+          MINISIGN_SECRET_KEY: ${{ secrets.MINISIGN_SECRET_KEY }}
+        run: |
+          if [ -z "${MINISIGN_SECRET_KEY}" ]; then
+            echo "::error::MINISIGN_SECRET_KEY secret is not set" >&2
+            exit 1
+          fi
+          printf '%s' "${MINISIGN_SECRET_KEY}" > minisign.key
+          # -W: secret key is unencrypted (no interactive passphrase prompt).
+          # Trusted comment carries the version so the box can sanity-check it.
+          minisign -S -W -s minisign.key \
+            -m cosift-linux-arm64 \
+            -t "cosift ${{ steps.ver.outputs.version }} linux/arm64"
+          rm -f minisign.key
+          # Verification against the embedded public key is done on the box;
+          # here we just confirm the .minisig was produced.
+          test -f cosift-linux-arm64.minisig
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: cosift-release-artifacts
+          path: |
+            cosift-linux-arm64
+            cosift-linux-arm64.sha256
+            cosift-linux-arm64.minisig
+          retention-days: 30
+
+  # (c) release — publish the signed binary as a GitHub Release asset.
+  # Only on a tag push (workflow_dispatch builds the artifact but does not
+  # cut a release).
+  release:
+    needs: build
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v7
+        with:
+          name: cosift-release-artifacts
+
+      - name: Create / update GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: cosift ${{ github.ref_name }}
+          generate_release_notes: true
+          fail_on_unmatched_files: true
+          files: |
+            cosift-linux-arm64
+            cosift-linux-arm64.sha256
+            cosift-linux-arm64.minisig
diff --git a/Makefile b/Makefile
index 1ef0533..df9ef4e 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ smoke:
 	@./scripts/smoke-test.sh
 
 build:
-	CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(VERSION) -X github.com/calinteodor/cosift/internal/server.Version=$(VERSION)" -o $(BINARY) $(PKG)
+	CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(VERSION) -X github.com/pilot-protocol/cosift/internal/server.Version=$(VERSION)" -o $(BINARY) $(PKG)
 
 # Light-touch verification: compile + vet + unit tests on packages that
 # don't need OPENAI/COHERE keys or live network. Catches latent compile
diff --git a/cmd/cosift/assets/chat.html b/cmd/cosift/assets/chat.html
index b8c1a0a..68a536a 100644
--- a/cmd/cosift/assets/chat.html
+++ b/cmd/cosift/assets/chat.html
@@ -213,6 +213,22 @@
   font-family: var(--mono); font-size: 11px;
   padding: 6px 8px; cursor: pointer;
 }
+.site-input-wrap {
+  display: flex; align-items: center; gap: 4px;
+  border: 1px solid var(--line); border-radius: 8px;
+  background: var(--bg-2); padding: 0 8px;
+}
+.site-input-label {
+  font-family: var(--mono); font-size: 10px; color: var(--ink-dim);
+  white-space: nowrap; user-select: none; flex-shrink: 0;
+}
+.site-input {
+  border: none; background: transparent; color: var(--ink);
+  font-family: var(--mono); font-size: 11px;
+  padding: 5px 0; width: 140px; min-width: 0;
+  outline: none;
+}
+.site-input::placeholder { color: var(--ink-dim); opacity: 0.6; }
 .send-btn {
   padding: 14px 22px;
   background: var(--accent); color: var(--accent-ink);
@@ -324,6 +340,10 @@ <h1>Chat with the corpus.</h1>
   <div class="composer-row">
     <textarea id="input" class="composer-input" rows="1" placeholder="Ask anything…" autofocus></textarea>
     <div class="composer-controls">
+      <div class="site-input-wrap" title="Restrict results to a specific domain, e.g. pilotprotocol.network or docs.example.com/api">
+        <span class="site-input-label">site:</span>
+        <input id="site-filter" class="site-input" type="text" placeholder="domain.com" autocomplete="off" spellcheck="false">
+      </div>
       <select id="mode" title="query = LLM-planned hybrid (default) · answer = direct RAG · research = multi-step planner">
         <option value="query" selected>query</option>
         <option value="answer">answer</option>
@@ -332,7 +352,7 @@ <h1>Chat with the corpus.</h1>
       <button id="send" class="send-btn">Send →</button>
     </div>
   </div>
-  <div class="composer-hint">Enter to send · Shift+Enter newline · /clear to reset</div>
+  <div class="composer-hint">Enter to send · Shift+Enter newline · /clear to reset · site:domain.com to scope</div>
 </div>
 
 <script>
@@ -349,9 +369,16 @@ <h1>Chat with the corpus.</h1>
   var inputEl = document.getElementById('input');
   var sendBtn = document.getElementById('send');
   var modeEl = document.getElementById('mode');
+  var siteEl = document.getElementById('site-filter');
   var emptyEl = document.getElementById('empty-state');
   var clearBtn = document.getElementById('clear-btn');
 
+  // Persist site filter across page loads
+  try { var _s = localStorage.getItem('cosift-site'); if (_s) siteEl.value = _s; } catch(e) {}
+  siteEl.addEventListener('change', function () {
+    try { localStorage.setItem('cosift-site', siteEl.value.trim()); } catch(e) {}
+  });
+
   function esc(s) { return String(s).replace(/[&<>"]/g, function(c){ return ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]); }); }
 
   function autoresize() {
@@ -519,7 +546,19 @@ <h1>Chat with the corpus.</h1>
       autoresize();
       return;
     }
-    appendMsg('user', q);
+
+    // Allow inline site:domain.com syntax — strip it from the query and
+    // populate the site filter input. e.g. "pilot tunnels site:pilotprotocol.network"
+    var siteMatch = q.match(/(?:^|\s)site:(\S+)/);
+    var site = siteEl.value.trim();
+    if (siteMatch) {
+      q = q.replace(siteMatch[0], ' ').trim();
+      site = siteMatch[1];
+      siteEl.value = site;
+      try { localStorage.setItem('cosift-site', site); } catch(e) {}
+    }
+
+    appendMsg('user', q + (site ? '  [site: ' + site + ']' : ''));
     inputEl.value = '';
     autoresize();
 
@@ -540,6 +579,7 @@ <h1>Chat with the corpus.</h1>
     // progress (planner sub-queries, per-expansion hit counts, judge
     // drops, synth tokens).
     var url = endpoint + '?q=' + encodeURIComponent(q) + '&k=5&stream=true';
+    if (site) url += '&site=' + encodeURIComponent(site);
 
     try {
       var resp = await fetch(url, { headers: { 'Accept': 'text/event-stream' } });
diff --git a/cmd/cosift/assets/landing.html b/cmd/cosift/assets/landing.html
index 9482b62..f11b1b4 100644
--- a/cmd/cosift/assets/landing.html
+++ b/cmd/cosift/assets/landing.html
@@ -340,6 +340,9 @@ <h2>Search the <em>live</em> <span class="accent">corpus.</span></h2>
         <button class="btn primary" type="submit">Search <span class="arr">→</span></button>
       </form>
       <div class="tryit-knobs">
+        <label>site
+          <input type="text" id="tryit-site" placeholder="domain.com" style="width:140px; font-family:monospace" title="Restrict results to a domain, e.g. pilotprotocol.network" autocomplete="off" spellcheck="false"/>
+        </label>
         <label>retriever
           <select id="tryit-retriever">
             <option value="bm25" selected>bm25</option>
@@ -561,6 +564,8 @@ <h2>Ask <em>anything.</em></h2>
     var p = new URLSearchParams();
     p.set('q', q);
     p.set('k', document.getElementById('tryit-k').value || '5');
+    var site = document.getElementById('tryit-site').value.trim();
+    if (site) p.set('site', site);
     var retr = document.getElementById('tryit-retriever').value;
     if (retr && retr !== 'bm25') p.set('retriever', retr);
     var mmr = document.getElementById('tryit-mmr').value;
diff --git a/cmd/cosift/backfill_host_postings.go b/cmd/cosift/backfill_host_postings.go
new file mode 100644
index 0000000..d168735
--- /dev/null
+++ b/cmd/cosift/backfill_host_postings.go
@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// runBackfillHostPostings rebuilds the 'P' host-partition family from existing
+// 'h' + 'p' data so site= queries can scan a single host's postings
+// (O(site_docs)) via SearchInHost instead of the global lists (O(corpus)).
+//
+// It re-uses the data already on disk — no re-tokenize, no embeddings — so it's
+// a one-time key reshuffle, typically ~1-2h on a multi-million-doc corpus. The
+// store opens read-write because it writes the new family, but it never touches
+// 'p'/'d'/'v', so it is safe to run alongside a live pebble-serve. After it
+// completes, set COSIFT_HOST_PARTITION_READ=1 on the server to route site=
+// queries to the partition (and COSIFT_HOST_PARTITION=1 to keep it fresh on
+// future crawls).
+//
+//	cosift backfill-host-postings -dir /home/ubuntu/cosift-data/pebble
+func runBackfillHostPostings(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("backfill-host-postings", flag.ExitOnError)
+	dir := fs.String("dir", "", "PebbleStore directory (required; same dir as pebble-serve -dir)")
+	host := fs.String("host", "", "backfill only this host (fast 'g'-index path); empty = full corpus single-pass")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *dir == "" {
+		return fmt.Errorf("-dir required")
+	}
+
+	ps, err := store.OpenPebble(*dir)
+	if err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	defer ps.Close()
+
+	start := time.Now()
+	if *host != "" {
+		fmt.Fprintf(os.Stderr, "backfill-host-postings: targeted backfill for host %q (fast 'g'-index path)…\n", *host)
+	} else {
+		fmt.Fprintf(os.Stderr, "backfill-host-postings: full-corpus single-pass 'p'→'P'…\n")
+	}
+	last := start
+	written, err := ps.BackfillHostPostings(ctx, *host, func(n int64) {
+		now := time.Now()
+		if now.Sub(last) >= 5*time.Second {
+			rate := float64(n) / time.Since(start).Seconds()
+			fmt.Fprintf(os.Stderr, "  %d host postings written (%.0f/s, %s elapsed)\n",
+				n, rate, time.Since(start).Round(time.Second))
+			last = now
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("backfill: %w (wrote %d before failing)", err, written)
+	}
+	fmt.Fprintf(os.Stderr, "backfill-host-postings: done — %d host postings in %s\n",
+		written, time.Since(start).Round(time.Second))
+	fmt.Fprintf(os.Stderr, "Next: set COSIFT_HOST_PARTITION_READ=1 (read) + COSIFT_HOST_PARTITION=1 (keep fresh) and restart pebble-serve.\n")
+	return nil
+}
diff --git a/cmd/cosift/cmd_admin.go b/cmd/cosift/cmd_admin.go
new file mode 100644
index 0000000..bd1aa4a
--- /dev/null
+++ b/cmd/cosift/cmd_admin.go
@@ -0,0 +1,878 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/server"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// runAdmin dispatches to a sub-operation under `cosift admin <op>`. Sub-ops
+// share bearer-token resolution: -token flag wins, then COSIFT_ADMIN_TOKEN env.
+// Empty token still hits the server (and gets a clear 401) — the CLI doesn't
+// pre-fail because the operator might be running an instance without admin
+// auth enabled (rare, but the failure path is the server's to own).
+// adminUsageError returns the multi-line help message printed when the
+// `cosift admin` parent is invoked without a subcommand. Listed alphabetically
+// by subcommand name so the output is stable across iterations.
+//
+// Extracted as a function (rather than a const) so tests can assert on the
+// listing's structure — every admin subcommand must appear here, in the same
+// alphabetic order it appears in runAdmin's dispatch switch.
+func adminUsageError() string {
+	return `admin: subcommand required.
+
+Available subcommands:
+  config                   Retriever defaults and capability flags.
+  recrawl <url...>         Re-enqueue specific URLs into the crawl frontier. Requires -y.
+  recrawl-domain <pattern> Bulk re-enqueue every doc matching the domain pattern. Requires -y or -dry-run.
+  reembed                  Re-embed every doc with the configured model (streams progress).
+                           Requires -y. Supports -drop-old and -since DATE.
+  stats                    Index counts, frontier breakdown, top domains, capability flags.
+                           Supports -summary for one-line output (monitoring / shell prompts).
+
+Common flags:
+  -server URL              cosift server URL (default: from local cosift.json)
+  -token TOKEN             admin bearer token (or COSIFT_ADMIN_TOKEN env)
+  -json                    raw JSON output instead of human-readable
+
+Usage: cosift admin <subcommand> [flags]
+`
+}
+
+func runAdmin(ctx context.Context, cfg *config.Config, op string, args []string) error {
+	switch op {
+	case "stats":
+		return runAdminStats(ctx, cfg, args)
+	case "config":
+		return runAdminConfig(ctx, cfg, args)
+	case "recrawl":
+		return runAdminRecrawl(ctx, cfg, args)
+	case "recrawl-domain":
+		return runAdminRecrawlDomain(ctx, cfg, args)
+	case "reembed":
+		return runAdminReembedCLI(ctx, cfg, args)
+	default:
+		return fmt.Errorf("unknown admin subcommand %q (want: stats | config | recrawl | recrawl-domain | reembed)", op)
+	}
+}
+
+// adminCommonFlags parses the -server/-token/-json flags shared by every admin
+// sub-operation. Returns parsed values + the leftover positional args (none for
+// stats/config; future ops like recrawl will have positional URL args).
+type adminCommonOpts struct {
+	server string
+	token  string
+	json   bool
+}
+
+func parseAdminCommon(name string, cfg *config.Config, args []string) (adminCommonOpts, []string, error) {
+	fs := flag.NewFlagSet(name, flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	server := fs.String("server", defaultServer, "cosift server URL")
+	// Token: -token flag overrides env. Env is the secure default — keeps the
+	// token out of shell history / process listings.
+	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
+	if err := fs.Parse(args); err != nil {
+		return adminCommonOpts{}, nil, err
+	}
+	resolvedToken := *token
+	if resolvedToken == "" {
+		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+	}
+	return adminCommonOpts{server: *server, token: resolvedToken, json: *jsonOut}, fs.Args(), nil
+}
+
+// adminRequest builds an authenticated request to the given /admin/* path. The
+// Authorization header is omitted when token is empty (server returns 401,
+// which is the right error to surface).
+func adminRequest(ctx context.Context, method, server, path, token string, body io.Reader) (*http.Request, error) {
+	endpoint := strings.TrimRight(server, "/") + path
+	req, err := http.NewRequestWithContext(ctx, method, endpoint, body)
+	if err != nil {
+		return nil, err
+	}
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	return req, nil
+}
+
+func runAdminStats(ctx context.Context, cfg *config.Config, args []string) error {
+	// inline flag parsing (vs parseAdminCommon) so we can add the
+	// -summary flag. Same shape as runAdminRecrawl's flag set —
+	// admin sub-ops with op-specific flags bypass the common helper.
+	fs := flag.NewFlagSet("admin stats", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response")
+	summary := fs.Bool("summary", false, "one-line compact output for monitoring / shell prompts")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *summary && *jsonOut {
+		return errors.New("-summary and -json are mutually exclusive")
+	}
+	resolvedToken := *token
+	if resolvedToken == "" {
+		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+	}
+
+	req, err := adminRequest(ctx, http.MethodGet, *serverURL, "/admin/stats", resolvedToken, nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	var st server.AdminStatsResponse
+	if err := json.Unmarshal(body, &st); err != nil {
+		return fmt.Errorf("decode: %w", err)
+	}
+	if *summary {
+		// Compact one-line format designed for: shell prompts, cron, status
+		// scripts. Always-present fields (docs/queued/errored). Embedder named
+		// only when configured (omitted with " · " to keep separator clean).
+		parts := []string{
+			fmt.Sprintf("%d docs", st.Documents),
+			fmt.Sprintf("%d queued", st.Frontier.Queued),
+			fmt.Sprintf("%d errored", st.Frontier.Errored),
+		}
+		if st.EmbedderModel != "" {
+			parts = append(parts, "embedder="+st.EmbedderModel)
+		}
+		fmt.Println(strings.Join(parts, " · "))
+		return nil
+	}
+	fmt.Println("=== Index ===")
+	fmt.Printf("  Documents:           %d\n", st.Documents)
+	fmt.Printf("  Terms:               %d\n", st.Terms)
+	fmt.Printf("  Passages:            %d\n", st.Passages)
+	if st.DocsWithPublishedAt > 0 {
+		pct := float64(st.DocsWithPublishedAt) / float64(st.Documents) * 100
+		fmt.Printf("  Docs w/ published_at: %d (%.1f%%)\n", st.DocsWithPublishedAt, pct)
+	}
+	// LLM-cache sizes. Show only when populated — a fresh
+	// deployment with no LLM cache yet shouldn't bloat the dashboard.
+	if st.Paraphrases > 0 || st.HyDECache > 0 {
+		fmt.Println()
+		fmt.Println("=== LLM caches ===")
+		if st.Paraphrases > 0 {
+			fmt.Printf("  Paraphrases:         %d\n", st.Paraphrases)
+		}
+		if st.HyDECache > 0 {
+			fmt.Printf("  HyDE passages:       %d\n", st.HyDECache)
+		}
+	}
+	fmt.Println()
+	fmt.Println("=== Frontier ===")
+	fmt.Printf("  Queued:    %d\n", st.Frontier.Queued)
+	fmt.Printf("  In-flight: %d\n", st.Frontier.InFlight)
+	fmt.Printf("  Done:      %d\n", st.Frontier.Done)
+	fmt.Printf("  Errored:   %d\n", st.Frontier.Errored)
+	fmt.Println()
+	fmt.Println("=== Capabilities ===")
+	fmt.Printf("  Dense:    %t\n", st.DenseEnabled)
+	fmt.Printf("  Answer:   %t\n", st.AnswerEnabled)
+	fmt.Printf("  Fetcher:  %t\n", st.FetcherEnabled)
+	if st.EmbedderModel != "" {
+		fmt.Printf("  Embedder: %s\n", st.EmbedderModel)
+	}
+	if st.ChatModel != "" {
+		fmt.Printf("  Chat:     %s\n", st.ChatModel)
+	}
+	if st.RerankerName != "" {
+		fmt.Printf("  Reranker: %s\n", st.RerankerName)
+	}
+	if len(st.TopDomains) > 0 {
+		fmt.Println()
+		fmt.Println("=== Top Domains ===")
+		// Map iteration order is random; sort by count desc, then domain asc.
+		type kv struct {
+			k string
+			v int64
+		}
+		pairs := make([]kv, 0, len(st.TopDomains))
+		for k, v := range st.TopDomains {
+			pairs = append(pairs, kv{k, v})
+		}
+		sort.Slice(pairs, func(i, j int) bool {
+			if pairs[i].v != pairs[j].v {
+				return pairs[i].v > pairs[j].v
+			}
+			return pairs[i].k < pairs[j].k
+		})
+		for _, p := range pairs {
+			fmt.Printf("  %-32s %d\n", p.k, p.v)
+		}
+	}
+	return nil
+}
+
+func runAdminConfig(ctx context.Context, cfg *config.Config, args []string) error {
+	opts, _, err := parseAdminCommon("admin config", cfg, args)
+	if err != nil {
+		return err
+	}
+	req, err := adminRequest(ctx, http.MethodGet, opts.server, "/admin/config", opts.token, nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", opts.server, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+	if opts.json {
+		fmt.Println(string(body))
+		return nil
+	}
+	var cfgResp server.AdminConfigResponse
+	if err := json.Unmarshal(body, &cfgResp); err != nil {
+		return fmt.Errorf("decode: %w", err)
+	}
+	fmt.Printf("Cosift version: %s\n\n", cfgResp.Version)
+	fmt.Println("=== Defaults ===")
+	d := cfgResp.Defaults
+	fmt.Printf("  Retriever: %s\n", d.Retriever)
+	fmt.Printf("  Expand:    %t\n", d.Expand)
+	fmt.Println()
+	fmt.Println("=== Capabilities ===")
+	c := cfgResp.Capabilities
+	fmt.Printf("  Dense:        %t\n", c.DenseEnabled)
+	fmt.Printf("  Chat:         %t\n", c.ChatEnabled)
+	fmt.Printf("  Rerank:       %t\n", c.RerankEnabled)
+	fmt.Printf("  Paraphraser:  %t\n", c.ParaphraserEnabled)
+	fmt.Printf("  Fetcher:      %t\n", c.FetcherEnabled)
+	fmt.Printf("  Admin:        %t\n", c.AdminEnabled)
+	if c.EmbedderModel != "" {
+		fmt.Printf("  Embedder model:  %s\n", c.EmbedderModel)
+	}
+	if c.ChatModel != "" {
+		fmt.Printf("  Chat model:      %s\n", c.ChatModel)
+	}
+	if c.RerankerName != "" {
+		fmt.Printf("  Reranker name:   %s\n", c.RerankerName)
+	}
+	return nil
+}
+
+// runAdminRecrawl re-enqueues the given URLs into the crawler frontier. First
+// destructive admin operation — requires `-y` to confirm. URLs come from
+// positional args, `-file`, or both (mirrors's `contents` CLI pattern).
+func runAdminRecrawl(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("admin recrawl", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
+	filePath := fs.String("file", "", "read URLs from FILE (one per line, # comments allowed)")
+	confirm := fs.Bool("y", false, "confirm the destructive operation (recrawl re-enqueues URLs into the frontier)")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	urls := fs.Args()
+	if *filePath != "" {
+		extra, err := readURLsFromFile(*filePath)
+		if err != nil {
+			return fmt.Errorf("read -file: %w", err)
+		}
+		urls = append(urls, extra...)
+	}
+	if len(urls) == 0 {
+		return errors.New("at least one URL or -file required")
+	}
+	if !*confirm {
+		return fmt.Errorf("admin recrawl is destructive (re-enqueues %d URL(s) into the crawl frontier). Add -y to confirm", len(urls))
+	}
+
+	resolvedToken := *token
+	if resolvedToken == "" {
+		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+	}
+
+	reqBody, err := json.Marshal(server.AdminRecrawlRequest{URLs: urls})
+	if err != nil {
+		return err
+	}
+	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/recrawl", resolvedToken, bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	client := &http.Client{Timeout: 60 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("post %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	var rr server.AdminRecrawlResponse
+	if err := json.Unmarshal(body, &rr); err != nil {
+		return fmt.Errorf("decode: %w", err)
+	}
+	fmt.Printf("Queued %d URL(s) for recrawl.\n", len(rr.Queued))
+	if len(rr.Errors) > 0 {
+		fmt.Printf("\n%d URL(s) failed to enqueue:\n", len(rr.Errors))
+		// Sort error URLs for stable output across runs.
+		keys := make([]string, 0, len(rr.Errors))
+		for k := range rr.Errors {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, u := range keys {
+			fmt.Printf("  %s: %s\n", u, rr.Errors[u])
+		}
+	}
+	return nil
+}
+
+// runAdminRecrawlDomain bulk-recrawls every doc whose domain matches `pattern`
+// (positional arg). Wraps's /admin/recrawl-by-domain endpoint. Requires
+// `-y` to commit or `-dry-run` to preview the matched count without queuing.
+// If both flags are set, `-dry-run` wins (safer-on-conflict).
+func runAdminRecrawlDomain(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("admin recrawl-domain", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
+	confirm := fs.Bool("y", false, "confirm the destructive operation (recrawl every matching URL)")
+	dryRun := fs.Bool("dry-run", false, "enumerate matched URLs without queuing — preview the blast radius")
+	// cap dry-run URL listing to avoid console spam on large matches.
+	// 0 = no list (count only — behavior). 20 = default safe cap.
+	// -1 = unlimited (operator explicitly wants the full list).
+	listLimit := fs.Int("limit-list", 20, "in dry-run mode, max URLs to print (0 = count only, -1 = unlimited)")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() < 1 {
+		return errors.New("domain pattern required (positional arg)")
+	}
+	if fs.NArg() > 1 {
+		return errors.New("only one domain pattern supported per call")
+	}
+	pattern := fs.Arg(0)
+
+	if !*confirm && !*dryRun {
+		return fmt.Errorf("admin recrawl-domain is destructive (would re-enqueue every URL matching %q). Add -y to commit or -dry-run to preview", pattern)
+	}
+
+	resolvedToken := *token
+	if resolvedToken == "" {
+		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+	}
+
+	// dry-run wins when both flags are set. Operator who typed both probably
+	// meant the safer option; the destructive interpretation requires they
+	// explicitly drop -dry-run.
+	effectiveDryRun := *dryRun
+
+	reqBody, err := json.Marshal(server.AdminRecrawlByDomainRequest{
+		Domain: pattern,
+		DryRun: effectiveDryRun,
+	})
+	if err != nil {
+		return err
+	}
+	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/recrawl-by-domain", resolvedToken, bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	client := &http.Client{Timeout: 120 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("post %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	var rr server.AdminRecrawlByDomainResponse
+	if err := json.Unmarshal(body, &rr); err != nil {
+		return fmt.Errorf("decode: %w", err)
+	}
+	if effectiveDryRun {
+		fmt.Printf("Dry-run: %d URL(s) match %q. Re-run with -y (not -dry-run) to queue them.\n", rr.Matched, rr.Domain)
+		// -limit-list 0 → count only. -limit-list -1 → unlimited (operator
+		// wants the full list, accepts console spam). Otherwise: print up
+		// to listLimit and append a "... (N more)" suffix.
+		if *listLimit != 0 && len(rr.URLs) > 0 {
+			n := *listLimit
+			if n < 0 || n > len(rr.URLs) {
+				n = len(rr.URLs)
+			}
+			for _, u := range rr.URLs[:n] {
+				fmt.Printf("  %s\n", u)
+			}
+			if remaining := len(rr.URLs) - n; remaining > 0 {
+				fmt.Printf("  ... (%d more — use -limit-list -1 to see all)\n", remaining)
+			}
+		}
+		return nil
+	}
+	fmt.Printf("Queued %d/%d URL(s) for recrawl matching %q.\n", rr.Queued, rr.Matched, rr.Domain)
+	if len(rr.Errors) > 0 {
+		fmt.Printf("\n%d URL(s) failed to enqueue:\n", len(rr.Errors))
+		keys := make([]string, 0, len(rr.Errors))
+		for k := range rr.Errors {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, u := range keys {
+			fmt.Printf("  %s: %s\n", u, rr.Errors[u])
+		}
+	}
+	return nil
+}
+
+// runAdminReembedCLI triggers a server-side reembed via the
+// `/admin/reembed` SSE endpoint. Requires `-y` to commit (reembed costs
+// LLM credits; same destructive-op guard pattern as other admin verbs).
+// `-drop-old` propagates to the server's `DropPassagesNotModel` step.
+func runAdminReembedCLI(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("admin reembed", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
+	dropOld := fs.Bool("drop-old", false, "also delete passages from other models after re-embedding")
+	// -since restricts reembed to docs published >= DATE. Client-side
+	// validation catches malformed dates before the HTTP call (saves a round-trip
+	// for the common typo case).
+	since := fs.String("since", "", "ISO date — only re-embed docs published >= this. Drops undated docs when set")
+	confirm := fs.Bool("y", false, "confirm the operation (reembed re-runs every embedding call; counts as LLM spend)")
+	// -dry-run shows the would-be-processed count without spending
+	// LLM credits. Either -y OR -dry-run is required.
+	dryRun := fs.Bool("dry-run", false, "preview the count of docs that would be re-embedded, without actually embedding")
+	jsonOut := fs.Bool("json", false, "emit raw SSE response instead of human-readable progress lines")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	// Validate -since client-side before checking -y, so operators see "bad
+	// date" before they've committed with -y (clearer iteration).
+	if _, err := parseExportDate(*since); err != nil {
+		return fmt.Errorf("-since: %w", err)
+	}
+	// dry-run wins when both flags set (safer-on-conflict, matches
+	// pattern on /admin/recrawl-by-domain).
+	effectiveDryRun := *dryRun
+	if !*confirm && !*dryRun {
+		guard := "every doc"
+		if *dropOld {
+			guard = "every doc AND drop other-model passages"
+		}
+		if *since != "" {
+			guard = "every doc published >= " + *since
+			if *dropOld {
+				guard += " AND drop other-model passages"
+			}
+		}
+		return fmt.Errorf("admin reembed will re-embed %s (counts as LLM spend). Add -y to commit or -dry-run to preview", guard)
+	}
+
+	resolvedToken := *token
+	if resolvedToken == "" {
+		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+	}
+
+	reqBody, err := json.Marshal(server.AdminReembedRequest{
+		DropOld: *dropOld,
+		Since:   *since,
+		DryRun:  effectiveDryRun,
+	})
+	if err != nil {
+		return err
+	}
+	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/reembed", resolvedToken, bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "text/event-stream")
+	// Reembed can run for tens of minutes on large corpora. Bump the HTTP
+	// client timeout well past the synth CLIs; the server emits keep-alive
+	// progress events every ~2s so the connection stays warm.
+	client := &http.Client{Timeout: 60 * time.Minute}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("post %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+		return fmt.Errorf("server %d: %s", resp.StatusCode, b)
+	}
+	if *jsonOut {
+		// In JSON mode, just stream the raw SSE bytes through to stdout. Useful
+		// for piping into a custom parser or log file. Different shape from the
+		// other admin CLIs' -json because reembed has no terminal JSON blob —
+		// the event stream IS the response.
+		_, err := io.Copy(os.Stdout, resp.Body)
+		return err
+	}
+	return consumeReembedSSE(resp.Body)
+}
+
+// consumeReembedSSE reads /admin/reembed events and renders progress lines.
+// third SSE consumer in the CLI (after research and
+// answer), reusing the forEachSSEEvent scanner.
+func consumeReembedSSE(body io.Reader) error {
+	var sawDone bool
+	// track the started event's total_docs and target_model so the
+	// dry-run done-event branch can reference them (the dry-run done payload
+	// reports zeros for processed/passages, but the meaningful count is in started).
+	var startedTotal int
+	var startedTarget string
+
+	err := forEachSSEEvent(body, func(event, data string) error {
+		switch event {
+		case "started":
+			var s struct {
+				TotalDocs   int    `json:"total_docs"`
+				TargetModel string `json:"target_model"`
+			}
+			if err := json.Unmarshal([]byte(data), &s); err != nil {
+				return nil // tolerate malformed mid-stream event
+			}
+			startedTotal = s.TotalDocs
+			startedTarget = s.TargetModel
+			fmt.Printf("[reembed started: %d docs, target=%s]\n", s.TotalDocs, s.TargetModel)
+		case "progress":
+			var p struct {
+				DocsProcessed   int `json:"docs_processed"`
+				PassagesWritten int `json:"passages_written"`
+			}
+			if err := json.Unmarshal([]byte(data), &p); err != nil {
+				return nil
+			}
+			fmt.Printf("[progress: %d docs, %d passages]\n", p.DocsProcessed, p.PassagesWritten)
+		case "done":
+			sawDone = true
+			var d struct {
+				DocsProcessed   int    `json:"docs_processed"`
+				PassagesWritten int    `json:"passages_written"`
+				DroppedOld      int64  `json:"dropped_old"`
+				DryRun          bool   `json:"dry_run,omitempty"`
+				Took            string `json:"took"`
+			}
+			if err := json.Unmarshal([]byte(data), &d); err != nil {
+				return fmt.Errorf("parse done event: %w", err)
+			}
+			if d.DryRun {
+				// dry-run summary — reference the started event's
+				// count (the meaningful "would-be-processed" number).
+				fmt.Printf("Dry-run: %d docs would be re-embedded with target=%s. Re-run without -dry-run (or with -y) to actually do it.\n",
+					startedTotal, startedTarget)
+			} else {
+				fmt.Printf("Done: %d docs reembedded, %d passages written, %d dropped, took %s.\n",
+					d.DocsProcessed, d.PassagesWritten, d.DroppedOld, d.Took)
+			}
+			return errSSEDone
+		case "error":
+			var e struct {
+				Detail string `json:"detail"`
+			}
+			_ = json.Unmarshal([]byte(data), &e)
+			return fmt.Errorf("server stream error: %s", e.Detail)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if !sawDone {
+		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
+	}
+	return nil
+}
+
+// peekWarnings extracts the warnings field from a server
+// response body without committing to a full typed decode. When the CLI hits
+// pebble-serve, the response carries a warnings[] slice naming silent no-ops
+// (unknown expand, rerank without reranker, malformed include_domains, etc).
+// Without surfacing these on stderr the human CLI mode swallows them.
+func peekWarnings(body []byte) []string {
+	var probe struct {
+		Warnings []string `json:"warnings"`
+	}
+	if err := json.Unmarshal(body, &probe); err != nil {
+		return nil
+	}
+	return probe.Warnings
+}
+
+// emitWarnings prints each warning to stderr with a 'cosift:' prefix.
+func emitWarnings(body []byte) {
+	for _, w := range peekWarnings(body) {
+		fmt.Fprintln(os.Stderr, "cosift: warning:", w)
+	}
+}
+
+func runStats(ctx context.Context, cfg *config.Config, args []string) error {
+	// -backend flag (-backend=pebble reads the Pebble subdir).
+	fs := flag.NewFlagSet("stats", flag.ExitOnError)
+	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	switch *backend {
+	case "sqlite", "":
+		s, err := store.Open(cfg.DataDir)
+		if err != nil {
+			return err
+		}
+		defer s.Close()
+		stats, err := s.Stats(ctx)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("documents: %d\nterms: %d\ndata_dir: %s\nbackend: sqlite\n",
+			stats.Documents, stats.Terms, cfg.DataDir)
+	case "pebble":
+		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
+		ps, err := openPebbleOrFriendlyErr(pebbleDir)
+		if err != nil {
+			return err
+		}
+		defer ps.Close()
+		stats, err := ps.Stats(ctx)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("documents: %d\ndata_dir: %s\nbackend: pebble\n",
+			stats.Documents, pebbleDir)
+	default:
+		return fmt.Errorf("stats: unknown -backend %q (want: sqlite | pebble)", *backend)
+	}
+	return nil
+}
+
+// runCrawlStatus prints an operator-friendly snapshot of an ongoing crawl:
+// counts, frontier breakdown, top hosts by indexed-doc count, recent error
+// classes, and rolling-window doc rates with a 1M-doc ETA. Safe to run
+// concurrently with a live `cosift crawl` — SQLite WAL mode allows readers
+// alongside the writer.
+// runStatusFile reads the crawl-status.json (the file the live
+// crawler writes every 10s) and pretty-prints it. Useful when an operator
+// can't run `cosift stats -backend=pebble` because Pebble's single-writer
+// lock is held by the crawl process.
+//
+// Default path: <cfg.DataDir>/crawl-status.json. -file flag overrides.
+func runStatusFile(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("status-file", flag.ExitOnError)
+	path := fs.String("file", "", "path to crawl-status.json (defaults to <data_dir>/crawl-status.json)")
+	asJSON := fs.Bool("json", false, "emit raw status JSON plus a derived 'age_seconds' field; suitable for jq / Prometheus")
+	target := fs.Int64("target", 0, "doc count target — show indexed/target progress percent (default 0 = no target line)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	p := *path
+	if p == "" {
+		p = filepath.Join(cfg.DataDir, "crawl-status.json")
+	}
+	buf, err := os.ReadFile(p)
+	if err != nil {
+		return fmt.Errorf("read %s: %w", p, err)
+	}
+	var d struct {
+		Queued             int64     `json:"frontier_queued"`
+		InFlight           int64     `json:"frontier_in_flight"`
+		Done               int64     `json:"frontier_done"`
+		Errored            int64     `json:"frontier_errored"`
+		IndexedDocs        int64     `json:"indexed_docs,omitempty"`
+		IndexedDocsAtStart int64     `json:"indexed_docs_at_start,omitempty"`
+		AvgDocLen          float64   `json:"avg_doc_len,omitempty"`
+		StartedAt          time.Time `json:"started_at,omitempty"`
+		WrittenAt          time.Time `json:"written_at"`
+	}
+	if err := json.Unmarshal(buf, &d); err != nil {
+		return fmt.Errorf("decode %s: %w", p, err)
+	}
+	age := time.Since(d.WrittenAt).Round(time.Second)
+	if *asJSON {
+		out := map[string]any{
+			"path":                  p,
+			"frontier_queued":       d.Queued,
+			"frontier_in_flight":    d.InFlight,
+			"frontier_done":         d.Done,
+			"frontier_errored":      d.Errored,
+			"indexed_docs":          d.IndexedDocs,
+			"indexed_docs_at_start": d.IndexedDocsAtStart,
+			"avg_doc_len":           d.AvgDocLen,
+			"started_at":            d.StartedAt,
+			"written_at":            d.WrittenAt,
+			"age_seconds":           int64(age.Seconds()),
+			"stale":                 age > 30*time.Second,
+		}
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		return enc.Encode(out)
+	}
+	fmt.Printf("status file: %s  (written %s ago)\n\n", p, age)
+	fmt.Printf("  queued:     %d\n", d.Queued)
+	fmt.Printf("  in_flight:  %d\n", d.InFlight)
+	fmt.Printf("  done:       %d\n", d.Done)
+	fmt.Printf("  errored:    %d\n", d.Errored)
+	if d.IndexedDocs > 0 {
+		fmt.Printf("  indexed:    %d (avg doc length: %.0f tokens)\n", d.IndexedDocs, d.AvgDocLen)
+	}
+	total := d.Queued + d.InFlight + d.Done + d.Errored
+	if total > 0 {
+		processed := d.Done + d.Errored
+		fmt.Printf("  processed:  %d / %d (%.1f%%)\n",
+			processed, total, float64(processed)/float64(total)*100)
+	}
+	// ?-target N → 'indexed/target (pct)' line for long crawls toward
+	// a known doc-count goal. No-op when target is unset or already met.
+	// ETA from started_at + indexed_docs_at_start fields.
+	// Rate is averaged since the dumper's first poll, not instantaneous.
+	if *target > 0 && d.IndexedDocs > 0 {
+		pct := float64(d.IndexedDocs) / float64(*target) * 100
+		// cap pct display at 100% and append a 'reached' marker when
+		// the crawl has met or exceeded the goal — operators watching a long
+		// crawl see the win line instead of '237.4%' growing confusingly.
+		switch {
+		case d.IndexedDocs >= *target:
+			fmt.Printf("  target:     %d / %d (100.0%%, reached)\n", d.IndexedDocs, *target)
+		default:
+			fmt.Printf("  target:     %d / %d (%.1f%%)\n", d.IndexedDocs, *target, pct)
+		}
+		gained := d.IndexedDocs - d.IndexedDocsAtStart
+		elapsed := d.WrittenAt.Sub(d.StartedAt)
+		if gained > 0 && elapsed > time.Second && d.IndexedDocs < *target {
+			rate := float64(gained) / elapsed.Seconds()
+			remaining := float64(*target - d.IndexedDocs)
+			eta := time.Duration(remaining/rate) * time.Second
+			fmt.Printf("  rate:       %.1f docs/sec  eta: %s\n", rate, eta.Round(time.Second))
+		}
+	}
+	if age > 30*time.Second {
+		fmt.Printf("\n  WARNING: status is %s old; crawler may have stopped\n", age)
+	}
+	return nil
+}
+
+func runCrawlStatus(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("crawl-status", flag.ExitOnError)
+	hostsN := fs.Int("hosts", 10, "show top N hosts by indexed-doc count")
+	errsN := fs.Int("errors", 8, "show top N distinct error classes")
+	target := fs.Int64("target", 1_000_000, "doc count target for ETA projection")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	r, err := s.CrawlStatus(ctx, *hostsN, *errsN)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("crawl status — %s\n\n", cfg.DataDir)
+	fmt.Printf("  documents:        %d\n", r.Documents)
+	fmt.Printf("  terms:            %d\n", r.Terms)
+	fmt.Printf("  unique hosts:     %d\n", r.UniqueHosts)
+	fmt.Println()
+	fmt.Println("  frontier:")
+	for _, fs := range r.FrontierByStatus {
+		fmt.Printf("    %-12s %d\n", fs.Status, fs.Count)
+	}
+	fmt.Println()
+	fmt.Println("  doc rate (rolling, by fetched_at):")
+	for _, win := range r.RateWindows {
+		ratePerMin := float64(win.Count) / float64(win.WindowSec) * 60.0
+		fmt.Printf("    last %2dmin: %5d docs (%7.0f /min)\n", win.WindowSec/60, win.Count, ratePerMin)
+	}
+	if len(r.RateWindows) > 0 {
+		// Use the longest available rolling window for ETA — most stable signal.
+		win := r.RateWindows[len(r.RateWindows)-1]
+		ratePerMin := float64(win.Count) / float64(win.WindowSec) * 60.0
+		if ratePerMin > 0 {
+			remaining := *target - r.Documents
+			if remaining < 0 {
+				remaining = 0
+			}
+			etaHours := float64(remaining) / ratePerMin / 60.0
+			fmt.Printf("    → ETA to %d docs: %.1f hours = %.2f days (at the %dmin rate)\n",
+				*target, etaHours, etaHours/24.0, win.WindowSec/60)
+		}
+	}
+	if len(r.TopHosts) > 0 {
+		fmt.Printf("\n  top %d hosts (by indexed doc count):\n", len(r.TopHosts))
+		for _, h := range r.TopHosts {
+			fmt.Printf("    %-50s %d\n", truncStr(h.Host, 50), h.Count)
+		}
+	}
+	if len(r.ErrorClasses) > 0 {
+		fmt.Printf("\n  top %d error classes:\n", len(r.ErrorClasses))
+		for _, e := range r.ErrorClasses {
+			fmt.Printf("    %-60s %d\n", truncStr(e.LastError, 60), e.Count)
+		}
+	}
+	return nil
+}
diff --git a/cmd/cosift/cmd_crawl.go b/cmd/cosift/cmd_crawl.go
new file mode 100644
index 0000000..c71b4b9
--- /dev/null
+++ b/cmd/cosift/cmd_crawl.go
@@ -0,0 +1,459 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/authority"
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/crawler"
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func runCrawl(ctx context.Context, cfg *config.Config, args []string) error {
+	// hoisted PebbleStore handle so the embedder wiring below can
+	// reach it to attach the HNSW bridge after the backend switch.
+	var pebbleStoreForCrawl *store.PebbleStore
+	fs := flag.NewFlagSet("crawl", flag.ExitOnError)
+	refresh := fs.Bool("refresh", false, "force re-crawl of URLs already in the frontier")
+	sitemap := fs.String("sitemap", "", "URL of a sitemap.xml (or sitemap index) to seed from")
+	seedsFile := fs.String("seeds-file", "", "path to a text file with one seed URL per line (blank lines and # comments ignored)")
+	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
+	duration := fs.Duration("duration", 0, "stop the crawl cleanly after this much time (0 = run until frontier empty or SIGTERM)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	urls := fs.Args()
+	// -seeds-file lets operators target specific websites in bulk
+	// without stuffing dozens of URLs on the command line. Each non-blank,
+	// non-comment line is treated as a seed.
+	if *seedsFile != "" {
+		buf, err := os.ReadFile(*seedsFile)
+		if err != nil {
+			return fmt.Errorf("read -seeds-file %s: %w", *seedsFile, err)
+		}
+		for _, line := range strings.Split(string(buf), "\n") {
+			s := strings.TrimSpace(line)
+			if s == "" || strings.HasPrefix(s, "#") {
+				continue
+			}
+			urls = append(urls, s)
+		}
+		log.Printf("crawler: loaded %d seeds from %s (after dedup of positional args, total queued = %d)", len(urls)-fs.NArg(), *seedsFile, len(urls))
+	}
+	if len(urls) == 0 && *sitemap == "" {
+		return errors.New("crawl: at least one URL, -seeds-file, or -sitemap is required")
+	}
+
+	// Wraps the caller's ctx with a
+	// timeout; workers see ctx.Err() != nil and exit cleanly. Pebble flushes
+	// on Close (via the deferred ps.Close() below) so durability is preserved.
+	if *duration > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, *duration)
+		defer cancel()
+		log.Printf("crawler: bounded run, will stop after %s", *duration)
+	}
+
+	var c *crawler.Crawler
+	switch *backend {
+	case "sqlite", "":
+		s, err := store.Open(cfg.DataDir)
+		if err != nil {
+			return err
+		}
+		defer s.Close()
+		c = crawler.New(cfg.Crawler, s)
+	case "pebble":
+		// route through the  Pebble path. The
+		// data dir layout under cfg.DataDir for Pebble: a sibling "pebble"
+		// subdir so SQLite and Pebble stores can coexist during migration.
+		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
+		var err error
+		pebbleStoreForCrawl, err = openPebbleOrFriendlyErr(pebbleDir)
+		if err != nil {
+			return err
+		}
+		defer pebbleStoreForCrawl.Close()
+		c = crawler.NewWithBackend(cfg.Crawler, pebbleStoreForCrawl, index.NewPebbleBM25(pebbleStoreForCrawl))
+		log.Printf("crawler: pebble backend at %s", pebbleDir)
+	default:
+		return fmt.Errorf("crawl: unknown -backend %q (want: sqlite | pebble)", *backend)
+	}
+
+	// Authority-weighted frontier priority: load Tranco / Majestic CSVs when
+	// configured. The scorer is read-only after loading — safe to share with
+	// the crawler's concurrent worker pool.
+	if cfg.Crawler.AuthorityPriority {
+		scorer := authority.New()
+		if cfg.Crawler.AuthorityTrancoCSV != "" {
+			f, ferr := os.Open(cfg.Crawler.AuthorityTrancoCSV)
+			if ferr != nil {
+				log.Printf("crawler: authority: open tranco csv %s: %v (skipping)", cfg.Crawler.AuthorityTrancoCSV, ferr)
+			} else {
+				n, lerr := scorer.LoadTranco(f)
+				_ = f.Close()
+				if lerr != nil {
+					log.Printf("crawler: authority: load tranco: %v (skipping)", lerr)
+				} else {
+					log.Printf("crawler: authority: loaded %d tranco entries from %s", n, cfg.Crawler.AuthorityTrancoCSV)
+				}
+			}
+		}
+		if cfg.Crawler.AuthorityMajesticCSV != "" {
+			f, ferr := os.Open(cfg.Crawler.AuthorityMajesticCSV)
+			if ferr != nil {
+				log.Printf("crawler: authority: open majestic csv %s: %v (skipping)", cfg.Crawler.AuthorityMajesticCSV, ferr)
+			} else {
+				n, lerr := scorer.LoadMajestic(f)
+				_ = f.Close()
+				if lerr != nil {
+					log.Printf("crawler: authority: load majestic: %v (skipping)", lerr)
+				} else {
+					log.Printf("crawler: authority: loaded %d majestic entries from %s", n, cfg.Crawler.AuthorityMajesticCSV)
+				}
+			}
+		}
+		// Even without CSV data the embedded whitelist in authority.New() gives
+		// known-good sites (arxiv, MDN, Wikipedia, kernel.org, …) a priority boost.
+		c = c.WithAuthority(scorer)
+		log.Printf("crawler: authority-weighted frontier priority enabled (tranco=%s majestic=%s)",
+			cfg.Crawler.AuthorityTrancoCSV, cfg.Crawler.AuthorityMajesticCSV)
+	}
+
+	// Auto-wire embedder when configured. For the Pebble backend, also
+	// build and persist an HNSW graph via the hnswPassageWriter bridge
+	// — that's the path /search?retriever=dense
+	// needs and that pre was a documented no-op.
+	// API key is OPTIONAL when cfg.Embeddings.URL points at a
+	// custom endpoint (Ollama / vLLM / TEI / etc — local self-hosted
+	// embedders don't need a Bearer token). Required only when hitting the
+	// default OpenAI endpoint.
+	if cfg.Embeddings.Model != "" {
+		apiKey := resolveEmbedAPIKey()
+		needsKey := cfg.Embeddings.URL == "" // default OpenAI requires a key
+		if apiKey != "" || !needsKey {
+			dim := cfg.Embeddings.Dim
+			if dim == 0 {
+				dim = 1536
+			}
+			emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
+			c = c.WithEmbedder(emb)
+			authStatus := "anonymous"
+			if apiKey != "" {
+				authStatus = "bearer-token"
+			}
+			log.Printf("crawler: dense embeddings enabled (model=%s, dim=%d, auth=%s)", cfg.Embeddings.Model, dim, authStatus)
+			if pebbleStoreForCrawl != nil {
+				h := index.NewHNSW(dim)
+				c = c.WithPassageWriter(&hnswPassageWriter{ps: pebbleStoreForCrawl, hnsw: h})
+				// periodic checkpoint every COSIFT_HNSW_CHECKPOINT_SEC
+				// (default 60s). Without this, a crash / SIGKILL / time-cap
+				// deadline mid-crawl loses all in-memory vectors because Persist
+				// only runs at deferred end-of-crawl. Goroutine exits when
+				// ctx.Done() fires (either deadline or SIGTERM).
+				checkpointEvery := 60 * time.Second
+				if v := os.Getenv("COSIFT_HNSW_CHECKPOINT_SEC"); v != "" {
+					if s, err := strconv.Atoi(v); err == nil && s >= 5 {
+						checkpointEvery = time.Duration(s) * time.Second
+					}
+				}
+				log.Printf("crawler: HNSW vector index attached (in-memory build, checkpoint every %s, final persist at crawl end)", checkpointEvery)
+				ckpDone := make(chan struct{})
+				go func() {
+					defer close(ckpDone)
+					t := time.NewTicker(checkpointEvery)
+					defer t.Stop()
+					var lastN int
+					for {
+						select {
+						case <-ctx.Done():
+							return
+						case <-t.C:
+							n := h.Len()
+							if n == 0 || n == lastN {
+								continue // nothing new to persist
+							}
+							log.Printf("crawler: HNSW checkpoint at %d nodes (delta=%d) ...", n, n-lastN)
+							if err := h.Persist(context.Background(), pebbleStoreForCrawl); err != nil {
+								log.Printf("crawler: HNSW checkpoint failed: %v", err)
+								continue
+							}
+							lastN = n
+						}
+					}
+				}()
+				defer func() {
+					<-ckpDone // ensure ticker goroutine exits before final persist
+					if h.Len() == 0 {
+						return
+					}
+					log.Printf("crawler: final HNSW persist (%d nodes, dim=%d) to Pebble...", h.Len(), dim)
+					if err := h.Persist(context.Background(), pebbleStoreForCrawl); err != nil {
+						log.Printf("crawler: HNSW persist failed: %v", err)
+					} else {
+						log.Printf("crawler: HNSW persist complete")
+					}
+				}()
+			}
+		} else {
+			log.Printf("warning: embeddings configured (default OpenAI endpoint) but no OPENAI_API_KEY in env; crawling BM25 only")
+		}
+	}
+	if *sitemap != "" {
+		n, err := c.SeedSitemap(ctx, *sitemap)
+		if err != nil {
+			return fmt.Errorf("sitemap %s: %w", *sitemap, err)
+		}
+		log.Printf("seeded %d URLs from sitemap %s", n, *sitemap)
+	}
+	for _, u := range urls {
+		if *refresh {
+			if err := c.Recrawl(ctx, u); err != nil {
+				return fmt.Errorf("recrawl %s: %w", u, err)
+			}
+		} else {
+			if err := c.Seed(u); err != nil {
+				return fmt.Errorf("seed %s: %w", u, err)
+			}
+		}
+	}
+	return c.Run(ctx)
+}
+
+// runRefreshDue re-queues URLs whose adaptive interval has elapsed.
+// Combined with conditional GET, a refresh-due pass over a stable corpus
+// mostly costs 304 round-trips — no body, no parse, no embed.
+//
+// One-shot by default. Pass `-interval` to loop forever — useful when running
+// as a systemd service or inside the Docker image (no extra cron needed).
+func runRefreshDue(ctx context.Context, cfg *config.Config, args []string) (err error) {
+	fs := flag.NewFlagSet("refresh-due", flag.ExitOnError)
+	minH := fs.Duration("min", 1*time.Hour, "minimum re-crawl interval")
+	maxH := fs.Duration("max", 30*24*time.Hour, "maximum re-crawl interval")
+	limit := fs.Int("limit", 100, "max URLs to enqueue per pass")
+	dryRun := fs.Bool("dry-run", false, "print URLs that would be refreshed, don't enqueue")
+	interval := fs.Duration("interval", 0, "loop with this delay between passes (0 = one-shot)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	pass := func() error {
+		due, err := s.DueForRefresh(ctx, *minH, *maxH, *limit)
+		if err != nil {
+			return err
+		}
+		if len(due) == 0 {
+			log.Printf("refresh-due: 0 URLs due")
+			return nil
+		}
+		log.Printf("refresh-due: %d URLs due (limit %d, min=%s max=%s)", len(due), *limit, *minH, *maxH)
+		if *dryRun {
+			for _, u := range due {
+				fmt.Println(u)
+			}
+			return nil
+		}
+		c := crawler.New(cfg.Crawler, s)
+		for _, u := range due {
+			if err := c.Recrawl(ctx, u); err != nil {
+				log.Printf("recrawl %s: %v", u, err)
+			}
+		}
+		return c.Run(ctx)
+	}
+
+	if *interval <= 0 {
+		return pass()
+	}
+	// Daemon mode: pass, sleep, pass, ... until ctx cancelled.
+	for {
+		if err := pass(); err != nil {
+			log.Printf("refresh-due pass: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(*interval):
+		}
+	}
+}
+
+// siteToHost extracts the hostname from a URL or returns the input verbatim if
+// it's already a bare host (e.g. "docs.example.com" vs "https://docs.example.com").
+// Used by `cosift init -site` to populate include_domains.
+func siteToHost(s string) (string, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "", errors.New("empty site")
+	}
+	// If no scheme, treat as bare host and validate.
+	if !strings.Contains(s, "://") {
+		// reject if it looks like a path or has whitespace
+		if strings.ContainsAny(s, " /\t\n") {
+			return "", fmt.Errorf("not a hostname: %q", s)
+		}
+		return s, nil
+	}
+	u, err := url.Parse(s)
+	if err != nil {
+		return "", err
+	}
+	if u.Host == "" {
+		return "", fmt.Errorf("could not extract host from %q", s)
+	}
+	return u.Host, nil
+}
+
+// runCheckRobots reports whether each URL is crawlable under the site's
+// robots.txt, plus any Crawl-delay. Lets operators plan crawls without
+// hand-inspecting the site's robots.txt.
+//
+// Wraps the existing internal/crawler.Robots so the CLI uses exactly the
+// same enforcement logic the real crawler would. No drift.
+func runCheckRobots(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("check-robots", flag.ExitOnError)
+	userAgent := fs.String("user-agent", cfg.Crawler.UserAgent, "User-Agent the check pretends to be (defaults to cfg.crawler.user_agent)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	urls := fs.Args()
+	if len(urls) == 0 {
+		return errors.New("usage: cosift check-robots [-user-agent UA] <url...>")
+	}
+	if *userAgent == "" {
+		*userAgent = "CosiftBot/0.0 (+https://github.com/pilot-protocol/cosift)"
+	}
+	httpClient := &http.Client{Timeout: 15 * time.Second}
+	r := crawler.NewRobots(httpClient, *userAgent)
+
+	fmt.Printf("user-agent: %s\n", *userAgent)
+
+	// Probe robots.txt for each unique host before running per-URL checks.
+	// Robots.Allowed returns (true, ...) when robots.txt is unreachable
+	// (graceful degradation suits the crawler), but the operator running
+	// check-robots wants to see "this host has no reachable robots.txt"
+	// explicitly. Group URLs by host so we only probe each host once.
+	hosts := uniqueHosts(urls)
+	fmt.Println()
+	for _, h := range hosts {
+		probeURL := h + "/robots.txt"
+		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, http.NoBody)
+		req.Header.Set("User-Agent", *userAgent)
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			fmt.Printf("robots.txt   %s — UNREACHABLE: %v (crawler will assume ALLOWED for everything on this host)\n", probeURL, err)
+			continue
+		}
+		_ = resp.Body.Close()
+		switch {
+		case resp.StatusCode == 200:
+			fmt.Printf("robots.txt   %s — OK (status 200)\n", probeURL)
+		case resp.StatusCode == 404:
+			fmt.Printf("robots.txt   %s — 404, no rules (crawler will assume ALLOWED)\n", probeURL)
+		default:
+			fmt.Printf("robots.txt   %s — status %d (crawler will assume ALLOWED on non-2xx/404)\n", probeURL, resp.StatusCode)
+		}
+		// surface Sitemap: directives from robots.txt. Modern
+		// crawlers auto-discover sitemaps this way; this output lets operators
+		// see what `cosift crawl -sitemap <URL>` could seed from.
+		if sitemaps := r.Sitemaps(ctx, h); len(sitemaps) > 0 {
+			for _, sm := range sitemaps {
+				fmt.Printf("  Sitemap:   %s\n", sm)
+			}
+		}
+	}
+	fmt.Println()
+	for _, u := range urls {
+		allowed, crawlDelay, err := r.Allowed(ctx, u)
+		if err != nil {
+			fmt.Printf("[ERR ]  %s — %v\n", u, err)
+			continue
+		}
+		mark := "[OK  ]"
+		if !allowed {
+			mark = "[DENY]"
+		}
+		extra := ""
+		if crawlDelay > 0 {
+			extra = fmt.Sprintf("  crawl-delay=%s", crawlDelay)
+		}
+		fmt.Printf("%s  %s%s\n", mark, u, extra)
+	}
+	return nil
+}
+
+// uniqueHosts extracts the scheme+host part from each URL, deduped + in order.
+// Used by check-robots to probe each host's robots.txt exactly once.
+func uniqueHosts(urls []string) []string {
+	seen := make(map[string]bool)
+	out := []string{}
+	for _, raw := range urls {
+		u, err := url.Parse(raw)
+		if err != nil || u.Host == "" {
+			continue
+		}
+		key := u.Scheme + "://" + u.Host
+		if !seen[key] {
+			seen[key] = true
+			out = append(out, key)
+		}
+	}
+	return out
+}
+
+// runCrawlErrors lists recently-errored frontier URLs with their failure reason.
+// operator diagnostic for "why are 142 URLs in error state?" without
+// requiring SQLite shell access. Pure read-only; no side effects on the index.
+//
+// The error column is's addition to the frontier schema; pre
+// errored URLs will have an empty reason.
+func runCrawlErrors(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("crawl-errors", flag.ExitOnError)
+	limit := fs.Int("limit", 50, "max number of errored URLs to list (most recent first)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	defer s.Close()
+
+	errs, err := s.ListErroredFrontier(ctx, *limit)
+	if err != nil {
+		return fmt.Errorf("list errored: %w", err)
+	}
+	if len(errs) == 0 {
+		fmt.Println("no errored frontier entries.")
+		return nil
+	}
+	fmt.Printf("%d errored frontier entries (most recent first):\n", len(errs))
+	fmt.Println()
+	for _, e := range errs {
+		reason := e.LastError
+		if reason == "" {
+			reason = "(no reason recorded — entry predates)"
+		}
+		fmt.Printf("  attempts=%d  %s\n    → %s\n", e.Attempts, e.URL, reason)
+	}
+	return nil
+}
diff --git a/cmd/cosift/cmd_eval.go b/cmd/cosift/cmd_eval.go
new file mode 100644
index 0000000..bc1d73c
--- /dev/null
+++ b/cmd/cosift/cmd_eval.go
@@ -0,0 +1,1842 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/crawler"
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/eval"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/rerank"
+	"github.com/pilot-protocol/cosift/internal/server"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// bm25Adapter adapts index.BM25 to the eval.Retriever interface (URL-only result list).
+type bm25Adapter struct{ inner *index.BM25 }
+
+func (a *bm25Adapter) Search(ctx context.Context, q string, k int) ([]string, error) {
+	hits, err := a.inner.Search(ctx, q, k)
+	if err != nil {
+		return nil, err
+	}
+	urls := make([]string, len(hits))
+	for i, h := range hits {
+		urls[i] = h.URL
+	}
+	return urls, nil
+}
+
+// denseAdapter is a Retriever backed by an in-memory VectorIndex + an Embedder.
+// The embedder is consulted once per query at search time.
+type denseAdapter struct {
+	idx *index.VectorIndex
+	emb embed.Embedder
+}
+
+func (a *denseAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
+	vecs, err := a.emb.Embed(ctx, []string{q})
+	if err != nil {
+		return nil, err
+	}
+	hits := a.idx.Search(ctx, vecs[0], k)
+	urls := make([]string, len(hits))
+	for i, h := range hits {
+		urls[i] = h.URL
+	}
+	return urls, nil
+}
+
+// hybridAdapter fans out to two retrievers and fuses with RRF (k=60).
+type hybridAdapter struct {
+	a, b eval.Retriever
+}
+
+func (h *hybridAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
+	// Each retriever gets to vote on 2k candidates so RRF has room to work.
+	cand := k * 2
+	if cand < 10 {
+		cand = 10
+	}
+	ra, errA := h.a.Search(ctx, q, cand)
+	if errA != nil {
+		return nil, errA
+	}
+	rb, errB := h.b.Search(ctx, q, cand)
+	if errB != nil {
+		return nil, errB
+	}
+	return index.RRF([][]string{ra, rb}, k, 60), nil
+}
+
+// rerankAdapter wraps any Retriever with a reranker. The inner retriever is
+// asked for `candidateK` candidates; the reranker reorders them; the top k come back.
+// textByURL provides the passage text the reranker reads.
+type rerankAdapter struct {
+	inner      eval.Retriever
+	reranker   rerank.Reranker
+	textByURL  map[string]string
+	candidateK int
+}
+
+func (a *rerankAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
+	candK := a.candidateK
+	if candK <= 0 || candK < k {
+		candK = k * 5
+	}
+	urls, err := a.inner.Search(ctx, q, candK)
+	if err != nil {
+		return nil, err
+	}
+	if len(urls) <= 1 {
+		return urls, nil
+	}
+	cands := make([]rerank.Candidate, 0, len(urls))
+	for _, u := range urls {
+		text := a.textByURL[u]
+		if text == "" {
+			text = u
+		}
+		cands = append(cands, rerank.Candidate{ID: u, Text: text})
+	}
+	reranked, err := a.reranker.Rerank(ctx, q, cands)
+	if err != nil {
+		return urls, nil // fall back to inner order
+	}
+	if len(reranked) > k {
+		reranked = reranked[:k]
+	}
+	return reranked, nil
+}
+
+func runEval(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("eval", flag.ExitOnError)
+	corpusPath := fs.String("corpus", "testdata/eval/corpus.json", "corpus JSON")
+	queriesPath := fs.String("queries", "testdata/eval/queries.json", "queries JSON")
+	savePath := fs.String("save", "", "save run summary")
+	baselinePath := fs.String("baseline", "", "compare against a prior summary")
+	retriever := fs.String("retriever", "bm25", "retriever: bm25 | dense | hybrid")
+	embModel := fs.String("embed-model", "text-embedding-3-small", "embedding model name")
+	embURL := fs.String("embed-url", "", "embedding endpoint URL (default OpenAI)")
+	embDim := fs.Int("embed-dim", 1536, "embedding dimensionality")
+	embCacheDir := fs.String("embed-cache", "./eval-embed-cache", "embedding cache dir (set empty to disable)")
+	useRerank := fs.Bool("rerank", false, "wrap retriever with the LLM listwise reranker")
+	rerankModel := fs.String("rerank-model", "gpt-4o-mini", "chat model for the reranker")
+	candK := fs.Int("rerank-k", 20, "candidates pulled from inner retriever before rerank")
+	apiURL := fs.String("api", "", "if set, query the remote cosift HTTP API instead of building a local index")
+	apiBearer := fs.String("api-bearer", "", "optional bearer token for the remote API")
+	distractors := fs.Int("distractors", 0, "inject N synthetic distractor docs into the index before scoring (tests robustness vs an inflated candidate set)")
+	distractorSeed := fs.Int64("distractor-seed", 42, "deterministic seed for distractor generation")
+	distractorMode := fs.String("distractor-mode", "diverse", "diverse (90-word neutral pool) | narrow (single-topic 7-word pool)")
+	embBatch := fs.Int("embed-batch", 256, "embedding batch size — lower for large-dim models (text-embedding-3-large returns malformed JSON at 1000)")
+	chunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = index.NewChunker default 320); A/B across runs to compare retrieval at different granularities")
+	chunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = default 64)")
+	autoParaphrase := fs.Bool("auto-paraphrase", false, "generate N paraphrases per query via the chat client at eval time + RRF-fuse results")
+	paraphraseN := fs.Int("paraphrase-n", 2, "number of paraphrases per query")
+	paraphraseModel := fs.String("paraphrase-model", "gpt-4o-mini", "chat model for paraphrase generation")
+	mainWeight := fs.Float64("main-weight", 0, "main-query weight in -auto-paraphrase OR -planner RRF fusion (paraphrases / sub-queries each weight 1.0); 0 = equal-weight (standard RRF); mirrors server-side Defaults.ExpandMainWeight for offline measurement")
+	usePlanner := fs.Bool("planner", false, "decompose each query into 2-3 sub-queries via the /research planner prompt + RRF-fuse results (mirror /research retrieval, no synth)")
+	plannerModel := fs.String("planner-model", "gpt-4o-mini", "chat model for the planner decomposition")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	qs, err := eval.LoadQuerySet(*queriesPath)
+	if err != nil {
+		return err
+	}
+
+	// --api short-circuit: run the queries straight against a deployed cosift's
+	// /search endpoint. Skip the local corpus/embed/index work entirely —
+	// the deployed instance owns its own index. The corpus.json's relevance
+	// labels still apply because they're declared independently of the index.
+	if *apiURL != "" {
+		ret := &httpAPIRetriever{
+			baseURL:   strings.TrimRight(*apiURL, "/"),
+			bearer:    *apiBearer,
+			retriever: *retriever,
+			rerank:    *useRerank,
+			http:      &http.Client{Timeout: 30 * time.Second},
+		}
+		summary, e := eval.Run(ctx, qs, ret)
+		if e != nil {
+			return e
+		}
+		summary.Name = fmt.Sprintf("%s/api(%s)", qs.Name, *apiURL)
+		fmt.Print(eval.PrintTable(summary))
+		if *baselinePath != "" {
+			base, e := eval.LoadSummary(*baselinePath)
+			if e != nil {
+				return fmt.Errorf("baseline: %w", e)
+			}
+			fmt.Printf("\nvs baseline (%s):\n%s", *baselinePath, eval.Diff(base, summary))
+		}
+		if *savePath != "" {
+			if e := eval.SaveSummary(summary, *savePath); e != nil {
+				return fmt.Errorf("save: %w", e)
+			}
+			fmt.Printf("\nsaved summary to %s\n", *savePath)
+		}
+		return nil
+	}
+
+	corpus, err := eval.LoadCorpus(*corpusPath)
+	if err != nil {
+		return err
+	}
+
+	// Build an ephemeral store in a temp dir — eval doesn't touch the user's index.
+	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-eval-%d", time.Now().UnixNano()))
+	s, err := store.Open(tmpDir)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		s.Close()
+		_ = os.RemoveAll(tmpDir)
+	}()
+
+	bm := index.NewBM25(s)
+	for _, d := range corpus.Docs {
+		id, e := s.UpsertDocument(ctx, &store.Document{
+			URL: d.URL, Title: d.Title, Text: d.Text,
+			Source: "eval", FetchedAt: time.Now(),
+		})
+		if e != nil {
+			return fmt.Errorf("ingest %s: %w", d.URL, e)
+		}
+		if e := bm.IndexDocument(ctx, id, d.Title, d.Text); e != nil {
+			return fmt.Errorf("index %s: %w", d.URL, e)
+		}
+	}
+
+	// BM25 distractor injection: synthetic noise to stress-test retrieval at
+	// higher index sizes than the 20-doc canonical set. Each distractor is 100
+	// random tokens from a neutral vocab disjoint from the eval queries' vocab —
+	// they should NEVER be relevant, so any showing up in top-K = measurable degradation.
+	// Dense distractors are embedded alongside the corpus in the dense block below.
+	var distractorTexts []string
+	if *distractors > 0 {
+		drng := rand.New(rand.NewSource(*distractorSeed))
+		neutralVocab := neutralVocabForDistractors()
+		// Narrow mode: clamp to the first topic group (pottery — 7 words).
+		// Tests the hypothesis that smaller-but-more-similar noise
+		// is harder for the reranker than larger-but-diverse noise.
+		if *distractorMode == "narrow" {
+			neutralVocab = neutralVocab[:7]
+		} else if *distractorMode != "diverse" {
+			return fmt.Errorf("unknown distractor-mode %q (diverse | narrow)", *distractorMode)
+		}
+		distractorTexts = make([]string, *distractors)
+		for i := 0; i < *distractors; i++ {
+			dURL := fmt.Sprintf("https://distractor.test/%d", i)
+			dTitle := fmt.Sprintf("Distractor %d", i)
+			dText := generateDistractorText(drng, neutralVocab, 100)
+			distractorTexts[i] = dText
+			id, e := s.UpsertDocument(ctx, &store.Document{
+				URL: dURL, Title: dTitle, Text: dText, Source: "distractor", FetchedAt: time.Now(),
+			})
+			if e != nil {
+				return fmt.Errorf("distractor %d: %w", i, e)
+			}
+			if e := bm.IndexDocument(ctx, id, dTitle, dText); e != nil {
+				return fmt.Errorf("index distractor %d: %w", i, e)
+			}
+		}
+		fmt.Printf("injected %d distractor docs into BM25 index\n", *distractors)
+	}
+
+	// Build the requested retriever.
+	var ret eval.Retriever
+	switch *retriever {
+	case "bm25":
+		ret = &bm25Adapter{inner: bm}
+	case "dense", "hybrid":
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI") // accept the short form too
+		}
+		if apiKey == "" {
+			return errors.New("OPENAI_API_KEY (or OPENAI) not set (put it in .env)")
+		}
+		oai := embed.NewOpenAIClient(apiKey, *embURL, *embModel, *embDim)
+		var emb embed.Embedder = oai
+		if *embCacheDir != "" {
+			emb = embed.NewCachedEmbedder(oai, *embCacheDir)
+		}
+		// Chunk corpus into passage windows, then batch-embed.
+		// Same chunker the crawler uses → eval and serve behave identically.
+		chunker := chunkerWith(*chunkSize, *chunkOverlap)
+		type passageRef struct {
+			docIdx int
+			chunk  index.Chunk
+		}
+		var allTexts []string
+		var refs []passageRef
+		for i, d := range corpus.Docs {
+			text := d.Title + "\n\n" + d.Text
+			for _, c := range chunker.Chunk(text) {
+				allTexts = append(allTexts, c.Text)
+				refs = append(refs, passageRef{docIdx: i, chunk: c})
+			}
+		}
+		fmt.Printf("embedding %d passages across %d docs with %s (batch=%d)...\n", len(allTexts), len(corpus.Docs), *embModel, *embBatch)
+		vecs, e := batchEmbed(ctx, emb, allTexts, *embBatch)
+		if e != nil {
+			return fmt.Errorf("embed corpus: %w", e)
+		}
+		vi := index.NewVectorIndex(*embDim)
+		for j, v := range vecs {
+			d := corpus.Docs[refs[j].docIdx]
+			vi.AddPassage(d.URL, d.Title, refs[j].chunk.Offset, refs[j].chunk.Length, v)
+		}
+		// Embed + add distractor passages into the SAME vector index.
+		// batchEmbed splits into ≤1000-input chunks for provider limits.
+		if len(distractorTexts) > 0 {
+			fmt.Printf("embedding %d distractor passages (batch=%d)...\n", len(distractorTexts), *embBatch)
+			dvecs, err := batchEmbed(ctx, emb, distractorTexts, *embBatch)
+			if err != nil {
+				return fmt.Errorf("embed distractors: %w", err)
+			}
+			for i, v := range dvecs {
+				vi.AddPassage(fmt.Sprintf("https://distractor.test/%d", i), "Distractor", 0, len(distractorTexts[i]), v)
+			}
+		}
+		denseRet := &denseAdapter{idx: vi, emb: emb}
+		if *retriever == "dense" {
+			ret = denseRet
+		} else {
+			ret = &hybridAdapter{a: &bm25Adapter{inner: bm}, b: denseRet}
+		}
+	default:
+		return fmt.Errorf("unknown retriever %q", *retriever)
+	}
+
+	// Apply paraphrase + rerank wrappers below.
+	suffix := *retriever
+	if *useRerank {
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI")
+		}
+		if apiKey == "" {
+			return errors.New("rerank: OPENAI_API_KEY (or OPENAI) not set")
+		}
+		chat := embed.NewOpenAIChat(apiKey, "", *rerankModel)
+		rr := rerank.NewLLMReranker(chat)
+		textByURL := make(map[string]string, len(corpus.Docs))
+		for _, d := range corpus.Docs {
+			textByURL[d.URL] = d.Title + "\n\n" + d.Text
+		}
+		ret = &rerankAdapter{inner: ret, reranker: rr, textByURL: textByURL, candidateK: *candK}
+		suffix = fmt.Sprintf("%s+rerank(%s)", *retriever, *rerankModel)
+	}
+
+	// Planner wrapper — applied BEFORE paraphrase so a "both" run looks like
+	// (rerank ⊂ planner ⊂ paraphrase). Mirrors /research's plan→retrieve pipeline
+	// without the synth step, so we measure planner *retrieval coverage* in isolation.
+	if *usePlanner {
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI")
+		}
+		if apiKey == "" {
+			return errors.New("planner: OPENAI_API_KEY (or OPENAI) not set")
+		}
+		chat := embed.NewOpenAIChat(apiKey, "", *plannerModel)
+		ret = &plannerRetriever{inner: ret, chat: chat, mainWeight: *mainWeight, cache: make(map[string][]string)}
+		if *mainWeight > 0 {
+			suffix = fmt.Sprintf("%s+planner(%s,mw=%.2f)", suffix, *plannerModel, *mainWeight)
+		} else {
+			suffix = fmt.Sprintf("%s+planner(%s)", suffix, *plannerModel)
+		}
+	}
+
+	// Auto-paraphrase wrapper — outermost so paraphrases hit the FULL retriever
+	// stack (including rerank, if enabled).
+	if *autoParaphrase {
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI")
+		}
+		if apiKey == "" {
+			return errors.New("auto-paraphrase: OPENAI_API_KEY (or OPENAI) not set")
+		}
+		chat := embed.NewOpenAIChat(apiKey, "", *paraphraseModel)
+		ret = &paraphraseRetriever{inner: ret, chat: chat, n: *paraphraseN, mainWeight: *mainWeight, cache: make(map[string][]string)}
+		if *mainWeight > 0 {
+			suffix = fmt.Sprintf("%s+paraphrase(%d/%s,mw=%.2f)", suffix, *paraphraseN, *paraphraseModel, *mainWeight)
+		} else {
+			suffix = fmt.Sprintf("%s+paraphrase(%d/%s)", suffix, *paraphraseN, *paraphraseModel)
+		}
+	}
+
+	summary, err := eval.Run(ctx, qs, ret)
+	if err != nil {
+		return err
+	}
+	summary.Name = fmt.Sprintf("%s/%s", qs.Name, suffix)
+	fmt.Print(eval.PrintTable(summary))
+
+	if *baselinePath != "" {
+		base, err := eval.LoadSummary(*baselinePath)
+		if err != nil {
+			return fmt.Errorf("baseline: %w", err)
+		}
+		fmt.Printf("\nvs baseline (%s):\n%s", *baselinePath, eval.Diff(base, summary))
+	}
+	if *savePath != "" {
+		if err := eval.SaveSummary(summary, *savePath); err != nil {
+			return fmt.Errorf("save: %w", err)
+		}
+		fmt.Printf("\nsaved summary to %s\n", *savePath)
+	}
+	return nil
+}
+
+// runBench measures search latency under synthetic load. No API calls, no disk
+// for the vector path (in-memory only); BM25 uses a temp SQLite.
+//
+// Use it to decide:
+//   - When to swap brute-force kNN for HNSW (when vector p50 > ~50 ms or
+//     p99 > ~200 ms at your target N).
+//   - When BM25 needs Tantivy or another inverted-index lib (when BM25 p50
+//     exceeds a comparable threshold).
+//
+// benchResult holds one mode's bench output. Each field maps to a JSON key
+// when `-json` is set; human-readable output uses the formatHuman method.
+type benchResult struct {
+	Mode           string  `json:"mode"`
+	N              int     `json:"n"`
+	Dim            int     `json:"dim,omitempty"`
+	Queries        int     `json:"queries,omitempty"`
+	P50Micros      int64   `json:"p50_us,omitempty"`
+	P95Micros      int64   `json:"p95_us,omitempty"`
+	P99Micros      int64   `json:"p99_us,omitempty"`
+	QPS            float64 `json:"qps,omitempty"`
+	ElapsedMicros  int64   `json:"elapsed_us,omitempty"`
+	Docs           int64   `json:"docs,omitempty"`
+	PagesPerSec    float64 `json:"pages_per_sec,omitempty"`
+	Terms          int64   `json:"terms,omitempty"`
+	PerHostDelayMs int     `json:"per_host_delay_ms,omitempty"`
+}
+
+func runBench(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("bench", flag.ExitOnError)
+	n := fs.Int("n", 10000, "number of passages (for vector/bm25 modes) or pages (for crawl mode)")
+	dim := fs.Int("dim", 384, "embedding dimensionality (vector mode)")
+	queries := fs.Int("queries", 100, "queries per run (vector/bm25 modes)")
+	mode := fs.String("mode", "vector", "vector | bm25 | crawl | all")
+	perHostDelay := fs.Int("per-host-delay", 0, "per-host delay in ms for crawl mode (default 0 — localhost throughput; set 100-1000 to measure under realistic politeness)")
+	jsonOut := fs.Bool("json", false, "emit one JSON object per mode instead of a human-readable line — useful for CI ingestion + dashboards")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	emit := func(r *benchResult) {
+		if *jsonOut {
+			b, _ := json.Marshal(r)
+			fmt.Println(string(b))
+		} else {
+			fmt.Println(r.formatHuman())
+		}
+	}
+
+	if *mode == "vector" || *mode == "all" || *mode == "both" {
+		emit(benchVector(*n, *dim, *queries))
+	}
+	if *mode == "bm25" || *mode == "all" || *mode == "both" {
+		r, err := benchBM25(ctx, *n, *queries)
+		if err != nil {
+			return err
+		}
+		emit(r)
+	}
+	if *mode == "crawl" || *mode == "all" {
+		// Crawl mode uses a smaller default — N=10k pages of an in-process site
+		// would slow the bench without proportional signal. 200 is plenty to
+		// measure pages/sec accurately on a typical host.
+		crawlN := *n
+		if crawlN > 1000 {
+			crawlN = 200
+		}
+		r, err := benchCrawl(ctx, crawlN, *perHostDelay)
+		if err != nil {
+			return err
+		}
+		emit(r)
+	}
+	if *mode == "storage" || *mode == "all" {
+		// SQLite vs Pebble head-to-head. Same N synthetic docs, same
+		// K queries, two backends. Emit one result per backend; consumers diff
+		// the two lines (or use cosift bench-compare on the saved JSON).
+		sq, err := benchBM25SQLite(ctx, *n, *queries)
+		if err != nil {
+			return err
+		}
+		emit(sq)
+		pb, err := benchBM25Pebble(ctx, *n, *queries)
+		if err != nil {
+			return err
+		}
+		emit(pb)
+	}
+	return nil
+}
+
+// formatHuman renders the result as the one-line format the bench used in iter
+// 1–70 — preserves the existing UX when `-json` isn't set.
+func (r *benchResult) formatHuman() string {
+	switch r.Mode {
+	case "vector":
+		return fmt.Sprintf("vector  n=%d dim=%d  p50=%-9s p95=%-9s p99=%-9s qps=%.0f",
+			r.N, r.Dim,
+			time.Duration(r.P50Micros)*time.Microsecond,
+			time.Duration(r.P95Micros)*time.Microsecond,
+			time.Duration(r.P99Micros)*time.Microsecond,
+			r.QPS)
+	case "bm25":
+		return fmt.Sprintf("bm25    n=%d  p50=%-9s p95=%-9s p99=%-9s qps=%.0f",
+			r.N,
+			time.Duration(r.P50Micros)*time.Microsecond,
+			time.Duration(r.P95Micros)*time.Microsecond,
+			time.Duration(r.P99Micros)*time.Microsecond,
+			r.QPS)
+	case "crawl":
+		delay := ""
+		if r.PerHostDelayMs > 0 {
+			delay = fmt.Sprintf(" per-host-delay=%dms", r.PerHostDelayMs)
+		}
+		return fmt.Sprintf("crawl   n=%-5d elapsed=%-10s docs=%-5d  pages/sec=%-6.0f  terms=%d%s",
+			r.N,
+			(time.Duration(r.ElapsedMicros) * time.Microsecond).Round(time.Millisecond),
+			r.Docs, r.PagesPerSec, r.Terms, delay)
+	}
+	return fmt.Sprintf("%+v", *r)
+}
+
+// loadBenchReport parses a `cosift bench -json` output file. Format: one
+// benchResult JSON object per line (NDJSON). Lines that don't parse as JSON
+// are skipped silently — this lets users pipe `bench -json` into a file
+// alongside other tool output if they want.
+func loadBenchReport(path string) (map[string]*benchResult, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	out := make(map[string]*benchResult)
+	for _, line := range strings.Split(string(b), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || !strings.HasPrefix(line, "{") {
+			continue
+		}
+		var r benchResult
+		if err := json.Unmarshal([]byte(line), &r); err != nil {
+			continue // skip unparseable lines rather than failing the whole report
+		}
+		if r.Mode == "" {
+			continue
+		}
+		out[r.Mode] = &r
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("%s: no parseable bench records (expecting NDJSON from `cosift bench -json`)", path)
+	}
+	return out, nil
+}
+
+// runBenchCompare diffs two saved bench reports side-by-side. Mirrors
+// answer-eval-compare for the bench surface — locks the JSON contract
+// as a stable interface tools can rely on.
+//
+// For each mode present in either report, prints the absolute numbers + deltas.
+// Reports a mode as "missing in BASELINE" or "missing in NEW" rather than
+// silently skipping — operators benefit from knowing the compare wasn't
+// fully apples-to-apples.
+func runBenchCompare(args []string) error {
+	fs := flag.NewFlagSet("bench-compare", flag.ExitOnError)
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	rest := fs.Args()
+	if len(rest) != 2 {
+		return errors.New("usage: cosift bench-compare BASELINE.json NEW.json")
+	}
+	a, err := loadBenchReport(rest[0])
+	if err != nil {
+		return fmt.Errorf("baseline: %w", err)
+	}
+	b, err := loadBenchReport(rest[1])
+	if err != nil {
+		return fmt.Errorf("new: %w", err)
+	}
+	fmt.Printf("BASELINE: %s  (modes: %v)\n", rest[0], sortedModes(a))
+	fmt.Printf("NEW:      %s  (modes: %v)\n", rest[1], sortedModes(b))
+	fmt.Println()
+
+	// Compare any mode present in either report. Stable order: vector, bm25, crawl.
+	allModes := union(a, b)
+	for _, mode := range allModes {
+		ar := a[mode]
+		br := b[mode]
+		switch {
+		case ar == nil:
+			fmt.Printf("%-7s (missing in BASELINE — new in this run)\n", mode)
+		case br == nil:
+			fmt.Printf("%-7s (missing in NEW — present in BASELINE only)\n", mode)
+		default:
+			printBenchDelta(mode, ar, br)
+		}
+	}
+	return nil
+}
+
+func sortedModes(m map[string]*benchResult) []string {
+	order := []string{"vector", "bm25", "crawl"}
+	out := make([]string, 0, len(m))
+	for _, k := range order {
+		if _, ok := m[k]; ok {
+			out = append(out, k)
+		}
+	}
+	// Catch any modes outside the known order too.
+	for k := range m {
+		if k != "vector" && k != "bm25" && k != "crawl" {
+			out = append(out, k)
+		}
+	}
+	return out
+}
+
+func union(a, b map[string]*benchResult) []string {
+	seen := make(map[string]bool)
+	order := []string{"vector", "bm25", "crawl"}
+	out := []string{}
+	for _, k := range order {
+		if _, ok := a[k]; ok {
+			seen[k] = true
+			out = append(out, k)
+		}
+		if _, ok := b[k]; ok && !seen[k] {
+			seen[k] = true
+			out = append(out, k)
+		}
+	}
+	for _, m := range []map[string]*benchResult{a, b} {
+		for k := range m {
+			if !seen[k] {
+				seen[k] = true
+				out = append(out, k)
+			}
+		}
+	}
+	return out
+}
+
+// printBenchDelta prints one mode's before/after side-by-side. Format
+// matches answer-eval-compare's row style.
+func printBenchDelta(mode string, a, b *benchResult) {
+	switch mode {
+	case "vector", "bm25":
+		// QPS is the headline; latency percentiles are signal too.
+		fmt.Printf("%-7s n=%d→%d  qps %.0f→%.0f (%+.0f)  p50 %dµs→%dµs (%+dµs)  p95 %dµs→%dµs (%+dµs)\n",
+			mode, a.N, b.N,
+			a.QPS, b.QPS, b.QPS-a.QPS,
+			a.P50Micros, b.P50Micros, b.P50Micros-a.P50Micros,
+			a.P95Micros, b.P95Micros, b.P95Micros-a.P95Micros,
+		)
+	case "crawl":
+		fmt.Printf("%-7s n=%d→%d  pages/sec %.0f→%.0f (%+.0f)  elapsed %dms→%dms  docs %d→%d\n",
+			mode, a.N, b.N,
+			a.PagesPerSec, b.PagesPerSec, b.PagesPerSec-a.PagesPerSec,
+			a.ElapsedMicros/1000, b.ElapsedMicros/1000,
+			a.Docs, b.Docs,
+		)
+	default:
+		fmt.Printf("%-7s (unknown mode — raw: a=%+v b=%+v)\n", mode, *a, *b)
+	}
+}
+
+func benchVector(n, dim, queries int) *benchResult {
+	rng := newSeededRand()
+	vi := index.NewVectorIndex(dim)
+	for i := 0; i < n; i++ {
+		v := randUnit(rng, dim)
+		vi.AddPassage(fmt.Sprintf("https://test/p%d", i), "doc", i*100, 100, v)
+	}
+	// Time `queries` searches.
+	q := make([][]float32, queries)
+	for i := range q {
+		q[i] = randUnit(rng, dim)
+	}
+	lats := make([]time.Duration, queries)
+	for i, qv := range q {
+		start := time.Now()
+		_ = vi.Search(context.Background(), qv, 10)
+		lats[i] = time.Since(start)
+	}
+	p50, p95, p99 := percentiles(lats)
+	return &benchResult{
+		Mode: "vector", N: n, Dim: dim, Queries: queries,
+		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
+		QPS: float64(queries) / sumDur(lats).Seconds(),
+	}
+}
+
+func benchBM25(ctx context.Context, n, queries int) (*benchResult, error) {
+	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-bench-%d", time.Now().UnixNano()))
+	s, err := store.Open(tmpDir)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		s.Close()
+		_ = os.RemoveAll(tmpDir)
+	}()
+	bm := index.NewBM25(s)
+
+	rng := newSeededRand()
+	vocab := []string{"alpha", "beta", "gamma", "delta", "epsilon", "go", "rust", "python",
+		"distributed", "consensus", "raft", "paxos", "quantum", "entanglement", "cell",
+		"mitochondria", "roman", "empire", "pacific", "midway", "tennis", "soccer",
+		"bitcoin", "blockchain", "transformer", "attention", "model", "vector", "index"}
+	// Build N synthetic docs of ~50 random vocab words each.
+	for i := 0; i < n; i++ {
+		var sb strings.Builder
+		for j := 0; j < 50; j++ {
+			if j > 0 {
+				sb.WriteByte(' ')
+			}
+			sb.WriteString(vocab[rng.Intn(len(vocab))])
+		}
+		id, err := s.UpsertDocument(ctx, &store.Document{
+			URL: fmt.Sprintf("https://test/d%d", i), Title: vocab[rng.Intn(len(vocab))],
+			Text: sb.String(), Source: "bench", FetchedAt: time.Now(),
+		})
+		if err != nil {
+			return nil, err
+		}
+		if err := bm.IndexDocument(ctx, id, "", sb.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	// Random queries of 2-4 vocab words.
+	qstrs := make([]string, queries)
+	for i := range qstrs {
+		k := 2 + rng.Intn(3)
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			if j > 0 {
+				sb.WriteByte(' ')
+			}
+			sb.WriteString(vocab[rng.Intn(len(vocab))])
+		}
+		qstrs[i] = sb.String()
+	}
+
+	lats := make([]time.Duration, queries)
+	for i, q := range qstrs {
+		start := time.Now()
+		_, err := bm.Search(ctx, q, 10)
+		if err != nil {
+			return nil, err
+		}
+		lats[i] = time.Since(start)
+	}
+	p50, p95, p99 := percentiles(lats)
+	return &benchResult{
+		Mode: "bm25", N: n, Queries: queries,
+		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
+		QPS: float64(queries) / sumDur(lats).Seconds(),
+	}, nil
+}
+
+// benchBM25SQLite is an alias of benchBM25 with an explicit "bm25-sqlite" Mode
+// label so storage-mode output is unambiguous when both backends are emitted.
+func benchBM25SQLite(ctx context.Context, n, queries int) (*benchResult, error) {
+	r, err := benchBM25(ctx, n, queries)
+	if err != nil {
+		return nil, err
+	}
+	r.Mode = "bm25-sqlite"
+	return r, nil
+}
+
+// benchBM25Pebble mirrors benchBM25 but runs against PebbleStore + PebbleBM25.
+// Deterministic vocab + RNG seed so SQLite and Pebble runs produce comparable
+// numbers (same N docs, same K queries, same query distribution).
+// empirical validation of the path-2 storage rework.
+func benchBM25Pebble(ctx context.Context, n, queries int) (*benchResult, error) {
+	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-pebble-bench-%d", time.Now().UnixNano()))
+	ps, err := store.OpenPebble(tmpDir)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		ps.Close()
+		_ = os.RemoveAll(tmpDir)
+	}()
+	bm := index.NewPebbleBM25(ps)
+
+	rng := newSeededRand()
+	vocab := []string{"alpha", "beta", "gamma", "delta", "epsilon", "go", "rust", "python",
+		"distributed", "consensus", "raft", "paxos", "quantum", "entanglement", "cell",
+		"mitochondria", "roman", "empire", "pacific", "midway", "tennis", "soccer",
+		"bitcoin", "blockchain", "transformer", "attention", "model", "vector", "index"}
+	for i := 0; i < n; i++ {
+		var sb strings.Builder
+		for j := 0; j < 50; j++ {
+			if j > 0 {
+				sb.WriteByte(' ')
+			}
+			sb.WriteString(vocab[rng.Intn(len(vocab))])
+		}
+		id, err := ps.UpsertDocument(ctx, &store.Document{
+			URL: fmt.Sprintf("https://test/d%d", i), Title: vocab[rng.Intn(len(vocab))],
+			Text: sb.String(), Source: "bench", FetchedAt: time.Now(),
+		})
+		if err != nil {
+			return nil, err
+		}
+		if err := bm.IndexDocument(ctx, id, "", sb.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	qstrs := make([]string, queries)
+	for i := range qstrs {
+		k := 2 + rng.Intn(3)
+		var sb strings.Builder
+		for j := 0; j < k; j++ {
+			if j > 0 {
+				sb.WriteByte(' ')
+			}
+			sb.WriteString(vocab[rng.Intn(len(vocab))])
+		}
+		qstrs[i] = sb.String()
+	}
+
+	lats := make([]time.Duration, queries)
+	for i, q := range qstrs {
+		start := time.Now()
+		_, err := bm.Search(ctx, q, 10)
+		if err != nil {
+			return nil, err
+		}
+		lats[i] = time.Since(start)
+	}
+	p50, p95, p99 := percentiles(lats)
+	return &benchResult{
+		Mode: "bm25-pebble", N: n, Queries: queries,
+		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
+		QPS: float64(queries) / sumDur(lats).Seconds(),
+	}, nil
+}
+
+func newSeededRand() *rand.Rand {
+	return rand.New(rand.NewSource(42)) // deterministic so runs are comparable
+}
+
+func randUnit(rng *rand.Rand, dim int) []float32 {
+	v := make([]float32, dim)
+	var norm float64
+	for i := range v {
+		v[i] = float32(rng.NormFloat64())
+		norm += float64(v[i]) * float64(v[i])
+	}
+	if norm == 0 {
+		v[0] = 1
+		return v
+	}
+	inv := float32(1.0 / sqrt(norm))
+	for i := range v {
+		v[i] *= inv
+	}
+	return v
+}
+
+func sqrt(x float64) float64 {
+	// stdlib math import already pulled through other packages; just do it here.
+	z := x / 2
+	for i := 0; i < 20; i++ {
+		if z == 0 {
+			break
+		}
+		z = (z + x/z) / 2
+	}
+	return z
+}
+
+// benchCrawl measures end-to-end crawler throughput against an in-process
+// website with N linked synthetic pages. Backs up the project directive's
+// "lightweight, on-par with modern crawlers" claim with measured pages/sec.
+//
+// The fixture: an httptest server serving N pages, each linking to the next
+// few via <a href="/pN+1">. The crawler is configured with per_host_delay=0
+// (we're hitting localhost), max_concurrent=8, robots disabled (would add a
+// constant ~1 fetch of /robots.txt unrelated to the throughput question).
+func benchCrawl(ctx context.Context, n, perHostDelayMs int) (*benchResult, error) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		// Parse "/pN" from request path.
+		if !strings.HasPrefix(r.URL.Path, "/p") {
+			http.NotFound(w, r)
+			return
+		}
+		idx, err := strconv.Atoi(r.URL.Path[2:])
+		if err != nil || idx < 0 || idx >= n {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, `<!doctype html><html><head><title>Page %d</title></head><body><h1>Page %d</h1>`, idx, idx)
+		fmt.Fprintf(w, `<p>Synthetic content for page %d. The crawler follows links from this page to nearby pages, building up the index.</p>`, idx)
+		// Link to next 3 pages so the crawler has a graph to walk.
+		for j := idx + 1; j < idx+4 && j < n; j++ {
+			fmt.Fprintf(w, `<a href="/p%d">page %d</a> `, j, j)
+		}
+		fmt.Fprint(w, `</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-bench-crawl-%d", time.Now().UnixNano()))
+	s, err := store.Open(tmpDir)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		s.Close()
+		_ = os.RemoveAll(tmpDir)
+	}()
+
+	cfg := config.Default().Crawler
+	// -per-host-delay propagates from the bench CLI flag. Default 0
+	// (localhost throughput); set to 100–1000 to measure under realistic politeness.
+	cfg.PerHostDelayMs = perHostDelayMs
+	// Lower concurrency than the default 8 — bench output gets noisy with
+	// SQLite BUSY contention messages at high parallelism, and the bench
+	// metric is meant to be repeatable. Real deployments crawling many hosts
+	// at once should keep the default 8.
+	cfg.MaxConcurrent = 4
+	cfg.MaxDepth = 100 // ensure no depth ceiling cuts the crawl short
+	cfg.RespectRobots = false
+	cfg.IncludeDomains = nil // accept the httptest host (port-bound)
+
+	c := crawler.New(cfg, s)
+	// Seed ALL pages — measures pure fetch+parse+index throughput, not the
+	// crawler's link-discovery rate. Avoids the 1.5s "frontier empty"
+	// terminator firing before link parsing has populated more work.
+	for i := 0; i < n; i++ {
+		if e := c.Seed(fmt.Sprintf("%s/p%d", srv.URL, i)); e != nil {
+			return nil, fmt.Errorf("seed p%d: %w", i, e)
+		}
+	}
+
+	start := time.Now()
+	// Cap the run at 60s in case something goes wrong — bench should never hang.
+	runCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+	if err := c.Run(runCtx); err != nil {
+		return nil, fmt.Errorf("run: %w", err)
+	}
+	elapsed := time.Since(start)
+
+	st, err := s.Stats(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &benchResult{
+		Mode: "crawl", N: n,
+		ElapsedMicros:  elapsed.Microseconds(),
+		Docs:           st.Documents,
+		PagesPerSec:    float64(st.Documents) / elapsed.Seconds(),
+		Terms:          st.Terms,
+		PerHostDelayMs: perHostDelayMs,
+	}, nil
+	// Note: the crawler's termination detector adds ~1.5s of polling at the
+	// end of every run. For small N this dominates elapsed and the pages/sec
+	// figure understates steady-state throughput. Larger N is more representative.
+}
+
+func percentiles(d []time.Duration) (p50, p95, p99 time.Duration) {
+	if len(d) == 0 {
+		return 0, 0, 0
+	}
+	sorted := make([]time.Duration, len(d))
+	copy(sorted, d)
+	// Simple sort — durations as int64 nanos.
+	for i := 1; i < len(sorted); i++ {
+		for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
+			sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
+		}
+	}
+	q := func(p float64) time.Duration {
+		idx := int(float64(len(sorted)-1) * p)
+		return sorted[idx]
+	}
+	return q(0.5), q(0.95), q(0.99)
+}
+
+func sumDur(d []time.Duration) time.Duration {
+	var s time.Duration
+	for _, x := range d {
+		s += x
+	}
+	return s
+}
+
+// paraphraseRetriever wraps any Retriever with on-the-fly query expansion via
+// a chat client. For each search, asks the LLM for N paraphrases, runs all of
+// them through the inner retriever, RRF-fuses the result lists.
+//
+// measured that hand-written paraphrases recover 0.02 nDCG at 10k
+// distractors. This is the auto-generated version — removes the hand-writing
+// requirement and makes the prescription deployable. Per-query overhead:
+// one extra chat call (~$0.0001 at gpt-4o-mini) + N extra inner-retriever calls.
+//
+// In-memory cache by exact query text — eval re-runs hit it; production /search
+// calls don't (different process). For production, the cache would move into
+// the SQLite store or a small Redis-shaped layer.
+type paraphraseRetriever struct {
+	inner      eval.Retriever
+	chat       embed.ChatClient
+	n          int
+	mainWeight float64 // 0 = equal-weight RRF (pre); >0 = main query gets this weight, paraphrases each 1.0
+	cache      map[string][]string
+	mu         sync.Mutex
+}
+
+const paraphraseSystem = `Generate paraphrases of a search query. Each paraphrase preserves the semantic intent but uses different vocabulary — different keywords that a target document might also use. Output ONLY a JSON array of strings.
+Example output for "go programming language": ["golang concurrent compiled language", "Google's systems programming language with goroutines"]`
+
+func (p *paraphraseRetriever) generateParaphrases(ctx context.Context, q string) []string {
+	p.mu.Lock()
+	if cached, ok := p.cache[q]; ok {
+		p.mu.Unlock()
+		return cached
+	}
+	p.mu.Unlock()
+
+	resp, err := p.chat.Chat(ctx, []embed.ChatMsg{
+		{Role: "system", Content: paraphraseSystem},
+		{Role: "user", Content: fmt.Sprintf("Generate %d paraphrases of: %s", p.n, q)},
+	})
+	if err != nil {
+		return nil
+	}
+	// Same robust JSON-array parsing as the research planner.
+	raw := strings.TrimSpace(resp)
+	for _, fence := range []string{"```json", "```"} {
+		raw = strings.TrimPrefix(raw, fence)
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	raw = strings.TrimSpace(raw)
+	start := strings.Index(raw, "[")
+	end := strings.LastIndex(raw, "]")
+	if start < 0 || end <= start {
+		return nil
+	}
+	var arr []string
+	if err := json.Unmarshal([]byte(raw[start:end+1]), &arr); err != nil {
+		return nil
+	}
+	if len(arr) > p.n {
+		arr = arr[:p.n]
+	}
+	p.mu.Lock()
+	p.cache[q] = arr
+	p.mu.Unlock()
+	return arr
+}
+
+func (p *paraphraseRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
+	main, err := p.inner.Search(ctx, q, k)
+	if err != nil {
+		return nil, err
+	}
+	paras := p.generateParaphrases(ctx, q)
+	if len(paras) == 0 {
+		return main, nil
+	}
+	lists := make([][]string, 0, 1+len(paras))
+	lists = append(lists, main)
+	for _, p2 := range paras {
+		extra, err := p.inner.Search(ctx, p2, k)
+		if err != nil {
+			continue
+		}
+		lists = append(lists, extra)
+	}
+	// Matches the server expand
+	// path's behavior so eval-time measurements use the same fusion
+	// math as production.
+	var weights []float64
+	if p.mainWeight > 0 {
+		weights = make([]float64, len(lists))
+		weights[0] = p.mainWeight
+		for i := 1; i < len(weights); i++ {
+			weights[i] = 1.0
+		}
+	}
+	return index.RRFWeighted(lists, weights, k, 60), nil
+}
+
+// plannerRetriever wraps any Retriever with the /research planner's
+// decomposition step: ask the LLM for 2-3 focused sub-queries, run each through
+// the inner retriever, RRF-fuse the results. Mirrors /research minus the synth.
+//
+// This exists so we can measure whether the planner's decomposition
+// already subsumes paraphrase expansion — that claim was asserted in
+// when we chose not to add ?expand=true to /research, but never measured.
+const plannerSystemPrompt = `Decompose the user's research question into 2-3 focused sub-queries that, taken together, would cover the answer. Output ONLY a JSON array of strings — no prose, no markdown. Example: ["sub-query 1", "sub-query 2"]`
+
+type plannerRetriever struct {
+	inner      eval.Retriever
+	chat       embed.ChatClient
+	mainWeight float64 // 0 = equal-weight RRF; >0 = main query gets this weight, sub-queries each 1.0
+	cache      map[string][]string
+	mu         sync.Mutex
+}
+
+func (p *plannerRetriever) plan(ctx context.Context, q string) []string {
+	p.mu.Lock()
+	if cached, ok := p.cache[q]; ok {
+		p.mu.Unlock()
+		return cached
+	}
+	p.mu.Unlock()
+
+	resp, err := p.chat.Chat(ctx, []embed.ChatMsg{
+		{Role: "system", Content: plannerSystemPrompt},
+		{Role: "user", Content: q},
+	})
+	if err != nil {
+		return nil
+	}
+	raw := strings.TrimSpace(resp)
+	for _, fence := range []string{"```json", "```"} {
+		raw = strings.TrimPrefix(raw, fence)
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	raw = strings.TrimSpace(raw)
+	start := strings.Index(raw, "[")
+	end := strings.LastIndex(raw, "]")
+	if start < 0 || end <= start {
+		return nil
+	}
+	var arr []string
+	if err := json.Unmarshal([]byte(raw[start:end+1]), &arr); err != nil {
+		return nil
+	}
+	if len(arr) > 5 {
+		arr = arr[:5] // hard cap matches /research planner
+	}
+	p.mu.Lock()
+	p.cache[q] = arr
+	p.mu.Unlock()
+	return arr
+}
+
+func (p *plannerRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
+	main, err := p.inner.Search(ctx, q, k)
+	if err != nil {
+		return nil, err
+	}
+	subs := p.plan(ctx, q)
+	if len(subs) == 0 {
+		return main, nil
+	}
+	lists := make([][]string, 0, 1+len(subs))
+	lists = append(lists, main)
+	for _, sq := range subs {
+		extra, err := p.inner.Search(ctx, sq, k)
+		if err != nil {
+			continue
+		}
+		lists = append(lists, extra)
+	}
+	// Same shape as
+	// paraphraseRetriever; main query gets mainWeight, sub-queries each 1.0.
+	var weights []float64
+	if p.mainWeight > 0 {
+		weights = make([]float64, len(lists))
+		weights[0] = p.mainWeight
+		for i := 1; i < len(weights); i++ {
+			weights[i] = 1.0
+		}
+	}
+	return index.RRFWeighted(lists, weights, k, 60), nil
+}
+
+// httpAPIRetriever queries a deployed cosift's /search endpoint and adapts the
+// JSON response to eval.Retriever. Lets `cosift eval -api <url>` validate a
+// remote deployment with the same metric harness used for local development.
+type httpAPIRetriever struct {
+	baseURL   string
+	bearer    string
+	retriever string
+	rerank    bool
+	http      *http.Client
+}
+
+func (r *httpAPIRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
+	v := url.Values{}
+	v.Set("q", q)
+	v.Set("k", strconv.Itoa(k))
+	if r.retriever != "" {
+		v.Set("retriever", r.retriever)
+	}
+	if r.rerank {
+		v.Set("rerank", "true")
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.baseURL+"/search?"+v.Encode(), http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+	if r.bearer != "" {
+		req.Header.Set("Authorization", "Bearer "+r.bearer)
+	}
+	resp, err := r.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return nil, fmt.Errorf("api %d: %s", resp.StatusCode, body)
+	}
+	var sr struct {
+		Hits []struct {
+			URL string `json:"url"`
+		} `json:"hits"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&sr); err != nil {
+		return nil, fmt.Errorf("decode: %w", err)
+	}
+	urls := make([]string, len(sr.Hits))
+	for i, h := range sr.Hits {
+		urls[i] = h.URL
+	}
+	return urls, nil
+}
+
+// neutralVocabForDistractors returns words that don't appear in the eval queries
+// or relevant docs, so any distractor returning in top-K is unambiguous noise.
+// Picked from common-but-eval-disjoint domains (pottery, lighthouses, knitting,
+// dentistry, beekeeping, etc.).
+func neutralVocabForDistractors() []string {
+	return []string{
+		"kiln", "pottery", "glaze", "ceramic", "wheel", "stoneware", "earthenware",
+		"lighthouse", "beacon", "keeper", "lantern", "rocky", "shoal", "fog",
+		"yarn", "knit", "purl", "stitch", "skein", "needle", "cable",
+		"dentistry", "molar", "incisor", "filling", "enamel", "gingival",
+		"beekeeping", "hive", "queen", "drone", "wax", "pollen", "swarm",
+		"bookbinding", "spine", "endpaper", "cloth", "headband", "signature",
+		"masonry", "mortar", "trowel", "course", "wythe", "lintel", "soffit",
+		"falconry", "raptor", "perch", "lure", "jess", "creance", "mews",
+		"weaving", "warp", "weft", "shuttle", "heddle", "treadle", "selvedge",
+		"smithing", "forge", "anvil", "tongs", "quench", "temper", "billet",
+		"glassblowing", "punty", "marver", "blowpipe", "annealing", "cane",
+		"watchmaking", "balance", "escapement", "mainspring", "barrel", "jewel",
+		"taxidermy", "tan", "mount", "armature", "habitat", "diorama",
+		"calligraphy", "nib", "ink", "stroke", "kerning", "ascender", "descender",
+		"thatching", "reed", "spar", "ridge", "rafter", "eaves",
+	}
+}
+
+// generateDistractorText assembles a short text out of the neutral vocab.
+func generateDistractorText(rng *rand.Rand, vocab []string, words int) string {
+	parts := make([]string, words)
+	for i := range parts {
+		parts[i] = vocab[rng.Intn(len(vocab))]
+	}
+	return strings.Join(parts, " ")
+}
+
+// runAnswerEval is's measurement of /research answer quality across
+// the two retrieval strategies (planner vs paraphrase). Iters 52-53 measured
+// *retrieval coverage* via nDCG and found paraphrase wins; added
+// ?strategy=paraphrase but kept planner as the default *because we hadn't
+// measured synthesis quality*. This subcommand fills that gap.
+//
+// Flow per query:
+//  1. Build an in-memory cosift HTTP server with corpus ingested, dense index
+//     built, chat + paraphraser configured.
+//  2. Call /research?strategy=planner and /research?strategy=paraphrase.
+//  3. Send (question, answer, sources) to a judge LLM with a different model.
+//  4. Parse two scores: coverage (1-5) and grounding (1-5). Comments are kept
+//     for the per-query report.
+//  5. Aggregate means per strategy and print a comparison table.
+//
+// Cost note: ~$0.005 per query at gpt-4o-mini synth + gpt-4o judge for the
+// 10 multi-faceted queries → roughly $0.10 total per run. Manageable but not
+// free; the -dry-run flag prints the plan without LLM calls.
+func runAnswerEval(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("answer-eval", flag.ExitOnError)
+	corpusPath := fs.String("corpus", "testdata/eval/corpus.json", "corpus JSON")
+	queriesPath := fs.String("queries", "testdata/eval/queries-multifaceted.json", "queries JSON (multi-faceted by default — these stress synthesis the most)")
+	synthModel := fs.String("synth-model", "gpt-4o-mini", "chat model for /research synthesis + planner/paraphraser")
+	judgeModel := fs.String("judge-model", "gpt-4o", "chat model for the judge (use a different + stronger model than synth to avoid self-preference)")
+	savePath := fs.String("save", "", "save full per-query report as JSON")
+	embModel := fs.String("embed-model", "text-embedding-3-small", "embedding model name")
+	embDim := fs.Int("embed-dim", 1536, "embedding dimensionality")
+	embCacheDir := fs.String("embed-cache", "./eval-embed-cache", "embedding cache dir (set empty to disable)")
+	embURL := fs.String("embed-url", "", "embedding endpoint URL (leave empty for OpenAI default)")
+	chatURL := fs.String("chat-url", "", "chat endpoint URL for synth+judge models")
+	answerChunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = default 320)")
+	answerChunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = default 64)")
+	dryRun := fs.Bool("dry-run", false, "build the harness + print the plan, but issue NO LLM calls")
+	useRerank := fs.Bool("rerank", false, "wire the LLM listwise reranker into the in-process server; tests whether rerank cuts the grounding=1 cases on single-doc")
+	rerankModel := fs.String("rerank-model", "gpt-4o-mini", "chat model for the reranker (kept default = synth-model so cost stays predictable)")
+	synthK := fs.Int("synth-k", 0, "cap synth-source count via WithDefaults({ResearchSynthK: N}); 0 = preserve built-in default (10). tests whether K=5 trades coverage for grounding on single-doc workloads.")
+	serverURL := fs.String("server", "", "live server base URL (e.g. https://cosift.pilotprotocol.network); skips in-process build, corpus+embed flags ignored")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	apiKey := os.Getenv("OPENAI_API_KEY")
+	if apiKey == "" {
+		apiKey = os.Getenv("OPENAI")
+	}
+	if apiKey == "" && *embURL == "" && *chatURL == "" && !*dryRun {
+		return errors.New("OPENAI_API_KEY (or OPENAI) not set; pass -embed-url/-chat-url for a local endpoint, or -dry-run to inspect")
+	}
+	if apiKey == "" {
+		apiKey = "local"
+	}
+
+	qs, err := eval.LoadQuerySet(*queriesPath)
+	if err != nil {
+		return fmt.Errorf("load queries: %w", err)
+	}
+
+	liveMode := *serverURL != ""
+	fmt.Printf("answer-eval: %d queries × 2 strategies × judge calls\n", len(qs.Queries))
+	if liveMode {
+		fmt.Printf("server = %s   judge model = %s   dry-run = %v\n", *serverURL, *judgeModel, *dryRun)
+	} else {
+		fmt.Printf("synth model = %s   judge model = %s   rerank = %v   synth_k = %d   dry-run = %v\n", *synthModel, *judgeModel, *useRerank, *synthK, *dryRun)
+	}
+	if *dryRun {
+		for _, q := range qs.Queries {
+			fmt.Printf("  - %s  (relevant: %d docs)\n", q.Text, len(q.Relevant))
+		}
+		return nil
+	}
+
+	// researchBaseURL is the base for /research calls — either live server or httptest.
+	var researchBaseURL string
+	judge := embed.NewOpenAIChat(apiKey, *chatURL, *judgeModel)
+
+	if liveMode {
+		researchBaseURL = strings.TrimRight(*serverURL, "/")
+	} else {
+		corpus, err := eval.LoadCorpus(*corpusPath)
+		if err != nil {
+			return fmt.Errorf("load corpus: %w", err)
+		}
+
+		// Build an ephemeral store + indexes (in-memory style, but SQLite needs a dir).
+		tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-answer-eval-%d", time.Now().UnixNano()))
+		st, err := store.Open(tmpDir)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			st.Close()
+			_ = os.RemoveAll(tmpDir)
+		}()
+
+		bm := index.NewBM25(st)
+		oai := embed.NewOpenAIClient(apiKey, *embURL, *embModel, *embDim)
+		var emb embed.Embedder = oai
+		if *embCacheDir != "" {
+			emb = embed.NewCachedEmbedder(oai, *embCacheDir)
+		}
+		chunker := chunkerWith(*answerChunkSize, *answerChunkOverlap)
+		var allTexts []string
+		for _, d := range corpus.Docs {
+			id, err := st.UpsertDocument(ctx, &store.Document{
+				URL: d.URL, Title: d.Title, Text: d.Text, Source: "answer-eval", FetchedAt: time.Now(),
+			})
+			if err != nil {
+				return err
+			}
+			if err := bm.IndexDocument(ctx, id, d.Title, d.Text); err != nil {
+				return err
+			}
+			text := d.Title + "\n\n" + d.Text
+			for _, c := range chunker.Chunk(text) {
+				allTexts = append(allTexts, c.Text)
+			}
+		}
+		fmt.Printf("embedding %d passages across %d docs...\n", len(allTexts), len(corpus.Docs))
+		vecs, err := batchEmbed(ctx, emb, allTexts, 256)
+		if err != nil {
+			return fmt.Errorf("embed: %w", err)
+		}
+		vi := index.NewVectorIndex(*embDim)
+		idx := 0
+		for _, d := range corpus.Docs {
+			text := d.Title + "\n\n" + d.Text
+			for _, c := range chunker.Chunk(text) {
+				if idx >= len(vecs) {
+					break
+				}
+				vi.AddPassage(d.URL, d.Title, c.Offset, c.Length, vecs[idx])
+				idx++
+			}
+		}
+
+		chat := embed.NewOpenAIChat(apiKey, *chatURL, *synthModel)
+
+		srv := server.New(st).
+			WithVector(vi, emb).
+			WithChat(chat).
+			WithParaphraser(chat, 2).
+			WithLLMLimiter(0, 0) // disable rate limiter — this is an in-process eval, not a public endpoint. caught this when the 10/min default 429'd after 5 queries.
+		if *useRerank {
+			rerankChat := chat
+			if *rerankModel != *synthModel {
+				rerankChat = embed.NewOpenAIChat(apiKey, "", *rerankModel)
+			}
+			srv = srv.WithReranker(rerank.NewLLMReranker(rerankChat), 20)
+		}
+		if *synthK > 0 {
+			srv = srv.WithDefaults(server.Defaults{ResearchSynthK: *synthK})
+		}
+		httpSrv := httptest.NewServer(srv.Handler())
+		defer httpSrv.Close()
+		researchBaseURL = httpSrv.URL
+	}
+
+	type strategyReport struct {
+		Strategy     string   `json:"strategy"`
+		Answer       string   `json:"answer"`
+		SourceURLs   []string `json:"source_urls"`
+		Coverage     int      `json:"coverage"`
+		Grounding    int      `json:"grounding"`
+		JudgeComment string   `json:"judge_comment"`
+	}
+	type queryReport struct {
+		Query   string           `json:"query"`
+		Gold    []string         `json:"gold_relevant"`
+		Reports []strategyReport `json:"reports"`
+	}
+
+	all := make([]queryReport, 0, len(qs.Queries))
+	httpClient := &http.Client{Timeout: 120 * time.Second}
+
+	for qi, q := range qs.Queries {
+		fmt.Printf("\n[%d/%d] %s\n", qi+1, len(qs.Queries), q.Text)
+		qr := queryReport{Query: q.Text, Gold: q.Relevant}
+
+		for _, strategy := range []string{"planner", "paraphrase"} {
+			// /research call.
+			u := researchBaseURL + "/research?strategy=" + strategy + "&q=" + url.QueryEscape(q.Text)
+			req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
+			resp, err := httpClient.Do(req)
+			if err != nil {
+				fmt.Printf("  %s: research call failed: %v\n", strategy, err)
+				continue
+			}
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			if resp.StatusCode != 200 {
+				fmt.Printf("  %s: research returned %d: %s\n", strategy, resp.StatusCode, string(body))
+				continue
+			}
+			var rr struct {
+				Query    string   `json:"query"`
+				Strategy string   `json:"strategy"`
+				Plan     []string `json:"plan"`
+				Answer   string   `json:"answer"`
+				Sources  []struct {
+					ID    int    `json:"id"`
+					URL   string `json:"url"`
+					Title string `json:"title"`
+				} `json:"sources"`
+			}
+			if err := json.Unmarshal(body, &rr); err != nil {
+				fmt.Printf("  %s: parse research: %v\n", strategy, err)
+				continue
+			}
+			sourceURLs := make([]string, len(rr.Sources))
+			for i, src := range rr.Sources {
+				sourceURLs[i] = src.URL
+			}
+			fmt.Printf("  %-10s answer=%dchars sources=%d  plan=%v\n", strategy, len(rr.Answer), len(rr.Sources), rr.Plan)
+
+			// Judge call.
+			score, err := judgeAnswer(ctx, judge, q.Text, rr.Answer, rr.Sources)
+			if err != nil {
+				fmt.Printf("    judge failed: %v\n", err)
+				continue
+			}
+			fmt.Printf("    judge: coverage=%d grounding=%d  — %s\n", score.Coverage, score.Grounding, score.Comment)
+			qr.Reports = append(qr.Reports, strategyReport{
+				Strategy:     strategy,
+				Answer:       rr.Answer,
+				SourceURLs:   sourceURLs,
+				Coverage:     score.Coverage,
+				Grounding:    score.Grounding,
+				JudgeComment: score.Comment,
+			})
+		}
+		all = append(all, qr)
+	}
+
+	// Aggregate. Exported field names matter — saved unexported agg
+	// fields which JSON-marshalled to `{}`. Now the summary in the saved JSON
+	// is actually inspectable by jq.
+	type strategySummary struct {
+		N             int     `json:"n"`
+		MeanCoverage  float64 `json:"mean_coverage"`
+		MeanGrounding float64 `json:"mean_grounding"`
+		Combined      float64 `json:"combined"`
+	}
+	rawAcc := map[string]struct{ cov, grd, n int }{"planner": {}, "paraphrase": {}}
+	for _, q := range all {
+		for _, sr := range q.Reports {
+			a, ok := rawAcc[sr.Strategy]
+			if !ok {
+				continue
+			}
+			a.cov += sr.Coverage
+			a.grd += sr.Grounding
+			a.n++
+			rawAcc[sr.Strategy] = a
+		}
+	}
+	summary := map[string]strategySummary{}
+	for _, name := range []string{"planner", "paraphrase"} {
+		a := rawAcc[name]
+		s := strategySummary{N: a.n}
+		if a.n > 0 {
+			s.MeanCoverage = float64(a.cov) / float64(a.n)
+			s.MeanGrounding = float64(a.grd) / float64(a.n)
+			s.Combined = s.MeanCoverage + s.MeanGrounding
+		}
+		summary[name] = s
+	}
+
+	fmt.Println("\n=== summary ===")
+	fmt.Printf("%-12s  N   mean coverage   mean grounding   combined\n", "strategy")
+	for _, name := range []string{"planner", "paraphrase"} {
+		s := summary[name]
+		if s.N == 0 {
+			fmt.Printf("%-12s  0   —                —                —\n", name)
+			continue
+		}
+		fmt.Printf("%-12s  %d   %.2f            %.2f             %.2f\n", name, s.N, s.MeanCoverage, s.MeanGrounding, s.Combined)
+	}
+
+	if *savePath != "" {
+		buf, _ := json.MarshalIndent(map[string]any{
+			"synth_model": *synthModel,
+			"judge_model": *judgeModel,
+			"queries":     *queriesPath,
+			"rerank":      *useRerank,
+			"synth_k":     *synthK,
+			"reports":     all,
+			"summary":     summary,
+			"when":        time.Now().UTC().Format(time.RFC3339),
+		}, "", "  ")
+		if err := os.WriteFile(*savePath, buf, 0o644); err != nil {
+			return fmt.Errorf("save report: %w", err)
+		}
+		fmt.Printf("\nsaved full report to %s\n", *savePath)
+	}
+	return nil
+}
+
+// judgeScore is what we parse out of the judge LLM's JSON response.
+type judgeScore struct {
+	Coverage  int    `json:"coverage"`
+	Grounding int    `json:"grounding"`
+	Comment   string `json:"comment"`
+}
+
+// judgeSystemPrompt was rewritten in after the trace revealed
+// the prior judge was evaluating the full source LIST relevance instead of the
+// CITED sources' accuracy. That bias deflated grounding scores in proportion
+// to how many tangential sources were in the list (paraphrase: 10, planner: 4-6).
+// The new prompt is explicit + the harness pre-filters sources to only those
+// the answer actually cites via [N].
+const judgeSystemPrompt = `You are an expert judge evaluating a research answer's quality. Score on TWO dimensions, each on a 1-5 integer scale:
+
+  - coverage: does the answer address every aspect of the question? (1 = misses major aspects, 5 = covers everything the question asked)
+  - grounding: do the sources the answer CITES via [N] actually support the claims those citations are attached to? (1 = cited sources don't support the claims they're attached to, 5 = every cited source genuinely supports the claim it appears next to)
+
+CRITICAL: Only consider sources the answer actually cites via [N]. Sources are listed by the IDs the answer used; uncited sources are not shown. Do not penalize the answer for not citing additional material.
+
+Output ONLY a single JSON object with this exact shape:
+{"coverage": <int 1-5>, "grounding": <int 1-5>, "comment": "<one sentence explaining the lower of the two scores>"}
+
+Do not output anything else — no preamble, no code fences, no markdown.`
+
+// citedSourceIDs extracts the integer ids referenced by [N] or [N,M,K] in the
+// answer text. fix for the judge-bias bug — we now pass only the
+// actually-cited sources to the judge, not the full source list.
+func citedSourceIDs(answer string) map[int]bool {
+	ids := make(map[int]bool)
+	// Match [1] or [1,2] or [1, 2, 3] inside square brackets.
+	for _, m := range citationRE.FindAllStringSubmatch(answer, -1) {
+		// m[1] is the contents inside the brackets.
+		for _, tok := range strings.Split(m[1], ",") {
+			n, err := strconv.Atoi(strings.TrimSpace(tok))
+			if err == nil {
+				ids[n] = true
+			}
+		}
+	}
+	return ids
+}
+
+var citationRE = regexp.MustCompile(`\[([0-9, ]+)\]`)
+
+// judgeAnswer asks the judge LLM to score a single (question, answer, sources)
+// triple. pre-filters sources to only those cited in the answer, so
+// the judge can't be confused by uncited tangential material in the list.
+// Returns a judgeScore or an error.
+func judgeAnswer(ctx context.Context, judge embed.ChatClient, question, answer string, sources []struct {
+	ID    int    `json:"id"`
+	URL   string `json:"url"`
+	Title string `json:"title"`
+}) (judgeScore, error) {
+	cited := citedSourceIDs(answer)
+	var sb strings.Builder
+	for _, src := range sources {
+		if !cited[src.ID] {
+			continue // skip uncited sources — judge only evaluates citation accuracy
+		}
+		fmt.Fprintf(&sb, "[%d] %s (%s)\n", src.ID, src.Title, src.URL)
+	}
+	citedBlock := sb.String()
+	if citedBlock == "" {
+		citedBlock = "(the answer did not cite any sources via [N])\n"
+	}
+	userMsg := fmt.Sprintf("Question: %s\n\nSources actually cited by the answer:\n%s\nAnswer:\n%s\n\nScore this answer.", question, citedBlock, answer)
+	raw, err := judge.Chat(ctx, []embed.ChatMsg{
+		{Role: "system", Content: judgeSystemPrompt},
+		{Role: "user", Content: userMsg},
+	})
+	if err != nil {
+		return judgeScore{}, fmt.Errorf("judge call: %w", err)
+	}
+	raw = strings.TrimSpace(raw)
+	for _, fence := range []string{"```json", "```"} {
+		raw = strings.TrimPrefix(raw, fence)
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	raw = strings.TrimSpace(raw)
+	startIdx := strings.Index(raw, "{")
+	endIdx := strings.LastIndex(raw, "}")
+	if startIdx < 0 || endIdx <= startIdx {
+		return judgeScore{}, fmt.Errorf("judge: no JSON object in response: %q", raw)
+	}
+	var sc judgeScore
+	if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &sc); err != nil {
+		return judgeScore{}, fmt.Errorf("judge: parse JSON: %w (raw=%q)", err, raw)
+	}
+	if sc.Coverage < 1 || sc.Coverage > 5 || sc.Grounding < 1 || sc.Grounding > 5 {
+		return judgeScore{}, fmt.Errorf("judge: scores out of 1-5 range: %+v", sc)
+	}
+	return sc, nil
+}
+
+// savedAnswerEvalReport mirrors the structure runAnswerEval writes. Used by
+// runAnswerEvalCompare to load and diff two prior reports without re-running
+// the eval. The shape is the same map[string]any savedJSON shape; this struct
+// carries only the fields we need for comparison.
+type savedAnswerEvalReport struct {
+	SynthModel string `json:"synth_model"`
+	JudgeModel string `json:"judge_model"`
+	Queries    string `json:"queries"`
+	Rerank     bool   `json:"rerank"`
+	SynthK     int    `json:"synth_k"`
+	When       string `json:"when"`
+	Summary    map[string]struct {
+		N             int     `json:"n"`
+		MeanCoverage  float64 `json:"mean_coverage"`
+		MeanGrounding float64 `json:"mean_grounding"`
+		Combined      float64 `json:"combined"`
+	} `json:"summary"`
+	Reports []struct {
+		Query   string `json:"query"`
+		Reports []struct {
+			Strategy     string `json:"strategy"`
+			Coverage     int    `json:"coverage"`
+			Grounding    int    `json:"grounding"`
+			JudgeComment string `json:"judge_comment"`
+		} `json:"reports"`
+	} `json:"reports"`
+}
+
+func loadAnswerEvalReport(path string) (*savedAnswerEvalReport, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	var r savedAnswerEvalReport
+	if err := json.Unmarshal(b, &r); err != nil {
+		return nil, fmt.Errorf("parse %s: %w", path, err)
+	}
+	// Old reports had an empty summary because the agg struct used
+	// unexported fields. Recompute the summary on the fly for those.
+	if len(r.Summary) == 0 || r.Summary["planner"].N == 0 {
+		r.Summary = recomputeSummary(r)
+	}
+	return &r, nil
+}
+
+func recomputeSummary(r savedAnswerEvalReport) map[string]struct {
+	N             int     `json:"n"`
+	MeanCoverage  float64 `json:"mean_coverage"`
+	MeanGrounding float64 `json:"mean_grounding"`
+	Combined      float64 `json:"combined"`
+} {
+	out := make(map[string]struct {
+		N             int     `json:"n"`
+		MeanCoverage  float64 `json:"mean_coverage"`
+		MeanGrounding float64 `json:"mean_grounding"`
+		Combined      float64 `json:"combined"`
+	})
+	for _, strategy := range []string{"planner", "paraphrase"} {
+		var covSum, grdSum, n int
+		for _, q := range r.Reports {
+			for _, sr := range q.Reports {
+				if sr.Strategy == strategy {
+					covSum += sr.Coverage
+					grdSum += sr.Grounding
+					n++
+				}
+			}
+		}
+		s := out[strategy]
+		s.N = n
+		if n > 0 {
+			s.MeanCoverage = float64(covSum) / float64(n)
+			s.MeanGrounding = float64(grdSum) / float64(n)
+			s.Combined = s.MeanCoverage + s.MeanGrounding
+		}
+		out[strategy] = s
+	}
+	return out
+}
+
+// runAnswerEvalCompare loads two saved answer-eval JSON reports and prints
+// side-by-side coverage/grounding/combined deltas + per-query moves above a
+// configurable threshold. No LLM calls; pure file-to-file diff —
+// first-class replacement for the ad-hoc python one-liners operators
+// previously chained for the same report.
+func runAnswerEvalCompare(_ context.Context, args []string) error {
+	fs := flag.NewFlagSet("answer-eval-compare", flag.ExitOnError)
+	threshold := fs.Int("query-threshold", 2, "report per-query moves of |Δgrounding| or |Δcoverage| ≥ this value (1-5 scale)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	rest := fs.Args()
+	if len(rest) != 2 {
+		return errors.New("usage: cosift answer-eval-compare [-query-threshold N] BASELINE.json NEW.json")
+	}
+	a, err := loadAnswerEvalReport(rest[0])
+	if err != nil {
+		return fmt.Errorf("baseline: %w", err)
+	}
+	b, err := loadAnswerEvalReport(rest[1])
+	if err != nil {
+		return fmt.Errorf("new: %w", err)
+	}
+	// Header — record what we're comparing.
+	fmt.Printf("BASELINE: %s\n", rest[0])
+	fmt.Printf("  synth=%s judge=%s rerank=%v synth_k=%d queries=%s\n", a.SynthModel, a.JudgeModel, a.Rerank, a.SynthK, a.Queries)
+	fmt.Printf("  when: %s\n", a.When)
+	fmt.Printf("NEW:      %s\n", rest[1])
+	fmt.Printf("  synth=%s judge=%s rerank=%v synth_k=%d queries=%s\n", b.SynthModel, b.JudgeModel, b.Rerank, b.SynthK, b.Queries)
+	fmt.Printf("  when: %s\n", b.When)
+	fmt.Println()
+
+	// Summary deltas.
+	fmt.Printf("%-12s   N     cov     grnd    comb   |   Δcov    Δgrnd   Δcomb\n", "strategy")
+	fmt.Println("-----------------------------------------------------------------------")
+	for _, strategy := range []string{"planner", "paraphrase"} {
+		ab, bb := a.Summary[strategy], b.Summary[strategy]
+		if ab.N == 0 && bb.N == 0 {
+			fmt.Printf("%-12s   (not present in either report)\n", strategy)
+			continue
+		}
+		fmt.Printf("%-12s %3d→%-3d %.2f→%.2f %.2f→%.2f %.2f→%.2f | %+.2f  %+.2f  %+.2f\n",
+			strategy, ab.N, bb.N,
+			ab.MeanCoverage, bb.MeanCoverage,
+			ab.MeanGrounding, bb.MeanGrounding,
+			ab.Combined, bb.Combined,
+			bb.MeanCoverage-ab.MeanCoverage,
+			bb.MeanGrounding-ab.MeanGrounding,
+			bb.Combined-ab.Combined,
+		)
+	}
+
+	// Per-query moves.
+	if *threshold > 0 {
+		// Index by (query, strategy) for fast lookup.
+		aIdx := make(map[string]struct{ cov, grd int })
+		for _, q := range a.Reports {
+			for _, sr := range q.Reports {
+				aIdx[q.Query+"\x00"+sr.Strategy] = struct{ cov, grd int }{sr.Coverage, sr.Grounding}
+			}
+		}
+		type move struct {
+			query, strategy string
+			oldCov, oldGrd  int
+			newCov, newGrd  int
+		}
+		var moves []move
+		for _, q := range b.Reports {
+			for _, sr := range q.Reports {
+				ab, ok := aIdx[q.Query+"\x00"+sr.Strategy]
+				if !ok {
+					continue
+				}
+				dc := abs(sr.Coverage - ab.cov)
+				dg := abs(sr.Grounding - ab.grd)
+				if dc >= *threshold || dg >= *threshold {
+					moves = append(moves, move{q.Query, sr.Strategy, ab.cov, ab.grd, sr.Coverage, sr.Grounding})
+				}
+			}
+		}
+		if len(moves) > 0 {
+			fmt.Printf("\nPer-query moves with |Δ| ≥ %d:\n", *threshold)
+			fmt.Printf("  %-10s  %-50s  cov   grnd\n", "strategy", "query")
+			for _, m := range moves {
+				fmt.Printf("  %-10s  %-50s  %d→%d  %d→%d\n",
+					m.strategy, truncate(m.query, 50),
+					m.oldCov, m.newCov,
+					m.oldGrd, m.newGrd,
+				)
+			}
+		} else {
+			fmt.Printf("\nNo per-query moves with |Δ| ≥ %d.\n", *threshold)
+		}
+	}
+	return nil
+}
diff --git a/cmd/cosift/cmd_index.go b/cmd/cosift/cmd_index.go
new file mode 100644
index 0000000..87ca882
--- /dev/null
+++ b/cmd/cosift/cmd_index.go
@@ -0,0 +1,819 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/eval"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// runIngest loads a corpus.json directly into the local store + indexes —
+// useful for evaluation, vertical deployments, and users with a pre-curated
+// dataset who don't need a crawler.
+//
+// The corpus shape matches eval.Corpus: {"docs": [{"url","title","text"}...]}.
+// If embeddings are configured, every doc is chunked + embedded + persisted
+// to the passages table the same way the crawler does.
+// progressReporter logs progress for a long-running CLI loop at most once per
+// configured interval. Time-based (vs every-N-iterations) so the cadence is
+// consistent regardless of per-item cost variance. Set interval to 0 to
+// disable. Designed for ingest but generic enough for other loops.
+type progressReporter struct {
+	label    string
+	total    int
+	interval time.Duration
+	last     time.Time
+}
+
+func newProgressReporter(label string, total int, interval time.Duration) *progressReporter {
+	return &progressReporter{
+		label:    label,
+		total:    total,
+		interval: interval,
+		// Initialize `last` to now so the first call after construction
+		// doesn't immediately log (avoids a noisy first-iter "0/N" line).
+		last: time.Now(),
+	}
+}
+
+// maybeLog emits a progress line only if `interval` has elapsed since the last
+// emission. Callers pass the 1-indexed current item count. No-op when interval
+// is <= 0 or when total is 0 (avoids divide-by-zero in the percentage).
+func (p *progressReporter) maybeLog(current int) {
+	if p.interval <= 0 || p.total <= 0 {
+		return
+	}
+	if time.Since(p.last) < p.interval {
+		return
+	}
+	pct := float64(current) / float64(p.total) * 100
+	log.Printf("%s: %d/%d (%.1f%%)", p.label, current, p.total, pct)
+	p.last = time.Now()
+}
+
+// loadCorpusJSONL reads a JSONL file (one eval.CorpusDoc per line) into a
+// Corpus. Empty lines are skipped (some tools emit trailing blanks). Pairs
+// with's `export -format jsonl` to close the round-trip for ML
+// pipelines.
+func loadCorpusJSONL(path string) (*eval.Corpus, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	// JSONL lines can be large (full doc text); raise the scanner buffer cap
+	// well above the default 64KB. 4MB matches's SSE parser ceiling.
+	scanner.Buffer(make([]byte, 0, 64<<10), 4<<20)
+	c := &eval.Corpus{}
+	lineNum := 0
+	for scanner.Scan() {
+		lineNum++
+		raw := strings.TrimSpace(scanner.Text())
+		if raw == "" {
+			continue
+		}
+		var d eval.CorpusDoc
+		if err := json.Unmarshal([]byte(raw), &d); err != nil {
+			return nil, fmt.Errorf("parse %s line %d: %w", path, lineNum, err)
+		}
+		c.Docs = append(c.Docs, d)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	return c, nil
+}
+
+func runIngest(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("ingest", flag.ExitOnError)
+	corpusPath := fs.String("corpus", "", "path to corpus file (required)")
+	source := fs.String("source", "ingest", "source tag for documents")
+	// -format selects the loader. Default "auto" infers from file
+	// extension (.jsonl → jsonl, anything else → json). Explicit override
+	// avoids ambiguity when files have non-standard extensions.
+	format := fs.String("format", "auto", "input format: auto (default — infer from extension) | json | jsonl")
+	// Operators ingesting large corpora need
+	// some signal that work is happening. 0 disables.
+	progressInterval := fs.Duration("progress", 5*time.Second, "log per-doc/per-passage progress every N (0 disables)")
+	// CLI override for chunker config (mirrors on eval/answer-eval).
+	// 0 → fall through to cfg.Crawler.{ChunkSize,ChunkOverlap}, then to NewChunker defaults.
+	ingestChunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = use cfg.Crawler.ChunkSize, then NewChunker default 320)")
+	ingestChunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = use cfg.Crawler.ChunkOverlap, then default 64)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *corpusPath == "" {
+		return errors.New("ingest: -corpus is required")
+	}
+	resolvedFormat := *format
+	if resolvedFormat == "auto" {
+		if strings.HasSuffix(strings.ToLower(*corpusPath), ".jsonl") {
+			resolvedFormat = "jsonl"
+		} else {
+			resolvedFormat = "json"
+		}
+	}
+	var corpus *eval.Corpus
+	var err error
+	switch resolvedFormat {
+	case "json":
+		corpus, err = eval.LoadCorpus(*corpusPath)
+	case "jsonl":
+		corpus, err = loadCorpusJSONL(*corpusPath)
+	default:
+		return fmt.Errorf("unknown -format %q (want: auto | json | jsonl)", *format)
+	}
+	if err != nil {
+		return err
+	}
+
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+	bm := index.NewBM25(s)
+
+	// Optional embedder — same auto-wire as runServe.
+	var emb embed.Embedder
+	if cfg.Embeddings.Model != "" {
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI")
+		}
+		if apiKey != "" {
+			dim := cfg.Embeddings.Dim
+			if dim == 0 {
+				dim = 1536
+			}
+			emb = embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
+		} else {
+			log.Printf("warning: embeddings configured but no API key; ingesting BM25 only")
+		}
+	}
+
+	// Pre-compute chunks for the whole corpus so we can batch-embed in one call.
+	// CLI flag wins over cfg.Crawler.ChunkSize (only when flag is set).
+	chunkSize := cfg.Crawler.ChunkSize
+	if *ingestChunkSize > 0 {
+		chunkSize = *ingestChunkSize
+	}
+	chunkOverlap := cfg.Crawler.ChunkOverlap
+	if *ingestChunkOverlap > 0 {
+		chunkOverlap = *ingestChunkOverlap
+	}
+	chunker := chunkerWith(chunkSize, chunkOverlap)
+	type passageRef struct {
+		docIdx int
+		chunk  index.Chunk
+	}
+	var allTexts []string
+	var refs []passageRef
+	if emb != nil {
+		for i, d := range corpus.Docs {
+			for _, c := range chunker.Chunk(d.Title + "\n\n" + d.Text) {
+				allTexts = append(allTexts, c.Text)
+				refs = append(refs, passageRef{docIdx: i, chunk: c})
+			}
+		}
+		log.Printf("ingest: embedding %d passages across %d docs with %s", len(allTexts), len(corpus.Docs), cfg.Embeddings.Model)
+	}
+
+	var vecs [][]float32
+	if emb != nil && len(allTexts) > 0 {
+		vecs, err = emb.Embed(ctx, allTexts)
+		if err != nil {
+			return fmt.Errorf("embed: %w", err)
+		}
+	}
+
+	// Map docIdx → docID for the passage writes.
+	docIDs := make([]int64, len(corpus.Docs))
+	docProgress := newProgressReporter("ingest docs", len(corpus.Docs), *progressInterval)
+	for i, d := range corpus.Docs {
+		sha := sha256.Sum256([]byte(d.Text))
+		id, err := s.UpsertDocument(ctx, &store.Document{
+			URL: d.URL, Title: d.Title, Text: d.Text,
+			Source: *source, Quality: 0.7, FetchedAt: time.Now(),
+			ContentSHA: sha[:],
+		})
+		if err != nil {
+			return fmt.Errorf("upsert %s: %w", d.URL, err)
+		}
+		if err := bm.IndexDocument(ctx, id, d.Title, d.Text); err != nil {
+			return fmt.Errorf("index %s: %w", d.URL, err)
+		}
+		docIDs[i] = id
+		docProgress.maybeLog(i + 1)
+	}
+
+	// Write passages.
+	if emb != nil && len(vecs) == len(refs) {
+		passageProgress := newProgressReporter("ingest passages", len(vecs), *progressInterval)
+		for j, v := range vecs {
+			ref := refs[j]
+			p := &store.Passage{
+				DocID:     docIDs[ref.docIdx],
+				Offset:    ref.chunk.Offset,
+				Length:    ref.chunk.Length,
+				Model:     emb.Model(),
+				Embedding: v,
+			}
+			if err := s.UpsertPassage(ctx, p); err != nil {
+				return fmt.Errorf("passage %s offset=%d: %w", corpus.Docs[ref.docIdx].URL, ref.chunk.Offset, err)
+			}
+			passageProgress.maybeLog(j + 1)
+		}
+	}
+
+	log.Printf("ingest: %d docs, %d passages", len(corpus.Docs), len(vecs))
+	return nil
+}
+
+// runExport writes the local store's documents to a portable corpus.json,
+// shape-compatible with `cosift ingest`. Round-trips cleanly:
+//
+//	cosift export -output corpus.json
+//	cosift -config other.json ingest -corpus corpus.json
+//
+// Documents only — passages (embeddings) aren't exported because the receiver
+// likely uses a different embedding model.
+// runMigrateToPebble copies a SQLite-backed cosift data directory into a
+// fresh Pebble store. Fifth piece of the path-2 storage rework.
+//
+// Migrates:
+//   - documents (URL, title, text, metadata) via PebbleStore.UpsertDocument
+//   - BM25 postings (re-tokenized + re-indexed to preserve title
+//     boost; the SQLite postings table is NOT copied directly because
+//     re-indexing through PebbleBM25 is the same code path that production
+//     uses going forward — eliminates the divergence risk of two
+//     posting-write paths)
+//
+// Does NOT migrate (deferred): frontier rows, query_outcomes feedback,
+// paraphrase/HyDE caches, vector embeddings. Operators starting from a
+// migrated Pebble store get a working BM25 index immediately; dense + LLM
+// caches rebuild from scratch on the new instance.
+//
+// The destination directory must be empty; the migration refuses to
+// overwrite existing data so a mistyped path can't clobber a working store.
+func runMigrateToPebble(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("migrate-to-pebble", flag.ExitOnError)
+	output := fs.String("output", "", "output directory for the Pebble store (required)")
+	progress := fs.Duration("progress", 5*time.Second, "log progress every N (0 disables)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *output == "" {
+		return errors.New("migrate-to-pebble: -output is required")
+	}
+	// Refuse to write into a directory that already contains a Pebble store.
+	if entries, err := os.ReadDir(*output); err == nil && len(entries) > 0 {
+		return fmt.Errorf("migrate-to-pebble: -output %s is non-empty; refusing to overwrite (move or remove first)", *output)
+	}
+
+	src, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return fmt.Errorf("open source SQLite store at %s: %w", cfg.DataDir, err)
+	}
+	defer src.Close()
+
+	dst, err := openPebbleOrFriendlyErr(*output)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	pidx := index.NewPebbleBM25(dst)
+
+	docs, err := src.ListDocuments(ctx, 0)
+	if err != nil {
+		return fmt.Errorf("list source documents: %w", err)
+	}
+	log.Printf("migrate-to-pebble: %d documents to copy (source: %s → destination: %s)",
+		len(docs), cfg.DataDir, *output)
+
+	reporter := newProgressReporter("migrate docs", len(docs), *progress)
+	for i, d := range docs {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		// Upsert the doc into Pebble — assigns a fresh Pebble-side ID.
+		newID, err := dst.UpsertDocument(ctx, d)
+		if err != nil {
+			return fmt.Errorf("upsert doc %s: %w", d.URL, err)
+		}
+		// Re-index in Pebble BM25. Uses the same Tokenize + TitleBoost that
+		// production reads through, so behavior of the migrated index matches
+		// what a fresh crawl would produce.
+		if err := pidx.IndexDocument(ctx, newID, d.Title, d.Text); err != nil {
+			return fmt.Errorf("index doc %s: %w", d.URL, err)
+		}
+		reporter.maybeLog(i + 1)
+	}
+
+	stats, err := dst.Stats(ctx)
+	if err != nil {
+		return err
+	}
+	log.Printf("migrate-to-pebble: %d documents indexed into Pebble at %s", stats.Documents, *output)
+	return nil
+}
+
+func runExport(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("export", flag.ExitOnError)
+	output := fs.String("output", "", "output path (default depends on -format: corpus-export.{json,jsonl,txt,md})")
+	limit := fs.Int("limit", 0, "max docs to export (0 = all, applied AFTER filters)")
+	// -format selects the on-disk shape.
+	//   json (default — eval.Corpus pretty-printed; backward-compatible with)
+	//   jsonl — one {url,title,text} per line; common for ML fine-tuning pipelines
+	//   text  — Title/URL header + body + --- separator; for RAG corpora / grep
+	//   md    — `# Title` + URL line + body + horizontal rule; for LLM prompt piping
+	format := fs.String("format", "json", "output format: json | jsonl | text | md")
+	// All applied client-side after ListDocuments.
+	includeDomains := fs.String("include-domains", "", "comma-separated allowlist of domains (suffix match on dot boundary)")
+	excludeDomains := fs.String("exclude-domains", "", "comma-separated denylist of domains")
+	since := fs.String("since", "", "ISO date — only docs with published_at on or after (drops undated docs)")
+	until := fs.String("until", "", "ISO date — only docs with published_at on or before (drops undated docs)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if err := validateExportFormat(*format); err != nil {
+		return err
+	}
+	if *output == "" {
+		*output = "corpus-export." + exportExtension(*format)
+	}
+
+	sinceT, err := parseExportDate(*since)
+	if err != nil {
+		return fmt.Errorf("-since: %w", err)
+	}
+	untilT, err := parseExportDate(*until)
+	if err != nil {
+		return fmt.Errorf("-until: %w", err)
+	}
+	includeList := splitDomainCSV(*includeDomains)
+	excludeList := splitDomainCSV(*excludeDomains)
+	hasDateFilter := !sinceT.IsZero() || !untilT.IsZero()
+
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	// limit applies AFTER filters now — operators expect "give me 100
+	// docs from example.com" to mean 100 example.com docs, not 100 docs total
+	// of which some are from example.com. Pass 0 (no limit) to ListDocuments
+	// and apply the limit client-side after filtering.
+	rawLimit := 0
+	if *limit > 0 && !hasDateFilter && len(includeList) == 0 && len(excludeList) == 0 {
+		// No filters → push the limit down to ListDocuments (faster, less memory).
+		rawLimit = *limit
+	}
+	docs, err := s.ListDocuments(ctx, rawLimit)
+	if err != nil {
+		return err
+	}
+
+	docs = filterExportDocs(docs, includeList, excludeList, sinceT, untilT)
+	if *limit > 0 && len(docs) > *limit {
+		docs = docs[:*limit]
+	}
+
+	f, cerr := os.Create(*output)
+	if cerr != nil {
+		return cerr
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil && err == nil {
+			err = fmt.Errorf("close output: %w", closeErr)
+		}
+	}()
+
+	switch *format {
+	case "json":
+		// wire shape: eval.Corpus pretty-printed. Preserved as default.
+		corpus := eval.Corpus{Docs: make([]eval.CorpusDoc, 0, len(docs))}
+		for _, d := range docs {
+			corpus.Docs = append(corpus.Docs, eval.CorpusDoc{URL: d.URL, Title: d.Title, Text: d.Text})
+		}
+		enc := json.NewEncoder(f)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(corpus); err != nil {
+			return fmt.Errorf("encode: %w", err)
+		}
+	case "jsonl":
+		// One {url,title,text} per line. No outer array — caller streams line-by-line.
+		enc := json.NewEncoder(f)
+		for _, d := range docs {
+			if err := enc.Encode(eval.CorpusDoc{URL: d.URL, Title: d.Title, Text: d.Text}); err != nil {
+				return fmt.Errorf("encode jsonl: %w", err)
+			}
+		}
+	case "text":
+		// Title:/URL: headers + blank + body + --- separator. Matches
+		// `contents -text` batch separator convention (LLMs interpret `---` as
+		// section breaks).
+		for i, d := range docs {
+			if i > 0 {
+				fmt.Fprintln(f, "---")
+				fmt.Fprintln(f)
+			}
+			if d.Title != "" {
+				fmt.Fprintf(f, "Title: %s\n", d.Title)
+			}
+			fmt.Fprintf(f, "URL: %s\n\n", d.URL)
+			fmt.Fprintln(f, d.Text)
+		}
+	case "md":
+		// `# Title` heading + italic URL line + body + horizontal rule. For
+		// piping into LLM prompt contexts that render markdown.
+		for i, d := range docs {
+			if i > 0 {
+				fmt.Fprintln(f)
+				fmt.Fprintln(f, "---")
+				fmt.Fprintln(f)
+			}
+			if d.Title != "" {
+				fmt.Fprintf(f, "# %s\n\n", d.Title)
+			}
+			fmt.Fprintf(f, "_%s_\n\n", d.URL)
+			fmt.Fprintln(f, d.Text)
+		}
+	}
+	log.Printf("export: %d docs → %s (%s)", len(docs), *output, *format)
+	return nil
+}
+
+// parseExportDate accepts YYYY-MM-DD or full RFC3339 (matches the
+// /search?since=/?until= conventions). Empty input returns zero time + nil
+// error (no filter).
+func parseExportDate(s string) (time.Time, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.RFC3339, s); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse("2006-01-02", s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("could not parse date %q", s)
+}
+
+// splitDomainCSV parses a comma-separated list of domains into lowercased
+// non-empty entries. (mirrors internal/server's unexported splitCSV).
+func splitDomainCSV(s string) []string {
+	if s == "" {
+		return nil
+	}
+	parts := strings.Split(s, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		t := strings.ToLower(strings.TrimSpace(p))
+		if t != "" {
+			out = append(out, t)
+		}
+	}
+	return out
+}
+
+// matchesDomainPattern is the strict suffix-on-dot-boundary match the
+// /search filter uses. `example.com` matches `blog.example.com` but NOT
+// `evilexample.com`.
+func matchesDomainPattern(host string, patterns []string) bool {
+	host = strings.ToLower(host)
+	for _, p := range patterns {
+		if host == p || strings.HasSuffix(host, "."+p) {
+			return true
+		}
+	}
+	return false
+}
+
+// hostFromURL extracts the lowercased host without port from a URL. Returns
+// empty string when parsing fails so callers can treat unparseable URLs
+// uniformly (typically: excluded from include-filtered exports).
+func hostFromURL(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(u.Hostname())
+}
+
+// filterExportDocs applies the export filters to a doc slice
+// in-memory. Returns the filtered slice (in input order — preserves whatever
+// ordering ListDocuments returned).
+func filterExportDocs(docs []*store.Document, include, exclude []string, since, until time.Time) []*store.Document {
+	hasDateFilter := !since.IsZero() || !until.IsZero()
+	hasDomainFilter := len(include) > 0 || len(exclude) > 0
+	if !hasDateFilter && !hasDomainFilter {
+		return docs
+	}
+	out := make([]*store.Document, 0, len(docs))
+	for _, d := range docs {
+		// Date filter: undated docs are excluded when ANY date bound is set
+		// (matches /search?since=/?until= semantics).
+		if hasDateFilter {
+			if d.PublishedAt.IsZero() {
+				continue
+			}
+			if !since.IsZero() && d.PublishedAt.Before(since) {
+				continue
+			}
+			if !until.IsZero() && d.PublishedAt.After(until) {
+				continue
+			}
+		}
+		if hasDomainFilter {
+			host := hostFromURL(d.URL)
+			if len(include) > 0 && !matchesDomainPattern(host, include) {
+				continue
+			}
+			if len(exclude) > 0 && matchesDomainPattern(host, exclude) {
+				continue
+			}
+		}
+		out = append(out, d)
+	}
+	return out
+}
+
+// validateExportFormat is the export-side counterpart to's
+// validateFormat. Different accept-list because export supports json/jsonl
+// (file-shaped) which the synth-output CLIs don't, and synth CLIs support
+// markdown aliases which export doesn't need (single canonical name reduces
+// confusion when picking a file extension).
+func validateExportFormat(v string) error {
+	switch v {
+	case "json", "jsonl", "text", "md":
+		return nil
+	default:
+		return fmt.Errorf("export format must be one of: json, jsonl, text, md (got %q)", v)
+	}
+}
+
+// exportExtension picks a conventional file extension for each export format.
+// Used to derive a default -output path when the operator doesn't specify one.
+func exportExtension(format string) string {
+	switch format {
+	case "jsonl":
+		return "jsonl"
+	case "text":
+		return "txt"
+	case "md":
+		return "md"
+	default: // json
+		return "json"
+	}
+}
+
+// runReembed walks every document, chunks it, embeds the chunks with the
+// configured model, and writes them as new passages. The new rows coexist with
+// the old (different model_id) until `-drop-old` is passed.
+//
+// Recommended sequence:
+//
+//	cosift reembed                       # write new passages alongside old
+//	# verify search quality on new model …
+//	cosift reembed -drop-old             # drop the others
+//
+// Or just `cosift reembed -drop-old` if you're confident.
+func runReembed(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("reembed", flag.ExitOnError)
+	dropOld := fs.Bool("drop-old", false, "after re-embed, delete passages with a different model_id")
+	batchSize := fs.Int("batch", 256, "embeddings per API call (provider-dependent cap)")
+	// time-based progress reporting (shared helper from).
+	progressInterval := fs.Duration("progress", 5*time.Second, "log re-embed progress every N (0 disables)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if cfg.Embeddings.Model == "" {
+		return errors.New("reembed: cfg.Embeddings.Model must be set")
+	}
+	apiKey := os.Getenv("OPENAI_API_KEY")
+	if apiKey == "" {
+		apiKey = os.Getenv("OPENAI")
+	}
+	if apiKey == "" {
+		return errors.New("reembed: no OPENAI key in env")
+	}
+	dim := cfg.Embeddings.Dim
+	if dim == 0 {
+		dim = 1536
+	}
+	emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
+
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	docs, err := s.ListDocuments(ctx, 0)
+	if err != nil {
+		return err
+	}
+	if len(docs) == 0 {
+		log.Printf("reembed: 0 docs in store — nothing to do")
+		return nil
+	}
+
+	// Pre-existing models (informational; useful for the operator).
+	models, _ := s.PassageModels(ctx)
+	log.Printf("reembed: %d docs · target model=%s · existing models=%v · batch=%d",
+		len(docs), cfg.Embeddings.Model, models, *batchSize)
+
+	chunker := chunkerWith(cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
+	type ref struct {
+		docID int64
+		chunk index.Chunk
+	}
+	var (
+		texts   []string
+		refs    []ref
+		total   int
+		flush   func() error
+		written int
+	)
+	flush = func() error {
+		if len(texts) == 0 {
+			return nil
+		}
+		vecs, err := emb.Embed(ctx, texts)
+		if err != nil {
+			return fmt.Errorf("embed batch (%d texts): %w", len(texts), err)
+		}
+		if len(vecs) != len(texts) {
+			return fmt.Errorf("embed count mismatch: %d vs %d", len(vecs), len(texts))
+		}
+		for i, v := range vecs {
+			if err := s.UpsertPassage(ctx, &store.Passage{
+				DocID:     refs[i].docID,
+				Offset:    refs[i].chunk.Offset,
+				Length:    refs[i].chunk.Length,
+				Model:     emb.Model(),
+				Embedding: v,
+			}); err != nil {
+				return fmt.Errorf("write passage: %w", err)
+			}
+			written++
+		}
+		texts = texts[:0]
+		refs = refs[:0]
+		return nil
+	}
+
+	docProgress := newProgressReporter("reembed docs", len(docs), *progressInterval)
+	for docIdx, d := range docs {
+		for _, c := range chunker.Chunk(d.Title + "\n\n" + d.Text) {
+			texts = append(texts, c.Text)
+			refs = append(refs, ref{docID: d.ID, chunk: c})
+			total++
+			if len(texts) >= *batchSize {
+				if err := flush(); err != nil {
+					return err
+				}
+			}
+		}
+		docProgress.maybeLog(docIdx + 1)
+	}
+	if err := flush(); err != nil {
+		return err
+	}
+	log.Printf("reembed: wrote %d passages with model=%s", written, cfg.Embeddings.Model)
+
+	if *dropOld {
+		n, err := s.DropPassagesNotModel(ctx, cfg.Embeddings.Model)
+		if err != nil {
+			return fmt.Errorf("drop old: %w", err)
+		}
+		log.Printf("reembed: dropped %d passages from other models", n)
+	} else if len(models) > 0 && !contains(models, cfg.Embeddings.Model) {
+		log.Printf("reembed: %d other-model passages remain — run with -drop-old to remove", len(models))
+	}
+	return nil
+}
+
+// runCompactIndex drops orphaned passages and terms, then VACUUMs.
+// Companion to `cosift gc` — gc cleans the frontier; compact-index cleans
+// the index proper.
+func runCompactIndex(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("compact-index", flag.ExitOnError)
+	vacuum := fs.Bool("vacuum", true, "run VACUUM after deletes to reclaim disk")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	dropped, err := s.DropOrphanedPassages(ctx)
+	if err != nil {
+		return fmt.Errorf("drop orphan passages: %w", err)
+	}
+	log.Printf("compact-index: dropped %d orphan passages", dropped)
+
+	terms, err := s.DropOrphanedTerms(ctx)
+	if err != nil {
+		return fmt.Errorf("drop orphan terms: %w", err)
+	}
+	log.Printf("compact-index: dropped %d orphan terms", terms)
+
+	if *vacuum {
+		log.Printf("compact-index: vacuuming...")
+		if err := s.Vacuum(ctx); err != nil {
+			return fmt.Errorf("vacuum: %w", err)
+		}
+		log.Printf("compact-index: vacuum complete")
+	}
+	return nil
+}
+
+// batchEmbed chunks a large text slice into provider-safe batches before
+// calling the embedder. OpenAI's text-embedding-3-* caps at 2048 inputs OR
+// 300k tokens per request; 1000 is a safe default that respects both at
+// our typical chunk sizes. Returns vectors in the input order.
+//
+// Between batches we sleep 200ms — keeps multi-batch runs under OpenAI's
+// 1M-TPM org limit for text-embedding-3-small, which hit at ~50k inputs.
+// Provider rate-limit errors get one bounded retry with the suggested delay.
+func batchEmbed(ctx context.Context, e embed.Embedder, texts []string, batchSize int) ([][]float32, error) {
+	if batchSize <= 0 {
+		batchSize = 256
+	}
+	// 1000 is the published OpenAI per-call cap, but large-dim models
+	// (text-embedding-3-large @ 3072) return malformed JSON well before that
+	// in practice. 256 is safe across the model lineup.
+	if batchSize > 1000 {
+		batchSize = 1000
+	}
+	if len(texts) <= batchSize {
+		return e.Embed(ctx, texts)
+	}
+	out := make([][]float32, 0, len(texts))
+	for i := 0; i < len(texts); i += batchSize {
+		end := i + batchSize
+		if end > len(texts) {
+			end = len(texts)
+		}
+		vecs, err := embedWithRetry(ctx, e, texts[i:end])
+		if err != nil {
+			return nil, fmt.Errorf("batch %d-%d: %w", i, end, err)
+		}
+		if len(vecs) != end-i {
+			return nil, fmt.Errorf("batch %d-%d: %d returned, expected %d", i, end, len(vecs), end-i)
+		}
+		out = append(out, vecs...)
+		if end < len(texts) {
+			select {
+			case <-ctx.Done():
+				return out, ctx.Err()
+			case <-time.After(200 * time.Millisecond):
+			}
+		}
+	}
+	return out, nil
+}
+
+// embedWithRetry retries once on rate-limit errors with a fixed backoff.
+// Crude but effective at our scale — keeps full distractor runs from failing
+// when a single batch slips past the TPM bucket. Production deployments with
+// higher request volumes want exponential backoff + provider-supplied delay
+// parsing; deferred until traffic warrants.
+func embedWithRetry(ctx context.Context, e embed.Embedder, texts []string) ([][]float32, error) {
+	vecs, err := e.Embed(ctx, texts)
+	if err == nil {
+		return vecs, nil
+	}
+	msg := err.Error()
+	if !strings.Contains(msg, "rate_limit") && !strings.Contains(msg, "429") {
+		return nil, err
+	}
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-time.After(2 * time.Second):
+	}
+	return e.Embed(ctx, texts)
+}
diff --git a/cmd/cosift/cmd_maintenance.go b/cmd/cosift/cmd_maintenance.go
new file mode 100644
index 0000000..457e0cf
--- /dev/null
+++ b/cmd/cosift/cmd_maintenance.go
@@ -0,0 +1,720 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/server"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// runGC drops persistently-errored frontier rows and (optionally) VACUUMs
+// the SQLite file. Designed for long-lived deployments where frontier garbage
+// would otherwise accumulate.
+func runGC(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("gc", flag.ExitOnError)
+	minAttempts := fs.Int("min-attempts", 5, "drop frontier rows with status='error' AND attempts >= N")
+	paraphraseTTL := fs.Duration("paraphrase-ttl", 0, "drop query_paraphrases rows older than this (0 = skip)")
+	vacuum := fs.Bool("vacuum", true, "run VACUUM after cleanup to reclaim disk")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	n, err := s.GCErroredFrontier(ctx, *minAttempts)
+	if err != nil {
+		return fmt.Errorf("gc frontier: %w", err)
+	}
+	log.Printf("gc: removed %d errored frontier rows (min_attempts=%d)", n, *minAttempts)
+
+	if *paraphraseTTL > 0 {
+		pn, err := s.PruneStaleParaphrases(ctx, *paraphraseTTL)
+		if err != nil {
+			return fmt.Errorf("gc paraphrases: %w", err)
+		}
+		log.Printf("gc: pruned %d stale paraphrases (ttl=%s)", pn, *paraphraseTTL)
+	}
+
+	if *vacuum {
+		log.Printf("gc: vacuuming...")
+		if err := s.Vacuum(ctx); err != nil {
+			return fmt.Errorf("vacuum: %w", err)
+		}
+		log.Printf("gc: vacuum complete")
+	}
+	return nil
+}
+
+// runFrontierTrim deletes completed (status='done') frontier rows older than a
+// configurable age and optionally VACUUMs the database.
+//
+// Background: the GC command (runGC) only drops status='error' rows. Completed
+// rows accumulate indefinitely — a 500k-URL crawl leaves 500k done rows that
+// serve no purpose once the URL is in the documents table. Over months of
+// continuous crawling this can reach tens of millions of rows, bloating the DB
+// by several GB and slowing ClaimFrontier's ORDER BY scan even with an index.
+//
+// Default -older-than of 168h (7 days) is safe for the default refresh cycle
+// (refresh-due at 24h): a URL that was crawled 7 days ago and whose done row
+// is deleted will re-enter the frontier on the next refresh-due pass, which is
+// the correct behavior. Operators with longer refresh cycles should raise this
+// value accordingly (e.g. -older-than=720h for monthly refreshes).
+func runFrontierTrim(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("frontier-trim", flag.ExitOnError)
+	olderThan := fs.Duration("older-than", 7*24*time.Hour, "delete done rows enqueued more than this long ago (e.g. 168h, 720h)")
+	dryRun := fs.Bool("dry-run", false, "print what would be deleted without actually deleting")
+	vacuum := fs.Bool("vacuum", true, "run VACUUM after deletion to reclaim disk space")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	stats, serr := s.GetFrontierStats(ctx)
+	if serr != nil {
+		return fmt.Errorf("frontier-trim: stats: %w", serr)
+	}
+	log.Printf("frontier-trim: before — queued=%d in_flight=%d done=%d error=%d",
+		stats.Queued, stats.InFlight, stats.Done, stats.Errored)
+
+	if *dryRun {
+		log.Printf("frontier-trim: dry-run — would delete done rows older than %s (skipping actual delete)", *olderThan)
+		return nil
+	}
+
+	n, derr := s.TrimDoneFrontier(ctx, *olderThan)
+	if derr != nil {
+		return fmt.Errorf("frontier-trim: %w", derr)
+	}
+	log.Printf("frontier-trim: removed %d done rows older than %s", n, *olderThan)
+
+	if *vacuum && n > 0 {
+		log.Printf("frontier-trim: vacuuming (this may take a while on large databases)…")
+		if verr := s.Vacuum(ctx); verr != nil {
+			return fmt.Errorf("frontier-trim: vacuum: %w", verr)
+		}
+		log.Printf("frontier-trim: vacuum complete")
+	}
+	return nil
+}
+
+// runSeedTranco reads a Tranco top-1M CSV and pushes the top-N root URLs into
+// the frontier as depth-0 seeds. This is the fastest way to bootstrap a broad
+// crawl from a high-quality, authority-ranked URL set without having to curate
+// a manual seeds file.
+//
+// Each Tranco entry becomes https://<domain>/ with a priority derived from its
+// rank: rank 1 → priority 1.0, rank 1M → priority 0.1, log-scale in between.
+// The INSERT OR IGNORE semantics of PushFrontier mean this is safe to run
+// repeatedly — it only adds URLs not already in the frontier (regardless of
+// their current status).
+//
+// Download the latest Tranco CSV from https://tranco-list.eu/ (updated weekly,
+// ~7 MB compressed). The format is "rank,hostname" with no header.
+//
+// Example:
+//
+//	cosift seed-tranco -tranco tranco-top1m.csv -top 10000
+func runSeedTranco(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("seed-tranco", flag.ExitOnError)
+	trancoPath := fs.String("tranco", "", "path to Tranco top-1M CSV (required)")
+	topN := fs.Int("top", 10000, "seed only the top-N domains (0 = all)")
+	scheme := fs.String("scheme", "https", "URL scheme to prepend (https or http)")
+	dryRun := fs.Bool("dry-run", false, "print URLs that would be seeded without writing to the frontier")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *trancoPath == "" {
+		return errors.New("seed-tranco: -tranco is required")
+	}
+	if *scheme != "https" && *scheme != "http" {
+		return fmt.Errorf("seed-tranco: -scheme must be https or http, got %q", *scheme)
+	}
+
+	f, ferr := os.Open(*trancoPath)
+	if ferr != nil {
+		return fmt.Errorf("seed-tranco: open %s: %w", *trancoPath, ferr)
+	}
+	defer f.Close()
+
+	// Parse the CSV into (rank, host) pairs up to topN.
+	type entry struct {
+		rank int
+		host string
+	}
+	entries := make([]entry, 0, 10000)
+	scanner := bufio.NewScanner(f)
+	scanner.Buffer(make([]byte, 0, 4096), 1<<20)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		var rankStr, host string
+		if i := strings.IndexAny(line, ",\t"); i > 0 {
+			rankStr = strings.TrimSpace(line[:i])
+			host = strings.ToLower(strings.TrimSpace(line[i+1:]))
+		} else {
+			continue
+		}
+		rank, err := strconv.Atoi(rankStr)
+		if err != nil || rank <= 0 {
+			continue
+		}
+		host = strings.TrimPrefix(host, "www.")
+		if host == "" {
+			continue
+		}
+		if *topN > 0 && rank > *topN {
+			break
+		}
+		entries = append(entries, entry{rank: rank, host: host})
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("seed-tranco: scan: %w", err)
+	}
+	log.Printf("seed-tranco: parsed %d entries from %s (top %d)", len(entries), *trancoPath, *topN)
+
+	if *dryRun {
+		limit := len(entries)
+		if limit > 20 {
+			limit = 20
+		}
+		for _, e := range entries[:limit] {
+			log.Printf("  [dry-run] rank=%d  %s://%s/", e.rank, *scheme, e.host)
+		}
+		if len(entries) > 20 {
+			log.Printf("  … and %d more", len(entries)-20)
+		}
+		return nil
+	}
+
+	s, serr := store.Open(cfg.DataDir)
+	if serr != nil {
+		return serr
+	}
+	defer s.Close()
+
+	// trancoPriority converts a Tranco rank to a frontier priority [0.1, 1.0].
+	// Uses decimal-digit counting as a cheap log10 proxy — good enough for
+	// priority bucketing. rank=1→1.0, rank=10→0.83, rank=100→0.67,
+	// rank=1k→0.50, rank=10k→0.33, rank=100k→0.17, rank=1M→0.10.
+	trancoPriority := func(rank int) float64 {
+		if rank <= 0 {
+			return 0.5
+		}
+		digits := 0
+		r := rank
+		for r >= 10 {
+			r /= 10
+			digits++
+		}
+		p := 1.0 - float64(digits)/6.0
+		if p < 0.1 {
+			p = 0.1
+		}
+		return p
+	}
+
+	pushed := 0
+	for _, e := range entries {
+		rawURL := fmt.Sprintf("%s://%s/", *scheme, e.host)
+		priority := trancoPriority(e.rank)
+		if err := s.PushFrontier(ctx, rawURL, 0, priority); err != nil {
+			log.Printf("seed-tranco: push %s: %v", rawURL, err)
+			continue
+		}
+		pushed++
+	}
+	log.Printf("seed-tranco: pushed %d URLs into frontier (INSERT OR IGNORE — already-present URLs unchanged)", pushed)
+	return nil
+}
+
+// runOutcomes exports query_outcomes for offline calibration work. JSON for
+// programmatic consumers, CSV for spreadsheet / pandas notebooks.
+func runOutcomes(ctx context.Context, cfg *config.Config, args []string) (err error) {
+	fs := flag.NewFlagSet("outcomes", flag.ExitOnError)
+	output := fs.String("output", "outcomes.json", "output path")
+	format := fs.String("format", "json", "json | csv")
+	limit := fs.Int("limit", 0, "max rows (0 = all)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *format != "json" && *format != "csv" {
+		return fmt.Errorf("unsupported format %q (use json or csv)", *format)
+	}
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	rows, err := s.ListOutcomes(ctx, *limit)
+	if err != nil {
+		return err
+	}
+	f, cerr := os.Create(*output)
+	if cerr != nil {
+		return cerr
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil && err == nil {
+			err = fmt.Errorf("close output: %w", closeErr)
+		}
+	}()
+
+	switch *format {
+	case "json":
+		enc := json.NewEncoder(f)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(rows); err != nil {
+			return err
+		}
+	case "csv":
+		w := csv.NewWriter(f)
+		_ = w.Write([]string{"id", "query", "url", "score", "useful", "source", "recorded_at"})
+		for _, r := range rows {
+			useful := "0"
+			if r.Useful {
+				useful = "1"
+			}
+			_ = w.Write([]string{
+				strconv.FormatInt(r.ID, 10),
+				r.Query, r.URL,
+				strconv.FormatFloat(r.Score, 'f', -1, 64),
+				useful, r.Source,
+				r.RecordedAt.Format(time.RFC3339),
+			})
+		}
+		w.Flush()
+		if err := w.Error(); err != nil {
+			return err
+		}
+	}
+	log.Printf("outcomes: %d rows → %s (%s)", len(rows), *output, *format)
+	return nil
+}
+
+// runDoctor checks the local install: data dir writable, schema open,
+// fixtures present, env recognized. Pure local checks — no network calls,
+// no API key spending. Use `cosift eval` for an end-to-end smoke against
+// real services.
+//
+// Exit code: 0 if everything is PASS or WARN; 1 if any FAIL.
+// doctorCheck is one row of the doctor report. Promoted from a locally-typed
+// struct so the defaults check can be tested without stdout capture.
+type doctorCheck struct {
+	Name   string `json:"name"`
+	Status string `json:"status"` // "PASS" | "WARN" | "FAIL" | "INFO" | "SKIP"
+	Detail string `json:"detail"`
+}
+
+// doctorDefaultsChecks cross-checks the defaults block against
+// configured capabilities. Returns the rows that would be appended to the
+// doctor report. Pure function — no IO, no stdout, no environment reads.
+// hasEmbed/hasChat reflect "is the capability available at all" (config
+// fields populated or env keys present); the caller resolves those before
+// calling.
+func doctorDefaultsChecks(d config.Defaults, hasEmbed, hasChat bool) []doctorCheck {
+	if d.Retriever == "" && !d.Expand && d.ResearchStrategy == "" && d.ResearchSynthK == 0 {
+		return []doctorCheck{{
+			Name:   "defaults",
+			Status: "PASS",
+			Detail: "no defaults set (per-request params win; planner is the /research default)",
+		}}
+	}
+	checks := []doctorCheck{{
+		Name:   "defaults",
+		Status: "PASS",
+		Detail: fmt.Sprintf("retriever=%q expand=%v research_strategy=%q research_synth_k=%d", d.Retriever, d.Expand, d.ResearchStrategy, d.ResearchSynthK),
+	}}
+	if d.ResearchSynthK < 0 {
+		checks = append(checks, doctorCheck{
+			Name:   "defaults.research_synth_k",
+			Status: "FAIL",
+			Detail: fmt.Sprintf("research_synth_k must be ≥ 0 (got %d). 0 = use built-in default (10); positive = cap", d.ResearchSynthK),
+		})
+	}
+	switch d.Retriever {
+	case "dense", "hybrid":
+		if !hasEmbed {
+			checks = append(checks, doctorCheck{
+				Name:   "defaults.retriever",
+				Status: "WARN",
+				Detail: fmt.Sprintf("%q default needs embeddings model — set cfg.Embeddings.Model or OPENAI_API_KEY", d.Retriever),
+			})
+		}
+	case "", "bm25":
+		// no-op; bm25 needs no capability beyond the SQLite index.
+	default:
+		checks = append(checks, doctorCheck{
+			Name:   "defaults.retriever",
+			Status: "FAIL",
+			Detail: fmt.Sprintf("unknown retriever %q (valid: bm25, dense, hybrid)", d.Retriever),
+		})
+	}
+	if d.Expand && !hasChat {
+		checks = append(checks, doctorCheck{
+			Name:   "defaults.expand",
+			Status: "WARN",
+			Detail: "expand=true needs a chat model for paraphrasing — set cfg.Chat.Model or OPENAI_API_KEY",
+		})
+	}
+	switch d.ResearchStrategy {
+	case "paraphrase":
+		if !hasChat {
+			checks = append(checks, doctorCheck{
+				Name:   "defaults.research_strategy",
+				Status: "WARN",
+				Detail: "paraphrase needs a chat model for the paraphraser — set cfg.Chat.Model or OPENAI_API_KEY",
+			})
+		}
+	case "", "planner":
+		// no-op; planner is the historic default.
+	default:
+		checks = append(checks, doctorCheck{
+			Name:   "defaults.research_strategy",
+			Status: "FAIL",
+			Detail: fmt.Sprintf("unknown strategy %q (valid: planner, paraphrase)", d.ResearchStrategy),
+		})
+	}
+	return checks
+}
+
+// runInit writes a sensible default cosift.json to the requested path. By
+// default it refuses to overwrite an existing file; -force opts in.
+//
+// The optional -site URL pre-populates `crawler.include_domains` so the user
+// can immediately `cosift crawl <url>` against a target site without editing
+// the config first. Lowers the onboarding cost for the self-hostable use case.
+func runInit(cfgPath string, args []string) error {
+	fs := flag.NewFlagSet("init", flag.ExitOnError)
+	force := fs.Bool("force", false, "overwrite an existing config file")
+	site := fs.String("site", "", "URL of a site to pre-populate crawler.include_domains with (e.g. https://docs.example.com)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(cfgPath); err == nil && !*force {
+		return fmt.Errorf("%s already exists; pass -force to overwrite", cfgPath)
+	}
+
+	cfg := config.Default()
+	if *site != "" {
+		host, err := siteToHost(*site)
+		if err != nil {
+			return fmt.Errorf("-site: %w", err)
+		}
+		cfg.Crawler.IncludeDomains = []string{host}
+	}
+
+	buf, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal: %w", err)
+	}
+	// cosift.json carries Server.AdminToken and may be the sink for
+	// LoadDotEnv-sourced secrets (OPENAI_API_KEY, etc.). Owner-only.
+	if err := os.WriteFile(cfgPath, buf, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", cfgPath, err)
+	}
+
+	fmt.Printf("wrote %s\n", cfgPath)
+	if cfg.Crawler.IncludeDomains != nil {
+		fmt.Printf("  crawler.include_domains: %v\n", cfg.Crawler.IncludeDomains)
+	}
+	fmt.Println()
+	fmt.Println("next steps:")
+	if *site != "" {
+		fmt.Printf("  1. cosift crawl %s\n", *site)
+	} else {
+		fmt.Println("  1. cosift crawl <url...>")
+	}
+	fmt.Println("  2. cosift serve")
+	fmt.Println()
+	fmt.Println("optional: set OPENAI_API_KEY in your environment (or .env) to enable")
+	fmt.Println("  dense/hybrid retrieval, /answer, /research, and ?expand=true.")
+	return nil
+}
+
+func runDoctor(ctx context.Context, cfg *config.Config, args []string) error {
+	// When -server is empty, doctor stays
+	// purely local. When -server is set, doctor also pings
+	// the server's /healthz and /stats; if -token is also set (or env), it
+	// validates the token by hitting /admin/config.
+	fs := flag.NewFlagSet("doctor", flag.ExitOnError)
+	remoteServer := fs.String("server", "", "remote cosift URL to additionally probe (empty = local checks only)")
+	remoteToken := fs.String("token", "", "admin bearer token for -server admin-endpoint check (defaults to COSIFT_ADMIN_TOKEN env)")
+	// Same checks, same
+	// non-zero exit on FAIL, just JSON instead of '[PASS] name — detail'.
+	asJSON := fs.Bool("json", false, "emit JSON instead of human-readable text (suitable for jq / CI)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	var checks []doctorCheck
+	add := func(name, status, detail string) { checks = append(checks, doctorCheck{name, status, detail}) }
+
+	// 1. Data dir writable.
+	probe := filepath.Join(cfg.DataDir, ".doctor.probe")
+	// Match store.Open's 0o700 — MkdirAll's perm only applies to NEW dirs,
+	// but doctor running first must not silently leave a looser mode behind.
+	if err := os.MkdirAll(cfg.DataDir, 0o700); err != nil {
+		add("data_dir writable", "FAIL", fmt.Sprintf("mkdir %s: %v", cfg.DataDir, err))
+	} else if err := os.WriteFile(probe, []byte("ok"), 0o600); err != nil {
+		add("data_dir writable", "FAIL", fmt.Sprintf("write %s: %v", probe, err))
+	} else {
+		_ = os.Remove(probe)
+		add("data_dir writable", "PASS", cfg.DataDir)
+	}
+
+	// 2. SQLite open + schema migrated.
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		add("sqlite open + schema", "FAIL", err.Error())
+	} else {
+		st, _ := s.Stats(ctx)
+		_ = s.Close()
+		add("sqlite open + schema", "PASS", fmt.Sprintf("%d docs, %d terms", st.Documents, st.Terms))
+	}
+
+	// 2b. Pebble open + counters. Pebble is optional; "absent" is
+	// not a failure, just a SKIP. When present, verify open + read counters.
+	// the open can fail because pebble-serve / a crawl holds the
+	// single-writer lock — that's a healthy state, not a FAIL. Distinguish
+	// the lock error from real corruption.
+	pebbleDir := filepath.Join(cfg.DataDir, "pebble")
+	if _, statErr := os.Stat(pebbleDir); statErr != nil {
+		add("pebble store", "SKIP", "no pebble dir (sqlite-only deployment)")
+	} else if ps, err := store.OpenPebble(pebbleDir); err != nil {
+		msg := err.Error()
+		if strings.Contains(msg, "lock") || strings.Contains(msg, "resource temporarily unavailable") {
+			add("pebble store", "INFO", "another process holds the writer lock (pebble-serve / crawl in flight)")
+		} else {
+			add("pebble store", "FAIL", msg)
+		}
+	} else {
+		_, n, _ := ps.CorpusStats(ctx)
+		// also report HNSW availability so operators can tell from
+		// `cosift doctor` whether dense / hybrid retrievers will work without
+		// firing pebble-info or pebble-serve /stats. Cheap — same 20-byte
+		// meta blob read pebble-info already does.
+		meta, hnswOK, _ := index.LoadHNSWMeta(ctx, ps)
+		_ = ps.Close()
+		add("pebble store", "PASS", fmt.Sprintf("%d indexed docs", n))
+		if hnswOK {
+			add("HNSW graph", "PASS", fmt.Sprintf("%d nodes, dim=%d — dense/hybrid available (load with COSIFT_LOAD_HNSW=true)", meta.NodeCount, meta.Dim))
+		} else {
+			add("HNSW graph", "INFO", "no vectors persisted — only bm25 / bm25-mlt retrievers (set cfg.Embeddings.Model and re-crawl to add dense)")
+		}
+	}
+
+	// 2c. COSIFT_* env vars. Lists which path-2 env overrides are
+	// set so operators can confirm 'is my override actually live'. Silent
+	// typos previously hid behind 'env unset → keep default' branches.
+	cosiftEnvs := []string{
+		"COSIFT_PEBBLE_CACHE_MB", "COSIFT_PEBBLE_MEMTABLE_MB", "COSIFT_PEBBLE_MEMTABLES", "COSIFT_PEBBLE_SYNC",
+		"COSIFT_BM25_K1", "COSIFT_BM25_B",
+		"COSIFT_HYDE_CACHE_SIZE", "COSIFT_PARA_CACHE_SIZE",
+		"COSIFT_LOAD_HNSW", //
+	}
+	var setEnvs []string
+	for _, name := range cosiftEnvs {
+		if v := os.Getenv(name); v != "" {
+			setEnvs = append(setEnvs, name+"="+v)
+		}
+	}
+	if len(setEnvs) == 0 {
+		add("COSIFT_* env", "INFO", "no path-2 overrides set (using defaults)")
+	} else {
+		add("COSIFT_* env", "INFO", strings.Join(setEnvs, ", "))
+	}
+
+	// 3. Config recognized.
+	add("config", "PASS", fmt.Sprintf("addr=%s, data_dir=%s", cfg.Server.Addr, cfg.DataDir))
+
+	// 4. API key env (warn — actually pinging is `cosift eval` territory).
+	// a custom embedder/chat URL (Ollama / vLLM / TEI) doesn't
+	// need a key; PASS when URL is set even without a key. Hosted defaults
+	// (empty URL) still require a key for the configured capability.
+	embedKey := resolveEmbedAPIKey()
+	chatKey := os.Getenv("OPENAI_API_KEY")
+	if chatKey == "" {
+		chatKey = os.Getenv("OPENAI")
+	}
+	embedConfigured := cfg.Embeddings.Model != ""
+	chatConfigured := cfg.Chat.Model != ""
+	embedLocal := cfg.Embeddings.URL != ""
+	chatLocal := cfg.Chat.URL != ""
+	switch {
+	case !embedConfigured && !chatConfigured:
+		add("embed/chat auth", "INFO", "no embed/chat models configured (fine for bm25-only)")
+	case (embedConfigured && (embedKey != "" || embedLocal)) && (!chatConfigured || chatKey != "" || chatLocal):
+		add("embed/chat auth", "PASS", fmt.Sprintf("embed=%s chat=%s", authStatus(embedConfigured, embedKey, embedLocal), authStatus(chatConfigured, chatKey, chatLocal)))
+	default:
+		var missing []string
+		if embedConfigured && embedKey == "" && !embedLocal {
+			missing = append(missing, "embed (no URL, no key)")
+		}
+		if chatConfigured && chatKey == "" && !chatLocal {
+			missing = append(missing, "chat (no URL, no key)")
+		}
+		add("embed/chat auth", "WARN", strings.Join(missing, ", ")+" — set COSIFT_EMBED_API_KEY / OPENAI_API_KEY, or point Embeddings.URL / Chat.URL at a local server")
+	}
+
+	// 5. Eval fixtures.
+	corpusPath := "testdata/eval/corpus.json"
+	queriesPath := "testdata/eval/queries.json"
+	if _, err := os.Stat(corpusPath); err != nil {
+		add("eval fixtures", "WARN", "testdata/eval/corpus.json missing (eval won't run)")
+	} else if _, err := os.Stat(queriesPath); err != nil {
+		add("eval fixtures", "WARN", "testdata/eval/queries.json missing")
+	} else {
+		add("eval fixtures", "PASS", "corpus.json + queries.json present")
+	}
+
+	// 6. Admin token configured if /admin/* is to be used.
+	if cfg.Server.AdminToken != "" {
+		add("admin_token", "PASS", "set (length redacted)")
+	} else {
+		add("admin_token", "WARN", "unset — /admin/* will return 403")
+	}
+
+	// 7. Defaults vs capabilities. The defaults block lets operators
+	// pre-configure retrieval behavior, but a default that requires capability
+	// the server isn't configured for (e.g. retriever=hybrid without an
+	// embedder) will silently fail at request time — 400 on every call. Catch
+	// the mismatch HERE instead of at first traffic.
+	hasEmbed := cfg.Embeddings.Model != "" || os.Getenv("OPENAI_API_KEY") != "" || os.Getenv("OPENAI") != ""
+	hasChat := cfg.Chat.Model != "" || os.Getenv("OPENAI_API_KEY") != "" || os.Getenv("OPENAI") != ""
+	checks = append(checks, doctorDefaultsChecks(cfg.Defaults, hasEmbed, hasChat)...)
+
+	// 8. Remote checks — only when -server is set. Token resolution
+	// mirrors admin CLIs: -token flag wins, then env.
+	if *remoteServer != "" {
+		resolvedToken := *remoteToken
+		if resolvedToken == "" {
+			resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
+		}
+		checks = append(checks, doctorRemoteChecks(ctx, *remoteServer, resolvedToken)...)
+	}
+
+	fail := 0
+	for _, c := range checks {
+		if c.Status == "FAIL" {
+			fail++
+		}
+	}
+	if *asJSON {
+		out := map[string]any{
+			"checks": checks,
+			"fail":   fail,
+			"ready":  fail == 0,
+		}
+		blob, _ := json.MarshalIndent(out, "", "  ")
+		fmt.Println(string(blob))
+		if fail > 0 {
+			return errors.New("doctor failures")
+		}
+		return nil
+	}
+	for _, c := range checks {
+		fmt.Printf("%-6s %s — %s\n", "["+c.Status+"]", c.Name, c.Detail)
+	}
+	fmt.Println()
+	if fail > 0 {
+		fmt.Printf("%d FAIL — fix the issues above before running `cosift serve`\n", fail)
+		return errors.New("doctor failures")
+	}
+	fmt.Println("ready.")
+	return nil
+}
+
+// doctorRemoteChecks probes a remote cosift server: /healthz reachable, /stats
+// returns a sensible shape, and (when token non-empty) /admin/config validates
+// the bearer token. Returns a slice of doctorCheck rows for the main runDoctor
+// loop to render.
+func doctorRemoteChecks(ctx context.Context, serverURL, token string) []doctorCheck {
+	var out []doctorCheck
+	base := strings.TrimRight(strings.Replace(serverURL, "0.0.0.0", "127.0.0.1", 1), "/")
+	client := &http.Client{Timeout: 10 * time.Second}
+
+	// /healthz
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, base+"/healthz", http.NoBody)
+	resp, err := client.Do(req)
+	if err != nil {
+		out = append(out, doctorCheck{"remote /healthz", "FAIL", fmt.Sprintf("%s: %v", serverURL, err)})
+		// If /healthz is unreachable, downstream checks will all fail too —
+		// emit one row and skip the cascade.
+		return out
+	}
+	resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		out = append(out, doctorCheck{"remote /healthz", "PASS", base})
+	} else {
+		out = append(out, doctorCheck{"remote /healthz", "FAIL", fmt.Sprintf("%s returned %d", base, resp.StatusCode)})
+	}
+
+	// /stats
+	req, _ = http.NewRequestWithContext(ctx, http.MethodGet, base+"/stats", http.NoBody)
+	resp, err = client.Do(req)
+	if err != nil {
+		out = append(out, doctorCheck{"remote /stats", "FAIL", err.Error()})
+	} else {
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			out = append(out, doctorCheck{"remote /stats", "FAIL", fmt.Sprintf("returned %d", resp.StatusCode)})
+		} else {
+			var st server.StatsResponse
+			body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+			if err := json.Unmarshal(body, &st); err != nil {
+				out = append(out, doctorCheck{"remote /stats", "WARN", fmt.Sprintf("decode: %v", err)})
+			} else {
+				out = append(out, doctorCheck{"remote /stats", "PASS", fmt.Sprintf("%d docs", st.Documents)})
+			}
+		}
+	}
+
+	// /admin/config — only when token is provided. No token → skip silently
+	// (not a failure; operator may not have admin auth enabled).
+	if token != "" {
+		req, _ = http.NewRequestWithContext(ctx, http.MethodGet, base+"/admin/config", http.NoBody)
+		req.Header.Set("Authorization", "Bearer "+token)
+		resp, err = client.Do(req)
+		if err != nil {
+			out = append(out, doctorCheck{"remote admin token", "FAIL", err.Error()})
+			return out
+		}
+		defer resp.Body.Close()
+		switch resp.StatusCode {
+		case http.StatusOK:
+			out = append(out, doctorCheck{"remote admin token", "PASS", "/admin/config accepted the token"})
+		case http.StatusUnauthorized, http.StatusForbidden:
+			out = append(out, doctorCheck{"remote admin token", "FAIL", fmt.Sprintf("token rejected (HTTP %d)", resp.StatusCode)})
+		default:
+			out = append(out, doctorCheck{"remote admin token", "WARN", fmt.Sprintf("unexpected status %d", resp.StatusCode)})
+		}
+	}
+	return out
+}
diff --git a/cmd/cosift/cmd_query.go b/cmd/cosift/cmd_query.go
new file mode 100644
index 0000000..b49b45b
--- /dev/null
+++ b/cmd/cosift/cmd_query.go
@@ -0,0 +1,1308 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/server"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func runQuery(ctx context.Context, cfg *config.Config, q string, args []string) error {
+	fs := flag.NewFlagSet("query", flag.ExitOnError)
+	k := fs.Int("k", 10, "max results")
+	jsonOut := fs.Bool("json", false, "emit JSON array instead of human-readable list — for shell pipelines")
+	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *k <= 0 || *k > 100 {
+		return errors.New("k must be in [1, 100]")
+	}
+
+	// -backend flag mirrors on crawl + stats. Both
+	// BM25 implementations expose the same Search(ctx, q, k) signature so
+	// the rest of runQuery is backend-agnostic.
+	type searcher interface {
+		Search(ctx context.Context, q string, k int) ([]index.Hit, error)
+	}
+	var idx searcher
+	switch *backend {
+	case "sqlite", "":
+		s, err := store.Open(cfg.DataDir)
+		if err != nil {
+			return err
+		}
+		defer s.Close()
+		idx = index.NewBM25(s)
+	case "pebble":
+		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
+		ps, err := openPebbleOrFriendlyErr(pebbleDir)
+		if err != nil {
+			return err
+		}
+		defer ps.Close()
+		// runQuery previously
+		// always used defaults; with env overrides set, score values diverged
+		// silently from pebble-serve's. Now they line up.
+		pidx := index.NewPebbleBM25(ps)
+		applyBM25EnvOverrides(pidx)
+		idx = pidx
+	default:
+		return fmt.Errorf("query: unknown -backend %q (want: sqlite | pebble)", *backend)
+	}
+	hits, err := idx.Search(ctx, q, *k)
+	if err != nil {
+		return err
+	}
+
+	if *jsonOut {
+		// Wire format matches the public /search response's per-hit shape
+		// (sans Highlight / Domain / Excerpt — those need a store lookup
+		// and `cosift query` is the no-server path).
+		type jsonHit struct {
+			URL   string  `json:"url"`
+			Title string  `json:"title"`
+			Score float64 `json:"score"`
+		}
+		out := make([]jsonHit, len(hits))
+		for i, h := range hits {
+			out[i] = jsonHit{URL: h.URL, Title: h.Title, Score: h.Score}
+		}
+		b, err := json.Marshal(out)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(b))
+		return nil
+	}
+	for i, h := range hits {
+		fmt.Printf("%2d. [%.3f] %s\n    %s\n", i+1, h.Score, h.Title, h.URL)
+	}
+	return nil
+}
+
+// runSearchCLI hits a running cosift server's /search endpoint over HTTP.
+// Unlike runQuery (local BM25 only), this exercises the full pipeline:
+// dense/hybrid retrieval, LLM rerank, paraphrase expansion, date/domain
+// filters, custom sort. Useful from operator shells without curl + jq.
+func runSearchCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
+	fs := flag.NewFlagSet("search", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	// 0.0.0.0 is a bind-only sentinel; clients must dial loopback.
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	k := fs.Int("k", 10, "max results")
+	retriever := fs.String("retriever", "", "bm25 | dense | hybrid (server default if empty)")
+	// MMR lambda passthrough for /search.
+	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (needs HNSW + embedder server-side)")
+	rerankFlag := fs.Bool("rerank", false, "wrap retrieval with LLM listwise reranker (server must have it configured)")
+	expand := fs.Bool("expand", false, "LLM paraphrase + RRF fusion (server must have a paraphraser)")
+	since := fs.String("since", "", "ISO date — only results published on or after")
+	until := fs.String("until", "", "ISO date — only results published on or before")
+	includeDomains := fs.String("include-domains", "", "comma-separated allowlist of result domains")
+	excludeDomains := fs.String("exclude-domains", "", "comma-separated denylist of result domains")
+	sortMode := fs.String("sort", "", "relevance | date_desc | date_asc (server default if empty)")
+	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable list")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *k <= 0 || *k > 100 {
+		return errors.New("k must be in [1, 100]")
+	}
+	if err := validateFormat(*format); err != nil {
+		return err
+	}
+
+	v := url.Values{}
+	v.Set("q", q)
+	v.Set("k", strconv.Itoa(*k))
+	if *retriever != "" {
+		v.Set("retriever", *retriever)
+	}
+	if *mmr != "" {
+		v.Set("mmr", *mmr)
+	}
+	if *rerankFlag {
+		v.Set("rerank", "true")
+	}
+	if *expand {
+		v.Set("expand", "true")
+	}
+	if *since != "" {
+		v.Set("since", *since)
+	}
+	if *until != "" {
+		v.Set("until", *until)
+	}
+	if *includeDomains != "" {
+		v.Set("include_domains", *includeDomains)
+	}
+	if *excludeDomains != "" {
+		v.Set("exclude_domains", *excludeDomains)
+	}
+	if *sortMode != "" {
+		v.Set("sort", *sortMode)
+	}
+
+	endpoint := strings.TrimRight(*serverURL, "/") + "/search?" + v.Encode()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+
+	if *jsonOut {
+		// Pass the server's JSON straight through — same shape as /search.
+		fmt.Println(string(body))
+		return nil
+	}
+	emitWarnings(body)
+	var sr server.SearchResponse
+	if err := json.Unmarshal(body, &sr); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if *format == "md" || *format == "markdown" {
+		fmt.Print(renderRankedMarkdown("Results: "+sr.Query, sr.Hits))
+		return nil
+	}
+	if len(sr.Hits) == 0 {
+		fmt.Println("(no results)")
+		return nil
+	}
+	for i, h := range sr.Hits {
+		date := ""
+		if h.PublishedAt != nil {
+			date = " " + h.PublishedAt.Format("2006-01-02")
+		}
+		domain := ""
+		if h.Domain != "" {
+			domain = " [" + h.Domain + "]"
+		}
+		fmt.Printf("%2d. [%.3f]%s%s %s\n    %s\n", i+1, h.Score, domain, date, h.Title, h.URL)
+		if h.Highlight != nil && h.Highlight.Text != "" {
+			fmt.Printf("    > %s\n", truncate(h.Highlight.Text, 200))
+		} else if h.Excerpt != "" {
+			fmt.Printf("    > %s\n", truncate(h.Excerpt, 200))
+		}
+	}
+	return nil
+}
+
+// runResearchCLI hits a running cosift server's /research endpoint over HTTP.
+// Sibling to runSearchCLI: same -server / -json affordance, but pulls
+// an LLM-synthesized answer with cited sources rather than a ranked URL list.
+// Non-streaming — the SSE mode (?stream=true) is a separate code path; this
+// returns the same JSON the non-streaming endpoint produces.
+// renderAnswerMarkdown formats a /research or /answer response as markdown,
+// suitable for piping into LLM contexts or markdown viewers. Used by both
+// runResearchCLI and runAnswerCLI via the shared `-format md|markdown` flag.
+// strategy/plan are research-only; pass "" / nil from /answer's callsite.
+func renderAnswerMarkdown(query, strategy string, plan []string, answer string, sources []server.AnswerSource) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "# %s\n\n", query)
+	if strategy != "" {
+		fmt.Fprintf(&b, "> Strategy: `%s`", strategy)
+		if len(plan) > 0 {
+			fmt.Fprintf(&b, " (plan: %s)", strings.Join(plan, " | "))
+		}
+		b.WriteString("\n\n")
+	}
+	b.WriteString(answer)
+	if !strings.HasSuffix(answer, "\n") {
+		b.WriteString("\n")
+	}
+	if len(sources) > 0 {
+		b.WriteString("\n## Sources\n\n")
+		for i, s := range sources {
+			// `N. [Title](URL)` — the leading N matches the citation IDs the
+			// LLM emits inline (e.g. `[1]` in the answer references source 1).
+			fmt.Fprintf(&b, "%d. [%s](%s)", sourceIDOf(i, s), s.Title, s.URL)
+			var trailing []string
+			if s.Domain != "" {
+				trailing = append(trailing, s.Domain)
+			}
+			if s.PublishedAt != nil {
+				trailing = append(trailing, s.PublishedAt.Format("2006-01-02"))
+			}
+			if len(trailing) > 0 {
+				fmt.Fprintf(&b, " — %s", strings.Join(trailing, ", "))
+			}
+			b.WriteString("\n")
+		}
+	}
+	return b.String()
+}
+
+// renderRankedMarkdown formats a ranked-hits response (search, find-similar)
+// as markdown for LLM piping or markdown viewers. `title` is the H1 text —
+// renderRankedMarkdown renders a ranked hit list as markdown. Callers
+// pass "Results: <query>" for /search or "Similar to: <url>" for
+// /find_similar; the hit-list shape is the same in both cases.
+func renderRankedMarkdown(title string, hits []server.SearchHit) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "# %s\n\n", title)
+	if len(hits) == 0 {
+		b.WriteString("_No results._\n")
+		return b.String()
+	}
+	for i, h := range hits {
+		fmt.Fprintf(&b, "## %d. [%s](%s)\n\n", i+1, h.Title, h.URL)
+		var meta []string
+		meta = append(meta, fmt.Sprintf("Score: %.3f", h.Score))
+		if h.Domain != "" {
+			meta = append(meta, h.Domain)
+		}
+		if h.PublishedAt != nil {
+			meta = append(meta, h.PublishedAt.Format("2006-01-02"))
+		}
+		fmt.Fprintf(&b, "_%s_\n", strings.Join(meta, " · "))
+		// Excerpt OR highlight as a blockquote. Highlight wins when both are
+		// present (it's the matched passage, more relevant than the prefix).
+		var excerpt string
+		if h.Highlight != nil && h.Highlight.Text != "" {
+			excerpt = h.Highlight.Text
+		} else if h.Excerpt != "" {
+			excerpt = h.Excerpt
+		}
+		if excerpt != "" {
+			b.WriteString("\n> ")
+			b.WriteString(truncate(excerpt, 240))
+			b.WriteString("\n")
+		}
+		// Trailing blank line between hits, but not after the last one.
+		if i < len(hits)-1 {
+			b.WriteString("\n")
+		}
+	}
+	return b.String()
+}
+
+// forEachSSEEvent reads an SSE response body and invokes handle(event, data)
+// for each `event: <name>\ndata: <json>\n\n` block. The handler can return
+// errSSEDone to signal terminal completion (the scanner returns nil), or any
+// other error to abort. Multi-line `data:` lines are concatenated with `\n`
+// per the SSE spec. Extracted from's consumeResearchSSE so
+// both research and answer CLI consumers share the framing logic.
+func forEachSSEEvent(body io.Reader, handle func(event, data string) error) error {
+	scanner := bufio.NewScanner(body)
+	// SSE events can be larger than the default 64KB scanner buffer (the
+	// terminal `done` blob in /research carries the full ResearchResponse).
+	scanner.Buffer(make([]byte, 0, 64<<10), 4<<20)
+
+	var event, data string
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			// Blank line terminates an event block — dispatch.
+			if event != "" {
+				if err := handle(event, data); err != nil {
+					if errors.Is(err, errSSEDone) {
+						return nil
+					}
+					return err
+				}
+			}
+			event, data = "", ""
+			continue
+		}
+		switch {
+		case strings.HasPrefix(line, "event:"):
+			event = strings.TrimSpace(strings.TrimPrefix(line, "event:"))
+		case strings.HasPrefix(line, "data:"):
+			d := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
+			if data == "" {
+				data = d
+			} else {
+				data = data + "\n" + d
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("read stream: %w", err)
+	}
+	// Stream ended without a terminal event — caller's responsibility to flag
+	// (this function just reports the natural end-of-stream as success).
+	return nil
+}
+
+// consumeResearchSSE reads a /research?stream=true response body and renders
+// each event as it arrives. Terminal events: "done" (full response) emits a
+// final newline + Sources section; "error" emits the detail and returns an
+// error. First streaming CLI in cosift. Refactored in to
+// use the generic forEachSSEEvent scanner.
+func consumeResearchSSE(body io.Reader, format string) error {
+	useMarkdown := format == "md" || format == "markdown"
+	var answerStarted, sawDone bool
+	// capture pebble-serve's sources event payload so we can render
+	// the final Sources section when pebble's minimal done event arrives.
+	var pebbleSources []server.AnswerSource
+
+	err := forEachSSEEvent(body, func(event, data string) error {
+		switch event {
+		case "warnings":
+			// pebble-serve emits this event when the
+			// request had silent no-ops. Surface to stderr like the sync CLI.
+			var w struct {
+				Warnings []string `json:"warnings"`
+			}
+			if err := json.Unmarshal([]byte(data), &w); err == nil {
+				for _, msg := range w.Warnings {
+					fmt.Fprintln(os.Stderr, "cosift: warning:", msg)
+				}
+			}
+		case "plan":
+			var p struct {
+				Strategy string   `json:"strategy"`
+				Variants []string `json:"variants"`
+				// pebble-serve uses 'plan' for the sub-query list and
+				// emits an 'expand' label instead of 'strategy'. Decode both
+				// shapes; fall back to "planner" when only plan is set.
+				Plan   []string `json:"plan"`
+				Expand string   `json:"expand"`
+			}
+			if err := json.Unmarshal([]byte(data), &p); err != nil {
+				return nil // tolerate malformed mid-stream event
+			}
+			strategy := p.Strategy
+			variants := p.Variants
+			if strategy == "" && len(p.Plan) > 0 {
+				strategy = "planner"
+				if p.Expand != "" {
+					strategy = "planner+" + p.Expand
+				}
+			}
+			if len(variants) == 0 {
+				variants = p.Plan
+			}
+			if useMarkdown {
+				fmt.Printf("> Strategy: `%s`", strategy)
+				if len(variants) > 0 {
+					fmt.Printf(" (plan: %s)", strings.Join(variants, " | "))
+				}
+				fmt.Println()
+				fmt.Println()
+			} else {
+				fmt.Printf("Strategy: %s", strategy)
+				if len(variants) > 0 {
+					fmt.Printf(" (plan: %s)", strings.Join(variants, " | "))
+				}
+				fmt.Println()
+			}
+		case "retrieved":
+			var r struct {
+				Variant string   `json:"variant"`
+				URLs    []string `json:"urls"`
+			}
+			if err := json.Unmarshal([]byte(data), &r); err != nil {
+				return nil
+			}
+			fmt.Printf("[retrieved %d url(s) for %q]\n", len(r.URLs), r.Variant)
+		case "synthesizing":
+			var s struct {
+				Sources int `json:"sources"`
+			}
+			if err := json.Unmarshal([]byte(data), &s); err != nil {
+				return nil
+			}
+			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", s.Sources)
+		case "sources":
+			// pebble-serve emits one 'sources' event after retrieval
+			// that combines what the SQLite server splits across
+			// retrieved+synthesizing. Render the same progress line.
+			// also capture the sources slice so the final 'done'
+			// event (pebble's is minimal) can still render the Sources block.
+			var s struct {
+				Sources []server.AnswerSource `json:"sources"`
+			}
+			if err := json.Unmarshal([]byte(data), &s); err != nil {
+				return nil
+			}
+			pebbleSources = s.Sources
+			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", len(s.Sources))
+		case "answer_chunk":
+			var c struct {
+				Text string `json:"text"`
+			}
+			if err := json.Unmarshal([]byte(data), &c); err != nil {
+				return nil
+			}
+			fmt.Print(c.Text)
+			answerStarted = true
+		case "done":
+			sawDone = true
+			if answerStarted {
+				fmt.Println()
+			}
+			var rr server.ResearchResponse
+			if err := json.Unmarshal([]byte(data), &rr); err != nil || len(rr.Sources) == 0 {
+				// pebble's done event has a minimal payload —
+				// fall back to the sources captured from the event.
+				renderStreamingSources(pebbleSources, useMarkdown)
+				return errSSEDone
+			}
+			renderStreamingSources(rr.Sources, useMarkdown)
+			return errSSEDone
+		case "error":
+			var e struct {
+				Detail string `json:"detail"`
+				// pebble-serve uses 'error' field instead of 'detail'.
+				Error string `json:"error"`
+				// pebble-serve tags errors with the phase that failed.
+				Phase string `json:"phase"`
+			}
+			_ = json.Unmarshal([]byte(data), &e)
+			msg := e.Detail
+			if msg == "" {
+				msg = e.Error
+			}
+			if e.Phase != "" {
+				return fmt.Errorf("server stream error (phase=%s): %s", e.Phase, msg)
+			}
+			return fmt.Errorf("server stream error: %s", msg)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if !sawDone {
+		// Stream ended without a terminal event — flag for operators but
+		// don't error (matches behavior).
+		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
+	}
+	return nil
+}
+
+// renderStreamingSources writes the terminal Sources section for both the
+// research and answer SSE consumers. Format matches the non-stream paths so
+// operators get consistent output regardless of whether they used -stream.
+// position-based ID fallback for source rendering when the
+// server didn't emit an id field (pebble-serve's source payload). SQLite-side
+// servers always emit non-zero IDs, so their output is unchanged.
+func sourceIDOf(i int, s server.AnswerSource) int {
+	if s.ID > 0 {
+		return s.ID
+	}
+	return i + 1
+}
+
+func renderStreamingSources(sources []server.AnswerSource, useMarkdown bool) {
+	if len(sources) == 0 {
+		return
+	}
+	idOf := sourceIDOf
+	if useMarkdown {
+		fmt.Println()
+		fmt.Println("## Sources")
+		fmt.Println()
+		for i, s := range sources {
+			fmt.Fprintf(os.Stdout, "%d. [%s](%s)", idOf(i, s), s.Title, s.URL)
+			var trailing []string
+			if s.Domain != "" {
+				trailing = append(trailing, s.Domain)
+			}
+			if s.PublishedAt != nil {
+				trailing = append(trailing, s.PublishedAt.Format("2006-01-02"))
+			}
+			if len(trailing) > 0 {
+				fmt.Printf(" — %s", strings.Join(trailing, ", "))
+			}
+			fmt.Println()
+		}
+		return
+	}
+	fmt.Println()
+	fmt.Println("Sources:")
+	for i, s := range sources {
+		date := ""
+		if s.PublishedAt != nil {
+			date = " " + s.PublishedAt.Format("2006-01-02")
+		}
+		domain := ""
+		if s.Domain != "" {
+			domain = " [" + s.Domain + "]"
+		}
+		fmt.Printf("  [%d]%s%s %s\n      %s\n", idOf(i, s), domain, date, s.Title, s.URL)
+	}
+}
+
+// consumeAnswerSSE reads an /answer?stream=true response and renders events as
+// they arrive. Same event vocabulary as consumeResearchSSE minus the `plan`
+// event (/answer has no expansion phase).
+func consumeAnswerSSE(body io.Reader, format string) error {
+	useMarkdown := format == "md" || format == "markdown"
+	var answerStarted, sawDone bool
+	var pebbleSources []server.AnswerSource //
+
+	err := forEachSSEEvent(body, func(event, data string) error {
+		switch event {
+		case "warnings":
+			// pebble-serve emits this event when the
+			// request had silent no-ops. Surface to stderr.
+			var w struct {
+				Warnings []string `json:"warnings"`
+			}
+			if err := json.Unmarshal([]byte(data), &w); err == nil {
+				for _, msg := range w.Warnings {
+					fmt.Fprintln(os.Stderr, "cosift: warning:", msg)
+				}
+			}
+		case "retrieved":
+			// /answer emits a single retrieved event with the k URLs (vs
+			// /research's one-per-variant).
+			var r struct {
+				URLs []string `json:"urls"`
+			}
+			if err := json.Unmarshal([]byte(data), &r); err != nil {
+				return nil
+			}
+			fmt.Printf("[retrieved %d source(s)]\n", len(r.URLs))
+		case "synthesizing":
+			var s struct {
+				Sources int `json:"sources"`
+			}
+			if err := json.Unmarshal([]byte(data), &s); err != nil {
+				return nil
+			}
+			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", s.Sources)
+		case "sources":
+			// pebble-serve emits 'sources' as the combined
+			// retrieved+synthesizing signal. Render the synthesizing line.
+			// capture for the final Sources block.
+			var s struct {
+				Sources []server.AnswerSource `json:"sources"`
+			}
+			if err := json.Unmarshal([]byte(data), &s); err != nil {
+				return nil
+			}
+			pebbleSources = s.Sources
+			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", len(s.Sources))
+		case "answer_chunk":
+			var c struct {
+				Text string `json:"text"`
+			}
+			if err := json.Unmarshal([]byte(data), &c); err != nil {
+				return nil
+			}
+			fmt.Print(c.Text)
+			answerStarted = true
+		case "done":
+			sawDone = true
+			if answerStarted {
+				fmt.Println()
+			}
+			var ar server.AnswerResponse
+			if err := json.Unmarshal([]byte(data), &ar); err != nil || len(ar.Sources) == 0 {
+				// tolerate pebble's minimal done payload; fall
+				// back to the sources captured from the sources event.
+				renderStreamingSources(pebbleSources, useMarkdown)
+				return errSSEDone
+			}
+			renderStreamingSources(ar.Sources, useMarkdown)
+			return errSSEDone
+		case "error":
+			var e struct {
+				Detail string `json:"detail"`
+				// pebble-serve uses 'error' field instead of 'detail'.
+				Error string `json:"error"`
+				// pebble-serve tags errors with the phase that failed.
+				Phase string `json:"phase"`
+			}
+			_ = json.Unmarshal([]byte(data), &e)
+			msg := e.Detail
+			if msg == "" {
+				msg = e.Error
+			}
+			if e.Phase != "" {
+				return fmt.Errorf("server stream error (phase=%s): %s", e.Phase, msg)
+			}
+			return fmt.Errorf("server stream error: %s", msg)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if !sawDone {
+		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
+	}
+	return nil
+}
+
+// errSSEDone signals consumeResearchSSE's outer loop that the terminal "done"
+// event was processed and the read loop should exit cleanly. Sentinel error,
+// not surfaced to the caller.
+var errSSEDone = errors.New("sse-done")
+
+// validateFormat returns nil if v is one of the accepted output-format values
+// for the synth CLIs (`-format`). "" is rejected — flag.String defaults to ""
+// when the user passes `-format` without a value, and we want that to fail
+// loudly rather than silently behave as text.
+func validateFormat(v string) error {
+	switch v {
+	case "text", "md", "markdown":
+		return nil
+	default:
+		return fmt.Errorf("format must be one of: text, markdown (got %q)", v)
+	}
+}
+
+func runResearchCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
+	fs := flag.NewFlagSet("research", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	strategy := fs.String("strategy", "", "planner | paraphrase (server default if empty)")
+	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
+	stream := fs.Bool("stream", false, "stream progress + token-by-token answer over SSE.")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable answer+sources")
+	// Expose the quality + scope flags on the CLI.
+	k := fs.Int("k", 0, "number of sources fed to synth (1-20, server default if 0)")
+	expand := fs.String("expand", "", "retrieval expansion: hyde | paraphrase (empty = no expansion)")
+	rerank := fs.Bool("rerank", false, "rerank retrieved sources before synth")
+	// per-sub-query retriever choice — applies to every sub-query
+	// the planner emits. Server uses bm25 by default if this is empty.
+	retriever := fs.String("retriever", "", "retriever: bm25 | dense | hybrid (server must have HNSW + embedder for dense/hybrid)")
+	// MMR lambda passthrough for /research.
+	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (needs HNSW + embedder server-side)")
+	since := fs.String("since", "", "ISO date — only sources published on or after")
+	until := fs.String("until", "", "ISO date — only sources published on or before")
+	includeDomains := fs.String("include-domains", "", "CSV allowlist of source domains")
+	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of source domains")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if err := validateFormat(*format); err != nil {
+		return err
+	}
+	if *stream && *jsonOut {
+		return errors.New("-stream and -json are mutually exclusive (stream is event-by-event, json is the final blob)")
+	}
+	if *k < 0 || *k > 20 {
+		return errors.New("k must be in [1, 20] or 0 to use server default")
+	}
+
+	v := url.Values{}
+	v.Set("q", q)
+	if *strategy != "" {
+		v.Set("strategy", *strategy)
+	}
+	if *k > 0 {
+		v.Set("k", strconv.Itoa(*k))
+	}
+	if *expand != "" {
+		v.Set("expand", *expand)
+	}
+	if *rerank {
+		v.Set("rerank", "true")
+	}
+	if *retriever != "" {
+		v.Set("retriever", *retriever)
+	}
+	if *mmr != "" {
+		v.Set("mmr", *mmr)
+	}
+	if *since != "" {
+		v.Set("since", *since)
+	}
+	if *until != "" {
+		v.Set("until", *until)
+	}
+	if *includeDomains != "" {
+		v.Set("include_domains", *includeDomains)
+	}
+	if *excludeDomains != "" {
+		v.Set("exclude_domains", *excludeDomains)
+	}
+	if *stream {
+		v.Set("stream", "true")
+	}
+
+	endpoint := strings.TrimRight(*serverURL, "/") + "/research?" + v.Encode()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
+	if err != nil {
+		return err
+	}
+	if *stream {
+		req.Header.Set("Accept", "text/event-stream")
+	}
+	// /research issues multiple LLM calls (expand + retrieve + synth); bump the
+	// client timeout from search's 30s so slow chat backends don't get cut off.
+	// In stream mode the timeout has to cover the whole response (multiple LLM
+	// chunks emitted incrementally), so it's still 120s — there's no per-event
+	// deadline; if needed in future, a custom transport timeout would do that.
+	client := &http.Client{Timeout: 120 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	if *stream {
+		if resp.StatusCode != http.StatusOK {
+			// Server may emit a JSON problem doc on non-200 even with SSE Accept.
+			b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+			return fmt.Errorf("server %d: %s", resp.StatusCode, b)
+		}
+		return consumeResearchSSE(resp.Body, *format)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	emitWarnings(body)
+	var rr server.ResearchResponse
+	if err := json.Unmarshal(body, &rr); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if *format == "md" || *format == "markdown" {
+		fmt.Print(renderAnswerMarkdown(rr.Query, rr.Strategy, rr.Plan, rr.Answer, rr.Sources))
+		return nil
+	}
+	fmt.Printf("Q: %s\n", rr.Query)
+	if rr.Strategy != "" {
+		fmt.Printf("Strategy: %s", rr.Strategy)
+		if len(rr.Plan) > 0 {
+			fmt.Printf(" (plan: %s)", strings.Join(rr.Plan, " | "))
+		}
+		fmt.Println()
+	}
+	fmt.Println()
+	fmt.Println(rr.Answer)
+	if len(rr.Sources) > 0 {
+		fmt.Println()
+		fmt.Println("Sources:")
+		for i, s := range rr.Sources {
+			date := ""
+			if s.PublishedAt != nil {
+				date = " " + s.PublishedAt.Format("2006-01-02")
+			}
+			domain := ""
+			if s.Domain != "" {
+				domain = " [" + s.Domain + "]"
+			}
+			fmt.Printf("  [%d]%s%s %s\n      %s\n", sourceIDOf(i, s), domain, date, s.Title, s.URL)
+		}
+	}
+	return nil
+}
+
+// runFindSimilarCLI hits a running cosift server's /find_similar endpoint.
+// Completes the  CLI matrix:
+//   - query        local BM25 only
+//   - search       /search (full retrieval pipeline)
+//   - research     /research (LLM synthesis)
+//   - find-similar /find_similar (dense neighbors of an indexed URL)
+func runFindSimilarCLI(ctx context.Context, cfg *config.Config, sourceURL string, args []string) error {
+	fs := flag.NewFlagSet("find-similar", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	k := fs.Int("k", 10, "max results")
+	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable list")
+	// alternative inputs — feed arbitrary text (or a file of text)
+	// instead of an indexed source URL. Mirrors /find_similar text mode.
+	textInput := fs.String("text", "", "arbitrary text for content-based MLT (no positional URL needed)")
+	textFile := fs.String("text-file", "", "read MLT source text from FILE")
+	textTitle := fs.String("text-title", "", "optional title (×3 boost) when using -text or -text-file")
+	// surface the filter + rerank flags on the CLI.
+	rerank := fs.Bool("rerank", false, "rerank neighbors against the MLT query (server must have rerank configured)")
+	// retriever choice — bm25-mlt (default) | dense | hybrid.
+	// Dense reads source vector from HNSW for URL-mode; hybrid RRF-fuses
+	// BM25-MLT and dense. Both require COSIFT_LOAD_HNSW=true server-side.
+	retriever := fs.String("retriever", "", "retriever: bm25 (BM25-MLT) | dense | hybrid (server must have HNSW; text-mode dense/hybrid also needs embedder)")
+	// MMR diversification for /find_similar.
+	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (URL-mode reuses graph vector — no embedder needed)")
+	since := fs.String("since", "", "ISO date — only neighbors published on or after")
+	until := fs.String("until", "", "ISO date — only neighbors published on or before")
+	includeDomains := fs.String("include-domains", "", "CSV allowlist of neighbor domains")
+	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of neighbor domains")
+	qExtra := fs.String("q", "", "extra query terms appended to the auto-derived MLT query")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *k <= 0 || *k > 100 {
+		return errors.New("k must be in [1, 100]")
+	}
+	if err := validateFormat(*format); err != nil {
+		return err
+	}
+
+	if *textFile != "" {
+		buf, err := os.ReadFile(*textFile)
+		if err != nil {
+			return fmt.Errorf("read -text-file %s: %w", *textFile, err)
+		}
+		*textInput = string(buf)
+	}
+	if sourceURL == "" && *textInput == "" {
+		return errors.New("find-similar: positional URL or -text/-text-file required")
+	}
+
+	v := url.Values{}
+	if sourceURL != "" {
+		v.Set("url", sourceURL)
+	}
+	if *textInput != "" {
+		v.Set("text", *textInput)
+	}
+	if *textTitle != "" {
+		v.Set("title", *textTitle)
+	}
+	v.Set("k", strconv.Itoa(*k))
+	// filter + rerank passthroughs.
+	if *rerank {
+		v.Set("rerank", "true")
+	}
+	if *retriever != "" {
+		v.Set("retriever", *retriever)
+	}
+	if *mmr != "" {
+		v.Set("mmr", *mmr)
+	}
+	if *since != "" {
+		v.Set("since", *since)
+	}
+	if *until != "" {
+		v.Set("until", *until)
+	}
+	if *includeDomains != "" {
+		v.Set("include_domains", *includeDomains)
+	}
+	if *excludeDomains != "" {
+		v.Set("exclude_domains", *excludeDomains)
+	}
+	if *qExtra != "" {
+		v.Set("q", *qExtra)
+	}
+
+	// switch GET → POST when -text is large (URL params have practical
+	// limits ~8 KB). The POST endpoint accepts the same flag set as a JSON body.
+	useMethod := http.MethodGet
+	endpoint := strings.TrimRight(*serverURL, "/") + "/find_similar?" + v.Encode()
+	var bodyBuf io.Reader
+	if len(*textInput) > 4096 {
+		useMethod = http.MethodPost
+		body := map[string]any{"k": *k}
+		if sourceURL != "" {
+			body["url"] = sourceURL
+		}
+		if *textInput != "" {
+			body["text"] = *textInput
+		}
+		if *textTitle != "" {
+			body["title"] = *textTitle
+		}
+		if *rerank {
+			body["rerank"] = true
+		}
+		if *retriever != "" {
+			body["retriever"] = *retriever
+		}
+		if *mmr != "" {
+			body["mmr"] = *mmr
+		}
+		if *since != "" {
+			body["since"] = *since
+		}
+		if *until != "" {
+			body["until"] = *until
+		}
+		if *includeDomains != "" {
+			body["include_domains"] = *includeDomains
+		}
+		if *excludeDomains != "" {
+			body["exclude_domains"] = *excludeDomains
+		}
+		if *qExtra != "" {
+			body["q"] = *qExtra
+		}
+		jb, _ := json.Marshal(body)
+		bodyBuf = bytes.NewReader(jb)
+		endpoint = strings.TrimRight(*serverURL, "/") + "/find_similar"
+	}
+	req, err := http.NewRequestWithContext(ctx, useMethod, endpoint, bodyBuf)
+	if err != nil {
+		return err
+	}
+	if useMethod == http.MethodPost {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	// /find_similar embeds the source doc (one embedder call) and does a vector
+	// search — 30s mirrors search's timeout.
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	emitWarnings(body)
+	var fr server.FindSimilarResponse
+	if err := json.Unmarshal(body, &fr); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if *format == "md" || *format == "markdown" {
+		fmt.Print(renderRankedMarkdown("Similar to: "+fr.URL, fr.Hits))
+		return nil
+	}
+	if len(fr.Hits) == 0 {
+		fmt.Println("(no similar documents)")
+		return nil
+	}
+	fmt.Printf("Similar to: %s\n\n", fr.URL)
+	for i, h := range fr.Hits {
+		date := ""
+		if h.PublishedAt != nil {
+			date = " " + h.PublishedAt.Format("2006-01-02")
+		}
+		domain := ""
+		if h.Domain != "" {
+			domain = " [" + h.Domain + "]"
+		}
+		fmt.Printf("%2d. [%.3f]%s%s %s\n    %s\n", i+1, h.Score, domain, date, h.Title, h.URL)
+	}
+	return nil
+}
+
+// runContentsCLI hits a running cosift server's /contents endpoint.
+// Single URL → GET /contents?url=<url>. Multiple URLs (positional or via
+// -file) → POST /contents with {urls: [...]} body.
+func runContentsCLI(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("contents", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	filePath := fs.String("file", "", "read URLs from FILE (one per line, # comments allowed)")
+	textOnly := fs.Bool("text", false, "print only the document text (no metadata) — useful for piping into LLMs")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable output")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	urls := fs.Args()
+	if *filePath != "" {
+		extra, err := readURLsFromFile(*filePath)
+		if err != nil {
+			return fmt.Errorf("read -file: %w", err)
+		}
+		urls = append(urls, extra...)
+	}
+	if len(urls) == 0 {
+		return errors.New("at least one URL or -file required")
+	}
+
+	base := strings.TrimRight(*serverURL, "/")
+	client := &http.Client{Timeout: 60 * time.Second}
+
+	if len(urls) == 1 {
+		// Single — GET /contents?url=<url>.
+		v := url.Values{}
+		v.Set("url", urls[0])
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/contents?"+v.Encode(), http.NoBody)
+		if err != nil {
+			return err
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			return fmt.Errorf("get %s: %w", *serverURL, err)
+		}
+		defer resp.Body.Close()
+		body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
+		if err != nil {
+			return fmt.Errorf("read response: %w", err)
+		}
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+		}
+		if *jsonOut {
+			fmt.Println(string(body))
+			return nil
+		}
+		var cr server.ContentsResponse
+		if err := json.Unmarshal(body, &cr); err != nil {
+			return fmt.Errorf("decode response: %w", err)
+		}
+		if *textOnly {
+			fmt.Println(cr.Text)
+			return nil
+		}
+		fmt.Printf("URL:        %s\n", cr.URL)
+		fmt.Printf("Title:      %s\n", cr.Title)
+		fmt.Printf("Cached:     %t\n", cr.Cached)
+		if !cr.FetchedAt.IsZero() {
+			fmt.Printf("FetchedAt:  %s\n", cr.FetchedAt.Format(time.RFC3339))
+		}
+		if cr.Lang != "" {
+			fmt.Printf("Lang:       %s\n", cr.Lang)
+		}
+		fmt.Println()
+		fmt.Println(cr.Text)
+		return nil
+	}
+
+	// Batch — POST /contents with {urls: [...]}.
+	reqBody, err := json.Marshal(server.ContentsBatchRequest{URLs: urls})
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/contents", bytes.NewReader(reqBody))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("post %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	var br server.ContentsBatchResponse
+	if err := json.Unmarshal(body, &br); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if *textOnly {
+		// Separator between docs lets downstream tools chunk by `---` if they want.
+		for i, item := range br.Results {
+			if i > 0 {
+				fmt.Println("---")
+			}
+			if !item.Found {
+				continue
+			}
+			fmt.Println(item.Text)
+		}
+		return nil
+	}
+	for _, item := range br.Results {
+		fmt.Printf("URL: %s\n", item.URL)
+		if !item.Found {
+			fmt.Printf("  (not found: %s)\n\n", item.Error)
+			continue
+		}
+		status := "fetched"
+		if item.Cached {
+			status = "cached"
+		}
+		fmt.Printf("  Title:      %s\n", item.Title)
+		fmt.Printf("  Status:     %s\n", status)
+		if !item.FetchedAt.IsZero() {
+			fmt.Printf("  FetchedAt:  %s\n", item.FetchedAt.Format(time.RFC3339))
+		}
+		if item.Lang != "" {
+			fmt.Printf("  Lang:       %s\n", item.Lang)
+		}
+		fmt.Printf("  Text:       %s\n\n", truncate(item.Text, 240))
+	}
+	return nil
+}
+
+// readURLsFromFile parses one URL per line. Blank lines and `#`-prefixed
+// comments are ignored. Used by `cosift contents -file`.
+func readURLsFromFile(path string) ([]string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var urls []string
+	for _, line := range strings.Split(string(b), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		urls = append(urls, line)
+	}
+	return urls, nil
+}
+
+// runAnswerCLI hits a running cosift server's /answer endpoint. Sibling to
+// runResearchCLI but no planner/paraphrase strategy surface —
+// /answer just retrieves k sources and synthesizes a cited answer. Smaller K
+// cap [1,20] mirrors the server's bound (vs /search's [1,100]).
+func runAnswerCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
+	fs := flag.NewFlagSet("answer", flag.ExitOnError)
+	defaultServer := "http://" + cfg.Server.Addr
+	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
+	serverURL := fs.String("server", defaultServer, "cosift server URL")
+	k := fs.Int("k", 5, "number of sources to retrieve (1-20)")
+	expand := fs.Bool("expand", false, "LLM paraphrase + RRF fusion of retrieval inputs (server must have a paraphraser)")
+	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
+	stream := fs.Bool("stream", false, "stream progress + token-by-token answer over SSE.")
+	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable answer+sources")
+	// Expose the quality + scope flags on the CLI.
+	rerank := fs.Bool("rerank", false, "rerank retrieved sources before synth (server must have rerank configured)")
+	// Empty
+	// passes nothing → server uses its default (bm25).
+	retriever := fs.String("retriever", "", "retriever: bm25 | dense | hybrid (server must have HNSW + embedder for dense/hybrid)")
+	// MMR lambda passthrough. Empty → no diversification.
+	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (0.7 = mostly relevance with some diversity; needs HNSW + embedder server-side)")
+	since := fs.String("since", "", "ISO date — only sources published on or after")
+	until := fs.String("until", "", "ISO date — only sources published on or before")
+	includeDomains := fs.String("include-domains", "", "CSV allowlist of source domains")
+	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of source domains")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *k <= 0 || *k > 20 {
+		return errors.New("k must be in [1, 20]")
+	}
+	if err := validateFormat(*format); err != nil {
+		return err
+	}
+	if *stream && *jsonOut {
+		return errors.New("-stream and -json are mutually exclusive (stream is event-by-event, json is the final blob)")
+	}
+
+	v := url.Values{}
+	v.Set("q", q)
+	v.Set("k", strconv.Itoa(*k))
+	if *expand {
+		v.Set("expand", "true")
+	}
+	if *rerank {
+		v.Set("rerank", "true")
+	}
+	if *retriever != "" {
+		v.Set("retriever", *retriever)
+	}
+	if *mmr != "" {
+		v.Set("mmr", *mmr)
+	}
+	if *since != "" {
+		v.Set("since", *since)
+	}
+	if *until != "" {
+		v.Set("until", *until)
+	}
+	if *includeDomains != "" {
+		v.Set("include_domains", *includeDomains)
+	}
+	if *excludeDomains != "" {
+		v.Set("exclude_domains", *excludeDomains)
+	}
+	if *stream {
+		v.Set("stream", "true")
+	}
+
+	endpoint := strings.TrimRight(*serverURL, "/") + "/answer?" + v.Encode()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
+	if err != nil {
+		return err
+	}
+	if *stream {
+		req.Header.Set("Accept", "text/event-stream")
+	}
+	// /answer issues retrieval + 1 synth LLM call; 120s mirrors /research.
+	client := &http.Client{Timeout: 120 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("get %s: %w", *serverURL, err)
+	}
+	defer resp.Body.Close()
+	if *stream {
+		if resp.StatusCode != http.StatusOK {
+			b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+			return fmt.Errorf("server %d: %s", resp.StatusCode, b)
+		}
+		return consumeAnswerSSE(resp.Body, *format)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
+	}
+
+	if *jsonOut {
+		fmt.Println(string(body))
+		return nil
+	}
+	emitWarnings(body)
+	var ar server.AnswerResponse
+	if err := json.Unmarshal(body, &ar); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if *format == "md" || *format == "markdown" {
+		// /answer has no strategy or plan — pass empty/nil so the shared
+		// renderer just skips the strategy line.
+		fmt.Print(renderAnswerMarkdown(ar.Query, "", nil, ar.Answer, ar.Sources))
+		return nil
+	}
+	fmt.Printf("Q: %s\n\n", ar.Query)
+	fmt.Println(ar.Answer)
+	if len(ar.Sources) > 0 {
+		fmt.Println()
+		fmt.Println("Sources:")
+		for i, s := range ar.Sources {
+			date := ""
+			if s.PublishedAt != nil {
+				date = " " + s.PublishedAt.Format("2006-01-02")
+			}
+			domain := ""
+			if s.Domain != "" {
+				domain = " [" + s.Domain + "]"
+			}
+			fmt.Printf("  [%d]%s%s %s\n      %s\n", sourceIDOf(i, s), domain, date, s.Title, s.URL)
+		}
+	}
+	return nil
+}
diff --git a/cmd/cosift/cmd_serve.go b/cmd/cosift/cmd_serve.go
new file mode 100644
index 0000000..7f52e25
--- /dev/null
+++ b/cmd/cosift/cmd_serve.go
@@ -0,0 +1,210 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/crawler"
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/rerank"
+	"github.com/pilot-protocol/cosift/internal/server"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// authStatus describes how a configured capability will authenticate to its
+// endpoint, for /doctor output.
+func authStatus(configured bool, key string, urlSet bool) string {
+	switch {
+	case !configured:
+		return "(none)"
+	case key != "":
+		return "bearer-token"
+	case urlSet:
+		return "anonymous(custom-url)"
+	default:
+		return "MISSING"
+	}
+}
+
+// hnswPassageWriter bridges the crawler's PassageWriter contract to
+// an in-memory index.HNSW graph. Each embedded passage looks up the doc's URL
+// and Title (via *PebbleStore.GetDocByID) and is inserted into the graph; the
+// graph is persisted to Pebble's 'v' family in a single Persist() call at the
+// end of the crawl.
+type hnswPassageWriter struct {
+	ps   *store.PebbleStore
+	hnsw *index.HNSW
+}
+
+func (w *hnswPassageWriter) UpsertPassage(ctx context.Context, p *store.Passage) error {
+	doc, err := w.ps.GetDocByID(ctx, p.DocID)
+	if err != nil {
+		return err
+	}
+	if doc == nil {
+		return nil // doc was deleted between index and embed — skip silently
+	}
+	w.hnsw.AddPassage(doc.URL, doc.Title, p.Offset, p.Length, p.Embedding)
+	return nil
+}
+
+// UpsertPassageBatch writes all passages in a single HNSW lock
+// acquisition — meaningful when a doc has many chunks. Implements the
+// optional crawler.PassageWriterBatch interface; the crawler will call
+// this when present instead of looping UpsertPassage per chunk.
+func (w *hnswPassageWriter) UpsertPassageBatch(ctx context.Context, ps []*store.Passage) error {
+	if len(ps) == 0 {
+		return nil
+	}
+	// All passages in a batch belong to the same DocID (the crawler hands
+	// us one doc's chunks). Resolve url + title once.
+	doc, err := w.ps.GetDocByID(ctx, ps[0].DocID)
+	if err != nil {
+		return err
+	}
+	if doc == nil {
+		return nil
+	}
+	items := make([]index.PassageInput, 0, len(ps))
+	for _, p := range ps {
+		items = append(items, index.PassageInput{
+			URL: doc.URL, Title: doc.Title, Offset: p.Offset, Length: p.Length, Vec: p.Embedding,
+		})
+	}
+	w.hnsw.AddPassageBatch(items)
+	return nil
+}
+
+// MarkURLInvalid satisfies the optional crawler.URLInvalidator interface.
+// Lets the crawler reclaim zombie passages (prior generations of chunks for
+// the same URL) before pushing a fresh chunk batch. Returns count zeroed.
+func (w *hnswPassageWriter) MarkURLInvalid(ctx context.Context, url string) (int, error) {
+	if err := ctx.Err(); err != nil {
+		return 0, err
+	}
+	return w.hnsw.MarkURLPassagesInvalid(url), nil
+}
+
+func runServe(ctx context.Context, cfg *config.Config) error {
+	s, err := store.Open(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	srv := server.New(s)
+	// Instance-wide retrieval defaults from cfg.Defaults. Per-request query
+	// params still override; this is just what the handler picks up when the
+	// caller doesn't specify.
+	srv = srv.WithDefaults(server.Defaults{
+		Retriever:         cfg.Defaults.Retriever,
+		Expand:            cfg.Defaults.Expand,
+		ResearchStrategy:  cfg.Defaults.ResearchStrategy,
+		ResearchSynthK:    cfg.Defaults.ResearchSynthK,
+		ExpandMainWeight:  cfg.Defaults.ExpandMainWeight,
+		HybridDenseWeight: cfg.Defaults.HybridDenseWeight,
+	})
+	if cfg.Defaults.Retriever != "" || cfg.Defaults.Expand || cfg.Defaults.ResearchStrategy != "" || cfg.Defaults.ResearchSynthK > 0 || cfg.Defaults.ExpandMainWeight > 0 || cfg.Defaults.HybridDenseWeight > 0 {
+		log.Printf("defaults: retriever=%q expand=%v research_strategy=%q research_synth_k=%d expand_main_weight=%v hybrid_dense_weight=%v",
+			cfg.Defaults.Retriever, cfg.Defaults.Expand, cfg.Defaults.ResearchStrategy, cfg.Defaults.ResearchSynthK, cfg.Defaults.ExpandMainWeight, cfg.Defaults.HybridDenseWeight)
+	}
+	// thread chunker config to /admin/reembed handler so re-embed
+	// produces passages with the same shape as crawl-time indexing.
+	if cfg.Crawler.ChunkSize > 0 || cfg.Crawler.ChunkOverlap > 0 {
+		srv = srv.WithChunker(cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
+		log.Printf("chunker: size=%d overlap=%d", cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
+	}
+	if tok := cfg.Server.AdminToken; tok != "" {
+		srv = srv.WithAdminToken(tok)
+		log.Printf("/admin/* endpoints enabled (bearer-auth)")
+	}
+	if len(cfg.Server.TrustedProxies) > 0 {
+		s2, err := srv.WithTrustedProxies(cfg.Server.TrustedProxies)
+		if err != nil {
+			return fmt.Errorf("trusted_proxies: %w", err)
+		}
+		srv = s2
+		log.Printf("X-Forwarded-For trusted from %v", cfg.Server.TrustedProxies)
+	}
+
+	// Auto-wire embeddings if model is set and an API key is in env.
+	if cfg.Embeddings.Model != "" {
+		apiKey := os.Getenv("OPENAI_API_KEY")
+		if apiKey == "" {
+			apiKey = os.Getenv("OPENAI")
+		}
+		if apiKey == "" {
+			log.Printf("warning: embeddings configured but no OPENAI_API_KEY in env; dense/hybrid disabled")
+		} else {
+			model := cfg.Embeddings.Model
+			dim := cfg.Embeddings.Dim
+			if dim == 0 {
+				dim = 1536
+			}
+			emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, model, dim)
+			vi, err := index.LoadVectorIndex(ctx, s, model, dim)
+			if err != nil {
+				return fmt.Errorf("load vector index: %w", err)
+			}
+			srv = srv.WithVector(vi, emb)
+			log.Printf("vector index loaded: %d passages, model=%s dim=%d", vi.Len(), model, dim)
+
+			// Chat is opt-in via cfg.Chat.Model. Reuses the same API key.
+			if cfg.Chat.Model != "" {
+				chat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Chat.Model)
+				srv = srv.WithChat(chat)
+				log.Printf("/answer enabled with chat model=%s", cfg.Chat.Model)
+				// Auto-enable /search?expand=true via the same chat client.
+				// measured +0.02 nDCG at 10k distractors for $0.004 per run.
+				srv = srv.WithParaphraser(chat, 2)
+				log.Printf("/search?expand=true enabled (auto-paraphrase via %s)", cfg.Chat.Model)
+			}
+
+			// Reranker: HTTP backend wins if URL is set; otherwise LLM backend
+			// (only if chat is configured). Either is opt-out via cfg.Rerank.Enabled=false
+			// even when configured.
+			var rr rerank.Reranker
+			rerankWanted := cfg.Rerank.URL != "" || cfg.Chat.Model != ""
+			if !cfg.Rerank.Enabled && rerankWanted {
+				// Default ON when wired; flip cfg.Rerank.Enabled false to skip.
+				cfg.Rerank.Enabled = true
+			}
+			if cfg.Rerank.Enabled && cfg.Rerank.URL != "" {
+				rerankKey := cfg.Rerank.APIKey
+				if rerankKey == "" {
+					rerankKey = firstEnv("COHERE_API_KEY", "VOYAGE_API_KEY")
+				}
+				rr = rerank.NewHTTPReranker(cfg.Rerank.URL, rerankKey, cfg.Rerank.Model)
+				log.Printf("rerank enabled with http endpoint=%s model=%s", cfg.Rerank.URL, cfg.Rerank.Model)
+			} else if cfg.Rerank.Enabled && cfg.Chat.Model != "" {
+				rerankModel := cfg.Rerank.Model
+				if rerankModel == "" {
+					rerankModel = cfg.Chat.Model
+				}
+				rerankChat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, rerankModel)
+				rr = rerank.NewLLMReranker(rerankChat)
+				log.Printf("rerank enabled with LLM model=%s (fallback path; consider an HTTP reranker for cost)", rerankModel)
+			}
+			if rr != nil {
+				srv = srv.WithReranker(rr, cfg.Rerank.CandidateK)
+			}
+		}
+	}
+
+	// On-demand /contents: use the crawler's FetchOne. Doesn't share the worker
+	// pool's rate gate or robots cache — callers should set a sensible UA + body cap.
+	srv = srv.WithFetcher(func(ctx context.Context, u string) (string, string, string, error) {
+		r, err := crawler.FetchOne(ctx, nil, cfg.Crawler.UserAgent, u, cfg.Crawler.MaxBodyBytes)
+		if err != nil {
+			return "", "", "", err
+		}
+		return r.Title, r.Text, r.Lang, nil
+	})
+
+	log.Printf("cosift serving on %s", cfg.Server.Addr)
+	return server.ListenAndServe(ctx, cfg.Server.Addr, srv.Handler())
+}
diff --git a/cmd/cosift/feedback.go b/cmd/cosift/feedback.go
new file mode 100644
index 0000000..16a9901
--- /dev/null
+++ b/cmd/cosift/feedback.go
@@ -0,0 +1,211 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Feedback — the usefulness signal. A caller (human UI or, in the Pilot app, the
+// calling agent) rates an answer by the query_id we returned via the
+// X-Cosift-Query-Id header. Ratings are appended to a JSONL log and joined to
+// the query log by qid to form a labeled dataset: (query, latency, empty?, rating).
+//
+// rating: -1 (penalize / unhelpful), +1 (reward / helpful). 0 accepted as neutral.
+// Feedback is weak, abusable signal (the endpoint is public) — treat it as
+// aggregated evidence, never absolute truth. caller is logged for dedup/abuse.
+
+type feedbackRec struct {
+	TS     string `json:"ts"`
+	Qid    string `json:"qid"`
+	Rating int    `json:"rating"`
+	Reason string `json:"reason,omitempty"`
+	Caller string `json:"caller,omitempty"`
+}
+
+// feedbackLogPath resolves the feedback log location: explicit env, else beside
+// the query log (so one COSIFT_QUERY_LOG setting enables both), else disabled.
+func feedbackLogPath() string {
+	if p := os.Getenv("COSIFT_FEEDBACK_LOG"); p != "" {
+		return p
+	}
+	if q := os.Getenv("COSIFT_QUERY_LOG"); q != "" {
+		return filepath.Join(filepath.Dir(q), "feedback-log.jsonl")
+	}
+	return ""
+}
+
+func (s *pebbleHTTP) writeFeedback(rec feedbackRec) {
+	if s.fbFile == nil {
+		return
+	}
+	b, err := json.Marshal(rec)
+	if err != nil {
+		return
+	}
+	s.fbMu.Lock()
+	defer s.fbMu.Unlock()
+	_, _ = s.fbFile.Write(append(b, '\n'))
+}
+
+// handleFeedback records a rating. Public (no admin token) like the query
+// endpoints — agents call it after using an answer. Body:
+// {"query_id":"...","rating":1,"reason":"..."}.
+func (s *pebbleHTTP) handleFeedback(w http.ResponseWriter, r *http.Request) {
+	if s.fbFile == nil {
+		writeProblem(w, http.StatusNotImplemented, "feedback disabled (COSIFT_QUERY_LOG/COSIFT_FEEDBACK_LOG unset)")
+		return
+	}
+	// Per-client rate limit (feedback is public/unauthed and abusable). Keyed on
+	// the real client via XFF, not the Caddy peer.
+	clientIP := r.Header.Get("X-Forwarded-For")
+	if i := strings.IndexByte(clientIP, ','); i >= 0 {
+		clientIP = clientIP[:i]
+	}
+	clientIP = strings.TrimSpace(clientIP)
+	if clientIP == "" {
+		clientIP = stripPort(r.RemoteAddr)
+	}
+	if s.fbRL != nil && !s.fbRL.allow(clientIP) {
+		w.Header().Set("Retry-After", "60")
+		writeProblem(w, http.StatusTooManyRequests, "feedback rate limit exceeded")
+		return
+	}
+	var req struct {
+		QueryID string `json:"query_id"`
+		Rating  int    `json:"rating"`
+		Reason  string `json:"reason"`
+	}
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 16<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.QueryID == "" {
+		writeProblem(w, http.StatusBadRequest, `expected {"query_id":"...","rating":-1|0|1,"reason":"..."}`)
+		return
+	}
+	if req.Rating < -1 || req.Rating > 1 {
+		writeProblem(w, http.StatusBadRequest, "rating must be -1, 0, or 1")
+		return
+	}
+	if len(req.Reason) > 1000 {
+		req.Reason = req.Reason[:1000]
+	}
+	s.writeFeedback(feedbackRec{
+		TS:     time.Now().UTC().Format(time.RFC3339),
+		Qid:    req.QueryID,
+		Rating: req.Rating,
+		Reason: req.Reason,
+		Caller: clientIP,
+	})
+	writeJSON(w, http.StatusOK, map[string]any{"ok": true, "query_id": req.QueryID, "rating": req.Rating})
+}
+
+// handleFeedbackList returns recent feedback (?n=N) or, with ?summary=1, an
+// aggregate that's immediately consumable: per-endpoint and overall
+// reward/penalty tallies joined to the query log by qid. This is the "make it
+// useful" surface — the labeled signal without an external pipeline.
+func (s *pebbleHTTP) handleFeedbackList(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		if r.Header.Get("Authorization") != "Bearer "+want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	fp := feedbackLogPath()
+	if fp == "" {
+		writeProblem(w, http.StatusNotImplemented, "feedback disabled")
+		return
+	}
+	if r.URL.Query().Get("summary") == "1" {
+		writeJSON(w, http.StatusOK, s.feedbackSummary(fp))
+		return
+	}
+	n := 100
+	if v := r.URL.Query().Get("n"); v != "" {
+		if k, err := strconv.Atoi(v); err == nil && k >= 1 && k <= 5000 {
+			n = k
+		}
+	}
+	lines := tailLines(fp, n)
+	recs := make([]json.RawMessage, 0, len(lines))
+	for _, ln := range lines {
+		recs = append(recs, json.RawMessage(ln))
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"count": len(recs), "entries": recs})
+}
+
+// feedbackSummary joins feedback ⋈ query-log on qid and tallies the signal:
+// total +/- by endpoint, plus the worst-rated recent queries (the penalize
+// targets). This is the consumable form of the reward/penalty signal.
+func (s *pebbleHTTP) feedbackSummary(fbPath string) map[string]any {
+	// qid -> query text (from the query log, recent window).
+	qtext := map[string]string{}
+	qep := map[string]string{}
+	for _, ln := range tailLines(os.Getenv("COSIFT_QUERY_LOG"), 20000) {
+		var qr queryLogRec
+		if json.Unmarshal([]byte(ln), &qr) == nil && qr.Qid != "" {
+			qtext[qr.Qid] = qr.Q
+			qep[qr.Qid] = qr.Ep
+		}
+	}
+	type tally struct{ Up, Down int }
+	byEp := map[string]*tally{}
+	byQuery := map[string]*tally{}
+	var up, down int
+	for _, ln := range tailLines(fbPath, 50000) {
+		var fr feedbackRec
+		if json.Unmarshal([]byte(ln), &fr) != nil {
+			continue
+		}
+		ep := qep[fr.Qid]
+		if ep == "" {
+			ep = "(unknown)"
+		}
+		if byEp[ep] == nil {
+			byEp[ep] = &tally{}
+		}
+		q := qtext[fr.Qid]
+		if q != "" {
+			if byQuery[q] == nil {
+				byQuery[q] = &tally{}
+			}
+		}
+		switch {
+		case fr.Rating > 0:
+			up++
+			byEp[ep].Up++
+			if q != "" {
+				byQuery[q].Up++
+			}
+		case fr.Rating < 0:
+			down++
+			byEp[ep].Down++
+			if q != "" {
+				byQuery[q].Down++
+			}
+		}
+	}
+	// Worst-rated queries = penalize targets (net negative).
+	type qScore struct {
+		Query string `json:"query"`
+		Up    int    `json:"up"`
+		Down  int    `json:"down"`
+		Net   int    `json:"net"`
+	}
+	var worst []qScore
+	for q, t := range byQuery {
+		net := t.Up - t.Down
+		if net < 0 {
+			worst = append(worst, qScore{Query: q, Up: t.Up, Down: t.Down, Net: net})
+		}
+	}
+	return map[string]any{
+		"total_up":     up,
+		"total_down":   down,
+		"by_endpoint":  byEp,
+		"penalize_top": worst, // queries with net-negative feedback
+	}
+}
diff --git a/cmd/cosift/find.go b/cmd/cosift/find.go
new file mode 100644
index 0000000..130faf7
--- /dev/null
+++ b/cmd/cosift/find.go
@@ -0,0 +1,270 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/embed"
+)
+
+// /find is the live resource-federation endpoint. Where /search and /research
+// query the INDEXED corpus (great for exploration, weak for known-item lookup),
+// /find federates LIVE queries to the authoritative source APIs — HuggingFace
+// Hub, GitHub, and PyPI — at request time, then synthesizes a cited answer with
+// real identifiers (HF model id, pip package, GitHub repo) and install commands.
+//
+// Rationale: a search engine over a sampled corpus can't beat hitting the source
+// index for "find me the specific installable thing for platform X". This closes
+// that gap by doing what a human does with WebFetch, in one call.
+
+type findHit struct {
+	Source  string `json:"source"` // huggingface | github | pypi
+	Title   string `json:"title"`
+	URL     string `json:"url"`
+	Summary string `json:"summary"`
+	Meta    string `json:"meta"` // "41 likes" | "3249 stars" | "v1.1.0"
+	rank    int    // source-local popularity for ordering
+}
+
+func findHTTPGet(ctx context.Context, u string, hdr map[string]string) ([]byte, int, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
+	if err != nil {
+		return nil, 0, err
+	}
+	req.Header.Set("User-Agent", "cosift-find/1.0")
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := (&http.Client{Timeout: 20 * time.Second}).Do(req)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	return b, resp.StatusCode, err
+}
+
+// federateHF queries the HF Hub search API across models, datasets, and spaces.
+func federateHF(ctx context.Context, q string) []findHit {
+	var out []findHit
+	kinds := []struct{ api, prefix string }{
+		{"models", "/"}, {"datasets", "/datasets/"}, {"spaces", "/spaces/"},
+	}
+	for _, k := range kinds {
+		u := fmt.Sprintf("https://huggingface.co/api/%s?search=%s&limit=5&sort=likes&direction=-1",
+			k.api, url.QueryEscape(q))
+		b, code, err := findHTTPGet(ctx, u, nil)
+		if err != nil || code != 200 {
+			continue
+		}
+		var items []struct {
+			ID        string `json:"id"`
+			Likes     int    `json:"likes"`
+			Downloads int    `json:"downloads"`
+		}
+		if json.Unmarshal(b, &items) != nil {
+			continue
+		}
+		for _, it := range items {
+			if it.ID == "" {
+				continue
+			}
+			out = append(out, findHit{
+				Source: "huggingface", Title: it.ID,
+				URL:  "https://huggingface.co" + k.prefix + it.ID,
+				Meta: fmt.Sprintf("%d likes, %d dl (%s)", it.Likes, it.Downloads, k.api),
+				rank: it.Likes,
+			})
+		}
+	}
+	return out
+}
+
+// federateGitHub queries the GitHub repo search API (by relevance/stars).
+func federateGitHub(ctx context.Context, q string) []findHit {
+	u := "https://api.github.com/search/repositories?q=" + url.QueryEscape(q) + "&sort=stars&order=desc&per_page=6"
+	b, code, err := findHTTPGet(ctx, u, map[string]string{"Accept": "application/vnd.github+json"})
+	if err != nil || code != 200 {
+		return nil
+	}
+	var res struct {
+		Items []struct {
+			FullName string `json:"full_name"`
+			HTMLURL  string `json:"html_url"`
+			Desc     string `json:"description"`
+			Stars    int    `json:"stargazers_count"`
+			Lang     string `json:"language"`
+		} `json:"items"`
+	}
+	if json.Unmarshal(b, &res) != nil {
+		return nil
+	}
+	var out []findHit
+	for _, it := range res.Items {
+		out = append(out, findHit{
+			Source: "github", Title: it.FullName, URL: it.HTMLURL,
+			Summary: it.Desc, Meta: fmt.Sprintf("%d stars, %s", it.Stars, it.Lang),
+			rank: it.Stars,
+		})
+	}
+	return out
+}
+
+var pypiNameRe = regexp.MustCompile(`package-snippet__name">([^<]+)<`)
+var pypiDescRe = regexp.MustCompile(`package-snippet__description">([^<]*)<`)
+var pypiVerRe = regexp.MustCompile(`package-snippet__version">([^<]+)<`)
+
+// federatePyPI parses the PyPI search results page (PyPI retired its search API,
+// but the HTML results page is stable and parseable). Best-effort.
+func federatePyPI(ctx context.Context, q string) []findHit {
+	u := "https://pypi.org/search/?q=" + url.QueryEscape(q)
+	b, code, err := findHTTPGet(ctx, u, nil)
+	if err != nil || code != 200 {
+		return nil
+	}
+	html := string(b)
+	names := pypiNameRe.FindAllStringSubmatch(html, 8)
+	descs := pypiDescRe.FindAllStringSubmatch(html, 8)
+	vers := pypiVerRe.FindAllStringSubmatch(html, 8)
+	var out []findHit
+	for i, n := range names {
+		name := strings.TrimSpace(n[1])
+		if name == "" {
+			continue
+		}
+		h := findHit{
+			Source: "pypi", Title: name,
+			URL: "https://pypi.org/project/" + name + "/",
+		}
+		if i < len(descs) {
+			h.Summary = strings.TrimSpace(descs[i][1])
+		}
+		if i < len(vers) {
+			h.Meta = strings.TrimSpace(vers[i][1])
+		}
+		out = append(out, h)
+	}
+	return out
+}
+
+const findSynthPrompt = `You are a technical resource finder for ML/software engineering. The user wants SPECIFIC, installable resources. Using ONLY the candidate resources provided (from HuggingFace, GitHub, and PyPI), recommend the most relevant ones.
+- Give EXACT identifiers: HF model/dataset id, pip package name, GitHub repo (owner/name).
+- Give the install/download command where applicable: ` + "`pip install <pkg>`" + `, ` + "`hf_hub_download(...)`" + `, ` + "`git clone`" + `.
+- If the user named a platform (e.g. Jetson/aarch64/JetPack), note compatibility ONLY if a source states it; otherwise say it must be verified on the resource's page.
+- Cite every recommendation by its numeric id, e.g. [2]. Do not invent resources or claims not in the sources.
+- Be concise and actionable: lead with the top 2-3 picks.`
+
+const findPlanPrompt = `Convert the user's resource request into 1-4 SHORT search phrases (2-4 words each) for searching model/package/repo catalogs (HuggingFace, GitHub, PyPI). These catalogs do AND-style matching, so long natural-language queries return nothing — extract the core nouns. Each phrase should target ONE resource the user needs (e.g. a detector, an OCR backend, a classifier). Drop platform/qualifier words (jetson, aarch64, model, library). Output ONLY a JSON array of strings. Example: ["license plate detection", "license plate OCR", "vehicle classifier"]`
+
+func (s *pebbleHTTP) handleFind(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	start := time.Now()
+
+	// Distill the NL request into short keyword phrases — the source search
+	// APIs AND-match, so a verbose query ("YOLOv8 LPR model for Jetson aarch64")
+	// returns zero, but "license plate detection" returns the right models.
+	phrases := []string{q}
+	if s.chat != nil {
+		if raw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+			{Role: "system", Content: findPlanPrompt},
+			{Role: "user", Content: q},
+		}); err == nil {
+			if p := parseSubQueries(raw, q); len(p) > 0 {
+				phrases = p
+			}
+		}
+	}
+	if len(phrases) > 4 {
+		phrases = phrases[:4]
+	}
+
+	// Federate each phrase across all three sources in parallel; each call
+	// degrades to empty on failure (e.g. GitHub rate-limit) so the others
+	// still answer. Dedup by URL afterward.
+	var mu sync.Mutex
+	var hits []findHit
+	var wg sync.WaitGroup
+	for _, ph := range phrases {
+		for _, fn := range []func(context.Context, string) []findHit{federateHF, federateGitHub, federatePyPI} {
+			ph, fn := ph, fn
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				h := fn(r.Context(), ph)
+				mu.Lock()
+				hits = append(hits, h...)
+				mu.Unlock()
+			}()
+		}
+	}
+	wg.Wait()
+
+	// Order: source-local popularity desc, grouped so the synth sees the
+	// strongest candidates first.
+	sort.SliceStable(hits, func(i, j int) bool { return hits[i].rank > hits[j].rank })
+
+	// Dedup by URL (keeps the highest-ranked occurrence since we sorted first),
+	// then cap the pool fed to synthesis.
+	seen := make(map[string]bool, len(hits))
+	deduped := hits[:0]
+	for _, h := range hits {
+		if h.URL == "" || seen[h.URL] {
+			continue
+		}
+		seen[h.URL] = true
+		deduped = append(deduped, h)
+	}
+	hits = deduped
+	const maxFindHits = 24
+	if len(hits) > maxFindHits {
+		hits = hits[:maxFindHits]
+	}
+
+	if len(hits) == 0 {
+		writeJSON(w, http.StatusOK, map[string]any{
+			"query": q, "sources": []findHit{}, "answer": "",
+			"note": "no resources found via HuggingFace/GitHub/PyPI federation",
+			"took": time.Since(start).String(),
+		})
+		return
+	}
+
+	// Build the cited sources block + synthesize (when chat is configured).
+	var sb strings.Builder
+	for i, h := range hits {
+		fmt.Fprintf(&sb, "[%d] (%s) %s — %s\n    %s%s\n", i+1, h.Source, h.Title, h.URL,
+			h.Summary, func() string {
+				if h.Meta != "" {
+					return " (" + h.Meta + ")"
+				}
+				return ""
+			}())
+	}
+	answer := ""
+	if s.chat != nil {
+		out, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+			{Role: "system", Content: findSynthPrompt},
+			{Role: "user", Content: "Request: " + q + "\n\nCandidate resources:\n" + sb.String()},
+		})
+		if err == nil {
+			answer = out
+		}
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"query": q, "answer": answer, "sources": hits,
+		"count": len(hits), "took": time.Since(start).String(),
+	})
+}
diff --git a/cmd/cosift/main.go b/cmd/cosift/main.go
index 0a59a3c..eff60a4 100644
--- a/cmd/cosift/main.go
+++ b/cmd/cosift/main.go
@@ -1,41 +1,19 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
 	"context"
-	"crypto/sha256"
-	"encoding/csv"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
-	"io"
-	"log"
-	"math/rand"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
 	"os"
 	"os/signal"
-	"path/filepath"
-	"regexp"
 	"runtime/debug"
-	"sort"
-	"strconv"
 	"strings"
-	"sync"
 	"syscall"
-	"time"
 
 	"github.com/pilot-protocol/cosift/internal/config"
 	"github.com/pilot-protocol/cosift/internal/crawler"
-	"github.com/pilot-protocol/cosift/internal/embed"
-	"github.com/pilot-protocol/cosift/internal/eval"
 	"github.com/pilot-protocol/cosift/internal/index"
-	"github.com/pilot-protocol/cosift/internal/rerank"
-	"github.com/pilot-protocol/cosift/internal/server"
-	"github.com/pilot-protocol/cosift/internal/store"
 )
 
 const usage = `cosift — self-hostable search + research
@@ -330,6 +308,14 @@ func run(cfgPath string) error {
 		if err := runFrontierClear(ctx, flag.Args()[1:]); err != nil {
 			return fmt.Errorf("frontier-clear: %w", err)
 		}
+	case "frontier-trim":
+		if err := runFrontierTrim(ctx, cfg, flag.Args()[1:]); err != nil {
+			return fmt.Errorf("frontier-trim: %w", err)
+		}
+	case "seed-tranco":
+		if err := runSeedTranco(ctx, cfg, flag.Args()[1:]); err != nil {
+			return fmt.Errorf("seed-tranco: %w", err)
+		}
 	case "domain-audit":
 		if err := runDomainAudit(ctx, flag.Args()[1:]); err != nil {
 			return fmt.Errorf("domain-audit: %w", err)
@@ -342,6 +328,10 @@ func run(cfgPath string) error {
 		if err := runPurgeDomain(ctx, flag.Args()[1:]); err != nil {
 			return fmt.Errorf("purge-domain: %w", err)
 		}
+	case "backfill-host-postings":
+		if err := runBackfillHostPostings(ctx, flag.Args()[1:]); err != nil {
+			return fmt.Errorf("backfill-host-postings: %w", err)
+		}
 	case "verify":
 		if err := runVerifyPebble(ctx, cfg, flag.Args()[1:]); err != nil {
 			return fmt.Errorf("verify: %w", err)
@@ -353,21 +343,6 @@ func run(cfgPath string) error {
 	return nil
 }
 
-// authStatus describes how a configured capability will authenticate to its
-// endpoint, for /doctor output.
-func authStatus(configured bool, key string, urlSet bool) string {
-	switch {
-	case !configured:
-		return "(none)"
-	case key != "":
-		return "bearer-token"
-	case urlSet:
-		return "anonymous(custom-url)"
-	default:
-		return "MISSING"
-	}
-}
-
 // resolveAPIKey returns the first non-empty API key from a slot-specific env
 // var, falling back to OPENAI_API_KEY / OPENAI. Lets operators using a
 // non-OpenAI embedder or chat endpoint name keys for what they
@@ -402,5852 +377,39 @@ func resolveAPIKey(slot string) string {
 // callers should use resolveAPIKey("embed").
 func resolveEmbedAPIKey() string { return resolveAPIKey("embed") }
 
-// hnswPassageWriter bridges the crawler's PassageWriter contract to
-// an in-memory index.HNSW graph. Each embedded passage looks up the doc's URL
-// and Title (via *PebbleStore.GetDocByID) and is inserted into the graph; the
-// graph is persisted to Pebble's 'v' family in a single Persist() call at the
-// end of the crawl.
-type hnswPassageWriter struct {
-	ps   *store.PebbleStore
-	hnsw *index.HNSW
-}
-
-func (w *hnswPassageWriter) UpsertPassage(ctx context.Context, p *store.Passage) error {
-	doc, err := w.ps.GetDocByID(ctx, p.DocID)
-	if err != nil {
-		return err
-	}
-	if doc == nil {
-		return nil // doc was deleted between index and embed — skip silently
-	}
-	w.hnsw.AddPassage(doc.URL, doc.Title, p.Offset, p.Length, p.Embedding)
-	return nil
-}
-
-// UpsertPassageBatch writes all passages in a single HNSW lock
-// acquisition — meaningful when a doc has many chunks. Implements the
-// optional crawler.PassageWriterBatch interface; the crawler will call
-// this when present instead of looping UpsertPassage per chunk.
-func (w *hnswPassageWriter) UpsertPassageBatch(ctx context.Context, ps []*store.Passage) error {
-	if len(ps) == 0 {
-		return nil
-	}
-	// All passages in a batch belong to the same DocID (the crawler hands
-	// us one doc's chunks). Resolve url + title once.
-	doc, err := w.ps.GetDocByID(ctx, ps[0].DocID)
-	if err != nil {
-		return err
-	}
-	if doc == nil {
-		return nil
-	}
-	items := make([]index.PassageInput, 0, len(ps))
-	for _, p := range ps {
-		items = append(items, index.PassageInput{
-			URL: doc.URL, Title: doc.Title, Offset: p.Offset, Length: p.Length, Vec: p.Embedding,
-		})
-	}
-	w.hnsw.AddPassageBatch(items)
-	return nil
-}
-
-// MarkURLInvalid satisfies the optional crawler.URLInvalidator interface.
-// Lets the crawler reclaim zombie passages (prior generations of chunks for
-// the same URL) before pushing a fresh chunk batch. Returns count zeroed.
-func (w *hnswPassageWriter) MarkURLInvalid(ctx context.Context, url string) (int, error) {
-	if err := ctx.Err(); err != nil {
-		return 0, err
-	}
-	return w.hnsw.MarkURLPassagesInvalid(url), nil
-}
-
-func runCrawl(ctx context.Context, cfg *config.Config, args []string) error {
-	// hoisted PebbleStore handle so the embedder wiring below can
-	// reach it to attach the HNSW bridge after the backend switch.
-	var pebbleStoreForCrawl *store.PebbleStore
-	fs := flag.NewFlagSet("crawl", flag.ExitOnError)
-	refresh := fs.Bool("refresh", false, "force re-crawl of URLs already in the frontier")
-	sitemap := fs.String("sitemap", "", "URL of a sitemap.xml (or sitemap index) to seed from")
-	seedsFile := fs.String("seeds-file", "", "path to a text file with one seed URL per line (blank lines and # comments ignored)")
-	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
-	duration := fs.Duration("duration", 0, "stop the crawl cleanly after this much time (0 = run until frontier empty or SIGTERM)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	urls := fs.Args()
-	// -seeds-file lets operators target specific websites in bulk
-	// without stuffing dozens of URLs on the command line. Each non-blank,
-	// non-comment line is treated as a seed.
-	if *seedsFile != "" {
-		buf, err := os.ReadFile(*seedsFile)
-		if err != nil {
-			return fmt.Errorf("read -seeds-file %s: %w", *seedsFile, err)
-		}
-		for _, line := range strings.Split(string(buf), "\n") {
-			s := strings.TrimSpace(line)
-			if s == "" || strings.HasPrefix(s, "#") {
-				continue
-			}
-			urls = append(urls, s)
-		}
-		log.Printf("crawler: loaded %d seeds from %s (after dedup of positional args, total queued = %d)", len(urls)-fs.NArg(), *seedsFile, len(urls))
-	}
-	if len(urls) == 0 && *sitemap == "" {
-		return errors.New("crawl: at least one URL, -seeds-file, or -sitemap is required")
-	}
-
-	// Wraps the caller's ctx with a
-	// timeout; workers see ctx.Err() != nil and exit cleanly. Pebble flushes
-	// on Close (via the deferred ps.Close() below) so durability is preserved.
-	if *duration > 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, *duration)
-		defer cancel()
-		log.Printf("crawler: bounded run, will stop after %s", *duration)
-	}
-
-	var c *crawler.Crawler
-	switch *backend {
-	case "sqlite", "":
-		s, err := store.Open(cfg.DataDir)
-		if err != nil {
-			return err
-		}
-		defer s.Close()
-		c = crawler.New(cfg.Crawler, s)
-	case "pebble":
-		// route through the  Pebble path. The
-		// data dir layout under cfg.DataDir for Pebble: a sibling "pebble"
-		// subdir so SQLite and Pebble stores can coexist during migration.
-		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
-		var err error
-		pebbleStoreForCrawl, err = openPebbleOrFriendlyErr(pebbleDir)
-		if err != nil {
-			return err
-		}
-		defer pebbleStoreForCrawl.Close()
-		c = crawler.NewWithBackend(cfg.Crawler, pebbleStoreForCrawl, index.NewPebbleBM25(pebbleStoreForCrawl))
-		log.Printf("crawler: pebble backend at %s", pebbleDir)
-	default:
-		return fmt.Errorf("crawl: unknown -backend %q (want: sqlite | pebble)", *backend)
-	}
-
-	// Auto-wire embedder when configured. For the Pebble backend, also
-	// build and persist an HNSW graph via the hnswPassageWriter bridge
-	// — that's the path /search?retriever=dense
-	// needs and that pre was a documented no-op.
-	// API key is OPTIONAL when cfg.Embeddings.URL points at a
-	// custom endpoint (Ollama / vLLM / TEI / etc — local self-hosted
-	// embedders don't need a Bearer token). Required only when hitting the
-	// default OpenAI endpoint.
-	if cfg.Embeddings.Model != "" {
-		apiKey := resolveEmbedAPIKey()
-		needsKey := cfg.Embeddings.URL == "" // default OpenAI requires a key
-		if apiKey != "" || !needsKey {
-			dim := cfg.Embeddings.Dim
-			if dim == 0 {
-				dim = 1536
-			}
-			emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
-			c = c.WithEmbedder(emb)
-			authStatus := "anonymous"
-			if apiKey != "" {
-				authStatus = "bearer-token"
-			}
-			log.Printf("crawler: dense embeddings enabled (model=%s, dim=%d, auth=%s)", cfg.Embeddings.Model, dim, authStatus)
-			if pebbleStoreForCrawl != nil {
-				h := index.NewHNSW(dim)
-				c = c.WithPassageWriter(&hnswPassageWriter{ps: pebbleStoreForCrawl, hnsw: h})
-				// periodic checkpoint every COSIFT_HNSW_CHECKPOINT_SEC
-				// (default 60s). Without this, a crash / SIGKILL / time-cap
-				// deadline mid-crawl loses all in-memory vectors because Persist
-				// only runs at deferred end-of-crawl. Goroutine exits when
-				// ctx.Done() fires (either deadline or SIGTERM).
-				checkpointEvery := 60 * time.Second
-				if v := os.Getenv("COSIFT_HNSW_CHECKPOINT_SEC"); v != "" {
-					if s, err := strconv.Atoi(v); err == nil && s >= 5 {
-						checkpointEvery = time.Duration(s) * time.Second
-					}
-				}
-				log.Printf("crawler: HNSW vector index attached (in-memory build, checkpoint every %s, final persist at crawl end)", checkpointEvery)
-				ckpDone := make(chan struct{})
-				go func() {
-					defer close(ckpDone)
-					t := time.NewTicker(checkpointEvery)
-					defer t.Stop()
-					var lastN int
-					for {
-						select {
-						case <-ctx.Done():
-							return
-						case <-t.C:
-							n := h.Len()
-							if n == 0 || n == lastN {
-								continue // nothing new to persist
-							}
-							log.Printf("crawler: HNSW checkpoint at %d nodes (delta=%d) ...", n, n-lastN)
-							if err := h.Persist(context.Background(), pebbleStoreForCrawl); err != nil {
-								log.Printf("crawler: HNSW checkpoint failed: %v", err)
-								continue
-							}
-							lastN = n
-						}
-					}
-				}()
-				defer func() {
-					<-ckpDone // ensure ticker goroutine exits before final persist
-					if h.Len() == 0 {
-						return
-					}
-					log.Printf("crawler: final HNSW persist (%d nodes, dim=%d) to Pebble...", h.Len(), dim)
-					if err := h.Persist(context.Background(), pebbleStoreForCrawl); err != nil {
-						log.Printf("crawler: HNSW persist failed: %v", err)
-					} else {
-						log.Printf("crawler: HNSW persist complete")
-					}
-				}()
-			}
-		} else {
-			log.Printf("warning: embeddings configured (default OpenAI endpoint) but no OPENAI_API_KEY in env; crawling BM25 only")
-		}
-	}
-	if *sitemap != "" {
-		n, err := c.SeedSitemap(ctx, *sitemap)
-		if err != nil {
-			return fmt.Errorf("sitemap %s: %w", *sitemap, err)
-		}
-		log.Printf("seeded %d URLs from sitemap %s", n, *sitemap)
-	}
-	for _, u := range urls {
-		if *refresh {
-			if err := c.Recrawl(ctx, u); err != nil {
-				return fmt.Errorf("recrawl %s: %w", u, err)
-			}
-		} else {
-			if err := c.Seed(u); err != nil {
-				return fmt.Errorf("seed %s: %w", u, err)
-			}
-		}
-	}
-	return c.Run(ctx)
-}
-
-func runQuery(ctx context.Context, cfg *config.Config, q string, args []string) error {
-	fs := flag.NewFlagSet("query", flag.ExitOnError)
-	k := fs.Int("k", 10, "max results")
-	jsonOut := fs.Bool("json", false, "emit JSON array instead of human-readable list — for shell pipelines")
-	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *k <= 0 || *k > 100 {
-		return errors.New("k must be in [1, 100]")
-	}
-
-	// -backend flag mirrors on crawl + stats. Both
-	// BM25 implementations expose the same Search(ctx, q, k) signature so
-	// the rest of runQuery is backend-agnostic.
-	type searcher interface {
-		Search(ctx context.Context, q string, k int) ([]index.Hit, error)
-	}
-	var idx searcher
-	switch *backend {
-	case "sqlite", "":
-		s, err := store.Open(cfg.DataDir)
-		if err != nil {
-			return err
-		}
-		defer s.Close()
-		idx = index.NewBM25(s)
-	case "pebble":
-		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
-		ps, err := openPebbleOrFriendlyErr(pebbleDir)
-		if err != nil {
-			return err
-		}
-		defer ps.Close()
-		// runQuery previously
-		// always used defaults; with env overrides set, score values diverged
-		// silently from pebble-serve's. Now they line up.
-		pidx := index.NewPebbleBM25(ps)
-		applyBM25EnvOverrides(pidx)
-		idx = pidx
-	default:
-		return fmt.Errorf("query: unknown -backend %q (want: sqlite | pebble)", *backend)
-	}
-	hits, err := idx.Search(ctx, q, *k)
-	if err != nil {
-		return err
-	}
-
-	if *jsonOut {
-		// Wire format matches the public /search response's per-hit shape
-		// (sans Highlight / Domain / Excerpt — those need a store lookup
-		// and `cosift query` is the no-server path).
-		type jsonHit struct {
-			URL   string  `json:"url"`
-			Title string  `json:"title"`
-			Score float64 `json:"score"`
-		}
-		out := make([]jsonHit, len(hits))
-		for i, h := range hits {
-			out[i] = jsonHit{URL: h.URL, Title: h.Title, Score: h.Score}
-		}
-		b, err := json.Marshal(out)
-		if err != nil {
-			return err
-		}
-		fmt.Println(string(b))
-		return nil
-	}
-	for i, h := range hits {
-		fmt.Printf("%2d. [%.3f] %s\n    %s\n", i+1, h.Score, h.Title, h.URL)
-	}
-	return nil
-}
-
-// runSearchCLI hits a running cosift server's /search endpoint over HTTP.
-// Unlike runQuery (local BM25 only), this exercises the full pipeline:
-// dense/hybrid retrieval, LLM rerank, paraphrase expansion, date/domain
-// filters, custom sort. Useful from operator shells without curl + jq.
-func runSearchCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
-	fs := flag.NewFlagSet("search", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	// 0.0.0.0 is a bind-only sentinel; clients must dial loopback.
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	k := fs.Int("k", 10, "max results")
-	retriever := fs.String("retriever", "", "bm25 | dense | hybrid (server default if empty)")
-	// MMR lambda passthrough for /search.
-	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (needs HNSW + embedder server-side)")
-	rerankFlag := fs.Bool("rerank", false, "wrap retrieval with LLM listwise reranker (server must have it configured)")
-	expand := fs.Bool("expand", false, "LLM paraphrase + RRF fusion (server must have a paraphraser)")
-	since := fs.String("since", "", "ISO date — only results published on or after")
-	until := fs.String("until", "", "ISO date — only results published on or before")
-	includeDomains := fs.String("include-domains", "", "comma-separated allowlist of result domains")
-	excludeDomains := fs.String("exclude-domains", "", "comma-separated denylist of result domains")
-	sortMode := fs.String("sort", "", "relevance | date_desc | date_asc (server default if empty)")
-	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable list")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *k <= 0 || *k > 100 {
-		return errors.New("k must be in [1, 100]")
-	}
-	if err := validateFormat(*format); err != nil {
-		return err
-	}
-
-	v := url.Values{}
-	v.Set("q", q)
-	v.Set("k", strconv.Itoa(*k))
-	if *retriever != "" {
-		v.Set("retriever", *retriever)
-	}
-	if *mmr != "" {
-		v.Set("mmr", *mmr)
-	}
-	if *rerankFlag {
-		v.Set("rerank", "true")
-	}
-	if *expand {
-		v.Set("expand", "true")
-	}
-	if *since != "" {
-		v.Set("since", *since)
-	}
-	if *until != "" {
-		v.Set("until", *until)
-	}
-	if *includeDomains != "" {
-		v.Set("include_domains", *includeDomains)
-	}
-	if *excludeDomains != "" {
-		v.Set("exclude_domains", *excludeDomains)
-	}
-	if *sortMode != "" {
-		v.Set("sort", *sortMode)
-	}
-
-	endpoint := strings.TrimRight(*serverURL, "/") + "/search?" + v.Encode()
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-
-	if *jsonOut {
-		// Pass the server's JSON straight through — same shape as /search.
-		fmt.Println(string(body))
-		return nil
-	}
-	emitWarnings(body)
-	var sr server.SearchResponse
-	if err := json.Unmarshal(body, &sr); err != nil {
-		return fmt.Errorf("decode response: %w", err)
-	}
-	if *format == "md" || *format == "markdown" {
-		fmt.Print(renderRankedMarkdown("Results: "+sr.Query, sr.Hits))
-		return nil
-	}
-	if len(sr.Hits) == 0 {
-		fmt.Println("(no results)")
-		return nil
-	}
-	for i, h := range sr.Hits {
-		date := ""
-		if h.PublishedAt != nil {
-			date = " " + h.PublishedAt.Format("2006-01-02")
-		}
-		domain := ""
-		if h.Domain != "" {
-			domain = " [" + h.Domain + "]"
-		}
-		fmt.Printf("%2d. [%.3f]%s%s %s\n    %s\n", i+1, h.Score, domain, date, h.Title, h.URL)
-		if h.Highlight != nil && h.Highlight.Text != "" {
-			fmt.Printf("    > %s\n", truncate(h.Highlight.Text, 200))
-		} else if h.Excerpt != "" {
-			fmt.Printf("    > %s\n", truncate(h.Excerpt, 200))
-		}
-	}
-	return nil
-}
-
-// runResearchCLI hits a running cosift server's /research endpoint over HTTP.
-// Sibling to runSearchCLI: same -server / -json affordance, but pulls
-// an LLM-synthesized answer with cited sources rather than a ranked URL list.
-// Non-streaming — the SSE mode (?stream=true) is a separate code path; this
-// returns the same JSON the non-streaming endpoint produces.
-// renderAnswerMarkdown formats a /research or /answer response as markdown,
-// suitable for piping into LLM contexts or markdown viewers. Used by both
-// runResearchCLI and runAnswerCLI via the shared `-format md|markdown` flag.
-// strategy/plan are research-only; pass "" / nil from /answer's callsite.
-func renderAnswerMarkdown(query, strategy string, plan []string, answer string, sources []server.AnswerSource) string {
-	var b strings.Builder
-	fmt.Fprintf(&b, "# %s\n\n", query)
-	if strategy != "" {
-		fmt.Fprintf(&b, "> Strategy: `%s`", strategy)
-		if len(plan) > 0 {
-			fmt.Fprintf(&b, " (plan: %s)", strings.Join(plan, " | "))
-		}
-		b.WriteString("\n\n")
-	}
-	b.WriteString(answer)
-	if !strings.HasSuffix(answer, "\n") {
-		b.WriteString("\n")
-	}
-	if len(sources) > 0 {
-		b.WriteString("\n## Sources\n\n")
-		for i, s := range sources {
-			// `N. [Title](URL)` — the leading N matches the citation IDs the
-			// LLM emits inline (e.g. `[1]` in the answer references source 1).
-			fmt.Fprintf(&b, "%d. [%s](%s)", sourceIDOf(i, s), s.Title, s.URL)
-			var trailing []string
-			if s.Domain != "" {
-				trailing = append(trailing, s.Domain)
-			}
-			if s.PublishedAt != nil {
-				trailing = append(trailing, s.PublishedAt.Format("2006-01-02"))
-			}
-			if len(trailing) > 0 {
-				fmt.Fprintf(&b, " — %s", strings.Join(trailing, ", "))
-			}
-			b.WriteString("\n")
-		}
-	}
-	return b.String()
-}
-
-// renderRankedMarkdown formats a ranked-hits response (search, find-similar)
-// as markdown for LLM piping or markdown viewers. `title` is the H1 text —
-// renderRankedMarkdown renders a ranked hit list as markdown. Callers
-// pass "Results: <query>" for /search or "Similar to: <url>" for
-// /find_similar; the hit-list shape is the same in both cases.
-func renderRankedMarkdown(title string, hits []server.SearchHit) string {
-	var b strings.Builder
-	fmt.Fprintf(&b, "# %s\n\n", title)
-	if len(hits) == 0 {
-		b.WriteString("_No results._\n")
-		return b.String()
-	}
-	for i, h := range hits {
-		fmt.Fprintf(&b, "## %d. [%s](%s)\n\n", i+1, h.Title, h.URL)
-		var meta []string
-		meta = append(meta, fmt.Sprintf("Score: %.3f", h.Score))
-		if h.Domain != "" {
-			meta = append(meta, h.Domain)
-		}
-		if h.PublishedAt != nil {
-			meta = append(meta, h.PublishedAt.Format("2006-01-02"))
-		}
-		fmt.Fprintf(&b, "_%s_\n", strings.Join(meta, " · "))
-		// Excerpt OR highlight as a blockquote. Highlight wins when both are
-		// present (it's the matched passage, more relevant than the prefix).
-		var excerpt string
-		if h.Highlight != nil && h.Highlight.Text != "" {
-			excerpt = h.Highlight.Text
-		} else if h.Excerpt != "" {
-			excerpt = h.Excerpt
-		}
-		if excerpt != "" {
-			b.WriteString("\n> ")
-			b.WriteString(truncate(excerpt, 240))
-			b.WriteString("\n")
-		}
-		// Trailing blank line between hits, but not after the last one.
-		if i < len(hits)-1 {
-			b.WriteString("\n")
-		}
-	}
-	return b.String()
-}
-
-// forEachSSEEvent reads an SSE response body and invokes handle(event, data)
-// for each `event: <name>\ndata: <json>\n\n` block. The handler can return
-// errSSEDone to signal terminal completion (the scanner returns nil), or any
-// other error to abort. Multi-line `data:` lines are concatenated with `\n`
-// per the SSE spec. Extracted from's consumeResearchSSE so
-// both research and answer CLI consumers share the framing logic.
-func forEachSSEEvent(body io.Reader, handle func(event, data string) error) error {
-	scanner := bufio.NewScanner(body)
-	// SSE events can be larger than the default 64KB scanner buffer (the
-	// terminal `done` blob in /research carries the full ResearchResponse).
-	scanner.Buffer(make([]byte, 0, 64<<10), 4<<20)
-
-	var event, data string
-	for scanner.Scan() {
-		line := scanner.Text()
-		if line == "" {
-			// Blank line terminates an event block — dispatch.
-			if event != "" {
-				if err := handle(event, data); err != nil {
-					if errors.Is(err, errSSEDone) {
-						return nil
-					}
-					return err
-				}
-			}
-			event, data = "", ""
-			continue
-		}
-		switch {
-		case strings.HasPrefix(line, "event:"):
-			event = strings.TrimSpace(strings.TrimPrefix(line, "event:"))
-		case strings.HasPrefix(line, "data:"):
-			d := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
-			if data == "" {
-				data = d
-			} else {
-				data = data + "\n" + d
-			}
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("read stream: %w", err)
-	}
-	// Stream ended without a terminal event — caller's responsibility to flag
-	// (this function just reports the natural end-of-stream as success).
-	return nil
-}
-
-// consumeResearchSSE reads a /research?stream=true response body and renders
-// each event as it arrives. Terminal events: "done" (full response) emits a
-// final newline + Sources section; "error" emits the detail and returns an
-// error. First streaming CLI in cosift. Refactored in to
-// use the generic forEachSSEEvent scanner.
-func consumeResearchSSE(body io.Reader, format string) error {
-	useMarkdown := format == "md" || format == "markdown"
-	var answerStarted, sawDone bool
-	// capture pebble-serve's sources event payload so we can render
-	// the final Sources section when pebble's minimal done event arrives.
-	var pebbleSources []server.AnswerSource
-
-	err := forEachSSEEvent(body, func(event, data string) error {
-		switch event {
-		case "warnings":
-			// pebble-serve emits this event when the
-			// request had silent no-ops. Surface to stderr like the sync CLI.
-			var w struct {
-				Warnings []string `json:"warnings"`
-			}
-			if err := json.Unmarshal([]byte(data), &w); err == nil {
-				for _, msg := range w.Warnings {
-					fmt.Fprintln(os.Stderr, "cosift: warning:", msg)
-				}
-			}
-		case "plan":
-			var p struct {
-				Strategy string   `json:"strategy"`
-				Variants []string `json:"variants"`
-				// pebble-serve uses 'plan' for the sub-query list and
-				// emits an 'expand' label instead of 'strategy'. Decode both
-				// shapes; fall back to "planner" when only plan is set.
-				Plan   []string `json:"plan"`
-				Expand string   `json:"expand"`
-			}
-			if err := json.Unmarshal([]byte(data), &p); err != nil {
-				return nil // tolerate malformed mid-stream event
-			}
-			strategy := p.Strategy
-			variants := p.Variants
-			if strategy == "" && len(p.Plan) > 0 {
-				strategy = "planner"
-				if p.Expand != "" {
-					strategy = "planner+" + p.Expand
-				}
-			}
-			if len(variants) == 0 {
-				variants = p.Plan
-			}
-			if useMarkdown {
-				fmt.Printf("> Strategy: `%s`", strategy)
-				if len(variants) > 0 {
-					fmt.Printf(" (plan: %s)", strings.Join(variants, " | "))
-				}
-				fmt.Println()
-				fmt.Println()
-			} else {
-				fmt.Printf("Strategy: %s", strategy)
-				if len(variants) > 0 {
-					fmt.Printf(" (plan: %s)", strings.Join(variants, " | "))
-				}
-				fmt.Println()
-			}
-		case "retrieved":
-			var r struct {
-				Variant string   `json:"variant"`
-				URLs    []string `json:"urls"`
-			}
-			if err := json.Unmarshal([]byte(data), &r); err != nil {
-				return nil
-			}
-			fmt.Printf("[retrieved %d url(s) for %q]\n", len(r.URLs), r.Variant)
-		case "synthesizing":
-			var s struct {
-				Sources int `json:"sources"`
-			}
-			if err := json.Unmarshal([]byte(data), &s); err != nil {
-				return nil
-			}
-			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", s.Sources)
-		case "sources":
-			// pebble-serve emits one 'sources' event after retrieval
-			// that combines what the SQLite server splits across
-			// retrieved+synthesizing. Render the same progress line.
-			// also capture the sources slice so the final 'done'
-			// event (pebble's is minimal) can still render the Sources block.
-			var s struct {
-				Sources []server.AnswerSource `json:"sources"`
-			}
-			if err := json.Unmarshal([]byte(data), &s); err != nil {
-				return nil
-			}
-			pebbleSources = s.Sources
-			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", len(s.Sources))
-		case "answer_chunk":
-			var c struct {
-				Text string `json:"text"`
-			}
-			if err := json.Unmarshal([]byte(data), &c); err != nil {
-				return nil
-			}
-			fmt.Print(c.Text)
-			answerStarted = true
-		case "done":
-			sawDone = true
-			if answerStarted {
-				fmt.Println()
-			}
-			var rr server.ResearchResponse
-			if err := json.Unmarshal([]byte(data), &rr); err != nil || len(rr.Sources) == 0 {
-				// pebble's done event has a minimal payload —
-				// fall back to the sources captured from the event.
-				renderStreamingSources(pebbleSources, useMarkdown)
-				return errSSEDone
-			}
-			renderStreamingSources(rr.Sources, useMarkdown)
-			return errSSEDone
-		case "error":
-			var e struct {
-				Detail string `json:"detail"`
-				// pebble-serve uses 'error' field instead of 'detail'.
-				Error string `json:"error"`
-				// pebble-serve tags errors with the phase that failed.
-				Phase string `json:"phase"`
-			}
-			_ = json.Unmarshal([]byte(data), &e)
-			msg := e.Detail
-			if msg == "" {
-				msg = e.Error
-			}
-			if e.Phase != "" {
-				return fmt.Errorf("server stream error (phase=%s): %s", e.Phase, msg)
-			}
-			return fmt.Errorf("server stream error: %s", msg)
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !sawDone {
-		// Stream ended without a terminal event — flag for operators but
-		// don't error (matches behavior).
-		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
-	}
-	return nil
-}
-
-// renderStreamingSources writes the terminal Sources section for both the
-// research and answer SSE consumers. Format matches the non-stream paths so
-// operators get consistent output regardless of whether they used -stream.
-// position-based ID fallback for source rendering when the
-// server didn't emit an id field (pebble-serve's source payload). SQLite-side
-// servers always emit non-zero IDs, so their output is unchanged.
-func sourceIDOf(i int, s server.AnswerSource) int {
-	if s.ID > 0 {
-		return s.ID
-	}
-	return i + 1
-}
-
-func renderStreamingSources(sources []server.AnswerSource, useMarkdown bool) {
-	if len(sources) == 0 {
-		return
-	}
-	idOf := sourceIDOf
-	if useMarkdown {
-		fmt.Println()
-		fmt.Println("## Sources")
-		fmt.Println()
-		for i, s := range sources {
-			fmt.Fprintf(os.Stdout, "%d. [%s](%s)", idOf(i, s), s.Title, s.URL)
-			var trailing []string
-			if s.Domain != "" {
-				trailing = append(trailing, s.Domain)
-			}
-			if s.PublishedAt != nil {
-				trailing = append(trailing, s.PublishedAt.Format("2006-01-02"))
-			}
-			if len(trailing) > 0 {
-				fmt.Printf(" — %s", strings.Join(trailing, ", "))
-			}
-			fmt.Println()
-		}
-		return
-	}
-	fmt.Println()
-	fmt.Println("Sources:")
-	for i, s := range sources {
-		date := ""
-		if s.PublishedAt != nil {
-			date = " " + s.PublishedAt.Format("2006-01-02")
-		}
-		domain := ""
-		if s.Domain != "" {
-			domain = " [" + s.Domain + "]"
+// firstEnv returns the first non-empty value among the named environment vars.
+func firstEnv(names ...string) string {
+	for _, n := range names {
+		if v := os.Getenv(n); v != "" {
+			return v
 		}
-		fmt.Printf("  [%d]%s%s %s\n      %s\n", idOf(i, s), domain, date, s.Title, s.URL)
 	}
+	return ""
 }
 
-// consumeAnswerSSE reads an /answer?stream=true response and renders events as
-// they arrive. Same event vocabulary as consumeResearchSSE minus the `plan`
-// event (/answer has no expansion phase).
-func consumeAnswerSSE(body io.Reader, format string) error {
-	useMarkdown := format == "md" || format == "markdown"
-	var answerStarted, sawDone bool
-	var pebbleSources []server.AnswerSource //
-
-	err := forEachSSEEvent(body, func(event, data string) error {
-		switch event {
-		case "warnings":
-			// pebble-serve emits this event when the
-			// request had silent no-ops. Surface to stderr.
-			var w struct {
-				Warnings []string `json:"warnings"`
-			}
-			if err := json.Unmarshal([]byte(data), &w); err == nil {
-				for _, msg := range w.Warnings {
-					fmt.Fprintln(os.Stderr, "cosift: warning:", msg)
-				}
-			}
-		case "retrieved":
-			// /answer emits a single retrieved event with the k URLs (vs
-			// /research's one-per-variant).
-			var r struct {
-				URLs []string `json:"urls"`
-			}
-			if err := json.Unmarshal([]byte(data), &r); err != nil {
-				return nil
-			}
-			fmt.Printf("[retrieved %d source(s)]\n", len(r.URLs))
-		case "synthesizing":
-			var s struct {
-				Sources int `json:"sources"`
-			}
-			if err := json.Unmarshal([]byte(data), &s); err != nil {
-				return nil
-			}
-			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", s.Sources)
-		case "sources":
-			// pebble-serve emits 'sources' as the combined
-			// retrieved+synthesizing signal. Render the synthesizing line.
-			// capture for the final Sources block.
-			var s struct {
-				Sources []server.AnswerSource `json:"sources"`
-			}
-			if err := json.Unmarshal([]byte(data), &s); err != nil {
-				return nil
-			}
-			pebbleSources = s.Sources
-			fmt.Printf("[synthesizing answer over %d source(s)]\n\n", len(s.Sources))
-		case "answer_chunk":
-			var c struct {
-				Text string `json:"text"`
-			}
-			if err := json.Unmarshal([]byte(data), &c); err != nil {
-				return nil
-			}
-			fmt.Print(c.Text)
-			answerStarted = true
-		case "done":
-			sawDone = true
-			if answerStarted {
-				fmt.Println()
-			}
-			var ar server.AnswerResponse
-			if err := json.Unmarshal([]byte(data), &ar); err != nil || len(ar.Sources) == 0 {
-				// tolerate pebble's minimal done payload; fall
-				// back to the sources captured from the sources event.
-				renderStreamingSources(pebbleSources, useMarkdown)
-				return errSSEDone
-			}
-			renderStreamingSources(ar.Sources, useMarkdown)
-			return errSSEDone
-		case "error":
-			var e struct {
-				Detail string `json:"detail"`
-				// pebble-serve uses 'error' field instead of 'detail'.
-				Error string `json:"error"`
-				// pebble-serve tags errors with the phase that failed.
-				Phase string `json:"phase"`
-			}
-			_ = json.Unmarshal([]byte(data), &e)
-			msg := e.Detail
-			if msg == "" {
-				msg = e.Error
-			}
-			if e.Phase != "" {
-				return fmt.Errorf("server stream error (phase=%s): %s", e.Phase, msg)
-			}
-			return fmt.Errorf("server stream error: %s", msg)
+func contains(ss []string, x string) bool {
+	for _, s := range ss {
+		if s == x {
+			return true
 		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !sawDone {
-		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
-	}
-	return nil
-}
-
-// errSSEDone signals consumeResearchSSE's outer loop that the terminal "done"
-// event was processed and the read loop should exit cleanly. Sentinel error,
-// not surfaced to the caller.
-var errSSEDone = errors.New("sse-done")
-
-// validateFormat returns nil if v is one of the accepted output-format values
-// for the synth CLIs (`-format`). "" is rejected — flag.String defaults to ""
-// when the user passes `-format` without a value, and we want that to fail
-// loudly rather than silently behave as text.
-func validateFormat(v string) error {
-	switch v {
-	case "text", "md", "markdown":
-		return nil
-	default:
-		return fmt.Errorf("format must be one of: text, markdown (got %q)", v)
 	}
+	return false
 }
 
-func runResearchCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
-	fs := flag.NewFlagSet("research", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	strategy := fs.String("strategy", "", "planner | paraphrase (server default if empty)")
-	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
-	stream := fs.Bool("stream", false, "stream progress + token-by-token answer over SSE.")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable answer+sources")
-	// Expose the quality + scope flags on the CLI.
-	k := fs.Int("k", 0, "number of sources fed to synth (1-20, server default if 0)")
-	expand := fs.String("expand", "", "retrieval expansion: hyde | paraphrase (empty = no expansion)")
-	rerank := fs.Bool("rerank", false, "rerank retrieved sources before synth")
-	// per-sub-query retriever choice — applies to every sub-query
-	// the planner emits. Server uses bm25 by default if this is empty.
-	retriever := fs.String("retriever", "", "retriever: bm25 | dense | hybrid (server must have HNSW + embedder for dense/hybrid)")
-	// MMR lambda passthrough for /research.
-	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (needs HNSW + embedder server-side)")
-	since := fs.String("since", "", "ISO date — only sources published on or after")
-	until := fs.String("until", "", "ISO date — only sources published on or before")
-	includeDomains := fs.String("include-domains", "", "CSV allowlist of source domains")
-	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of source domains")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if err := validateFormat(*format); err != nil {
-		return err
-	}
-	if *stream && *jsonOut {
-		return errors.New("-stream and -json are mutually exclusive (stream is event-by-event, json is the final blob)")
-	}
-	if *k < 0 || *k > 20 {
-		return errors.New("k must be in [1, 20] or 0 to use server default")
-	}
-
-	v := url.Values{}
-	v.Set("q", q)
-	if *strategy != "" {
-		v.Set("strategy", *strategy)
-	}
-	if *k > 0 {
-		v.Set("k", strconv.Itoa(*k))
-	}
-	if *expand != "" {
-		v.Set("expand", *expand)
-	}
-	if *rerank {
-		v.Set("rerank", "true")
-	}
-	if *retriever != "" {
-		v.Set("retriever", *retriever)
-	}
-	if *mmr != "" {
-		v.Set("mmr", *mmr)
-	}
-	if *since != "" {
-		v.Set("since", *since)
-	}
-	if *until != "" {
-		v.Set("until", *until)
-	}
-	if *includeDomains != "" {
-		v.Set("include_domains", *includeDomains)
-	}
-	if *excludeDomains != "" {
-		v.Set("exclude_domains", *excludeDomains)
-	}
-	if *stream {
-		v.Set("stream", "true")
-	}
-
-	endpoint := strings.TrimRight(*serverURL, "/") + "/research?" + v.Encode()
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
-	if err != nil {
-		return err
-	}
-	if *stream {
-		req.Header.Set("Accept", "text/event-stream")
-	}
-	// /research issues multiple LLM calls (expand + retrieve + synth); bump the
-	// client timeout from search's 30s so slow chat backends don't get cut off.
-	// In stream mode the timeout has to cover the whole response (multiple LLM
-	// chunks emitted incrementally), so it's still 120s — there's no per-event
-	// deadline; if needed in future, a custom transport timeout would do that.
-	client := &http.Client{Timeout: 120 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	if *stream {
-		if resp.StatusCode != http.StatusOK {
-			// Server may emit a JSON problem doc on non-200 even with SSE Accept.
-			b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
-			return fmt.Errorf("server %d: %s", resp.StatusCode, b)
-		}
-		return consumeResearchSSE(resp.Body, *format)
-	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	emitWarnings(body)
-	var rr server.ResearchResponse
-	if err := json.Unmarshal(body, &rr); err != nil {
-		return fmt.Errorf("decode response: %w", err)
-	}
-	if *format == "md" || *format == "markdown" {
-		fmt.Print(renderAnswerMarkdown(rr.Query, rr.Strategy, rr.Plan, rr.Answer, rr.Sources))
-		return nil
-	}
-	fmt.Printf("Q: %s\n", rr.Query)
-	if rr.Strategy != "" {
-		fmt.Printf("Strategy: %s", rr.Strategy)
-		if len(rr.Plan) > 0 {
-			fmt.Printf(" (plan: %s)", strings.Join(rr.Plan, " | "))
-		}
-		fmt.Println()
-	}
-	fmt.Println()
-	fmt.Println(rr.Answer)
-	if len(rr.Sources) > 0 {
-		fmt.Println()
-		fmt.Println("Sources:")
-		for i, s := range rr.Sources {
-			date := ""
-			if s.PublishedAt != nil {
-				date = " " + s.PublishedAt.Format("2006-01-02")
-			}
-			domain := ""
-			if s.Domain != "" {
-				domain = " [" + s.Domain + "]"
-			}
-			fmt.Printf("  [%d]%s%s %s\n      %s\n", sourceIDOf(i, s), domain, date, s.Title, s.URL)
-		}
+func abs(n int) int {
+	if n < 0 {
+		return -n
 	}
-	return nil
+	return n
 }
 
-// runFindSimilarCLI hits a running cosift server's /find_similar endpoint.
-// Completes the  CLI matrix:
-//   - query        local BM25 only
-//   - search       /search (full retrieval pipeline)
-//   - research     /research (LLM synthesis)
-//   - find-similar /find_similar (dense neighbors of an indexed URL)
-func runFindSimilarCLI(ctx context.Context, cfg *config.Config, sourceURL string, args []string) error {
-	fs := flag.NewFlagSet("find-similar", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	k := fs.Int("k", 10, "max results")
-	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable list")
-	// alternative inputs — feed arbitrary text (or a file of text)
-	// instead of an indexed source URL. Mirrors /find_similar text mode.
-	textInput := fs.String("text", "", "arbitrary text for content-based MLT (no positional URL needed)")
-	textFile := fs.String("text-file", "", "read MLT source text from FILE")
-	textTitle := fs.String("text-title", "", "optional title (×3 boost) when using -text or -text-file")
-	// surface the filter + rerank flags on the CLI.
-	rerank := fs.Bool("rerank", false, "rerank neighbors against the MLT query (server must have rerank configured)")
-	// retriever choice — bm25-mlt (default) | dense | hybrid.
-	// Dense reads source vector from HNSW for URL-mode; hybrid RRF-fuses
-	// BM25-MLT and dense. Both require COSIFT_LOAD_HNSW=true server-side.
-	retriever := fs.String("retriever", "", "retriever: bm25 (BM25-MLT) | dense | hybrid (server must have HNSW; text-mode dense/hybrid also needs embedder)")
-	// MMR diversification for /find_similar.
-	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (URL-mode reuses graph vector — no embedder needed)")
-	since := fs.String("since", "", "ISO date — only neighbors published on or after")
-	until := fs.String("until", "", "ISO date — only neighbors published on or before")
-	includeDomains := fs.String("include-domains", "", "CSV allowlist of neighbor domains")
-	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of neighbor domains")
-	qExtra := fs.String("q", "", "extra query terms appended to the auto-derived MLT query")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *k <= 0 || *k > 100 {
-		return errors.New("k must be in [1, 100]")
-	}
-	if err := validateFormat(*format); err != nil {
-		return err
-	}
-
-	if *textFile != "" {
-		buf, err := os.ReadFile(*textFile)
-		if err != nil {
-			return fmt.Errorf("read -text-file %s: %w", *textFile, err)
-		}
-		*textInput = string(buf)
-	}
-	if sourceURL == "" && *textInput == "" {
-		return errors.New("find-similar: positional URL or -text/-text-file required")
-	}
-
-	v := url.Values{}
-	if sourceURL != "" {
-		v.Set("url", sourceURL)
-	}
-	if *textInput != "" {
-		v.Set("text", *textInput)
-	}
-	if *textTitle != "" {
-		v.Set("title", *textTitle)
-	}
-	v.Set("k", strconv.Itoa(*k))
-	// filter + rerank passthroughs.
-	if *rerank {
-		v.Set("rerank", "true")
-	}
-	if *retriever != "" {
-		v.Set("retriever", *retriever)
-	}
-	if *mmr != "" {
-		v.Set("mmr", *mmr)
-	}
-	if *since != "" {
-		v.Set("since", *since)
-	}
-	if *until != "" {
-		v.Set("until", *until)
-	}
-	if *includeDomains != "" {
-		v.Set("include_domains", *includeDomains)
-	}
-	if *excludeDomains != "" {
-		v.Set("exclude_domains", *excludeDomains)
-	}
-	if *qExtra != "" {
-		v.Set("q", *qExtra)
-	}
-
-	// switch GET → POST when -text is large (URL params have practical
-	// limits ~8 KB). The POST endpoint accepts the same flag set as a JSON body.
-	useMethod := http.MethodGet
-	endpoint := strings.TrimRight(*serverURL, "/") + "/find_similar?" + v.Encode()
-	var bodyBuf io.Reader
-	if len(*textInput) > 4096 {
-		useMethod = http.MethodPost
-		body := map[string]any{"k": *k}
-		if sourceURL != "" {
-			body["url"] = sourceURL
-		}
-		if *textInput != "" {
-			body["text"] = *textInput
-		}
-		if *textTitle != "" {
-			body["title"] = *textTitle
-		}
-		if *rerank {
-			body["rerank"] = true
-		}
-		if *retriever != "" {
-			body["retriever"] = *retriever
-		}
-		if *mmr != "" {
-			body["mmr"] = *mmr
-		}
-		if *since != "" {
-			body["since"] = *since
-		}
-		if *until != "" {
-			body["until"] = *until
-		}
-		if *includeDomains != "" {
-			body["include_domains"] = *includeDomains
-		}
-		if *excludeDomains != "" {
-			body["exclude_domains"] = *excludeDomains
-		}
-		if *qExtra != "" {
-			body["q"] = *qExtra
-		}
-		jb, _ := json.Marshal(body)
-		bodyBuf = bytes.NewReader(jb)
-		endpoint = strings.TrimRight(*serverURL, "/") + "/find_similar"
-	}
-	req, err := http.NewRequestWithContext(ctx, useMethod, endpoint, bodyBuf)
-	if err != nil {
-		return err
-	}
-	if useMethod == http.MethodPost {
-		req.Header.Set("Content-Type", "application/json")
-	}
-	// /find_similar embeds the source doc (one embedder call) and does a vector
-	// search — 30s mirrors search's timeout.
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	emitWarnings(body)
-	var fr server.FindSimilarResponse
-	if err := json.Unmarshal(body, &fr); err != nil {
-		return fmt.Errorf("decode response: %w", err)
-	}
-	if *format == "md" || *format == "markdown" {
-		fmt.Print(renderRankedMarkdown("Similar to: "+fr.URL, fr.Hits))
-		return nil
-	}
-	if len(fr.Hits) == 0 {
-		fmt.Println("(no similar documents)")
-		return nil
-	}
-	fmt.Printf("Similar to: %s\n\n", fr.URL)
-	for i, h := range fr.Hits {
-		date := ""
-		if h.PublishedAt != nil {
-			date = " " + h.PublishedAt.Format("2006-01-02")
-		}
-		domain := ""
-		if h.Domain != "" {
-			domain = " [" + h.Domain + "]"
-		}
-		fmt.Printf("%2d. [%.3f]%s%s %s\n    %s\n", i+1, h.Score, domain, date, h.Title, h.URL)
-	}
-	return nil
-}
-
-// runContentsCLI hits a running cosift server's /contents endpoint.
-// Single URL → GET /contents?url=<url>. Multiple URLs (positional or via
-// -file) → POST /contents with {urls: [...]} body.
-func runContentsCLI(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("contents", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	filePath := fs.String("file", "", "read URLs from FILE (one per line, # comments allowed)")
-	textOnly := fs.Bool("text", false, "print only the document text (no metadata) — useful for piping into LLMs")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable output")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	urls := fs.Args()
-	if *filePath != "" {
-		extra, err := readURLsFromFile(*filePath)
-		if err != nil {
-			return fmt.Errorf("read -file: %w", err)
-		}
-		urls = append(urls, extra...)
-	}
-	if len(urls) == 0 {
-		return errors.New("at least one URL or -file required")
-	}
-
-	base := strings.TrimRight(*serverURL, "/")
-	client := &http.Client{Timeout: 60 * time.Second}
-
-	if len(urls) == 1 {
-		// Single — GET /contents?url=<url>.
-		v := url.Values{}
-		v.Set("url", urls[0])
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/contents?"+v.Encode(), http.NoBody)
-		if err != nil {
-			return err
-		}
-		resp, err := client.Do(req)
-		if err != nil {
-			return fmt.Errorf("get %s: %w", *serverURL, err)
-		}
-		defer resp.Body.Close()
-		body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
-		if err != nil {
-			return fmt.Errorf("read response: %w", err)
-		}
-		if resp.StatusCode != http.StatusOK {
-			return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-		}
-		if *jsonOut {
-			fmt.Println(string(body))
-			return nil
-		}
-		var cr server.ContentsResponse
-		if err := json.Unmarshal(body, &cr); err != nil {
-			return fmt.Errorf("decode response: %w", err)
-		}
-		if *textOnly {
-			fmt.Println(cr.Text)
-			return nil
-		}
-		fmt.Printf("URL:        %s\n", cr.URL)
-		fmt.Printf("Title:      %s\n", cr.Title)
-		fmt.Printf("Cached:     %t\n", cr.Cached)
-		if !cr.FetchedAt.IsZero() {
-			fmt.Printf("FetchedAt:  %s\n", cr.FetchedAt.Format(time.RFC3339))
-		}
-		if cr.Lang != "" {
-			fmt.Printf("Lang:       %s\n", cr.Lang)
-		}
-		fmt.Println()
-		fmt.Println(cr.Text)
-		return nil
-	}
-
-	// Batch — POST /contents with {urls: [...]}.
-	reqBody, err := json.Marshal(server.ContentsBatchRequest{URLs: urls})
-	if err != nil {
-		return err
-	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/contents", bytes.NewReader(reqBody))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("post %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	var br server.ContentsBatchResponse
-	if err := json.Unmarshal(body, &br); err != nil {
-		return fmt.Errorf("decode response: %w", err)
-	}
-	if *textOnly {
-		// Separator between docs lets downstream tools chunk by `---` if they want.
-		for i, item := range br.Results {
-			if i > 0 {
-				fmt.Println("---")
-			}
-			if !item.Found {
-				continue
-			}
-			fmt.Println(item.Text)
-		}
-		return nil
-	}
-	for _, item := range br.Results {
-		fmt.Printf("URL: %s\n", item.URL)
-		if !item.Found {
-			fmt.Printf("  (not found: %s)\n\n", item.Error)
-			continue
-		}
-		status := "fetched"
-		if item.Cached {
-			status = "cached"
-		}
-		fmt.Printf("  Title:      %s\n", item.Title)
-		fmt.Printf("  Status:     %s\n", status)
-		if !item.FetchedAt.IsZero() {
-			fmt.Printf("  FetchedAt:  %s\n", item.FetchedAt.Format(time.RFC3339))
-		}
-		if item.Lang != "" {
-			fmt.Printf("  Lang:       %s\n", item.Lang)
-		}
-		fmt.Printf("  Text:       %s\n\n", truncate(item.Text, 240))
-	}
-	return nil
-}
-
-// readURLsFromFile parses one URL per line. Blank lines and `#`-prefixed
-// comments are ignored. Used by `cosift contents -file`.
-func readURLsFromFile(path string) ([]string, error) {
-	b, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	var urls []string
-	for _, line := range strings.Split(string(b), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		urls = append(urls, line)
-	}
-	return urls, nil
-}
-
-// runAnswerCLI hits a running cosift server's /answer endpoint. Sibling to
-// runResearchCLI but no planner/paraphrase strategy surface —
-// /answer just retrieves k sources and synthesizes a cited answer. Smaller K
-// cap [1,20] mirrors the server's bound (vs /search's [1,100]).
-func runAnswerCLI(ctx context.Context, cfg *config.Config, q string, args []string) error {
-	fs := flag.NewFlagSet("answer", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	k := fs.Int("k", 5, "number of sources to retrieve (1-20)")
-	expand := fs.Bool("expand", false, "LLM paraphrase + RRF fusion of retrieval inputs (server must have a paraphraser)")
-	format := fs.String("format", "text", "human-output format: text | markdown (or md).")
-	stream := fs.Bool("stream", false, "stream progress + token-by-token answer over SSE.")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable answer+sources")
-	// Expose the quality + scope flags on the CLI.
-	rerank := fs.Bool("rerank", false, "rerank retrieved sources before synth (server must have rerank configured)")
-	// Empty
-	// passes nothing → server uses its default (bm25).
-	retriever := fs.String("retriever", "", "retriever: bm25 | dense | hybrid (server must have HNSW + embedder for dense/hybrid)")
-	// MMR lambda passthrough. Empty → no diversification.
-	mmr := fs.String("mmr", "", "MMR diversification lambda in [0,1] (0.7 = mostly relevance with some diversity; needs HNSW + embedder server-side)")
-	since := fs.String("since", "", "ISO date — only sources published on or after")
-	until := fs.String("until", "", "ISO date — only sources published on or before")
-	includeDomains := fs.String("include-domains", "", "CSV allowlist of source domains")
-	excludeDomains := fs.String("exclude-domains", "", "CSV denylist of source domains")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *k <= 0 || *k > 20 {
-		return errors.New("k must be in [1, 20]")
-	}
-	if err := validateFormat(*format); err != nil {
-		return err
-	}
-	if *stream && *jsonOut {
-		return errors.New("-stream and -json are mutually exclusive (stream is event-by-event, json is the final blob)")
-	}
-
-	v := url.Values{}
-	v.Set("q", q)
-	v.Set("k", strconv.Itoa(*k))
-	if *expand {
-		v.Set("expand", "true")
-	}
-	if *rerank {
-		v.Set("rerank", "true")
-	}
-	if *retriever != "" {
-		v.Set("retriever", *retriever)
-	}
-	if *mmr != "" {
-		v.Set("mmr", *mmr)
-	}
-	if *since != "" {
-		v.Set("since", *since)
-	}
-	if *until != "" {
-		v.Set("until", *until)
-	}
-	if *includeDomains != "" {
-		v.Set("include_domains", *includeDomains)
-	}
-	if *excludeDomains != "" {
-		v.Set("exclude_domains", *excludeDomains)
-	}
-	if *stream {
-		v.Set("stream", "true")
-	}
-
-	endpoint := strings.TrimRight(*serverURL, "/") + "/answer?" + v.Encode()
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
-	if err != nil {
-		return err
-	}
-	if *stream {
-		req.Header.Set("Accept", "text/event-stream")
-	}
-	// /answer issues retrieval + 1 synth LLM call; 120s mirrors /research.
-	client := &http.Client{Timeout: 120 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	if *stream {
-		if resp.StatusCode != http.StatusOK {
-			b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
-			return fmt.Errorf("server %d: %s", resp.StatusCode, b)
-		}
-		return consumeAnswerSSE(resp.Body, *format)
-	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	emitWarnings(body)
-	var ar server.AnswerResponse
-	if err := json.Unmarshal(body, &ar); err != nil {
-		return fmt.Errorf("decode response: %w", err)
-	}
-	if *format == "md" || *format == "markdown" {
-		// /answer has no strategy or plan — pass empty/nil so the shared
-		// renderer just skips the strategy line.
-		fmt.Print(renderAnswerMarkdown(ar.Query, "", nil, ar.Answer, ar.Sources))
-		return nil
-	}
-	fmt.Printf("Q: %s\n\n", ar.Query)
-	fmt.Println(ar.Answer)
-	if len(ar.Sources) > 0 {
-		fmt.Println()
-		fmt.Println("Sources:")
-		for i, s := range ar.Sources {
-			date := ""
-			if s.PublishedAt != nil {
-				date = " " + s.PublishedAt.Format("2006-01-02")
-			}
-			domain := ""
-			if s.Domain != "" {
-				domain = " [" + s.Domain + "]"
-			}
-			fmt.Printf("  [%d]%s%s %s\n      %s\n", sourceIDOf(i, s), domain, date, s.Title, s.URL)
-		}
-	}
-	return nil
-}
-
-// runAdmin dispatches to a sub-operation under `cosift admin <op>`. Sub-ops
-// share bearer-token resolution: -token flag wins, then COSIFT_ADMIN_TOKEN env.
-// Empty token still hits the server (and gets a clear 401) — the CLI doesn't
-// pre-fail because the operator might be running an instance without admin
-// auth enabled (rare, but the failure path is the server's to own).
-// adminUsageError returns the multi-line help message printed when the
-// `cosift admin` parent is invoked without a subcommand. Listed alphabetically
-// by subcommand name so the output is stable across iterations.
-//
-// Extracted as a function (rather than a const) so tests can assert on the
-// listing's structure — every admin subcommand must appear here, in the same
-// alphabetic order it appears in runAdmin's dispatch switch.
-func adminUsageError() string {
-	return `admin: subcommand required.
-
-Available subcommands:
-  config                   Retriever defaults and capability flags.
-  recrawl <url...>         Re-enqueue specific URLs into the crawl frontier. Requires -y.
-  recrawl-domain <pattern> Bulk re-enqueue every doc matching the domain pattern. Requires -y or -dry-run.
-  reembed                  Re-embed every doc with the configured model (streams progress).
-                           Requires -y. Supports -drop-old and -since DATE.
-  stats                    Index counts, frontier breakdown, top domains, capability flags.
-                           Supports -summary for one-line output (monitoring / shell prompts).
-
-Common flags:
-  -server URL              cosift server URL (default: from local cosift.json)
-  -token TOKEN             admin bearer token (or COSIFT_ADMIN_TOKEN env)
-  -json                    raw JSON output instead of human-readable
-
-Usage: cosift admin <subcommand> [flags]
-`
-}
-
-func runAdmin(ctx context.Context, cfg *config.Config, op string, args []string) error {
-	switch op {
-	case "stats":
-		return runAdminStats(ctx, cfg, args)
-	case "config":
-		return runAdminConfig(ctx, cfg, args)
-	case "recrawl":
-		return runAdminRecrawl(ctx, cfg, args)
-	case "recrawl-domain":
-		return runAdminRecrawlDomain(ctx, cfg, args)
-	case "reembed":
-		return runAdminReembedCLI(ctx, cfg, args)
-	default:
-		return fmt.Errorf("unknown admin subcommand %q (want: stats | config | recrawl | recrawl-domain | reembed)", op)
-	}
-}
-
-// adminCommonFlags parses the -server/-token/-json flags shared by every admin
-// sub-operation. Returns parsed values + the leftover positional args (none for
-// stats/config; future ops like recrawl will have positional URL args).
-type adminCommonOpts struct {
-	server string
-	token  string
-	json   bool
-}
-
-func parseAdminCommon(name string, cfg *config.Config, args []string) (adminCommonOpts, []string, error) {
-	fs := flag.NewFlagSet(name, flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	server := fs.String("server", defaultServer, "cosift server URL")
-	// Token: -token flag overrides env. Env is the secure default — keeps the
-	// token out of shell history / process listings.
-	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
-	if err := fs.Parse(args); err != nil {
-		return adminCommonOpts{}, nil, err
-	}
-	resolvedToken := *token
-	if resolvedToken == "" {
-		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-	}
-	return adminCommonOpts{server: *server, token: resolvedToken, json: *jsonOut}, fs.Args(), nil
-}
-
-// adminRequest builds an authenticated request to the given /admin/* path. The
-// Authorization header is omitted when token is empty (server returns 401,
-// which is the right error to surface).
-func adminRequest(ctx context.Context, method, server, path, token string, body io.Reader) (*http.Request, error) {
-	endpoint := strings.TrimRight(server, "/") + path
-	req, err := http.NewRequestWithContext(ctx, method, endpoint, body)
-	if err != nil {
-		return nil, err
-	}
-	if token != "" {
-		req.Header.Set("Authorization", "Bearer "+token)
-	}
-	return req, nil
-}
-
-func runAdminStats(ctx context.Context, cfg *config.Config, args []string) error {
-	// inline flag parsing (vs parseAdminCommon) so we can add the
-	// -summary flag. Same shape as runAdminRecrawl's flag set —
-	// admin sub-ops with op-specific flags bypass the common helper.
-	fs := flag.NewFlagSet("admin stats", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response")
-	summary := fs.Bool("summary", false, "one-line compact output for monitoring / shell prompts")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *summary && *jsonOut {
-		return errors.New("-summary and -json are mutually exclusive")
-	}
-	resolvedToken := *token
-	if resolvedToken == "" {
-		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-	}
-
-	req, err := adminRequest(ctx, http.MethodGet, *serverURL, "/admin/stats", resolvedToken, nil)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	var st server.AdminStatsResponse
-	if err := json.Unmarshal(body, &st); err != nil {
-		return fmt.Errorf("decode: %w", err)
-	}
-	if *summary {
-		// Compact one-line format designed for: shell prompts, cron, status
-		// scripts. Always-present fields (docs/queued/errored). Embedder named
-		// only when configured (omitted with " · " to keep separator clean).
-		parts := []string{
-			fmt.Sprintf("%d docs", st.Documents),
-			fmt.Sprintf("%d queued", st.Frontier.Queued),
-			fmt.Sprintf("%d errored", st.Frontier.Errored),
-		}
-		if st.EmbedderModel != "" {
-			parts = append(parts, "embedder="+st.EmbedderModel)
-		}
-		fmt.Println(strings.Join(parts, " · "))
-		return nil
-	}
-	fmt.Println("=== Index ===")
-	fmt.Printf("  Documents:           %d\n", st.Documents)
-	fmt.Printf("  Terms:               %d\n", st.Terms)
-	fmt.Printf("  Passages:            %d\n", st.Passages)
-	if st.DocsWithPublishedAt > 0 {
-		pct := float64(st.DocsWithPublishedAt) / float64(st.Documents) * 100
-		fmt.Printf("  Docs w/ published_at: %d (%.1f%%)\n", st.DocsWithPublishedAt, pct)
-	}
-	// LLM-cache sizes. Show only when populated — a fresh
-	// deployment with no LLM cache yet shouldn't bloat the dashboard.
-	if st.Paraphrases > 0 || st.HyDECache > 0 {
-		fmt.Println()
-		fmt.Println("=== LLM caches ===")
-		if st.Paraphrases > 0 {
-			fmt.Printf("  Paraphrases:         %d\n", st.Paraphrases)
-		}
-		if st.HyDECache > 0 {
-			fmt.Printf("  HyDE passages:       %d\n", st.HyDECache)
-		}
-	}
-	fmt.Println()
-	fmt.Println("=== Frontier ===")
-	fmt.Printf("  Queued:    %d\n", st.Frontier.Queued)
-	fmt.Printf("  In-flight: %d\n", st.Frontier.InFlight)
-	fmt.Printf("  Done:      %d\n", st.Frontier.Done)
-	fmt.Printf("  Errored:   %d\n", st.Frontier.Errored)
-	fmt.Println()
-	fmt.Println("=== Capabilities ===")
-	fmt.Printf("  Dense:    %t\n", st.DenseEnabled)
-	fmt.Printf("  Answer:   %t\n", st.AnswerEnabled)
-	fmt.Printf("  Fetcher:  %t\n", st.FetcherEnabled)
-	if st.EmbedderModel != "" {
-		fmt.Printf("  Embedder: %s\n", st.EmbedderModel)
-	}
-	if st.ChatModel != "" {
-		fmt.Printf("  Chat:     %s\n", st.ChatModel)
-	}
-	if st.RerankerName != "" {
-		fmt.Printf("  Reranker: %s\n", st.RerankerName)
-	}
-	if len(st.TopDomains) > 0 {
-		fmt.Println()
-		fmt.Println("=== Top Domains ===")
-		// Map iteration order is random; sort by count desc, then domain asc.
-		type kv struct {
-			k string
-			v int64
-		}
-		pairs := make([]kv, 0, len(st.TopDomains))
-		for k, v := range st.TopDomains {
-			pairs = append(pairs, kv{k, v})
-		}
-		sort.Slice(pairs, func(i, j int) bool {
-			if pairs[i].v != pairs[j].v {
-				return pairs[i].v > pairs[j].v
-			}
-			return pairs[i].k < pairs[j].k
-		})
-		for _, p := range pairs {
-			fmt.Printf("  %-32s %d\n", p.k, p.v)
-		}
-	}
-	return nil
-}
-
-func runAdminConfig(ctx context.Context, cfg *config.Config, args []string) error {
-	opts, _, err := parseAdminCommon("admin config", cfg, args)
-	if err != nil {
-		return err
-	}
-	req, err := adminRequest(ctx, http.MethodGet, opts.server, "/admin/config", opts.token, nil)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("get %s: %w", opts.server, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-	if opts.json {
-		fmt.Println(string(body))
-		return nil
-	}
-	var cfgResp server.AdminConfigResponse
-	if err := json.Unmarshal(body, &cfgResp); err != nil {
-		return fmt.Errorf("decode: %w", err)
-	}
-	fmt.Printf("Cosift version: %s\n\n", cfgResp.Version)
-	fmt.Println("=== Defaults ===")
-	d := cfgResp.Defaults
-	fmt.Printf("  Retriever: %s\n", d.Retriever)
-	fmt.Printf("  Expand:    %t\n", d.Expand)
-	fmt.Println()
-	fmt.Println("=== Capabilities ===")
-	c := cfgResp.Capabilities
-	fmt.Printf("  Dense:        %t\n", c.DenseEnabled)
-	fmt.Printf("  Chat:         %t\n", c.ChatEnabled)
-	fmt.Printf("  Rerank:       %t\n", c.RerankEnabled)
-	fmt.Printf("  Paraphraser:  %t\n", c.ParaphraserEnabled)
-	fmt.Printf("  Fetcher:      %t\n", c.FetcherEnabled)
-	fmt.Printf("  Admin:        %t\n", c.AdminEnabled)
-	if c.EmbedderModel != "" {
-		fmt.Printf("  Embedder model:  %s\n", c.EmbedderModel)
-	}
-	if c.ChatModel != "" {
-		fmt.Printf("  Chat model:      %s\n", c.ChatModel)
-	}
-	if c.RerankerName != "" {
-		fmt.Printf("  Reranker name:   %s\n", c.RerankerName)
-	}
-	return nil
-}
-
-// runAdminRecrawl re-enqueues the given URLs into the crawler frontier. First
-// destructive admin operation — requires `-y` to confirm. URLs come from
-// positional args, `-file`, or both (mirrors's `contents` CLI pattern).
-func runAdminRecrawl(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("admin recrawl", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
-	filePath := fs.String("file", "", "read URLs from FILE (one per line, # comments allowed)")
-	confirm := fs.Bool("y", false, "confirm the destructive operation (recrawl re-enqueues URLs into the frontier)")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	urls := fs.Args()
-	if *filePath != "" {
-		extra, err := readURLsFromFile(*filePath)
-		if err != nil {
-			return fmt.Errorf("read -file: %w", err)
-		}
-		urls = append(urls, extra...)
-	}
-	if len(urls) == 0 {
-		return errors.New("at least one URL or -file required")
-	}
-	if !*confirm {
-		return fmt.Errorf("admin recrawl is destructive (re-enqueues %d URL(s) into the crawl frontier). Add -y to confirm", len(urls))
-	}
-
-	resolvedToken := *token
-	if resolvedToken == "" {
-		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-	}
-
-	reqBody, err := json.Marshal(server.AdminRecrawlRequest{URLs: urls})
-	if err != nil {
-		return err
-	}
-	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/recrawl", resolvedToken, bytes.NewReader(reqBody))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	client := &http.Client{Timeout: 60 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("post %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	var rr server.AdminRecrawlResponse
-	if err := json.Unmarshal(body, &rr); err != nil {
-		return fmt.Errorf("decode: %w", err)
-	}
-	fmt.Printf("Queued %d URL(s) for recrawl.\n", len(rr.Queued))
-	if len(rr.Errors) > 0 {
-		fmt.Printf("\n%d URL(s) failed to enqueue:\n", len(rr.Errors))
-		// Sort error URLs for stable output across runs.
-		keys := make([]string, 0, len(rr.Errors))
-		for k := range rr.Errors {
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-		for _, u := range keys {
-			fmt.Printf("  %s: %s\n", u, rr.Errors[u])
-		}
-	}
-	return nil
-}
-
-// runAdminRecrawlDomain bulk-recrawls every doc whose domain matches `pattern`
-// (positional arg). Wraps's /admin/recrawl-by-domain endpoint. Requires
-// `-y` to commit or `-dry-run` to preview the matched count without queuing.
-// If both flags are set, `-dry-run` wins (safer-on-conflict).
-func runAdminRecrawlDomain(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("admin recrawl-domain", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
-	confirm := fs.Bool("y", false, "confirm the destructive operation (recrawl every matching URL)")
-	dryRun := fs.Bool("dry-run", false, "enumerate matched URLs without queuing — preview the blast radius")
-	// cap dry-run URL listing to avoid console spam on large matches.
-	// 0 = no list (count only — behavior). 20 = default safe cap.
-	// -1 = unlimited (operator explicitly wants the full list).
-	listLimit := fs.Int("limit-list", 20, "in dry-run mode, max URLs to print (0 = count only, -1 = unlimited)")
-	jsonOut := fs.Bool("json", false, "emit raw JSON response instead of human-readable summary")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if fs.NArg() < 1 {
-		return errors.New("domain pattern required (positional arg)")
-	}
-	if fs.NArg() > 1 {
-		return errors.New("only one domain pattern supported per call")
-	}
-	pattern := fs.Arg(0)
-
-	if !*confirm && !*dryRun {
-		return fmt.Errorf("admin recrawl-domain is destructive (would re-enqueue every URL matching %q). Add -y to commit or -dry-run to preview", pattern)
-	}
-
-	resolvedToken := *token
-	if resolvedToken == "" {
-		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-	}
-
-	// dry-run wins when both flags are set. Operator who typed both probably
-	// meant the safer option; the destructive interpretation requires they
-	// explicitly drop -dry-run.
-	effectiveDryRun := *dryRun
-
-	reqBody, err := json.Marshal(server.AdminRecrawlByDomainRequest{
-		Domain: pattern,
-		DryRun: effectiveDryRun,
-	})
-	if err != nil {
-		return err
-	}
-	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/recrawl-by-domain", resolvedToken, bytes.NewReader(reqBody))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	client := &http.Client{Timeout: 120 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("post %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-	if err != nil {
-		return fmt.Errorf("read response: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("server %d: %s", resp.StatusCode, body)
-	}
-	if *jsonOut {
-		fmt.Println(string(body))
-		return nil
-	}
-	var rr server.AdminRecrawlByDomainResponse
-	if err := json.Unmarshal(body, &rr); err != nil {
-		return fmt.Errorf("decode: %w", err)
-	}
-	if effectiveDryRun {
-		fmt.Printf("Dry-run: %d URL(s) match %q. Re-run with -y (not -dry-run) to queue them.\n", rr.Matched, rr.Domain)
-		// -limit-list 0 → count only. -limit-list -1 → unlimited (operator
-		// wants the full list, accepts console spam). Otherwise: print up
-		// to listLimit and append a "... (N more)" suffix.
-		if *listLimit != 0 && len(rr.URLs) > 0 {
-			n := *listLimit
-			if n < 0 || n > len(rr.URLs) {
-				n = len(rr.URLs)
-			}
-			for _, u := range rr.URLs[:n] {
-				fmt.Printf("  %s\n", u)
-			}
-			if remaining := len(rr.URLs) - n; remaining > 0 {
-				fmt.Printf("  ... (%d more — use -limit-list -1 to see all)\n", remaining)
-			}
-		}
-		return nil
-	}
-	fmt.Printf("Queued %d/%d URL(s) for recrawl matching %q.\n", rr.Queued, rr.Matched, rr.Domain)
-	if len(rr.Errors) > 0 {
-		fmt.Printf("\n%d URL(s) failed to enqueue:\n", len(rr.Errors))
-		keys := make([]string, 0, len(rr.Errors))
-		for k := range rr.Errors {
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-		for _, u := range keys {
-			fmt.Printf("  %s: %s\n", u, rr.Errors[u])
-		}
-	}
-	return nil
-}
-
-// runAdminReembedCLI triggers a server-side reembed via the
-// `/admin/reembed` SSE endpoint. Requires `-y` to commit (reembed costs
-// LLM credits; same destructive-op guard pattern as other admin verbs).
-// `-drop-old` propagates to the server's `DropPassagesNotModel` step.
-func runAdminReembedCLI(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("admin reembed", flag.ExitOnError)
-	defaultServer := "http://" + cfg.Server.Addr
-	defaultServer = strings.Replace(defaultServer, "0.0.0.0", "127.0.0.1", 1)
-	serverURL := fs.String("server", defaultServer, "cosift server URL")
-	token := fs.String("token", "", "admin bearer token (defaults to COSIFT_ADMIN_TOKEN env)")
-	dropOld := fs.Bool("drop-old", false, "also delete passages from other models after re-embedding")
-	// -since restricts reembed to docs published >= DATE. Client-side
-	// validation catches malformed dates before the HTTP call (saves a round-trip
-	// for the common typo case).
-	since := fs.String("since", "", "ISO date — only re-embed docs published >= this. Drops undated docs when set")
-	confirm := fs.Bool("y", false, "confirm the operation (reembed re-runs every embedding call; counts as LLM spend)")
-	// -dry-run shows the would-be-processed count without spending
-	// LLM credits. Either -y OR -dry-run is required.
-	dryRun := fs.Bool("dry-run", false, "preview the count of docs that would be re-embedded, without actually embedding")
-	jsonOut := fs.Bool("json", false, "emit raw SSE response instead of human-readable progress lines")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	// Validate -since client-side before checking -y, so operators see "bad
-	// date" before they've committed with -y (clearer iteration).
-	if _, err := parseExportDate(*since); err != nil {
-		return fmt.Errorf("-since: %w", err)
-	}
-	// dry-run wins when both flags set (safer-on-conflict, matches
-	// pattern on /admin/recrawl-by-domain).
-	effectiveDryRun := *dryRun
-	if !*confirm && !*dryRun {
-		guard := "every doc"
-		if *dropOld {
-			guard = "every doc AND drop other-model passages"
-		}
-		if *since != "" {
-			guard = "every doc published >= " + *since
-			if *dropOld {
-				guard += " AND drop other-model passages"
-			}
-		}
-		return fmt.Errorf("admin reembed will re-embed %s (counts as LLM spend). Add -y to commit or -dry-run to preview", guard)
-	}
-
-	resolvedToken := *token
-	if resolvedToken == "" {
-		resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-	}
-
-	reqBody, err := json.Marshal(server.AdminReembedRequest{
-		DropOld: *dropOld,
-		Since:   *since,
-		DryRun:  effectiveDryRun,
-	})
-	if err != nil {
-		return err
-	}
-	req, err := adminRequest(ctx, http.MethodPost, *serverURL, "/admin/reembed", resolvedToken, bytes.NewReader(reqBody))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Accept", "text/event-stream")
-	// Reembed can run for tens of minutes on large corpora. Bump the HTTP
-	// client timeout well past the synth CLIs; the server emits keep-alive
-	// progress events every ~2s so the connection stays warm.
-	client := &http.Client{Timeout: 60 * time.Minute}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("post %s: %w", *serverURL, err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
-		return fmt.Errorf("server %d: %s", resp.StatusCode, b)
-	}
-	if *jsonOut {
-		// In JSON mode, just stream the raw SSE bytes through to stdout. Useful
-		// for piping into a custom parser or log file. Different shape from the
-		// other admin CLIs' -json because reembed has no terminal JSON blob —
-		// the event stream IS the response.
-		_, err := io.Copy(os.Stdout, resp.Body)
-		return err
-	}
-	return consumeReembedSSE(resp.Body)
-}
-
-// consumeReembedSSE reads /admin/reembed events and renders progress lines.
-// third SSE consumer in the CLI (after research and
-// answer), reusing the forEachSSEEvent scanner.
-func consumeReembedSSE(body io.Reader) error {
-	var sawDone bool
-	// track the started event's total_docs and target_model so the
-	// dry-run done-event branch can reference them (the dry-run done payload
-	// reports zeros for processed/passages, but the meaningful count is in started).
-	var startedTotal int
-	var startedTarget string
-
-	err := forEachSSEEvent(body, func(event, data string) error {
-		switch event {
-		case "started":
-			var s struct {
-				TotalDocs   int    `json:"total_docs"`
-				TargetModel string `json:"target_model"`
-			}
-			if err := json.Unmarshal([]byte(data), &s); err != nil {
-				return nil // tolerate malformed mid-stream event
-			}
-			startedTotal = s.TotalDocs
-			startedTarget = s.TargetModel
-			fmt.Printf("[reembed started: %d docs, target=%s]\n", s.TotalDocs, s.TargetModel)
-		case "progress":
-			var p struct {
-				DocsProcessed   int `json:"docs_processed"`
-				PassagesWritten int `json:"passages_written"`
-			}
-			if err := json.Unmarshal([]byte(data), &p); err != nil {
-				return nil
-			}
-			fmt.Printf("[progress: %d docs, %d passages]\n", p.DocsProcessed, p.PassagesWritten)
-		case "done":
-			sawDone = true
-			var d struct {
-				DocsProcessed   int    `json:"docs_processed"`
-				PassagesWritten int    `json:"passages_written"`
-				DroppedOld      int64  `json:"dropped_old"`
-				DryRun          bool   `json:"dry_run,omitempty"`
-				Took            string `json:"took"`
-			}
-			if err := json.Unmarshal([]byte(data), &d); err != nil {
-				return fmt.Errorf("parse done event: %w", err)
-			}
-			if d.DryRun {
-				// dry-run summary — reference the started event's
-				// count (the meaningful "would-be-processed" number).
-				fmt.Printf("Dry-run: %d docs would be re-embedded with target=%s. Re-run without -dry-run (or with -y) to actually do it.\n",
-					startedTotal, startedTarget)
-			} else {
-				fmt.Printf("Done: %d docs reembedded, %d passages written, %d dropped, took %s.\n",
-					d.DocsProcessed, d.PassagesWritten, d.DroppedOld, d.Took)
-			}
-			return errSSEDone
-		case "error":
-			var e struct {
-				Detail string `json:"detail"`
-			}
-			_ = json.Unmarshal([]byte(data), &e)
-			return fmt.Errorf("server stream error: %s", e.Detail)
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	if !sawDone {
-		fmt.Fprintln(os.Stderr, "(stream ended without `done` event)")
-	}
-	return nil
-}
-
-// bm25Adapter adapts index.BM25 to the eval.Retriever interface (URL-only result list).
-type bm25Adapter struct{ inner *index.BM25 }
-
-func (a *bm25Adapter) Search(ctx context.Context, q string, k int) ([]string, error) {
-	hits, err := a.inner.Search(ctx, q, k)
-	if err != nil {
-		return nil, err
-	}
-	urls := make([]string, len(hits))
-	for i, h := range hits {
-		urls[i] = h.URL
-	}
-	return urls, nil
-}
-
-// denseAdapter is a Retriever backed by an in-memory VectorIndex + an Embedder.
-// The embedder is consulted once per query at search time.
-type denseAdapter struct {
-	idx *index.VectorIndex
-	emb embed.Embedder
-}
-
-func (a *denseAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
-	vecs, err := a.emb.Embed(ctx, []string{q})
-	if err != nil {
-		return nil, err
-	}
-	hits := a.idx.Search(ctx, vecs[0], k)
-	urls := make([]string, len(hits))
-	for i, h := range hits {
-		urls[i] = h.URL
-	}
-	return urls, nil
-}
-
-// hybridAdapter fans out to two retrievers and fuses with RRF (k=60).
-type hybridAdapter struct {
-	a, b eval.Retriever
-}
-
-func (h *hybridAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
-	// Each retriever gets to vote on 2k candidates so RRF has room to work.
-	cand := k * 2
-	if cand < 10 {
-		cand = 10
-	}
-	ra, errA := h.a.Search(ctx, q, cand)
-	if errA != nil {
-		return nil, errA
-	}
-	rb, errB := h.b.Search(ctx, q, cand)
-	if errB != nil {
-		return nil, errB
-	}
-	return index.RRF([][]string{ra, rb}, k, 60), nil
-}
-
-// rerankAdapter wraps any Retriever with a reranker. The inner retriever is
-// asked for `candidateK` candidates; the reranker reorders them; the top k come back.
-// textByURL provides the passage text the reranker reads.
-type rerankAdapter struct {
-	inner      eval.Retriever
-	reranker   rerank.Reranker
-	textByURL  map[string]string
-	candidateK int
-}
-
-func (a *rerankAdapter) Search(ctx context.Context, q string, k int) ([]string, error) {
-	candK := a.candidateK
-	if candK <= 0 || candK < k {
-		candK = k * 5
-	}
-	urls, err := a.inner.Search(ctx, q, candK)
-	if err != nil {
-		return nil, err
-	}
-	if len(urls) <= 1 {
-		return urls, nil
-	}
-	cands := make([]rerank.Candidate, 0, len(urls))
-	for _, u := range urls {
-		text := a.textByURL[u]
-		if text == "" {
-			text = u
-		}
-		cands = append(cands, rerank.Candidate{ID: u, Text: text})
-	}
-	reranked, err := a.reranker.Rerank(ctx, q, cands)
-	if err != nil {
-		return urls, nil // fall back to inner order
-	}
-	if len(reranked) > k {
-		reranked = reranked[:k]
-	}
-	return reranked, nil
-}
-
-func runEval(ctx context.Context, args []string) error {
-	fs := flag.NewFlagSet("eval", flag.ExitOnError)
-	corpusPath := fs.String("corpus", "testdata/eval/corpus.json", "corpus JSON")
-	queriesPath := fs.String("queries", "testdata/eval/queries.json", "queries JSON")
-	savePath := fs.String("save", "", "save run summary")
-	baselinePath := fs.String("baseline", "", "compare against a prior summary")
-	retriever := fs.String("retriever", "bm25", "retriever: bm25 | dense | hybrid")
-	embModel := fs.String("embed-model", "text-embedding-3-small", "embedding model name")
-	embURL := fs.String("embed-url", "", "embedding endpoint URL (default OpenAI)")
-	embDim := fs.Int("embed-dim", 1536, "embedding dimensionality")
-	embCacheDir := fs.String("embed-cache", "./eval-embed-cache", "embedding cache dir (set empty to disable)")
-	useRerank := fs.Bool("rerank", false, "wrap retriever with the LLM listwise reranker")
-	rerankModel := fs.String("rerank-model", "gpt-4o-mini", "chat model for the reranker")
-	candK := fs.Int("rerank-k", 20, "candidates pulled from inner retriever before rerank")
-	apiURL := fs.String("api", "", "if set, query the remote cosift HTTP API instead of building a local index")
-	apiBearer := fs.String("api-bearer", "", "optional bearer token for the remote API")
-	distractors := fs.Int("distractors", 0, "inject N synthetic distractor docs into the index before scoring (tests robustness vs an inflated candidate set)")
-	distractorSeed := fs.Int64("distractor-seed", 42, "deterministic seed for distractor generation")
-	distractorMode := fs.String("distractor-mode", "diverse", "diverse (90-word neutral pool) | narrow (single-topic 7-word pool)")
-	embBatch := fs.Int("embed-batch", 256, "embedding batch size — lower for large-dim models (text-embedding-3-large returns malformed JSON at 1000)")
-	chunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = index.NewChunker default 320); A/B across runs to compare retrieval at different granularities")
-	chunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = default 64)")
-	autoParaphrase := fs.Bool("auto-paraphrase", false, "generate N paraphrases per query via the chat client at eval time + RRF-fuse results")
-	paraphraseN := fs.Int("paraphrase-n", 2, "number of paraphrases per query")
-	paraphraseModel := fs.String("paraphrase-model", "gpt-4o-mini", "chat model for paraphrase generation")
-	mainWeight := fs.Float64("main-weight", 0, "main-query weight in -auto-paraphrase OR -planner RRF fusion (paraphrases / sub-queries each weight 1.0); 0 = equal-weight (standard RRF); mirrors server-side Defaults.ExpandMainWeight for offline measurement")
-	usePlanner := fs.Bool("planner", false, "decompose each query into 2-3 sub-queries via the /research planner prompt + RRF-fuse results (mirror /research retrieval, no synth)")
-	plannerModel := fs.String("planner-model", "gpt-4o-mini", "chat model for the planner decomposition")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	qs, err := eval.LoadQuerySet(*queriesPath)
-	if err != nil {
-		return err
-	}
-
-	// --api short-circuit: run the queries straight against a deployed cosift's
-	// /search endpoint. Skip the local corpus/embed/index work entirely —
-	// the deployed instance owns its own index. The corpus.json's relevance
-	// labels still apply because they're declared independently of the index.
-	if *apiURL != "" {
-		ret := &httpAPIRetriever{
-			baseURL:   strings.TrimRight(*apiURL, "/"),
-			bearer:    *apiBearer,
-			retriever: *retriever,
-			rerank:    *useRerank,
-			http:      &http.Client{Timeout: 30 * time.Second},
-		}
-		summary, e := eval.Run(ctx, qs, ret)
-		if e != nil {
-			return e
-		}
-		summary.Name = fmt.Sprintf("%s/api(%s)", qs.Name, *apiURL)
-		fmt.Print(eval.PrintTable(summary))
-		if *baselinePath != "" {
-			base, e := eval.LoadSummary(*baselinePath)
-			if e != nil {
-				return fmt.Errorf("baseline: %w", e)
-			}
-			fmt.Printf("\nvs baseline (%s):\n%s", *baselinePath, eval.Diff(base, summary))
-		}
-		if *savePath != "" {
-			if e := eval.SaveSummary(summary, *savePath); e != nil {
-				return fmt.Errorf("save: %w", e)
-			}
-			fmt.Printf("\nsaved summary to %s\n", *savePath)
-		}
-		return nil
-	}
-
-	corpus, err := eval.LoadCorpus(*corpusPath)
-	if err != nil {
-		return err
-	}
-
-	// Build an ephemeral store in a temp dir — eval doesn't touch the user's index.
-	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-eval-%d", time.Now().UnixNano()))
-	s, err := store.Open(tmpDir)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		s.Close()
-		_ = os.RemoveAll(tmpDir)
-	}()
-
-	bm := index.NewBM25(s)
-	for _, d := range corpus.Docs {
-		id, e := s.UpsertDocument(ctx, &store.Document{
-			URL: d.URL, Title: d.Title, Text: d.Text,
-			Source: "eval", FetchedAt: time.Now(),
-		})
-		if e != nil {
-			return fmt.Errorf("ingest %s: %w", d.URL, e)
-		}
-		if e := bm.IndexDocument(ctx, id, d.Title, d.Text); e != nil {
-			return fmt.Errorf("index %s: %w", d.URL, e)
-		}
-	}
-
-	// BM25 distractor injection: synthetic noise to stress-test retrieval at
-	// higher index sizes than the 20-doc canonical set. Each distractor is 100
-	// random tokens from a neutral vocab disjoint from the eval queries' vocab —
-	// they should NEVER be relevant, so any showing up in top-K = measurable degradation.
-	// Dense distractors are embedded alongside the corpus in the dense block below.
-	var distractorTexts []string
-	if *distractors > 0 {
-		drng := rand.New(rand.NewSource(*distractorSeed))
-		neutralVocab := neutralVocabForDistractors()
-		// Narrow mode: clamp to the first topic group (pottery — 7 words).
-		// Tests the hypothesis that smaller-but-more-similar noise
-		// is harder for the reranker than larger-but-diverse noise.
-		if *distractorMode == "narrow" {
-			neutralVocab = neutralVocab[:7]
-		} else if *distractorMode != "diverse" {
-			return fmt.Errorf("unknown distractor-mode %q (diverse | narrow)", *distractorMode)
-		}
-		distractorTexts = make([]string, *distractors)
-		for i := 0; i < *distractors; i++ {
-			dURL := fmt.Sprintf("https://distractor.test/%d", i)
-			dTitle := fmt.Sprintf("Distractor %d", i)
-			dText := generateDistractorText(drng, neutralVocab, 100)
-			distractorTexts[i] = dText
-			id, e := s.UpsertDocument(ctx, &store.Document{
-				URL: dURL, Title: dTitle, Text: dText, Source: "distractor", FetchedAt: time.Now(),
-			})
-			if e != nil {
-				return fmt.Errorf("distractor %d: %w", i, e)
-			}
-			if e := bm.IndexDocument(ctx, id, dTitle, dText); e != nil {
-				return fmt.Errorf("index distractor %d: %w", i, e)
-			}
-		}
-		fmt.Printf("injected %d distractor docs into BM25 index\n", *distractors)
-	}
-
-	// Build the requested retriever.
-	var ret eval.Retriever
-	switch *retriever {
-	case "bm25":
-		ret = &bm25Adapter{inner: bm}
-	case "dense", "hybrid":
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI") // accept the short form too
-		}
-		if apiKey == "" {
-			return errors.New("OPENAI_API_KEY (or OPENAI) not set (put it in .env)")
-		}
-		oai := embed.NewOpenAIClient(apiKey, *embURL, *embModel, *embDim)
-		var emb embed.Embedder = oai
-		if *embCacheDir != "" {
-			emb = embed.NewCachedEmbedder(oai, *embCacheDir)
-		}
-		// Chunk corpus into passage windows, then batch-embed.
-		// Same chunker the crawler uses → eval and serve behave identically.
-		chunker := chunkerWith(*chunkSize, *chunkOverlap)
-		type passageRef struct {
-			docIdx int
-			chunk  index.Chunk
-		}
-		var allTexts []string
-		var refs []passageRef
-		for i, d := range corpus.Docs {
-			text := d.Title + "\n\n" + d.Text
-			for _, c := range chunker.Chunk(text) {
-				allTexts = append(allTexts, c.Text)
-				refs = append(refs, passageRef{docIdx: i, chunk: c})
-			}
-		}
-		fmt.Printf("embedding %d passages across %d docs with %s (batch=%d)...\n", len(allTexts), len(corpus.Docs), *embModel, *embBatch)
-		vecs, e := batchEmbed(ctx, emb, allTexts, *embBatch)
-		if e != nil {
-			return fmt.Errorf("embed corpus: %w", e)
-		}
-		vi := index.NewVectorIndex(*embDim)
-		for j, v := range vecs {
-			d := corpus.Docs[refs[j].docIdx]
-			vi.AddPassage(d.URL, d.Title, refs[j].chunk.Offset, refs[j].chunk.Length, v)
-		}
-		// Embed + add distractor passages into the SAME vector index.
-		// batchEmbed splits into ≤1000-input chunks for provider limits.
-		if len(distractorTexts) > 0 {
-			fmt.Printf("embedding %d distractor passages (batch=%d)...\n", len(distractorTexts), *embBatch)
-			dvecs, err := batchEmbed(ctx, emb, distractorTexts, *embBatch)
-			if err != nil {
-				return fmt.Errorf("embed distractors: %w", err)
-			}
-			for i, v := range dvecs {
-				vi.AddPassage(fmt.Sprintf("https://distractor.test/%d", i), "Distractor", 0, len(distractorTexts[i]), v)
-			}
-		}
-		denseRet := &denseAdapter{idx: vi, emb: emb}
-		if *retriever == "dense" {
-			ret = denseRet
-		} else {
-			ret = &hybridAdapter{a: &bm25Adapter{inner: bm}, b: denseRet}
-		}
-	default:
-		return fmt.Errorf("unknown retriever %q", *retriever)
-	}
-
-	// Apply paraphrase + rerank wrappers below.
-	suffix := *retriever
-	if *useRerank {
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI")
-		}
-		if apiKey == "" {
-			return errors.New("rerank: OPENAI_API_KEY (or OPENAI) not set")
-		}
-		chat := embed.NewOpenAIChat(apiKey, "", *rerankModel)
-		rr := rerank.NewLLMReranker(chat)
-		textByURL := make(map[string]string, len(corpus.Docs))
-		for _, d := range corpus.Docs {
-			textByURL[d.URL] = d.Title + "\n\n" + d.Text
-		}
-		ret = &rerankAdapter{inner: ret, reranker: rr, textByURL: textByURL, candidateK: *candK}
-		suffix = fmt.Sprintf("%s+rerank(%s)", *retriever, *rerankModel)
-	}
-
-	// Planner wrapper — applied BEFORE paraphrase so a "both" run looks like
-	// (rerank ⊂ planner ⊂ paraphrase). Mirrors /research's plan→retrieve pipeline
-	// without the synth step, so we measure planner *retrieval coverage* in isolation.
-	if *usePlanner {
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI")
-		}
-		if apiKey == "" {
-			return errors.New("planner: OPENAI_API_KEY (or OPENAI) not set")
-		}
-		chat := embed.NewOpenAIChat(apiKey, "", *plannerModel)
-		ret = &plannerRetriever{inner: ret, chat: chat, mainWeight: *mainWeight, cache: make(map[string][]string)}
-		if *mainWeight > 0 {
-			suffix = fmt.Sprintf("%s+planner(%s,mw=%.2f)", suffix, *plannerModel, *mainWeight)
-		} else {
-			suffix = fmt.Sprintf("%s+planner(%s)", suffix, *plannerModel)
-		}
-	}
-
-	// Auto-paraphrase wrapper — outermost so paraphrases hit the FULL retriever
-	// stack (including rerank, if enabled).
-	if *autoParaphrase {
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI")
-		}
-		if apiKey == "" {
-			return errors.New("auto-paraphrase: OPENAI_API_KEY (or OPENAI) not set")
-		}
-		chat := embed.NewOpenAIChat(apiKey, "", *paraphraseModel)
-		ret = &paraphraseRetriever{inner: ret, chat: chat, n: *paraphraseN, mainWeight: *mainWeight, cache: make(map[string][]string)}
-		if *mainWeight > 0 {
-			suffix = fmt.Sprintf("%s+paraphrase(%d/%s,mw=%.2f)", suffix, *paraphraseN, *paraphraseModel, *mainWeight)
-		} else {
-			suffix = fmt.Sprintf("%s+paraphrase(%d/%s)", suffix, *paraphraseN, *paraphraseModel)
-		}
-	}
-
-	summary, err := eval.Run(ctx, qs, ret)
-	if err != nil {
-		return err
-	}
-	summary.Name = fmt.Sprintf("%s/%s", qs.Name, suffix)
-	fmt.Print(eval.PrintTable(summary))
-
-	if *baselinePath != "" {
-		base, err := eval.LoadSummary(*baselinePath)
-		if err != nil {
-			return fmt.Errorf("baseline: %w", err)
-		}
-		fmt.Printf("\nvs baseline (%s):\n%s", *baselinePath, eval.Diff(base, summary))
-	}
-	if *savePath != "" {
-		if err := eval.SaveSummary(summary, *savePath); err != nil {
-			return fmt.Errorf("save: %w", err)
-		}
-		fmt.Printf("\nsaved summary to %s\n", *savePath)
-	}
-	return nil
-}
-
-// firstEnv returns the first non-empty value among the named environment vars.
-func firstEnv(names ...string) string {
-	for _, n := range names {
-		if v := os.Getenv(n); v != "" {
-			return v
-		}
-	}
-	return ""
-}
-
-func runServe(ctx context.Context, cfg *config.Config) error {
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	srv := server.New(s)
-	// Instance-wide retrieval defaults from cfg.Defaults. Per-request query
-	// params still override; this is just what the handler picks up when the
-	// caller doesn't specify.
-	srv = srv.WithDefaults(server.Defaults{
-		Retriever:         cfg.Defaults.Retriever,
-		Expand:            cfg.Defaults.Expand,
-		ResearchStrategy:  cfg.Defaults.ResearchStrategy,
-		ResearchSynthK:    cfg.Defaults.ResearchSynthK,
-		ExpandMainWeight:  cfg.Defaults.ExpandMainWeight,
-		HybridDenseWeight: cfg.Defaults.HybridDenseWeight,
-	})
-	if cfg.Defaults.Retriever != "" || cfg.Defaults.Expand || cfg.Defaults.ResearchStrategy != "" || cfg.Defaults.ResearchSynthK > 0 || cfg.Defaults.ExpandMainWeight > 0 || cfg.Defaults.HybridDenseWeight > 0 {
-		log.Printf("defaults: retriever=%q expand=%v research_strategy=%q research_synth_k=%d expand_main_weight=%v hybrid_dense_weight=%v",
-			cfg.Defaults.Retriever, cfg.Defaults.Expand, cfg.Defaults.ResearchStrategy, cfg.Defaults.ResearchSynthK, cfg.Defaults.ExpandMainWeight, cfg.Defaults.HybridDenseWeight)
-	}
-	// thread chunker config to /admin/reembed handler so re-embed
-	// produces passages with the same shape as crawl-time indexing.
-	if cfg.Crawler.ChunkSize > 0 || cfg.Crawler.ChunkOverlap > 0 {
-		srv = srv.WithChunker(cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
-		log.Printf("chunker: size=%d overlap=%d", cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
-	}
-	if tok := cfg.Server.AdminToken; tok != "" {
-		srv = srv.WithAdminToken(tok)
-		log.Printf("/admin/* endpoints enabled (bearer-auth)")
-	}
-	if len(cfg.Server.TrustedProxies) > 0 {
-		s2, err := srv.WithTrustedProxies(cfg.Server.TrustedProxies)
-		if err != nil {
-			return fmt.Errorf("trusted_proxies: %w", err)
-		}
-		srv = s2
-		log.Printf("X-Forwarded-For trusted from %v", cfg.Server.TrustedProxies)
-	}
-
-	// Auto-wire embeddings if model is set and an API key is in env.
-	if cfg.Embeddings.Model != "" {
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI")
-		}
-		if apiKey == "" {
-			log.Printf("warning: embeddings configured but no OPENAI_API_KEY in env; dense/hybrid disabled")
-		} else {
-			model := cfg.Embeddings.Model
-			dim := cfg.Embeddings.Dim
-			if dim == 0 {
-				dim = 1536
-			}
-			emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, model, dim)
-			vi, err := index.LoadVectorIndex(ctx, s, model, dim)
-			if err != nil {
-				return fmt.Errorf("load vector index: %w", err)
-			}
-			srv = srv.WithVector(vi, emb)
-			log.Printf("vector index loaded: %d passages, model=%s dim=%d", vi.Len(), model, dim)
-
-			// Chat is opt-in via cfg.Chat.Model. Reuses the same API key.
-			if cfg.Chat.Model != "" {
-				chat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Chat.Model)
-				srv = srv.WithChat(chat)
-				log.Printf("/answer enabled with chat model=%s", cfg.Chat.Model)
-				// Auto-enable /search?expand=true via the same chat client.
-				// measured +0.02 nDCG at 10k distractors for $0.004 per run.
-				srv = srv.WithParaphraser(chat, 2)
-				log.Printf("/search?expand=true enabled (auto-paraphrase via %s)", cfg.Chat.Model)
-			}
-
-			// Reranker: HTTP backend wins if URL is set; otherwise LLM backend
-			// (only if chat is configured). Either is opt-out via cfg.Rerank.Enabled=false
-			// even when configured.
-			var rr rerank.Reranker
-			rerankWanted := cfg.Rerank.URL != "" || cfg.Chat.Model != ""
-			if !cfg.Rerank.Enabled && rerankWanted {
-				// Default ON when wired; flip cfg.Rerank.Enabled false to skip.
-				cfg.Rerank.Enabled = true
-			}
-			if cfg.Rerank.Enabled && cfg.Rerank.URL != "" {
-				rerankKey := cfg.Rerank.APIKey
-				if rerankKey == "" {
-					rerankKey = firstEnv("COHERE_API_KEY", "VOYAGE_API_KEY")
-				}
-				rr = rerank.NewHTTPReranker(cfg.Rerank.URL, rerankKey, cfg.Rerank.Model)
-				log.Printf("rerank enabled with http endpoint=%s model=%s", cfg.Rerank.URL, cfg.Rerank.Model)
-			} else if cfg.Rerank.Enabled && cfg.Chat.Model != "" {
-				rerankModel := cfg.Rerank.Model
-				if rerankModel == "" {
-					rerankModel = cfg.Chat.Model
-				}
-				rerankChat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, rerankModel)
-				rr = rerank.NewLLMReranker(rerankChat)
-				log.Printf("rerank enabled with LLM model=%s (fallback path; consider an HTTP reranker for cost)", rerankModel)
-			}
-			if rr != nil {
-				srv = srv.WithReranker(rr, cfg.Rerank.CandidateK)
-			}
-		}
-	}
-
-	// On-demand /contents: use the crawler's FetchOne. Doesn't share the worker
-	// pool's rate gate or robots cache — callers should set a sensible UA + body cap.
-	srv = srv.WithFetcher(func(ctx context.Context, u string) (string, string, string, error) {
-		r, err := crawler.FetchOne(ctx, nil, cfg.Crawler.UserAgent, u, cfg.Crawler.MaxBodyBytes)
-		if err != nil {
-			return "", "", "", err
-		}
-		return r.Title, r.Text, r.Lang, nil
-	})
-
-	log.Printf("cosift serving on %s", cfg.Server.Addr)
-	return server.ListenAndServe(ctx, cfg.Server.Addr, srv.Handler())
-}
-
-// runIngest loads a corpus.json directly into the local store + indexes —
-// useful for evaluation, vertical deployments, and users with a pre-curated
-// dataset who don't need a crawler.
-//
-// The corpus shape matches eval.Corpus: {"docs": [{"url","title","text"}...]}.
-// If embeddings are configured, every doc is chunked + embedded + persisted
-// to the passages table the same way the crawler does.
-// progressReporter logs progress for a long-running CLI loop at most once per
-// configured interval. Time-based (vs every-N-iterations) so the cadence is
-// consistent regardless of per-item cost variance. Set interval to 0 to
-// disable. Designed for ingest but generic enough for other loops.
-type progressReporter struct {
-	label    string
-	total    int
-	interval time.Duration
-	last     time.Time
-}
-
-func newProgressReporter(label string, total int, interval time.Duration) *progressReporter {
-	return &progressReporter{
-		label:    label,
-		total:    total,
-		interval: interval,
-		// Initialize `last` to now so the first call after construction
-		// doesn't immediately log (avoids a noisy first-iter "0/N" line).
-		last: time.Now(),
-	}
-}
-
-// maybeLog emits a progress line only if `interval` has elapsed since the last
-// emission. Callers pass the 1-indexed current item count. No-op when interval
-// is <= 0 or when total is 0 (avoids divide-by-zero in the percentage).
-func (p *progressReporter) maybeLog(current int) {
-	if p.interval <= 0 || p.total <= 0 {
-		return
-	}
-	if time.Since(p.last) < p.interval {
-		return
-	}
-	pct := float64(current) / float64(p.total) * 100
-	log.Printf("%s: %d/%d (%.1f%%)", p.label, current, p.total, pct)
-	p.last = time.Now()
-}
-
-// loadCorpusJSONL reads a JSONL file (one eval.CorpusDoc per line) into a
-// Corpus. Empty lines are skipped (some tools emit trailing blanks). Pairs
-// with's `export -format jsonl` to close the round-trip for ML
-// pipelines.
-func loadCorpusJSONL(path string) (*eval.Corpus, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("read %s: %w", path, err)
-	}
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	// JSONL lines can be large (full doc text); raise the scanner buffer cap
-	// well above the default 64KB. 4MB matches's SSE parser ceiling.
-	scanner.Buffer(make([]byte, 0, 64<<10), 4<<20)
-	c := &eval.Corpus{}
-	lineNum := 0
-	for scanner.Scan() {
-		lineNum++
-		raw := strings.TrimSpace(scanner.Text())
-		if raw == "" {
-			continue
-		}
-		var d eval.CorpusDoc
-		if err := json.Unmarshal([]byte(raw), &d); err != nil {
-			return nil, fmt.Errorf("parse %s line %d: %w", path, lineNum, err)
-		}
-		c.Docs = append(c.Docs, d)
-	}
-	if err := scanner.Err(); err != nil {
-		return nil, fmt.Errorf("read %s: %w", path, err)
-	}
-	return c, nil
-}
-
-func runIngest(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("ingest", flag.ExitOnError)
-	corpusPath := fs.String("corpus", "", "path to corpus file (required)")
-	source := fs.String("source", "ingest", "source tag for documents")
-	// -format selects the loader. Default "auto" infers from file
-	// extension (.jsonl → jsonl, anything else → json). Explicit override
-	// avoids ambiguity when files have non-standard extensions.
-	format := fs.String("format", "auto", "input format: auto (default — infer from extension) | json | jsonl")
-	// Operators ingesting large corpora need
-	// some signal that work is happening. 0 disables.
-	progressInterval := fs.Duration("progress", 5*time.Second, "log per-doc/per-passage progress every N (0 disables)")
-	// CLI override for chunker config (mirrors on eval/answer-eval).
-	// 0 → fall through to cfg.Crawler.{ChunkSize,ChunkOverlap}, then to NewChunker defaults.
-	ingestChunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = use cfg.Crawler.ChunkSize, then NewChunker default 320)")
-	ingestChunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = use cfg.Crawler.ChunkOverlap, then default 64)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *corpusPath == "" {
-		return errors.New("ingest: -corpus is required")
-	}
-	resolvedFormat := *format
-	if resolvedFormat == "auto" {
-		if strings.HasSuffix(strings.ToLower(*corpusPath), ".jsonl") {
-			resolvedFormat = "jsonl"
-		} else {
-			resolvedFormat = "json"
-		}
-	}
-	var corpus *eval.Corpus
-	var err error
-	switch resolvedFormat {
-	case "json":
-		corpus, err = eval.LoadCorpus(*corpusPath)
-	case "jsonl":
-		corpus, err = loadCorpusJSONL(*corpusPath)
-	default:
-		return fmt.Errorf("unknown -format %q (want: auto | json | jsonl)", *format)
-	}
-	if err != nil {
-		return err
-	}
-
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-	bm := index.NewBM25(s)
-
-	// Optional embedder — same auto-wire as runServe.
-	var emb embed.Embedder
-	if cfg.Embeddings.Model != "" {
-		apiKey := os.Getenv("OPENAI_API_KEY")
-		if apiKey == "" {
-			apiKey = os.Getenv("OPENAI")
-		}
-		if apiKey != "" {
-			dim := cfg.Embeddings.Dim
-			if dim == 0 {
-				dim = 1536
-			}
-			emb = embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
-		} else {
-			log.Printf("warning: embeddings configured but no API key; ingesting BM25 only")
-		}
-	}
-
-	// Pre-compute chunks for the whole corpus so we can batch-embed in one call.
-	// CLI flag wins over cfg.Crawler.ChunkSize (only when flag is set).
-	chunkSize := cfg.Crawler.ChunkSize
-	if *ingestChunkSize > 0 {
-		chunkSize = *ingestChunkSize
-	}
-	chunkOverlap := cfg.Crawler.ChunkOverlap
-	if *ingestChunkOverlap > 0 {
-		chunkOverlap = *ingestChunkOverlap
-	}
-	chunker := chunkerWith(chunkSize, chunkOverlap)
-	type passageRef struct {
-		docIdx int
-		chunk  index.Chunk
-	}
-	var allTexts []string
-	var refs []passageRef
-	if emb != nil {
-		for i, d := range corpus.Docs {
-			for _, c := range chunker.Chunk(d.Title + "\n\n" + d.Text) {
-				allTexts = append(allTexts, c.Text)
-				refs = append(refs, passageRef{docIdx: i, chunk: c})
-			}
-		}
-		log.Printf("ingest: embedding %d passages across %d docs with %s", len(allTexts), len(corpus.Docs), cfg.Embeddings.Model)
-	}
-
-	var vecs [][]float32
-	if emb != nil && len(allTexts) > 0 {
-		vecs, err = emb.Embed(ctx, allTexts)
-		if err != nil {
-			return fmt.Errorf("embed: %w", err)
-		}
-	}
-
-	// Map docIdx → docID for the passage writes.
-	docIDs := make([]int64, len(corpus.Docs))
-	docProgress := newProgressReporter("ingest docs", len(corpus.Docs), *progressInterval)
-	for i, d := range corpus.Docs {
-		sha := sha256.Sum256([]byte(d.Text))
-		id, err := s.UpsertDocument(ctx, &store.Document{
-			URL: d.URL, Title: d.Title, Text: d.Text,
-			Source: *source, Quality: 0.7, FetchedAt: time.Now(),
-			ContentSHA: sha[:],
-		})
-		if err != nil {
-			return fmt.Errorf("upsert %s: %w", d.URL, err)
-		}
-		if err := bm.IndexDocument(ctx, id, d.Title, d.Text); err != nil {
-			return fmt.Errorf("index %s: %w", d.URL, err)
-		}
-		docIDs[i] = id
-		docProgress.maybeLog(i + 1)
-	}
-
-	// Write passages.
-	if emb != nil && len(vecs) == len(refs) {
-		passageProgress := newProgressReporter("ingest passages", len(vecs), *progressInterval)
-		for j, v := range vecs {
-			ref := refs[j]
-			p := &store.Passage{
-				DocID:     docIDs[ref.docIdx],
-				Offset:    ref.chunk.Offset,
-				Length:    ref.chunk.Length,
-				Model:     emb.Model(),
-				Embedding: v,
-			}
-			if err := s.UpsertPassage(ctx, p); err != nil {
-				return fmt.Errorf("passage %s offset=%d: %w", corpus.Docs[ref.docIdx].URL, ref.chunk.Offset, err)
-			}
-			passageProgress.maybeLog(j + 1)
-		}
-	}
-
-	log.Printf("ingest: %d docs, %d passages", len(corpus.Docs), len(vecs))
-	return nil
-}
-
-// runBench measures search latency under synthetic load. No API calls, no disk
-// for the vector path (in-memory only); BM25 uses a temp SQLite.
-//
-// Use it to decide:
-//   - When to swap brute-force kNN for HNSW (when vector p50 > ~50 ms or
-//     p99 > ~200 ms at your target N).
-//   - When BM25 needs Tantivy or another inverted-index lib (when BM25 p50
-//     exceeds a comparable threshold).
-//
-// benchResult holds one mode's bench output. Each field maps to a JSON key
-// when `-json` is set; human-readable output uses the formatHuman method.
-type benchResult struct {
-	Mode           string  `json:"mode"`
-	N              int     `json:"n"`
-	Dim            int     `json:"dim,omitempty"`
-	Queries        int     `json:"queries,omitempty"`
-	P50Micros      int64   `json:"p50_us,omitempty"`
-	P95Micros      int64   `json:"p95_us,omitempty"`
-	P99Micros      int64   `json:"p99_us,omitempty"`
-	QPS            float64 `json:"qps,omitempty"`
-	ElapsedMicros  int64   `json:"elapsed_us,omitempty"`
-	Docs           int64   `json:"docs,omitempty"`
-	PagesPerSec    float64 `json:"pages_per_sec,omitempty"`
-	Terms          int64   `json:"terms,omitempty"`
-	PerHostDelayMs int     `json:"per_host_delay_ms,omitempty"`
-}
-
-func runBench(ctx context.Context, args []string) error {
-	fs := flag.NewFlagSet("bench", flag.ExitOnError)
-	n := fs.Int("n", 10000, "number of passages (for vector/bm25 modes) or pages (for crawl mode)")
-	dim := fs.Int("dim", 384, "embedding dimensionality (vector mode)")
-	queries := fs.Int("queries", 100, "queries per run (vector/bm25 modes)")
-	mode := fs.String("mode", "vector", "vector | bm25 | crawl | all")
-	perHostDelay := fs.Int("per-host-delay", 0, "per-host delay in ms for crawl mode (default 0 — localhost throughput; set 100-1000 to measure under realistic politeness)")
-	jsonOut := fs.Bool("json", false, "emit one JSON object per mode instead of a human-readable line — useful for CI ingestion + dashboards")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	emit := func(r *benchResult) {
-		if *jsonOut {
-			b, _ := json.Marshal(r)
-			fmt.Println(string(b))
-		} else {
-			fmt.Println(r.formatHuman())
-		}
-	}
-
-	if *mode == "vector" || *mode == "all" || *mode == "both" {
-		emit(benchVector(*n, *dim, *queries))
-	}
-	if *mode == "bm25" || *mode == "all" || *mode == "both" {
-		r, err := benchBM25(ctx, *n, *queries)
-		if err != nil {
-			return err
-		}
-		emit(r)
-	}
-	if *mode == "crawl" || *mode == "all" {
-		// Crawl mode uses a smaller default — N=10k pages of an in-process site
-		// would slow the bench without proportional signal. 200 is plenty to
-		// measure pages/sec accurately on a typical host.
-		crawlN := *n
-		if crawlN > 1000 {
-			crawlN = 200
-		}
-		r, err := benchCrawl(ctx, crawlN, *perHostDelay)
-		if err != nil {
-			return err
-		}
-		emit(r)
-	}
-	if *mode == "storage" || *mode == "all" {
-		// SQLite vs Pebble head-to-head. Same N synthetic docs, same
-		// K queries, two backends. Emit one result per backend; consumers diff
-		// the two lines (or use cosift bench-compare on the saved JSON).
-		sq, err := benchBM25SQLite(ctx, *n, *queries)
-		if err != nil {
-			return err
-		}
-		emit(sq)
-		pb, err := benchBM25Pebble(ctx, *n, *queries)
-		if err != nil {
-			return err
-		}
-		emit(pb)
-	}
-	return nil
-}
-
-// formatHuman renders the result as the one-line format the bench used in iter
-// 1–70 — preserves the existing UX when `-json` isn't set.
-func (r *benchResult) formatHuman() string {
-	switch r.Mode {
-	case "vector":
-		return fmt.Sprintf("vector  n=%d dim=%d  p50=%-9s p95=%-9s p99=%-9s qps=%.0f",
-			r.N, r.Dim,
-			time.Duration(r.P50Micros)*time.Microsecond,
-			time.Duration(r.P95Micros)*time.Microsecond,
-			time.Duration(r.P99Micros)*time.Microsecond,
-			r.QPS)
-	case "bm25":
-		return fmt.Sprintf("bm25    n=%d  p50=%-9s p95=%-9s p99=%-9s qps=%.0f",
-			r.N,
-			time.Duration(r.P50Micros)*time.Microsecond,
-			time.Duration(r.P95Micros)*time.Microsecond,
-			time.Duration(r.P99Micros)*time.Microsecond,
-			r.QPS)
-	case "crawl":
-		delay := ""
-		if r.PerHostDelayMs > 0 {
-			delay = fmt.Sprintf(" per-host-delay=%dms", r.PerHostDelayMs)
-		}
-		return fmt.Sprintf("crawl   n=%-5d elapsed=%-10s docs=%-5d  pages/sec=%-6.0f  terms=%d%s",
-			r.N,
-			(time.Duration(r.ElapsedMicros) * time.Microsecond).Round(time.Millisecond),
-			r.Docs, r.PagesPerSec, r.Terms, delay)
-	}
-	return fmt.Sprintf("%+v", *r)
-}
-
-// loadBenchReport parses a `cosift bench -json` output file. Format: one
-// benchResult JSON object per line (NDJSON). Lines that don't parse as JSON
-// are skipped silently — this lets users pipe `bench -json` into a file
-// alongside other tool output if they want.
-func loadBenchReport(path string) (map[string]*benchResult, error) {
-	b, err := os.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("read %s: %w", path, err)
-	}
-	out := make(map[string]*benchResult)
-	for _, line := range strings.Split(string(b), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || !strings.HasPrefix(line, "{") {
-			continue
-		}
-		var r benchResult
-		if err := json.Unmarshal([]byte(line), &r); err != nil {
-			continue // skip unparseable lines rather than failing the whole report
-		}
-		if r.Mode == "" {
-			continue
-		}
-		out[r.Mode] = &r
-	}
-	if len(out) == 0 {
-		return nil, fmt.Errorf("%s: no parseable bench records (expecting NDJSON from `cosift bench -json`)", path)
-	}
-	return out, nil
-}
-
-// runBenchCompare diffs two saved bench reports side-by-side. Mirrors
-// answer-eval-compare for the bench surface — locks the JSON contract
-// as a stable interface tools can rely on.
-//
-// For each mode present in either report, prints the absolute numbers + deltas.
-// Reports a mode as "missing in BASELINE" or "missing in NEW" rather than
-// silently skipping — operators benefit from knowing the compare wasn't
-// fully apples-to-apples.
-func runBenchCompare(args []string) error {
-	fs := flag.NewFlagSet("bench-compare", flag.ExitOnError)
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	rest := fs.Args()
-	if len(rest) != 2 {
-		return errors.New("usage: cosift bench-compare BASELINE.json NEW.json")
-	}
-	a, err := loadBenchReport(rest[0])
-	if err != nil {
-		return fmt.Errorf("baseline: %w", err)
-	}
-	b, err := loadBenchReport(rest[1])
-	if err != nil {
-		return fmt.Errorf("new: %w", err)
-	}
-	fmt.Printf("BASELINE: %s  (modes: %v)\n", rest[0], sortedModes(a))
-	fmt.Printf("NEW:      %s  (modes: %v)\n", rest[1], sortedModes(b))
-	fmt.Println()
-
-	// Compare any mode present in either report. Stable order: vector, bm25, crawl.
-	allModes := union(a, b)
-	for _, mode := range allModes {
-		ar := a[mode]
-		br := b[mode]
-		switch {
-		case ar == nil:
-			fmt.Printf("%-7s (missing in BASELINE — new in this run)\n", mode)
-		case br == nil:
-			fmt.Printf("%-7s (missing in NEW — present in BASELINE only)\n", mode)
-		default:
-			printBenchDelta(mode, ar, br)
-		}
-	}
-	return nil
-}
-
-func sortedModes(m map[string]*benchResult) []string {
-	order := []string{"vector", "bm25", "crawl"}
-	out := make([]string, 0, len(m))
-	for _, k := range order {
-		if _, ok := m[k]; ok {
-			out = append(out, k)
-		}
-	}
-	// Catch any modes outside the known order too.
-	for k := range m {
-		if k != "vector" && k != "bm25" && k != "crawl" {
-			out = append(out, k)
-		}
-	}
-	return out
-}
-
-func union(a, b map[string]*benchResult) []string {
-	seen := make(map[string]bool)
-	order := []string{"vector", "bm25", "crawl"}
-	out := []string{}
-	for _, k := range order {
-		if _, ok := a[k]; ok {
-			seen[k] = true
-			out = append(out, k)
-		}
-		if _, ok := b[k]; ok && !seen[k] {
-			seen[k] = true
-			out = append(out, k)
-		}
-	}
-	for _, m := range []map[string]*benchResult{a, b} {
-		for k := range m {
-			if !seen[k] {
-				seen[k] = true
-				out = append(out, k)
-			}
-		}
-	}
-	return out
-}
-
-// printBenchDelta prints one mode's before/after side-by-side. Format
-// matches answer-eval-compare's row style.
-func printBenchDelta(mode string, a, b *benchResult) {
-	switch mode {
-	case "vector", "bm25":
-		// QPS is the headline; latency percentiles are signal too.
-		fmt.Printf("%-7s n=%d→%d  qps %.0f→%.0f (%+.0f)  p50 %dµs→%dµs (%+dµs)  p95 %dµs→%dµs (%+dµs)\n",
-			mode, a.N, b.N,
-			a.QPS, b.QPS, b.QPS-a.QPS,
-			a.P50Micros, b.P50Micros, b.P50Micros-a.P50Micros,
-			a.P95Micros, b.P95Micros, b.P95Micros-a.P95Micros,
-		)
-	case "crawl":
-		fmt.Printf("%-7s n=%d→%d  pages/sec %.0f→%.0f (%+.0f)  elapsed %dms→%dms  docs %d→%d\n",
-			mode, a.N, b.N,
-			a.PagesPerSec, b.PagesPerSec, b.PagesPerSec-a.PagesPerSec,
-			a.ElapsedMicros/1000, b.ElapsedMicros/1000,
-			a.Docs, b.Docs,
-		)
-	default:
-		fmt.Printf("%-7s (unknown mode — raw: a=%+v b=%+v)\n", mode, *a, *b)
-	}
-}
-
-func benchVector(n, dim, queries int) *benchResult {
-	rng := newSeededRand()
-	vi := index.NewVectorIndex(dim)
-	for i := 0; i < n; i++ {
-		v := randUnit(rng, dim)
-		vi.AddPassage(fmt.Sprintf("https://test/p%d", i), "doc", i*100, 100, v)
-	}
-	// Time `queries` searches.
-	q := make([][]float32, queries)
-	for i := range q {
-		q[i] = randUnit(rng, dim)
-	}
-	lats := make([]time.Duration, queries)
-	for i, qv := range q {
-		start := time.Now()
-		_ = vi.Search(context.Background(), qv, 10)
-		lats[i] = time.Since(start)
-	}
-	p50, p95, p99 := percentiles(lats)
-	return &benchResult{
-		Mode: "vector", N: n, Dim: dim, Queries: queries,
-		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
-		QPS: float64(queries) / sumDur(lats).Seconds(),
-	}
-}
-
-func benchBM25(ctx context.Context, n, queries int) (*benchResult, error) {
-	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-bench-%d", time.Now().UnixNano()))
-	s, err := store.Open(tmpDir)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		s.Close()
-		_ = os.RemoveAll(tmpDir)
-	}()
-	bm := index.NewBM25(s)
-
-	rng := newSeededRand()
-	vocab := []string{"alpha", "beta", "gamma", "delta", "epsilon", "go", "rust", "python",
-		"distributed", "consensus", "raft", "paxos", "quantum", "entanglement", "cell",
-		"mitochondria", "roman", "empire", "pacific", "midway", "tennis", "soccer",
-		"bitcoin", "blockchain", "transformer", "attention", "model", "vector", "index"}
-	// Build N synthetic docs of ~50 random vocab words each.
-	for i := 0; i < n; i++ {
-		var sb strings.Builder
-		for j := 0; j < 50; j++ {
-			if j > 0 {
-				sb.WriteByte(' ')
-			}
-			sb.WriteString(vocab[rng.Intn(len(vocab))])
-		}
-		id, err := s.UpsertDocument(ctx, &store.Document{
-			URL: fmt.Sprintf("https://test/d%d", i), Title: vocab[rng.Intn(len(vocab))],
-			Text: sb.String(), Source: "bench", FetchedAt: time.Now(),
-		})
-		if err != nil {
-			return nil, err
-		}
-		if err := bm.IndexDocument(ctx, id, "", sb.String()); err != nil {
-			return nil, err
-		}
-	}
-
-	// Random queries of 2-4 vocab words.
-	qstrs := make([]string, queries)
-	for i := range qstrs {
-		k := 2 + rng.Intn(3)
-		var sb strings.Builder
-		for j := 0; j < k; j++ {
-			if j > 0 {
-				sb.WriteByte(' ')
-			}
-			sb.WriteString(vocab[rng.Intn(len(vocab))])
-		}
-		qstrs[i] = sb.String()
-	}
-
-	lats := make([]time.Duration, queries)
-	for i, q := range qstrs {
-		start := time.Now()
-		_, err := bm.Search(ctx, q, 10)
-		if err != nil {
-			return nil, err
-		}
-		lats[i] = time.Since(start)
-	}
-	p50, p95, p99 := percentiles(lats)
-	return &benchResult{
-		Mode: "bm25", N: n, Queries: queries,
-		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
-		QPS: float64(queries) / sumDur(lats).Seconds(),
-	}, nil
-}
-
-// benchBM25SQLite is an alias of benchBM25 with an explicit "bm25-sqlite" Mode
-// label so storage-mode output is unambiguous when both backends are emitted.
-func benchBM25SQLite(ctx context.Context, n, queries int) (*benchResult, error) {
-	r, err := benchBM25(ctx, n, queries)
-	if err != nil {
-		return nil, err
-	}
-	r.Mode = "bm25-sqlite"
-	return r, nil
-}
-
-// benchBM25Pebble mirrors benchBM25 but runs against PebbleStore + PebbleBM25.
-// Deterministic vocab + RNG seed so SQLite and Pebble runs produce comparable
-// numbers (same N docs, same K queries, same query distribution).
-// empirical validation of the path-2 storage rework.
-func benchBM25Pebble(ctx context.Context, n, queries int) (*benchResult, error) {
-	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-pebble-bench-%d", time.Now().UnixNano()))
-	ps, err := store.OpenPebble(tmpDir)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		ps.Close()
-		_ = os.RemoveAll(tmpDir)
-	}()
-	bm := index.NewPebbleBM25(ps)
-
-	rng := newSeededRand()
-	vocab := []string{"alpha", "beta", "gamma", "delta", "epsilon", "go", "rust", "python",
-		"distributed", "consensus", "raft", "paxos", "quantum", "entanglement", "cell",
-		"mitochondria", "roman", "empire", "pacific", "midway", "tennis", "soccer",
-		"bitcoin", "blockchain", "transformer", "attention", "model", "vector", "index"}
-	for i := 0; i < n; i++ {
-		var sb strings.Builder
-		for j := 0; j < 50; j++ {
-			if j > 0 {
-				sb.WriteByte(' ')
-			}
-			sb.WriteString(vocab[rng.Intn(len(vocab))])
-		}
-		id, err := ps.UpsertDocument(ctx, &store.Document{
-			URL: fmt.Sprintf("https://test/d%d", i), Title: vocab[rng.Intn(len(vocab))],
-			Text: sb.String(), Source: "bench", FetchedAt: time.Now(),
-		})
-		if err != nil {
-			return nil, err
-		}
-		if err := bm.IndexDocument(ctx, id, "", sb.String()); err != nil {
-			return nil, err
-		}
-	}
-
-	qstrs := make([]string, queries)
-	for i := range qstrs {
-		k := 2 + rng.Intn(3)
-		var sb strings.Builder
-		for j := 0; j < k; j++ {
-			if j > 0 {
-				sb.WriteByte(' ')
-			}
-			sb.WriteString(vocab[rng.Intn(len(vocab))])
-		}
-		qstrs[i] = sb.String()
-	}
-
-	lats := make([]time.Duration, queries)
-	for i, q := range qstrs {
-		start := time.Now()
-		_, err := bm.Search(ctx, q, 10)
-		if err != nil {
-			return nil, err
-		}
-		lats[i] = time.Since(start)
-	}
-	p50, p95, p99 := percentiles(lats)
-	return &benchResult{
-		Mode: "bm25-pebble", N: n, Queries: queries,
-		P50Micros: p50.Microseconds(), P95Micros: p95.Microseconds(), P99Micros: p99.Microseconds(),
-		QPS: float64(queries) / sumDur(lats).Seconds(),
-	}, nil
-}
-
-func newSeededRand() *rand.Rand {
-	return rand.New(rand.NewSource(42)) // deterministic so runs are comparable
-}
-
-func randUnit(rng *rand.Rand, dim int) []float32 {
-	v := make([]float32, dim)
-	var norm float64
-	for i := range v {
-		v[i] = float32(rng.NormFloat64())
-		norm += float64(v[i]) * float64(v[i])
-	}
-	if norm == 0 {
-		v[0] = 1
-		return v
-	}
-	inv := float32(1.0 / sqrt(norm))
-	for i := range v {
-		v[i] *= inv
-	}
-	return v
-}
-
-func sqrt(x float64) float64 {
-	// stdlib math import already pulled through other packages; just do it here.
-	z := x / 2
-	for i := 0; i < 20; i++ {
-		if z == 0 {
-			break
-		}
-		z = (z + x/z) / 2
-	}
-	return z
-}
-
-// benchCrawl measures end-to-end crawler throughput against an in-process
-// website with N linked synthetic pages. Backs up the project directive's
-// "lightweight, on-par with modern crawlers" claim with measured pages/sec.
-//
-// The fixture: an httptest server serving N pages, each linking to the next
-// few via <a href="/pN+1">. The crawler is configured with per_host_delay=0
-// (we're hitting localhost), max_concurrent=8, robots disabled (would add a
-// constant ~1 fetch of /robots.txt unrelated to the throughput question).
-func benchCrawl(ctx context.Context, n, perHostDelayMs int) (*benchResult, error) {
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		// Parse "/pN" from request path.
-		if !strings.HasPrefix(r.URL.Path, "/p") {
-			http.NotFound(w, r)
-			return
-		}
-		idx, err := strconv.Atoi(r.URL.Path[2:])
-		if err != nil || idx < 0 || idx >= n {
-			http.NotFound(w, r)
-			return
-		}
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, `<!doctype html><html><head><title>Page %d</title></head><body><h1>Page %d</h1>`, idx, idx)
-		fmt.Fprintf(w, `<p>Synthetic content for page %d. The crawler follows links from this page to nearby pages, building up the index.</p>`, idx)
-		// Link to next 3 pages so the crawler has a graph to walk.
-		for j := idx + 1; j < idx+4 && j < n; j++ {
-			fmt.Fprintf(w, `<a href="/p%d">page %d</a> `, j, j)
-		}
-		fmt.Fprint(w, `</body></html>`)
-	})
-	srv := httptest.NewServer(mux)
-	defer srv.Close()
-
-	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-bench-crawl-%d", time.Now().UnixNano()))
-	s, err := store.Open(tmpDir)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		s.Close()
-		_ = os.RemoveAll(tmpDir)
-	}()
-
-	cfg := config.Default().Crawler
-	// -per-host-delay propagates from the bench CLI flag. Default 0
-	// (localhost throughput); set to 100–1000 to measure under realistic politeness.
-	cfg.PerHostDelayMs = perHostDelayMs
-	// Lower concurrency than the default 8 — bench output gets noisy with
-	// SQLite BUSY contention messages at high parallelism, and the bench
-	// metric is meant to be repeatable. Real deployments crawling many hosts
-	// at once should keep the default 8.
-	cfg.MaxConcurrent = 4
-	cfg.MaxDepth = 100 // ensure no depth ceiling cuts the crawl short
-	cfg.RespectRobots = false
-	cfg.IncludeDomains = nil // accept the httptest host (port-bound)
-
-	c := crawler.New(cfg, s)
-	// Seed ALL pages — measures pure fetch+parse+index throughput, not the
-	// crawler's link-discovery rate. Avoids the 1.5s "frontier empty"
-	// terminator firing before link parsing has populated more work.
-	for i := 0; i < n; i++ {
-		if e := c.Seed(fmt.Sprintf("%s/p%d", srv.URL, i)); e != nil {
-			return nil, fmt.Errorf("seed p%d: %w", i, e)
-		}
-	}
-
-	start := time.Now()
-	// Cap the run at 60s in case something goes wrong — bench should never hang.
-	runCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
-	defer cancel()
-	if err := c.Run(runCtx); err != nil {
-		return nil, fmt.Errorf("run: %w", err)
-	}
-	elapsed := time.Since(start)
-
-	st, err := s.Stats(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return &benchResult{
-		Mode: "crawl", N: n,
-		ElapsedMicros:  elapsed.Microseconds(),
-		Docs:           st.Documents,
-		PagesPerSec:    float64(st.Documents) / elapsed.Seconds(),
-		Terms:          st.Terms,
-		PerHostDelayMs: perHostDelayMs,
-	}, nil
-	// Note: the crawler's termination detector adds ~1.5s of polling at the
-	// end of every run. For small N this dominates elapsed and the pages/sec
-	// figure understates steady-state throughput. Larger N is more representative.
-}
-
-func percentiles(d []time.Duration) (p50, p95, p99 time.Duration) {
-	if len(d) == 0 {
-		return 0, 0, 0
-	}
-	sorted := make([]time.Duration, len(d))
-	copy(sorted, d)
-	// Simple sort — durations as int64 nanos.
-	for i := 1; i < len(sorted); i++ {
-		for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- {
-			sorted[j], sorted[j-1] = sorted[j-1], sorted[j]
-		}
-	}
-	q := func(p float64) time.Duration {
-		idx := int(float64(len(sorted)-1) * p)
-		return sorted[idx]
-	}
-	return q(0.5), q(0.95), q(0.99)
-}
-
-func sumDur(d []time.Duration) time.Duration {
-	var s time.Duration
-	for _, x := range d {
-		s += x
-	}
-	return s
-}
-
-// runRefreshDue re-queues URLs whose adaptive interval has elapsed.
-// Combined with conditional GET, a refresh-due pass over a stable corpus
-// mostly costs 304 round-trips — no body, no parse, no embed.
-//
-// One-shot by default. Pass `-interval` to loop forever — useful when running
-// as a systemd service or inside the Docker image (no extra cron needed).
-func runRefreshDue(ctx context.Context, cfg *config.Config, args []string) (err error) {
-	fs := flag.NewFlagSet("refresh-due", flag.ExitOnError)
-	minH := fs.Duration("min", 1*time.Hour, "minimum re-crawl interval")
-	maxH := fs.Duration("max", 30*24*time.Hour, "maximum re-crawl interval")
-	limit := fs.Int("limit", 100, "max URLs to enqueue per pass")
-	dryRun := fs.Bool("dry-run", false, "print URLs that would be refreshed, don't enqueue")
-	interval := fs.Duration("interval", 0, "loop with this delay between passes (0 = one-shot)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	pass := func() error {
-		due, err := s.DueForRefresh(ctx, *minH, *maxH, *limit)
-		if err != nil {
-			return err
-		}
-		if len(due) == 0 {
-			log.Printf("refresh-due: 0 URLs due")
-			return nil
-		}
-		log.Printf("refresh-due: %d URLs due (limit %d, min=%s max=%s)", len(due), *limit, *minH, *maxH)
-		if *dryRun {
-			for _, u := range due {
-				fmt.Println(u)
-			}
-			return nil
-		}
-		c := crawler.New(cfg.Crawler, s)
-		for _, u := range due {
-			if err := c.Recrawl(ctx, u); err != nil {
-				log.Printf("recrawl %s: %v", u, err)
-			}
-		}
-		return c.Run(ctx)
-	}
-
-	if *interval <= 0 {
-		return pass()
-	}
-	// Daemon mode: pass, sleep, pass, ... until ctx cancelled.
-	for {
-		if err := pass(); err != nil {
-			log.Printf("refresh-due pass: %v", err)
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(*interval):
-		}
-	}
-}
-
-// runExport writes the local store's documents to a portable corpus.json,
-// shape-compatible with `cosift ingest`. Round-trips cleanly:
-//
-//	cosift export -output corpus.json
-//	cosift -config other.json ingest -corpus corpus.json
-//
-// Documents only — passages (embeddings) aren't exported because the receiver
-// likely uses a different embedding model.
-// runMigrateToPebble copies a SQLite-backed cosift data directory into a
-// fresh Pebble store. Fifth piece of the path-2 storage rework.
-//
-// Migrates:
-//   - documents (URL, title, text, metadata) via PebbleStore.UpsertDocument
-//   - BM25 postings (re-tokenized + re-indexed to preserve title
-//     boost; the SQLite postings table is NOT copied directly because
-//     re-indexing through PebbleBM25 is the same code path that production
-//     uses going forward — eliminates the divergence risk of two
-//     posting-write paths)
-//
-// Does NOT migrate (deferred): frontier rows, query_outcomes feedback,
-// paraphrase/HyDE caches, vector embeddings. Operators starting from a
-// migrated Pebble store get a working BM25 index immediately; dense + LLM
-// caches rebuild from scratch on the new instance.
-//
-// The destination directory must be empty; the migration refuses to
-// overwrite existing data so a mistyped path can't clobber a working store.
-func runMigrateToPebble(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("migrate-to-pebble", flag.ExitOnError)
-	output := fs.String("output", "", "output directory for the Pebble store (required)")
-	progress := fs.Duration("progress", 5*time.Second, "log progress every N (0 disables)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *output == "" {
-		return errors.New("migrate-to-pebble: -output is required")
-	}
-	// Refuse to write into a directory that already contains a Pebble store.
-	if entries, err := os.ReadDir(*output); err == nil && len(entries) > 0 {
-		return fmt.Errorf("migrate-to-pebble: -output %s is non-empty; refusing to overwrite (move or remove first)", *output)
-	}
-
-	src, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return fmt.Errorf("open source SQLite store at %s: %w", cfg.DataDir, err)
-	}
-	defer src.Close()
-
-	dst, err := openPebbleOrFriendlyErr(*output)
-	if err != nil {
-		return err
-	}
-	defer dst.Close()
-
-	pidx := index.NewPebbleBM25(dst)
-
-	docs, err := src.ListDocuments(ctx, 0)
-	if err != nil {
-		return fmt.Errorf("list source documents: %w", err)
-	}
-	log.Printf("migrate-to-pebble: %d documents to copy (source: %s → destination: %s)",
-		len(docs), cfg.DataDir, *output)
-
-	reporter := newProgressReporter("migrate docs", len(docs), *progress)
-	for i, d := range docs {
-		if err := ctx.Err(); err != nil {
-			return err
-		}
-		// Upsert the doc into Pebble — assigns a fresh Pebble-side ID.
-		newID, err := dst.UpsertDocument(ctx, d)
-		if err != nil {
-			return fmt.Errorf("upsert doc %s: %w", d.URL, err)
-		}
-		// Re-index in Pebble BM25. Uses the same Tokenize + TitleBoost that
-		// production reads through, so behavior of the migrated index matches
-		// what a fresh crawl would produce.
-		if err := pidx.IndexDocument(ctx, newID, d.Title, d.Text); err != nil {
-			return fmt.Errorf("index doc %s: %w", d.URL, err)
-		}
-		reporter.maybeLog(i + 1)
-	}
-
-	stats, err := dst.Stats(ctx)
-	if err != nil {
-		return err
-	}
-	log.Printf("migrate-to-pebble: %d documents indexed into Pebble at %s", stats.Documents, *output)
-	return nil
-}
-
-func runExport(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("export", flag.ExitOnError)
-	output := fs.String("output", "", "output path (default depends on -format: corpus-export.{json,jsonl,txt,md})")
-	limit := fs.Int("limit", 0, "max docs to export (0 = all, applied AFTER filters)")
-	// -format selects the on-disk shape.
-	//   json (default — eval.Corpus pretty-printed; backward-compatible with)
-	//   jsonl — one {url,title,text} per line; common for ML fine-tuning pipelines
-	//   text  — Title/URL header + body + --- separator; for RAG corpora / grep
-	//   md    — `# Title` + URL line + body + horizontal rule; for LLM prompt piping
-	format := fs.String("format", "json", "output format: json | jsonl | text | md")
-	// All applied client-side after ListDocuments.
-	includeDomains := fs.String("include-domains", "", "comma-separated allowlist of domains (suffix match on dot boundary)")
-	excludeDomains := fs.String("exclude-domains", "", "comma-separated denylist of domains")
-	since := fs.String("since", "", "ISO date — only docs with published_at on or after (drops undated docs)")
-	until := fs.String("until", "", "ISO date — only docs with published_at on or before (drops undated docs)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if err := validateExportFormat(*format); err != nil {
-		return err
-	}
-	if *output == "" {
-		*output = "corpus-export." + exportExtension(*format)
-	}
-
-	sinceT, err := parseExportDate(*since)
-	if err != nil {
-		return fmt.Errorf("-since: %w", err)
-	}
-	untilT, err := parseExportDate(*until)
-	if err != nil {
-		return fmt.Errorf("-until: %w", err)
-	}
-	includeList := splitDomainCSV(*includeDomains)
-	excludeList := splitDomainCSV(*excludeDomains)
-	hasDateFilter := !sinceT.IsZero() || !untilT.IsZero()
-
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	// limit applies AFTER filters now — operators expect "give me 100
-	// docs from example.com" to mean 100 example.com docs, not 100 docs total
-	// of which some are from example.com. Pass 0 (no limit) to ListDocuments
-	// and apply the limit client-side after filtering.
-	rawLimit := 0
-	if *limit > 0 && !hasDateFilter && len(includeList) == 0 && len(excludeList) == 0 {
-		// No filters → push the limit down to ListDocuments (faster, less memory).
-		rawLimit = *limit
-	}
-	docs, err := s.ListDocuments(ctx, rawLimit)
-	if err != nil {
-		return err
-	}
-
-	docs = filterExportDocs(docs, includeList, excludeList, sinceT, untilT)
-	if *limit > 0 && len(docs) > *limit {
-		docs = docs[:*limit]
-	}
-
-	f, cerr := os.Create(*output)
-	if cerr != nil {
-		return cerr
-	}
-	defer func() {
-		if closeErr := f.Close(); closeErr != nil && err == nil {
-			err = fmt.Errorf("close output: %w", closeErr)
-		}
-	}()
-
-	switch *format {
-	case "json":
-		// wire shape: eval.Corpus pretty-printed. Preserved as default.
-		corpus := eval.Corpus{Docs: make([]eval.CorpusDoc, 0, len(docs))}
-		for _, d := range docs {
-			corpus.Docs = append(corpus.Docs, eval.CorpusDoc{URL: d.URL, Title: d.Title, Text: d.Text})
-		}
-		enc := json.NewEncoder(f)
-		enc.SetIndent("", "  ")
-		if err := enc.Encode(corpus); err != nil {
-			return fmt.Errorf("encode: %w", err)
-		}
-	case "jsonl":
-		// One {url,title,text} per line. No outer array — caller streams line-by-line.
-		enc := json.NewEncoder(f)
-		for _, d := range docs {
-			if err := enc.Encode(eval.CorpusDoc{URL: d.URL, Title: d.Title, Text: d.Text}); err != nil {
-				return fmt.Errorf("encode jsonl: %w", err)
-			}
-		}
-	case "text":
-		// Title:/URL: headers + blank + body + --- separator. Matches
-		// `contents -text` batch separator convention (LLMs interpret `---` as
-		// section breaks).
-		for i, d := range docs {
-			if i > 0 {
-				fmt.Fprintln(f, "---")
-				fmt.Fprintln(f)
-			}
-			if d.Title != "" {
-				fmt.Fprintf(f, "Title: %s\n", d.Title)
-			}
-			fmt.Fprintf(f, "URL: %s\n\n", d.URL)
-			fmt.Fprintln(f, d.Text)
-		}
-	case "md":
-		// `# Title` heading + italic URL line + body + horizontal rule. For
-		// piping into LLM prompt contexts that render markdown.
-		for i, d := range docs {
-			if i > 0 {
-				fmt.Fprintln(f)
-				fmt.Fprintln(f, "---")
-				fmt.Fprintln(f)
-			}
-			if d.Title != "" {
-				fmt.Fprintf(f, "# %s\n\n", d.Title)
-			}
-			fmt.Fprintf(f, "_%s_\n\n", d.URL)
-			fmt.Fprintln(f, d.Text)
-		}
-	}
-	log.Printf("export: %d docs → %s (%s)", len(docs), *output, *format)
-	return nil
-}
-
-// parseExportDate accepts YYYY-MM-DD or full RFC3339 (matches the
-// /search?since=/?until= conventions). Empty input returns zero time + nil
-// error (no filter).
-func parseExportDate(s string) (time.Time, error) {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return time.Time{}, nil
-	}
-	if t, err := time.Parse(time.RFC3339, s); err == nil {
-		return t, nil
-	}
-	if t, err := time.Parse("2006-01-02", s); err == nil {
-		return t, nil
-	}
-	return time.Time{}, fmt.Errorf("could not parse date %q", s)
-}
-
-// splitDomainCSV parses a comma-separated list of domains into lowercased
-// non-empty entries. (mirrors internal/server's unexported splitCSV).
-func splitDomainCSV(s string) []string {
-	if s == "" {
-		return nil
-	}
-	parts := strings.Split(s, ",")
-	out := make([]string, 0, len(parts))
-	for _, p := range parts {
-		t := strings.ToLower(strings.TrimSpace(p))
-		if t != "" {
-			out = append(out, t)
-		}
-	}
-	return out
-}
-
-// matchesDomainPattern is the strict suffix-on-dot-boundary match the
-// /search filter uses. `example.com` matches `blog.example.com` but NOT
-// `evilexample.com`.
-func matchesDomainPattern(host string, patterns []string) bool {
-	host = strings.ToLower(host)
-	for _, p := range patterns {
-		if host == p || strings.HasSuffix(host, "."+p) {
-			return true
-		}
-	}
-	return false
-}
-
-// hostFromURL extracts the lowercased host without port from a URL. Returns
-// empty string when parsing fails so callers can treat unparseable URLs
-// uniformly (typically: excluded from include-filtered exports).
-func hostFromURL(rawURL string) string {
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return ""
-	}
-	return strings.ToLower(u.Hostname())
-}
-
-// filterExportDocs applies the export filters to a doc slice
-// in-memory. Returns the filtered slice (in input order — preserves whatever
-// ordering ListDocuments returned).
-func filterExportDocs(docs []*store.Document, include, exclude []string, since, until time.Time) []*store.Document {
-	hasDateFilter := !since.IsZero() || !until.IsZero()
-	hasDomainFilter := len(include) > 0 || len(exclude) > 0
-	if !hasDateFilter && !hasDomainFilter {
-		return docs
-	}
-	out := make([]*store.Document, 0, len(docs))
-	for _, d := range docs {
-		// Date filter: undated docs are excluded when ANY date bound is set
-		// (matches /search?since=/?until= semantics).
-		if hasDateFilter {
-			if d.PublishedAt.IsZero() {
-				continue
-			}
-			if !since.IsZero() && d.PublishedAt.Before(since) {
-				continue
-			}
-			if !until.IsZero() && d.PublishedAt.After(until) {
-				continue
-			}
-		}
-		if hasDomainFilter {
-			host := hostFromURL(d.URL)
-			if len(include) > 0 && !matchesDomainPattern(host, include) {
-				continue
-			}
-			if len(exclude) > 0 && matchesDomainPattern(host, exclude) {
-				continue
-			}
-		}
-		out = append(out, d)
-	}
-	return out
-}
-
-// validateExportFormat is the export-side counterpart to's
-// validateFormat. Different accept-list because export supports json/jsonl
-// (file-shaped) which the synth-output CLIs don't, and synth CLIs support
-// markdown aliases which export doesn't need (single canonical name reduces
-// confusion when picking a file extension).
-func validateExportFormat(v string) error {
-	switch v {
-	case "json", "jsonl", "text", "md":
-		return nil
-	default:
-		return fmt.Errorf("export format must be one of: json, jsonl, text, md (got %q)", v)
-	}
-}
-
-// exportExtension picks a conventional file extension for each export format.
-// Used to derive a default -output path when the operator doesn't specify one.
-func exportExtension(format string) string {
-	switch format {
-	case "jsonl":
-		return "jsonl"
-	case "text":
-		return "txt"
-	case "md":
-		return "md"
-	default: // json
-		return "json"
-	}
-}
-
-// runGC drops persistently-errored frontier rows and (optionally) VACUUMs
-// the SQLite file. Designed for long-lived deployments where frontier garbage
-// would otherwise accumulate.
-func runGC(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("gc", flag.ExitOnError)
-	minAttempts := fs.Int("min-attempts", 5, "drop frontier rows with status='error' AND attempts >= N")
-	paraphraseTTL := fs.Duration("paraphrase-ttl", 0, "drop query_paraphrases rows older than this (0 = skip)")
-	vacuum := fs.Bool("vacuum", true, "run VACUUM after cleanup to reclaim disk")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	n, err := s.GCErroredFrontier(ctx, *minAttempts)
-	if err != nil {
-		return fmt.Errorf("gc frontier: %w", err)
-	}
-	log.Printf("gc: removed %d errored frontier rows (min_attempts=%d)", n, *minAttempts)
-
-	if *paraphraseTTL > 0 {
-		pn, err := s.PruneStaleParaphrases(ctx, *paraphraseTTL)
-		if err != nil {
-			return fmt.Errorf("gc paraphrases: %w", err)
-		}
-		log.Printf("gc: pruned %d stale paraphrases (ttl=%s)", pn, *paraphraseTTL)
-	}
-
-	if *vacuum {
-		log.Printf("gc: vacuuming...")
-		if err := s.Vacuum(ctx); err != nil {
-			return fmt.Errorf("vacuum: %w", err)
-		}
-		log.Printf("gc: vacuum complete")
-	}
-	return nil
-}
-
-// runOutcomes exports query_outcomes for offline calibration work. JSON for
-// programmatic consumers, CSV for spreadsheet / pandas notebooks.
-func runOutcomes(ctx context.Context, cfg *config.Config, args []string) (err error) {
-	fs := flag.NewFlagSet("outcomes", flag.ExitOnError)
-	output := fs.String("output", "outcomes.json", "output path")
-	format := fs.String("format", "json", "json | csv")
-	limit := fs.Int("limit", 0, "max rows (0 = all)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *format != "json" && *format != "csv" {
-		return fmt.Errorf("unsupported format %q (use json or csv)", *format)
-	}
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	rows, err := s.ListOutcomes(ctx, *limit)
-	if err != nil {
-		return err
-	}
-	f, cerr := os.Create(*output)
-	if cerr != nil {
-		return cerr
-	}
-	defer func() {
-		if closeErr := f.Close(); closeErr != nil && err == nil {
-			err = fmt.Errorf("close output: %w", closeErr)
-		}
-	}()
-
-	switch *format {
-	case "json":
-		enc := json.NewEncoder(f)
-		enc.SetIndent("", "  ")
-		if err := enc.Encode(rows); err != nil {
-			return err
-		}
-	case "csv":
-		w := csv.NewWriter(f)
-		_ = w.Write([]string{"id", "query", "url", "score", "useful", "source", "recorded_at"})
-		for _, r := range rows {
-			useful := "0"
-			if r.Useful {
-				useful = "1"
-			}
-			_ = w.Write([]string{
-				strconv.FormatInt(r.ID, 10),
-				r.Query, r.URL,
-				strconv.FormatFloat(r.Score, 'f', -1, 64),
-				useful, r.Source,
-				r.RecordedAt.Format(time.RFC3339),
-			})
-		}
-		w.Flush()
-		if err := w.Error(); err != nil {
-			return err
-		}
-	}
-	log.Printf("outcomes: %d rows → %s (%s)", len(rows), *output, *format)
-	return nil
-}
-
-// runDoctor checks the local install: data dir writable, schema open,
-// fixtures present, env recognized. Pure local checks — no network calls,
-// no API key spending. Use `cosift eval` for an end-to-end smoke against
-// real services.
-//
-// Exit code: 0 if everything is PASS or WARN; 1 if any FAIL.
-// doctorCheck is one row of the doctor report. Promoted from a locally-typed
-// struct so the defaults check can be tested without stdout capture.
-type doctorCheck struct {
-	Name   string `json:"name"`
-	Status string `json:"status"` // "PASS" | "WARN" | "FAIL" | "INFO" | "SKIP"
-	Detail string `json:"detail"`
-}
-
-// doctorDefaultsChecks cross-checks the defaults block against
-// configured capabilities. Returns the rows that would be appended to the
-// doctor report. Pure function — no IO, no stdout, no environment reads.
-// hasEmbed/hasChat reflect "is the capability available at all" (config
-// fields populated or env keys present); the caller resolves those before
-// calling.
-func doctorDefaultsChecks(d config.Defaults, hasEmbed, hasChat bool) []doctorCheck {
-	if d.Retriever == "" && !d.Expand && d.ResearchStrategy == "" && d.ResearchSynthK == 0 {
-		return []doctorCheck{{
-			Name:   "defaults",
-			Status: "PASS",
-			Detail: "no defaults set (per-request params win; planner is the /research default)",
-		}}
-	}
-	checks := []doctorCheck{{
-		Name:   "defaults",
-		Status: "PASS",
-		Detail: fmt.Sprintf("retriever=%q expand=%v research_strategy=%q research_synth_k=%d", d.Retriever, d.Expand, d.ResearchStrategy, d.ResearchSynthK),
-	}}
-	if d.ResearchSynthK < 0 {
-		checks = append(checks, doctorCheck{
-			Name:   "defaults.research_synth_k",
-			Status: "FAIL",
-			Detail: fmt.Sprintf("research_synth_k must be ≥ 0 (got %d). 0 = use built-in default (10); positive = cap", d.ResearchSynthK),
-		})
-	}
-	switch d.Retriever {
-	case "dense", "hybrid":
-		if !hasEmbed {
-			checks = append(checks, doctorCheck{
-				Name:   "defaults.retriever",
-				Status: "WARN",
-				Detail: fmt.Sprintf("%q default needs embeddings model — set cfg.Embeddings.Model or OPENAI_API_KEY", d.Retriever),
-			})
-		}
-	case "", "bm25":
-		// no-op; bm25 needs no capability beyond the SQLite index.
-	default:
-		checks = append(checks, doctorCheck{
-			Name:   "defaults.retriever",
-			Status: "FAIL",
-			Detail: fmt.Sprintf("unknown retriever %q (valid: bm25, dense, hybrid)", d.Retriever),
-		})
-	}
-	if d.Expand && !hasChat {
-		checks = append(checks, doctorCheck{
-			Name:   "defaults.expand",
-			Status: "WARN",
-			Detail: "expand=true needs a chat model for paraphrasing — set cfg.Chat.Model or OPENAI_API_KEY",
-		})
-	}
-	switch d.ResearchStrategy {
-	case "paraphrase":
-		if !hasChat {
-			checks = append(checks, doctorCheck{
-				Name:   "defaults.research_strategy",
-				Status: "WARN",
-				Detail: "paraphrase needs a chat model for the paraphraser — set cfg.Chat.Model or OPENAI_API_KEY",
-			})
-		}
-	case "", "planner":
-		// no-op; planner is the historic default.
-	default:
-		checks = append(checks, doctorCheck{
-			Name:   "defaults.research_strategy",
-			Status: "FAIL",
-			Detail: fmt.Sprintf("unknown strategy %q (valid: planner, paraphrase)", d.ResearchStrategy),
-		})
-	}
-	return checks
-}
-
-// runInit writes a sensible default cosift.json to the requested path. By
-// default it refuses to overwrite an existing file; -force opts in.
-//
-// The optional -site URL pre-populates `crawler.include_domains` so the user
-// can immediately `cosift crawl <url>` against a target site without editing
-// the config first. Lowers the onboarding cost for the self-hostable use case.
-func runInit(cfgPath string, args []string) error {
-	fs := flag.NewFlagSet("init", flag.ExitOnError)
-	force := fs.Bool("force", false, "overwrite an existing config file")
-	site := fs.String("site", "", "URL of a site to pre-populate crawler.include_domains with (e.g. https://docs.example.com)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	if _, err := os.Stat(cfgPath); err == nil && !*force {
-		return fmt.Errorf("%s already exists; pass -force to overwrite", cfgPath)
-	}
-
-	cfg := config.Default()
-	if *site != "" {
-		host, err := siteToHost(*site)
-		if err != nil {
-			return fmt.Errorf("-site: %w", err)
-		}
-		cfg.Crawler.IncludeDomains = []string{host}
-	}
-
-	buf, err := json.MarshalIndent(cfg, "", "  ")
-	if err != nil {
-		return fmt.Errorf("marshal: %w", err)
-	}
-	// cosift.json carries Server.AdminToken and may be the sink for
-	// LoadDotEnv-sourced secrets (OPENAI_API_KEY, etc.). Owner-only.
-	if err := os.WriteFile(cfgPath, buf, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", cfgPath, err)
-	}
-
-	fmt.Printf("wrote %s\n", cfgPath)
-	if cfg.Crawler.IncludeDomains != nil {
-		fmt.Printf("  crawler.include_domains: %v\n", cfg.Crawler.IncludeDomains)
-	}
-	fmt.Println()
-	fmt.Println("next steps:")
-	if *site != "" {
-		fmt.Printf("  1. cosift crawl %s\n", *site)
-	} else {
-		fmt.Println("  1. cosift crawl <url...>")
-	}
-	fmt.Println("  2. cosift serve")
-	fmt.Println()
-	fmt.Println("optional: set OPENAI_API_KEY in your environment (or .env) to enable")
-	fmt.Println("  dense/hybrid retrieval, /answer, /research, and ?expand=true.")
-	return nil
-}
-
-// siteToHost extracts the hostname from a URL or returns the input verbatim if
-// it's already a bare host (e.g. "docs.example.com" vs "https://docs.example.com").
-// Used by `cosift init -site` to populate include_domains.
-func siteToHost(s string) (string, error) {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return "", errors.New("empty site")
-	}
-	// If no scheme, treat as bare host and validate.
-	if !strings.Contains(s, "://") {
-		// reject if it looks like a path or has whitespace
-		if strings.ContainsAny(s, " /\t\n") {
-			return "", fmt.Errorf("not a hostname: %q", s)
-		}
-		return s, nil
-	}
-	u, err := url.Parse(s)
-	if err != nil {
-		return "", err
-	}
-	if u.Host == "" {
-		return "", fmt.Errorf("could not extract host from %q", s)
-	}
-	return u.Host, nil
-}
-
-// runCheckRobots reports whether each URL is crawlable under the site's
-// robots.txt, plus any Crawl-delay. Lets operators plan crawls without
-// hand-inspecting the site's robots.txt.
-//
-// Wraps the existing internal/crawler.Robots so the CLI uses exactly the
-// same enforcement logic the real crawler would. No drift.
-func runCheckRobots(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("check-robots", flag.ExitOnError)
-	userAgent := fs.String("user-agent", cfg.Crawler.UserAgent, "User-Agent the check pretends to be (defaults to cfg.crawler.user_agent)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	urls := fs.Args()
-	if len(urls) == 0 {
-		return errors.New("usage: cosift check-robots [-user-agent UA] <url...>")
-	}
-	if *userAgent == "" {
-		*userAgent = "CosiftBot/0.0 (+https://github.com/pilot-protocol/cosift)"
-	}
-	httpClient := &http.Client{Timeout: 15 * time.Second}
-	r := crawler.NewRobots(httpClient, *userAgent)
-
-	fmt.Printf("user-agent: %s\n", *userAgent)
-
-	// Probe robots.txt for each unique host before running per-URL checks.
-	// Robots.Allowed returns (true, ...) when robots.txt is unreachable
-	// (graceful degradation suits the crawler), but the operator running
-	// check-robots wants to see "this host has no reachable robots.txt"
-	// explicitly. Group URLs by host so we only probe each host once.
-	hosts := uniqueHosts(urls)
-	fmt.Println()
-	for _, h := range hosts {
-		probeURL := h + "/robots.txt"
-		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, http.NoBody)
-		req.Header.Set("User-Agent", *userAgent)
-		resp, err := httpClient.Do(req)
-		if err != nil {
-			fmt.Printf("robots.txt   %s — UNREACHABLE: %v (crawler will assume ALLOWED for everything on this host)\n", probeURL, err)
-			continue
-		}
-		_ = resp.Body.Close()
-		switch {
-		case resp.StatusCode == 200:
-			fmt.Printf("robots.txt   %s — OK (status 200)\n", probeURL)
-		case resp.StatusCode == 404:
-			fmt.Printf("robots.txt   %s — 404, no rules (crawler will assume ALLOWED)\n", probeURL)
-		default:
-			fmt.Printf("robots.txt   %s — status %d (crawler will assume ALLOWED on non-2xx/404)\n", probeURL, resp.StatusCode)
-		}
-		// surface Sitemap: directives from robots.txt. Modern
-		// crawlers auto-discover sitemaps this way; this output lets operators
-		// see what `cosift crawl -sitemap <URL>` could seed from.
-		if sitemaps := r.Sitemaps(ctx, h); len(sitemaps) > 0 {
-			for _, sm := range sitemaps {
-				fmt.Printf("  Sitemap:   %s\n", sm)
-			}
-		}
-	}
-	fmt.Println()
-	for _, u := range urls {
-		allowed, crawlDelay, err := r.Allowed(ctx, u)
-		if err != nil {
-			fmt.Printf("[ERR ]  %s — %v\n", u, err)
-			continue
-		}
-		mark := "[OK  ]"
-		if !allowed {
-			mark = "[DENY]"
-		}
-		extra := ""
-		if crawlDelay > 0 {
-			extra = fmt.Sprintf("  crawl-delay=%s", crawlDelay)
-		}
-		fmt.Printf("%s  %s%s\n", mark, u, extra)
-	}
-	return nil
-}
-
-// uniqueHosts extracts the scheme+host part from each URL, deduped + in order.
-// Used by check-robots to probe each host's robots.txt exactly once.
-func uniqueHosts(urls []string) []string {
-	seen := make(map[string]bool)
-	out := []string{}
-	for _, raw := range urls {
-		u, err := url.Parse(raw)
-		if err != nil || u.Host == "" {
-			continue
-		}
-		key := u.Scheme + "://" + u.Host
-		if !seen[key] {
-			seen[key] = true
-			out = append(out, key)
-		}
-	}
-	return out
-}
-
-// runCrawlErrors lists recently-errored frontier URLs with their failure reason.
-// operator diagnostic for "why are 142 URLs in error state?" without
-// requiring SQLite shell access. Pure read-only; no side effects on the index.
-//
-// The error column is's addition to the frontier schema; pre
-// errored URLs will have an empty reason.
-func runCrawlErrors(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("crawl-errors", flag.ExitOnError)
-	limit := fs.Int("limit", 50, "max number of errored URLs to list (most recent first)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return fmt.Errorf("open store: %w", err)
-	}
-	defer s.Close()
-
-	errs, err := s.ListErroredFrontier(ctx, *limit)
-	if err != nil {
-		return fmt.Errorf("list errored: %w", err)
-	}
-	if len(errs) == 0 {
-		fmt.Println("no errored frontier entries.")
-		return nil
-	}
-	fmt.Printf("%d errored frontier entries (most recent first):\n", len(errs))
-	fmt.Println()
-	for _, e := range errs {
-		reason := e.LastError
-		if reason == "" {
-			reason = "(no reason recorded — entry predates)"
-		}
-		fmt.Printf("  attempts=%d  %s\n    → %s\n", e.Attempts, e.URL, reason)
-	}
-	return nil
-}
-
-func runDoctor(ctx context.Context, cfg *config.Config, args []string) error {
-	// When -server is empty, doctor stays
-	// purely local. When -server is set, doctor also pings
-	// the server's /healthz and /stats; if -token is also set (or env), it
-	// validates the token by hitting /admin/config.
-	fs := flag.NewFlagSet("doctor", flag.ExitOnError)
-	remoteServer := fs.String("server", "", "remote cosift URL to additionally probe (empty = local checks only)")
-	remoteToken := fs.String("token", "", "admin bearer token for -server admin-endpoint check (defaults to COSIFT_ADMIN_TOKEN env)")
-	// Same checks, same
-	// non-zero exit on FAIL, just JSON instead of '[PASS] name — detail'.
-	asJSON := fs.Bool("json", false, "emit JSON instead of human-readable text (suitable for jq / CI)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	var checks []doctorCheck
-	add := func(name, status, detail string) { checks = append(checks, doctorCheck{name, status, detail}) }
-
-	// 1. Data dir writable.
-	probe := filepath.Join(cfg.DataDir, ".doctor.probe")
-	// Match store.Open's 0o700 — MkdirAll's perm only applies to NEW dirs,
-	// but doctor running first must not silently leave a looser mode behind.
-	if err := os.MkdirAll(cfg.DataDir, 0o700); err != nil {
-		add("data_dir writable", "FAIL", fmt.Sprintf("mkdir %s: %v", cfg.DataDir, err))
-	} else if err := os.WriteFile(probe, []byte("ok"), 0o600); err != nil {
-		add("data_dir writable", "FAIL", fmt.Sprintf("write %s: %v", probe, err))
-	} else {
-		_ = os.Remove(probe)
-		add("data_dir writable", "PASS", cfg.DataDir)
-	}
-
-	// 2. SQLite open + schema migrated.
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		add("sqlite open + schema", "FAIL", err.Error())
-	} else {
-		st, _ := s.Stats(ctx)
-		_ = s.Close()
-		add("sqlite open + schema", "PASS", fmt.Sprintf("%d docs, %d terms", st.Documents, st.Terms))
-	}
-
-	// 2b. Pebble open + counters. Pebble is optional; "absent" is
-	// not a failure, just a SKIP. When present, verify open + read counters.
-	// the open can fail because pebble-serve / a crawl holds the
-	// single-writer lock — that's a healthy state, not a FAIL. Distinguish
-	// the lock error from real corruption.
-	pebbleDir := filepath.Join(cfg.DataDir, "pebble")
-	if _, statErr := os.Stat(pebbleDir); statErr != nil {
-		add("pebble store", "SKIP", "no pebble dir (sqlite-only deployment)")
-	} else if ps, err := store.OpenPebble(pebbleDir); err != nil {
-		msg := err.Error()
-		if strings.Contains(msg, "lock") || strings.Contains(msg, "resource temporarily unavailable") {
-			add("pebble store", "INFO", "another process holds the writer lock (pebble-serve / crawl in flight)")
-		} else {
-			add("pebble store", "FAIL", msg)
-		}
-	} else {
-		_, n, _ := ps.CorpusStats(ctx)
-		// also report HNSW availability so operators can tell from
-		// `cosift doctor` whether dense / hybrid retrievers will work without
-		// firing pebble-info or pebble-serve /stats. Cheap — same 20-byte
-		// meta blob read pebble-info already does.
-		meta, hnswOK, _ := index.LoadHNSWMeta(ctx, ps)
-		_ = ps.Close()
-		add("pebble store", "PASS", fmt.Sprintf("%d indexed docs", n))
-		if hnswOK {
-			add("HNSW graph", "PASS", fmt.Sprintf("%d nodes, dim=%d — dense/hybrid available (load with COSIFT_LOAD_HNSW=true)", meta.NodeCount, meta.Dim))
-		} else {
-			add("HNSW graph", "INFO", "no vectors persisted — only bm25 / bm25-mlt retrievers (set cfg.Embeddings.Model and re-crawl to add dense)")
-		}
-	}
-
-	// 2c. COSIFT_* env vars. Lists which path-2 env overrides are
-	// set so operators can confirm 'is my override actually live'. Silent
-	// typos previously hid behind 'env unset → keep default' branches.
-	cosiftEnvs := []string{
-		"COSIFT_PEBBLE_CACHE_MB", "COSIFT_PEBBLE_MEMTABLE_MB", "COSIFT_PEBBLE_MEMTABLES", "COSIFT_PEBBLE_SYNC",
-		"COSIFT_BM25_K1", "COSIFT_BM25_B",
-		"COSIFT_HYDE_CACHE_SIZE", "COSIFT_PARA_CACHE_SIZE",
-		"COSIFT_LOAD_HNSW", //
-	}
-	var setEnvs []string
-	for _, name := range cosiftEnvs {
-		if v := os.Getenv(name); v != "" {
-			setEnvs = append(setEnvs, name+"="+v)
-		}
-	}
-	if len(setEnvs) == 0 {
-		add("COSIFT_* env", "INFO", "no path-2 overrides set (using defaults)")
-	} else {
-		add("COSIFT_* env", "INFO", strings.Join(setEnvs, ", "))
-	}
-
-	// 3. Config recognized.
-	add("config", "PASS", fmt.Sprintf("addr=%s, data_dir=%s", cfg.Server.Addr, cfg.DataDir))
-
-	// 4. API key env (warn — actually pinging is `cosift eval` territory).
-	// a custom embedder/chat URL (Ollama / vLLM / TEI) doesn't
-	// need a key; PASS when URL is set even without a key. Hosted defaults
-	// (empty URL) still require a key for the configured capability.
-	embedKey := resolveEmbedAPIKey()
-	chatKey := os.Getenv("OPENAI_API_KEY")
-	if chatKey == "" {
-		chatKey = os.Getenv("OPENAI")
-	}
-	embedConfigured := cfg.Embeddings.Model != ""
-	chatConfigured := cfg.Chat.Model != ""
-	embedLocal := cfg.Embeddings.URL != ""
-	chatLocal := cfg.Chat.URL != ""
-	switch {
-	case !embedConfigured && !chatConfigured:
-		add("embed/chat auth", "INFO", "no embed/chat models configured (fine for bm25-only)")
-	case (embedConfigured && (embedKey != "" || embedLocal)) && (!chatConfigured || chatKey != "" || chatLocal):
-		add("embed/chat auth", "PASS", fmt.Sprintf("embed=%s chat=%s", authStatus(embedConfigured, embedKey, embedLocal), authStatus(chatConfigured, chatKey, chatLocal)))
-	default:
-		var missing []string
-		if embedConfigured && embedKey == "" && !embedLocal {
-			missing = append(missing, "embed (no URL, no key)")
-		}
-		if chatConfigured && chatKey == "" && !chatLocal {
-			missing = append(missing, "chat (no URL, no key)")
-		}
-		add("embed/chat auth", "WARN", strings.Join(missing, ", ")+" — set COSIFT_EMBED_API_KEY / OPENAI_API_KEY, or point Embeddings.URL / Chat.URL at a local server")
-	}
-
-	// 5. Eval fixtures.
-	corpusPath := "testdata/eval/corpus.json"
-	queriesPath := "testdata/eval/queries.json"
-	if _, err := os.Stat(corpusPath); err != nil {
-		add("eval fixtures", "WARN", "testdata/eval/corpus.json missing (eval won't run)")
-	} else if _, err := os.Stat(queriesPath); err != nil {
-		add("eval fixtures", "WARN", "testdata/eval/queries.json missing")
-	} else {
-		add("eval fixtures", "PASS", "corpus.json + queries.json present")
-	}
-
-	// 6. Admin token configured if /admin/* is to be used.
-	if cfg.Server.AdminToken != "" {
-		add("admin_token", "PASS", "set (length redacted)")
-	} else {
-		add("admin_token", "WARN", "unset — /admin/* will return 403")
-	}
-
-	// 7. Defaults vs capabilities. The defaults block lets operators
-	// pre-configure retrieval behavior, but a default that requires capability
-	// the server isn't configured for (e.g. retriever=hybrid without an
-	// embedder) will silently fail at request time — 400 on every call. Catch
-	// the mismatch HERE instead of at first traffic.
-	hasEmbed := cfg.Embeddings.Model != "" || os.Getenv("OPENAI_API_KEY") != "" || os.Getenv("OPENAI") != ""
-	hasChat := cfg.Chat.Model != "" || os.Getenv("OPENAI_API_KEY") != "" || os.Getenv("OPENAI") != ""
-	checks = append(checks, doctorDefaultsChecks(cfg.Defaults, hasEmbed, hasChat)...)
-
-	// 8. Remote checks — only when -server is set. Token resolution
-	// mirrors admin CLIs: -token flag wins, then env.
-	if *remoteServer != "" {
-		resolvedToken := *remoteToken
-		if resolvedToken == "" {
-			resolvedToken = os.Getenv("COSIFT_ADMIN_TOKEN")
-		}
-		checks = append(checks, doctorRemoteChecks(ctx, *remoteServer, resolvedToken)...)
-	}
-
-	fail := 0
-	for _, c := range checks {
-		if c.Status == "FAIL" {
-			fail++
-		}
-	}
-	if *asJSON {
-		out := map[string]any{
-			"checks": checks,
-			"fail":   fail,
-			"ready":  fail == 0,
-		}
-		blob, _ := json.MarshalIndent(out, "", "  ")
-		fmt.Println(string(blob))
-		if fail > 0 {
-			return errors.New("doctor failures")
-		}
-		return nil
-	}
-	for _, c := range checks {
-		fmt.Printf("%-6s %s — %s\n", "["+c.Status+"]", c.Name, c.Detail)
-	}
-	fmt.Println()
-	if fail > 0 {
-		fmt.Printf("%d FAIL — fix the issues above before running `cosift serve`\n", fail)
-		return errors.New("doctor failures")
-	}
-	fmt.Println("ready.")
-	return nil
-}
-
-// doctorRemoteChecks probes a remote cosift server: /healthz reachable, /stats
-// returns a sensible shape, and (when token non-empty) /admin/config validates
-// the bearer token. Returns a slice of doctorCheck rows for the main runDoctor
-// loop to render.
-func doctorRemoteChecks(ctx context.Context, serverURL, token string) []doctorCheck {
-	var out []doctorCheck
-	base := strings.TrimRight(strings.Replace(serverURL, "0.0.0.0", "127.0.0.1", 1), "/")
-	client := &http.Client{Timeout: 10 * time.Second}
-
-	// /healthz
-	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, base+"/healthz", http.NoBody)
-	resp, err := client.Do(req)
-	if err != nil {
-		out = append(out, doctorCheck{"remote /healthz", "FAIL", fmt.Sprintf("%s: %v", serverURL, err)})
-		// If /healthz is unreachable, downstream checks will all fail too —
-		// emit one row and skip the cascade.
-		return out
-	}
-	resp.Body.Close()
-	if resp.StatusCode == http.StatusOK {
-		out = append(out, doctorCheck{"remote /healthz", "PASS", base})
-	} else {
-		out = append(out, doctorCheck{"remote /healthz", "FAIL", fmt.Sprintf("%s returned %d", base, resp.StatusCode)})
-	}
-
-	// /stats
-	req, _ = http.NewRequestWithContext(ctx, http.MethodGet, base+"/stats", http.NoBody)
-	resp, err = client.Do(req)
-	if err != nil {
-		out = append(out, doctorCheck{"remote /stats", "FAIL", err.Error()})
-	} else {
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			out = append(out, doctorCheck{"remote /stats", "FAIL", fmt.Sprintf("returned %d", resp.StatusCode)})
-		} else {
-			var st server.StatsResponse
-			body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
-			if err := json.Unmarshal(body, &st); err != nil {
-				out = append(out, doctorCheck{"remote /stats", "WARN", fmt.Sprintf("decode: %v", err)})
-			} else {
-				out = append(out, doctorCheck{"remote /stats", "PASS", fmt.Sprintf("%d docs", st.Documents)})
-			}
-		}
-	}
-
-	// /admin/config — only when token is provided. No token → skip silently
-	// (not a failure; operator may not have admin auth enabled).
-	if token != "" {
-		req, _ = http.NewRequestWithContext(ctx, http.MethodGet, base+"/admin/config", http.NoBody)
-		req.Header.Set("Authorization", "Bearer "+token)
-		resp, err = client.Do(req)
-		if err != nil {
-			out = append(out, doctorCheck{"remote admin token", "FAIL", err.Error()})
-			return out
-		}
-		defer resp.Body.Close()
-		switch resp.StatusCode {
-		case http.StatusOK:
-			out = append(out, doctorCheck{"remote admin token", "PASS", "/admin/config accepted the token"})
-		case http.StatusUnauthorized, http.StatusForbidden:
-			out = append(out, doctorCheck{"remote admin token", "FAIL", fmt.Sprintf("token rejected (HTTP %d)", resp.StatusCode)})
-		default:
-			out = append(out, doctorCheck{"remote admin token", "WARN", fmt.Sprintf("unexpected status %d", resp.StatusCode)})
-		}
-	}
-	return out
-}
-
-// runReembed walks every document, chunks it, embeds the chunks with the
-// configured model, and writes them as new passages. The new rows coexist with
-// the old (different model_id) until `-drop-old` is passed.
-//
-// Recommended sequence:
-//
-//	cosift reembed                       # write new passages alongside old
-//	# verify search quality on new model …
-//	cosift reembed -drop-old             # drop the others
-//
-// Or just `cosift reembed -drop-old` if you're confident.
-func runReembed(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("reembed", flag.ExitOnError)
-	dropOld := fs.Bool("drop-old", false, "after re-embed, delete passages with a different model_id")
-	batchSize := fs.Int("batch", 256, "embeddings per API call (provider-dependent cap)")
-	// time-based progress reporting (shared helper from).
-	progressInterval := fs.Duration("progress", 5*time.Second, "log re-embed progress every N (0 disables)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if cfg.Embeddings.Model == "" {
-		return errors.New("reembed: cfg.Embeddings.Model must be set")
-	}
-	apiKey := os.Getenv("OPENAI_API_KEY")
-	if apiKey == "" {
-		apiKey = os.Getenv("OPENAI")
-	}
-	if apiKey == "" {
-		return errors.New("reembed: no OPENAI key in env")
-	}
-	dim := cfg.Embeddings.Dim
-	if dim == 0 {
-		dim = 1536
-	}
-	emb := embed.NewOpenAIClient(apiKey, cfg.Embeddings.URL, cfg.Embeddings.Model, dim)
-
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	docs, err := s.ListDocuments(ctx, 0)
-	if err != nil {
-		return err
-	}
-	if len(docs) == 0 {
-		log.Printf("reembed: 0 docs in store — nothing to do")
-		return nil
-	}
-
-	// Pre-existing models (informational; useful for the operator).
-	models, _ := s.PassageModels(ctx)
-	log.Printf("reembed: %d docs · target model=%s · existing models=%v · batch=%d",
-		len(docs), cfg.Embeddings.Model, models, *batchSize)
-
-	chunker := chunkerWith(cfg.Crawler.ChunkSize, cfg.Crawler.ChunkOverlap)
-	type ref struct {
-		docID int64
-		chunk index.Chunk
-	}
-	var (
-		texts   []string
-		refs    []ref
-		total   int
-		flush   func() error
-		written int
-	)
-	flush = func() error {
-		if len(texts) == 0 {
-			return nil
-		}
-		vecs, err := emb.Embed(ctx, texts)
-		if err != nil {
-			return fmt.Errorf("embed batch (%d texts): %w", len(texts), err)
-		}
-		if len(vecs) != len(texts) {
-			return fmt.Errorf("embed count mismatch: %d vs %d", len(vecs), len(texts))
-		}
-		for i, v := range vecs {
-			if err := s.UpsertPassage(ctx, &store.Passage{
-				DocID:     refs[i].docID,
-				Offset:    refs[i].chunk.Offset,
-				Length:    refs[i].chunk.Length,
-				Model:     emb.Model(),
-				Embedding: v,
-			}); err != nil {
-				return fmt.Errorf("write passage: %w", err)
-			}
-			written++
-		}
-		texts = texts[:0]
-		refs = refs[:0]
-		return nil
-	}
-
-	docProgress := newProgressReporter("reembed docs", len(docs), *progressInterval)
-	for docIdx, d := range docs {
-		for _, c := range chunker.Chunk(d.Title + "\n\n" + d.Text) {
-			texts = append(texts, c.Text)
-			refs = append(refs, ref{docID: d.ID, chunk: c})
-			total++
-			if len(texts) >= *batchSize {
-				if err := flush(); err != nil {
-					return err
-				}
-			}
-		}
-		docProgress.maybeLog(docIdx + 1)
-	}
-	if err := flush(); err != nil {
-		return err
-	}
-	log.Printf("reembed: wrote %d passages with model=%s", written, cfg.Embeddings.Model)
-
-	if *dropOld {
-		n, err := s.DropPassagesNotModel(ctx, cfg.Embeddings.Model)
-		if err != nil {
-			return fmt.Errorf("drop old: %w", err)
-		}
-		log.Printf("reembed: dropped %d passages from other models", n)
-	} else if len(models) > 0 && !contains(models, cfg.Embeddings.Model) {
-		log.Printf("reembed: %d other-model passages remain — run with -drop-old to remove", len(models))
-	}
-	return nil
-}
-
-func contains(ss []string, x string) bool {
-	for _, s := range ss {
-		if s == x {
-			return true
-		}
-	}
-	return false
-}
-
-// runCompactIndex drops orphaned passages and terms, then VACUUMs.
-// Companion to `cosift gc` — gc cleans the frontier; compact-index cleans
-// the index proper.
-func runCompactIndex(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("compact-index", flag.ExitOnError)
-	vacuum := fs.Bool("vacuum", true, "run VACUUM after deletes to reclaim disk")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	dropped, err := s.DropOrphanedPassages(ctx)
-	if err != nil {
-		return fmt.Errorf("drop orphan passages: %w", err)
-	}
-	log.Printf("compact-index: dropped %d orphan passages", dropped)
-
-	terms, err := s.DropOrphanedTerms(ctx)
-	if err != nil {
-		return fmt.Errorf("drop orphan terms: %w", err)
-	}
-	log.Printf("compact-index: dropped %d orphan terms", terms)
-
-	if *vacuum {
-		log.Printf("compact-index: vacuuming...")
-		if err := s.Vacuum(ctx); err != nil {
-			return fmt.Errorf("vacuum: %w", err)
-		}
-		log.Printf("compact-index: vacuum complete")
-	}
-	return nil
-}
-
-// batchEmbed chunks a large text slice into provider-safe batches before
-// calling the embedder. OpenAI's text-embedding-3-* caps at 2048 inputs OR
-// 300k tokens per request; 1000 is a safe default that respects both at
-// our typical chunk sizes. Returns vectors in the input order.
-//
-// Between batches we sleep 200ms — keeps multi-batch runs under OpenAI's
-// 1M-TPM org limit for text-embedding-3-small, which hit at ~50k inputs.
-// Provider rate-limit errors get one bounded retry with the suggested delay.
-func batchEmbed(ctx context.Context, e embed.Embedder, texts []string, batchSize int) ([][]float32, error) {
-	if batchSize <= 0 {
-		batchSize = 256
-	}
-	// 1000 is the published OpenAI per-call cap, but large-dim models
-	// (text-embedding-3-large @ 3072) return malformed JSON well before that
-	// in practice. 256 is safe across the model lineup.
-	if batchSize > 1000 {
-		batchSize = 1000
-	}
-	if len(texts) <= batchSize {
-		return e.Embed(ctx, texts)
-	}
-	out := make([][]float32, 0, len(texts))
-	for i := 0; i < len(texts); i += batchSize {
-		end := i + batchSize
-		if end > len(texts) {
-			end = len(texts)
-		}
-		vecs, err := embedWithRetry(ctx, e, texts[i:end])
-		if err != nil {
-			return nil, fmt.Errorf("batch %d-%d: %w", i, end, err)
-		}
-		if len(vecs) != end-i {
-			return nil, fmt.Errorf("batch %d-%d: %d returned, expected %d", i, end, len(vecs), end-i)
-		}
-		out = append(out, vecs...)
-		if end < len(texts) {
-			select {
-			case <-ctx.Done():
-				return out, ctx.Err()
-			case <-time.After(200 * time.Millisecond):
-			}
-		}
-	}
-	return out, nil
-}
-
-// embedWithRetry retries once on rate-limit errors with a fixed backoff.
-// Crude but effective at our scale — keeps full distractor runs from failing
-// when a single batch slips past the TPM bucket. Production deployments with
-// higher request volumes want exponential backoff + provider-supplied delay
-// parsing; deferred until traffic warrants.
-func embedWithRetry(ctx context.Context, e embed.Embedder, texts []string) ([][]float32, error) {
-	vecs, err := e.Embed(ctx, texts)
-	if err == nil {
-		return vecs, nil
-	}
-	msg := err.Error()
-	if !strings.Contains(msg, "rate_limit") && !strings.Contains(msg, "429") {
-		return nil, err
-	}
-	select {
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	case <-time.After(2 * time.Second):
-	}
-	return e.Embed(ctx, texts)
-}
-
-// paraphraseRetriever wraps any Retriever with on-the-fly query expansion via
-// a chat client. For each search, asks the LLM for N paraphrases, runs all of
-// them through the inner retriever, RRF-fuses the result lists.
-//
-// measured that hand-written paraphrases recover 0.02 nDCG at 10k
-// distractors. This is the auto-generated version — removes the hand-writing
-// requirement and makes the prescription deployable. Per-query overhead:
-// one extra chat call (~$0.0001 at gpt-4o-mini) + N extra inner-retriever calls.
-//
-// In-memory cache by exact query text — eval re-runs hit it; production /search
-// calls don't (different process). For production, the cache would move into
-// the SQLite store or a small Redis-shaped layer.
-type paraphraseRetriever struct {
-	inner      eval.Retriever
-	chat       embed.ChatClient
-	n          int
-	mainWeight float64 // 0 = equal-weight RRF (pre); >0 = main query gets this weight, paraphrases each 1.0
-	cache      map[string][]string
-	mu         sync.Mutex
-}
-
-const paraphraseSystem = `Generate paraphrases of a search query. Each paraphrase preserves the semantic intent but uses different vocabulary — different keywords that a target document might also use. Output ONLY a JSON array of strings.
-Example output for "go programming language": ["golang concurrent compiled language", "Google's systems programming language with goroutines"]`
-
-func (p *paraphraseRetriever) generateParaphrases(ctx context.Context, q string) []string {
-	p.mu.Lock()
-	if cached, ok := p.cache[q]; ok {
-		p.mu.Unlock()
-		return cached
-	}
-	p.mu.Unlock()
-
-	resp, err := p.chat.Chat(ctx, []embed.ChatMsg{
-		{Role: "system", Content: paraphraseSystem},
-		{Role: "user", Content: fmt.Sprintf("Generate %d paraphrases of: %s", p.n, q)},
-	})
-	if err != nil {
-		return nil
-	}
-	// Same robust JSON-array parsing as the research planner.
-	raw := strings.TrimSpace(resp)
-	for _, fence := range []string{"```json", "```"} {
-		raw = strings.TrimPrefix(raw, fence)
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	raw = strings.TrimSpace(raw)
-	start := strings.Index(raw, "[")
-	end := strings.LastIndex(raw, "]")
-	if start < 0 || end <= start {
-		return nil
-	}
-	var arr []string
-	if err := json.Unmarshal([]byte(raw[start:end+1]), &arr); err != nil {
-		return nil
-	}
-	if len(arr) > p.n {
-		arr = arr[:p.n]
-	}
-	p.mu.Lock()
-	p.cache[q] = arr
-	p.mu.Unlock()
-	return arr
-}
-
-func (p *paraphraseRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
-	main, err := p.inner.Search(ctx, q, k)
-	if err != nil {
-		return nil, err
-	}
-	paras := p.generateParaphrases(ctx, q)
-	if len(paras) == 0 {
-		return main, nil
-	}
-	lists := make([][]string, 0, 1+len(paras))
-	lists = append(lists, main)
-	for _, p2 := range paras {
-		extra, err := p.inner.Search(ctx, p2, k)
-		if err != nil {
-			continue
-		}
-		lists = append(lists, extra)
-	}
-	// Matches the server expand
-	// path's behavior so eval-time measurements use the same fusion
-	// math as production.
-	var weights []float64
-	if p.mainWeight > 0 {
-		weights = make([]float64, len(lists))
-		weights[0] = p.mainWeight
-		for i := 1; i < len(weights); i++ {
-			weights[i] = 1.0
-		}
-	}
-	return index.RRFWeighted(lists, weights, k, 60), nil
-}
-
-// plannerRetriever wraps any Retriever with the /research planner's
-// decomposition step: ask the LLM for 2-3 focused sub-queries, run each through
-// the inner retriever, RRF-fuse the results. Mirrors /research minus the synth.
-//
-// This exists so we can measure whether the planner's decomposition
-// already subsumes paraphrase expansion — that claim was asserted in
-// when we chose not to add ?expand=true to /research, but never measured.
-const plannerSystemPrompt = `Decompose the user's research question into 2-3 focused sub-queries that, taken together, would cover the answer. Output ONLY a JSON array of strings — no prose, no markdown. Example: ["sub-query 1", "sub-query 2"]`
-
-type plannerRetriever struct {
-	inner      eval.Retriever
-	chat       embed.ChatClient
-	mainWeight float64 // 0 = equal-weight RRF; >0 = main query gets this weight, sub-queries each 1.0
-	cache      map[string][]string
-	mu         sync.Mutex
-}
-
-func (p *plannerRetriever) plan(ctx context.Context, q string) []string {
-	p.mu.Lock()
-	if cached, ok := p.cache[q]; ok {
-		p.mu.Unlock()
-		return cached
-	}
-	p.mu.Unlock()
-
-	resp, err := p.chat.Chat(ctx, []embed.ChatMsg{
-		{Role: "system", Content: plannerSystemPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		return nil
-	}
-	raw := strings.TrimSpace(resp)
-	for _, fence := range []string{"```json", "```"} {
-		raw = strings.TrimPrefix(raw, fence)
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	raw = strings.TrimSpace(raw)
-	start := strings.Index(raw, "[")
-	end := strings.LastIndex(raw, "]")
-	if start < 0 || end <= start {
-		return nil
-	}
-	var arr []string
-	if err := json.Unmarshal([]byte(raw[start:end+1]), &arr); err != nil {
-		return nil
-	}
-	if len(arr) > 5 {
-		arr = arr[:5] // hard cap matches /research planner
-	}
-	p.mu.Lock()
-	p.cache[q] = arr
-	p.mu.Unlock()
-	return arr
-}
-
-func (p *plannerRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
-	main, err := p.inner.Search(ctx, q, k)
-	if err != nil {
-		return nil, err
-	}
-	subs := p.plan(ctx, q)
-	if len(subs) == 0 {
-		return main, nil
-	}
-	lists := make([][]string, 0, 1+len(subs))
-	lists = append(lists, main)
-	for _, sq := range subs {
-		extra, err := p.inner.Search(ctx, sq, k)
-		if err != nil {
-			continue
-		}
-		lists = append(lists, extra)
-	}
-	// Same shape as
-	// paraphraseRetriever; main query gets mainWeight, sub-queries each 1.0.
-	var weights []float64
-	if p.mainWeight > 0 {
-		weights = make([]float64, len(lists))
-		weights[0] = p.mainWeight
-		for i := 1; i < len(weights); i++ {
-			weights[i] = 1.0
-		}
-	}
-	return index.RRFWeighted(lists, weights, k, 60), nil
-}
-
-// httpAPIRetriever queries a deployed cosift's /search endpoint and adapts the
-// JSON response to eval.Retriever. Lets `cosift eval -api <url>` validate a
-// remote deployment with the same metric harness used for local development.
-type httpAPIRetriever struct {
-	baseURL   string
-	bearer    string
-	retriever string
-	rerank    bool
-	http      *http.Client
-}
-
-func (r *httpAPIRetriever) Search(ctx context.Context, q string, k int) ([]string, error) {
-	v := url.Values{}
-	v.Set("q", q)
-	v.Set("k", strconv.Itoa(k))
-	if r.retriever != "" {
-		v.Set("retriever", r.retriever)
-	}
-	if r.rerank {
-		v.Set("rerank", "true")
-	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.baseURL+"/search?"+v.Encode(), http.NoBody)
-	if err != nil {
-		return nil, err
-	}
-	if r.bearer != "" {
-		req.Header.Set("Authorization", "Bearer "+r.bearer)
-	}
-	resp, err := r.http.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
-		return nil, fmt.Errorf("api %d: %s", resp.StatusCode, body)
-	}
-	var sr struct {
-		Hits []struct {
-			URL string `json:"url"`
-		} `json:"hits"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&sr); err != nil {
-		return nil, fmt.Errorf("decode: %w", err)
-	}
-	urls := make([]string, len(sr.Hits))
-	for i, h := range sr.Hits {
-		urls[i] = h.URL
-	}
-	return urls, nil
-}
-
-// neutralVocabForDistractors returns words that don't appear in the eval queries
-// or relevant docs, so any distractor returning in top-K is unambiguous noise.
-// Picked from common-but-eval-disjoint domains (pottery, lighthouses, knitting,
-// dentistry, beekeeping, etc.).
-func neutralVocabForDistractors() []string {
-	return []string{
-		"kiln", "pottery", "glaze", "ceramic", "wheel", "stoneware", "earthenware",
-		"lighthouse", "beacon", "keeper", "lantern", "rocky", "shoal", "fog",
-		"yarn", "knit", "purl", "stitch", "skein", "needle", "cable",
-		"dentistry", "molar", "incisor", "filling", "enamel", "gingival",
-		"beekeeping", "hive", "queen", "drone", "wax", "pollen", "swarm",
-		"bookbinding", "spine", "endpaper", "cloth", "headband", "signature",
-		"masonry", "mortar", "trowel", "course", "wythe", "lintel", "soffit",
-		"falconry", "raptor", "perch", "lure", "jess", "creance", "mews",
-		"weaving", "warp", "weft", "shuttle", "heddle", "treadle", "selvedge",
-		"smithing", "forge", "anvil", "tongs", "quench", "temper", "billet",
-		"glassblowing", "punty", "marver", "blowpipe", "annealing", "cane",
-		"watchmaking", "balance", "escapement", "mainspring", "barrel", "jewel",
-		"taxidermy", "tan", "mount", "armature", "habitat", "diorama",
-		"calligraphy", "nib", "ink", "stroke", "kerning", "ascender", "descender",
-		"thatching", "reed", "spar", "ridge", "rafter", "eaves",
-	}
-}
-
-// generateDistractorText assembles a short text out of the neutral vocab.
-func generateDistractorText(rng *rand.Rand, vocab []string, words int) string {
-	parts := make([]string, words)
-	for i := range parts {
-		parts[i] = vocab[rng.Intn(len(vocab))]
-	}
-	return strings.Join(parts, " ")
-}
-
-// runAnswerEval is's measurement of /research answer quality across
-// the two retrieval strategies (planner vs paraphrase). Iters 52-53 measured
-// *retrieval coverage* via nDCG and found paraphrase wins; added
-// ?strategy=paraphrase but kept planner as the default *because we hadn't
-// measured synthesis quality*. This subcommand fills that gap.
-//
-// Flow per query:
-//  1. Build an in-memory cosift HTTP server with corpus ingested, dense index
-//     built, chat + paraphraser configured.
-//  2. Call /research?strategy=planner and /research?strategy=paraphrase.
-//  3. Send (question, answer, sources) to a judge LLM with a different model.
-//  4. Parse two scores: coverage (1-5) and grounding (1-5). Comments are kept
-//     for the per-query report.
-//  5. Aggregate means per strategy and print a comparison table.
-//
-// Cost note: ~$0.005 per query at gpt-4o-mini synth + gpt-4o judge for the
-// 10 multi-faceted queries → roughly $0.10 total per run. Manageable but not
-// free; the -dry-run flag prints the plan without LLM calls.
-func runAnswerEval(ctx context.Context, args []string) error {
-	fs := flag.NewFlagSet("answer-eval", flag.ExitOnError)
-	corpusPath := fs.String("corpus", "testdata/eval/corpus.json", "corpus JSON")
-	queriesPath := fs.String("queries", "testdata/eval/queries-multifaceted.json", "queries JSON (multi-faceted by default — these stress synthesis the most)")
-	synthModel := fs.String("synth-model", "gpt-4o-mini", "chat model for /research synthesis + planner/paraphraser")
-	judgeModel := fs.String("judge-model", "gpt-4o", "chat model for the judge (use a different + stronger model than synth to avoid self-preference)")
-	savePath := fs.String("save", "", "save full per-query report as JSON")
-	embModel := fs.String("embed-model", "text-embedding-3-small", "embedding model name")
-	embDim := fs.Int("embed-dim", 1536, "embedding dimensionality")
-	embCacheDir := fs.String("embed-cache", "./eval-embed-cache", "embedding cache dir (set empty to disable)")
-	answerChunkSize := fs.Int("chunk-size", 0, "passage chunker target words (0 = default 320)")
-	answerChunkOverlap := fs.Int("chunk-overlap", 0, "passage chunker overlap words (0 = default 64)")
-	dryRun := fs.Bool("dry-run", false, "build the harness + print the plan, but issue NO LLM calls")
-	useRerank := fs.Bool("rerank", false, "wire the LLM listwise reranker into the in-process server; tests whether rerank cuts the grounding=1 cases on single-doc")
-	rerankModel := fs.String("rerank-model", "gpt-4o-mini", "chat model for the reranker (kept default = synth-model so cost stays predictable)")
-	synthK := fs.Int("synth-k", 0, "cap synth-source count via WithDefaults({ResearchSynthK: N}); 0 = preserve built-in default (10). tests whether K=5 trades coverage for grounding on single-doc workloads.")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-
-	apiKey := os.Getenv("OPENAI_API_KEY")
-	if apiKey == "" {
-		apiKey = os.Getenv("OPENAI")
-	}
-	if apiKey == "" && !*dryRun {
-		return errors.New("OPENAI_API_KEY (or OPENAI) not set; pass -dry-run to inspect the plan without spending")
-	}
-
-	corpus, err := eval.LoadCorpus(*corpusPath)
-	if err != nil {
-		return fmt.Errorf("load corpus: %w", err)
-	}
-	qs, err := eval.LoadQuerySet(*queriesPath)
-	if err != nil {
-		return fmt.Errorf("load queries: %w", err)
-	}
-
-	fmt.Printf("answer-eval: %d queries × 2 strategies × judge calls\n", len(qs.Queries))
-	fmt.Printf("synth model = %s   judge model = %s   rerank = %v   synth_k = %d   dry-run = %v\n", *synthModel, *judgeModel, *useRerank, *synthK, *dryRun)
-	if *dryRun {
-		for _, q := range qs.Queries {
-			fmt.Printf("  - %s  (relevant: %d docs)\n", q.Text, len(q.Relevant))
-		}
-		return nil
-	}
-
-	// Build an ephemeral store + indexes (in-memory style, but SQLite needs a dir).
-	tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("cosift-answer-eval-%d", time.Now().UnixNano()))
-	st, err := store.Open(tmpDir)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		st.Close()
-		_ = os.RemoveAll(tmpDir)
-	}()
-
-	bm := index.NewBM25(st)
-	oai := embed.NewOpenAIClient(apiKey, "", *embModel, *embDim)
-	var emb embed.Embedder = oai
-	if *embCacheDir != "" {
-		emb = embed.NewCachedEmbedder(oai, *embCacheDir)
-	}
-	chunker := chunkerWith(*answerChunkSize, *answerChunkOverlap)
-	var allTexts []string
-	for _, d := range corpus.Docs {
-		id, err := st.UpsertDocument(ctx, &store.Document{
-			URL: d.URL, Title: d.Title, Text: d.Text, Source: "answer-eval", FetchedAt: time.Now(),
-		})
-		if err != nil {
-			return err
-		}
-		if err := bm.IndexDocument(ctx, id, d.Title, d.Text); err != nil {
-			return err
-		}
-		text := d.Title + "\n\n" + d.Text
-		for _, c := range chunker.Chunk(text) {
-			allTexts = append(allTexts, c.Text)
-		}
-	}
-	fmt.Printf("embedding %d passages across %d docs...\n", len(allTexts), len(corpus.Docs))
-	vecs, err := batchEmbed(ctx, emb, allTexts, 256)
-	if err != nil {
-		return fmt.Errorf("embed: %w", err)
-	}
-	vi := index.NewVectorIndex(*embDim)
-	idx := 0
-	for _, d := range corpus.Docs {
-		text := d.Title + "\n\n" + d.Text
-		for _, c := range chunker.Chunk(text) {
-			if idx >= len(vecs) {
-				break
-			}
-			vi.AddPassage(d.URL, d.Title, c.Offset, c.Length, vecs[idx])
-			idx++
-		}
-	}
-
-	chat := embed.NewOpenAIChat(apiKey, "", *synthModel)
-	judge := embed.NewOpenAIChat(apiKey, "", *judgeModel)
-
-	srv := server.New(st).
-		WithVector(vi, emb).
-		WithChat(chat).
-		WithParaphraser(chat, 2).
-		WithLLMLimiter(0, 0) // disable rate limiter — this is an in-process eval, not a public endpoint. caught this when the 10/min default 429'd after 5 queries.
-	if *useRerank {
-		rerankChat := chat
-		if *rerankModel != *synthModel {
-			rerankChat = embed.NewOpenAIChat(apiKey, "", *rerankModel)
-		}
-		srv = srv.WithReranker(rerank.NewLLMReranker(rerankChat), 20)
-	}
-	if *synthK > 0 {
-		srv = srv.WithDefaults(server.Defaults{ResearchSynthK: *synthK})
-	}
-	httpSrv := httptest.NewServer(srv.Handler())
-	defer httpSrv.Close()
-
-	type strategyReport struct {
-		Strategy     string   `json:"strategy"`
-		Answer       string   `json:"answer"`
-		SourceURLs   []string `json:"source_urls"`
-		Coverage     int      `json:"coverage"`
-		Grounding    int      `json:"grounding"`
-		JudgeComment string   `json:"judge_comment"`
-	}
-	type queryReport struct {
-		Query   string           `json:"query"`
-		Gold    []string         `json:"gold_relevant"`
-		Reports []strategyReport `json:"reports"`
-	}
-
-	all := make([]queryReport, 0, len(qs.Queries))
-	httpClient := &http.Client{Timeout: 120 * time.Second}
-
-	for qi, q := range qs.Queries {
-		fmt.Printf("\n[%d/%d] %s\n", qi+1, len(qs.Queries), q.Text)
-		qr := queryReport{Query: q.Text, Gold: q.Relevant}
-
-		for _, strategy := range []string{"planner", "paraphrase"} {
-			// /research call.
-			u := httpSrv.URL + "/research?strategy=" + strategy + "&q=" + url.QueryEscape(q.Text)
-			req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
-			resp, err := httpClient.Do(req)
-			if err != nil {
-				fmt.Printf("  %s: research call failed: %v\n", strategy, err)
-				continue
-			}
-			body, _ := io.ReadAll(resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode != 200 {
-				fmt.Printf("  %s: research returned %d: %s\n", strategy, resp.StatusCode, string(body))
-				continue
-			}
-			var rr struct {
-				Query    string   `json:"query"`
-				Strategy string   `json:"strategy"`
-				Plan     []string `json:"plan"`
-				Answer   string   `json:"answer"`
-				Sources  []struct {
-					ID    int    `json:"id"`
-					URL   string `json:"url"`
-					Title string `json:"title"`
-				} `json:"sources"`
-			}
-			if err := json.Unmarshal(body, &rr); err != nil {
-				fmt.Printf("  %s: parse research: %v\n", strategy, err)
-				continue
-			}
-			sourceURLs := make([]string, len(rr.Sources))
-			for i, src := range rr.Sources {
-				sourceURLs[i] = src.URL
-			}
-			fmt.Printf("  %-10s answer=%dchars sources=%d  plan=%v\n", strategy, len(rr.Answer), len(rr.Sources), rr.Plan)
-
-			// Judge call.
-			score, err := judgeAnswer(ctx, judge, q.Text, rr.Answer, rr.Sources)
-			if err != nil {
-				fmt.Printf("    judge failed: %v\n", err)
-				continue
-			}
-			fmt.Printf("    judge: coverage=%d grounding=%d  — %s\n", score.Coverage, score.Grounding, score.Comment)
-			qr.Reports = append(qr.Reports, strategyReport{
-				Strategy:     strategy,
-				Answer:       rr.Answer,
-				SourceURLs:   sourceURLs,
-				Coverage:     score.Coverage,
-				Grounding:    score.Grounding,
-				JudgeComment: score.Comment,
-			})
-		}
-		all = append(all, qr)
-	}
-
-	// Aggregate. Exported field names matter — saved unexported agg
-	// fields which JSON-marshalled to `{}`. Now the summary in the saved JSON
-	// is actually inspectable by jq.
-	type strategySummary struct {
-		N             int     `json:"n"`
-		MeanCoverage  float64 `json:"mean_coverage"`
-		MeanGrounding float64 `json:"mean_grounding"`
-		Combined      float64 `json:"combined"`
-	}
-	rawAcc := map[string]struct{ cov, grd, n int }{"planner": {}, "paraphrase": {}}
-	for _, q := range all {
-		for _, sr := range q.Reports {
-			a, ok := rawAcc[sr.Strategy]
-			if !ok {
-				continue
-			}
-			a.cov += sr.Coverage
-			a.grd += sr.Grounding
-			a.n++
-			rawAcc[sr.Strategy] = a
-		}
-	}
-	summary := map[string]strategySummary{}
-	for _, name := range []string{"planner", "paraphrase"} {
-		a := rawAcc[name]
-		s := strategySummary{N: a.n}
-		if a.n > 0 {
-			s.MeanCoverage = float64(a.cov) / float64(a.n)
-			s.MeanGrounding = float64(a.grd) / float64(a.n)
-			s.Combined = s.MeanCoverage + s.MeanGrounding
-		}
-		summary[name] = s
-	}
-
-	fmt.Println("\n=== summary ===")
-	fmt.Printf("%-12s  N   mean coverage   mean grounding   combined\n", "strategy")
-	for _, name := range []string{"planner", "paraphrase"} {
-		s := summary[name]
-		if s.N == 0 {
-			fmt.Printf("%-12s  0   —                —                —\n", name)
-			continue
-		}
-		fmt.Printf("%-12s  %d   %.2f            %.2f             %.2f\n", name, s.N, s.MeanCoverage, s.MeanGrounding, s.Combined)
-	}
-
-	if *savePath != "" {
-		buf, _ := json.MarshalIndent(map[string]any{
-			"synth_model": *synthModel,
-			"judge_model": *judgeModel,
-			"queries":     *queriesPath,
-			"rerank":      *useRerank,
-			"synth_k":     *synthK,
-			"reports":     all,
-			"summary":     summary,
-			"when":        time.Now().UTC().Format(time.RFC3339),
-		}, "", "  ")
-		if err := os.WriteFile(*savePath, buf, 0o644); err != nil {
-			return fmt.Errorf("save report: %w", err)
-		}
-		fmt.Printf("\nsaved full report to %s\n", *savePath)
-	}
-	return nil
-}
-
-// judgeScore is what we parse out of the judge LLM's JSON response.
-type judgeScore struct {
-	Coverage  int    `json:"coverage"`
-	Grounding int    `json:"grounding"`
-	Comment   string `json:"comment"`
-}
-
-// judgeSystemPrompt was rewritten in after the trace revealed
-// the prior judge was evaluating the full source LIST relevance instead of the
-// CITED sources' accuracy. That bias deflated grounding scores in proportion
-// to how many tangential sources were in the list (paraphrase: 10, planner: 4-6).
-// The new prompt is explicit + the harness pre-filters sources to only those
-// the answer actually cites via [N].
-const judgeSystemPrompt = `You are an expert judge evaluating a research answer's quality. Score on TWO dimensions, each on a 1-5 integer scale:
-
-  - coverage: does the answer address every aspect of the question? (1 = misses major aspects, 5 = covers everything the question asked)
-  - grounding: do the sources the answer CITES via [N] actually support the claims those citations are attached to? (1 = cited sources don't support the claims they're attached to, 5 = every cited source genuinely supports the claim it appears next to)
-
-CRITICAL: Only consider sources the answer actually cites via [N]. Sources are listed by the IDs the answer used; uncited sources are not shown. Do not penalize the answer for not citing additional material.
-
-Output ONLY a single JSON object with this exact shape:
-{"coverage": <int 1-5>, "grounding": <int 1-5>, "comment": "<one sentence explaining the lower of the two scores>"}
-
-Do not output anything else — no preamble, no code fences, no markdown.`
-
-// citedSourceIDs extracts the integer ids referenced by [N] or [N,M,K] in the
-// answer text. fix for the judge-bias bug — we now pass only the
-// actually-cited sources to the judge, not the full source list.
-func citedSourceIDs(answer string) map[int]bool {
-	ids := make(map[int]bool)
-	// Match [1] or [1,2] or [1, 2, 3] inside square brackets.
-	for _, m := range citationRE.FindAllStringSubmatch(answer, -1) {
-		// m[1] is the contents inside the brackets.
-		for _, tok := range strings.Split(m[1], ",") {
-			n, err := strconv.Atoi(strings.TrimSpace(tok))
-			if err == nil {
-				ids[n] = true
-			}
-		}
-	}
-	return ids
-}
-
-var citationRE = regexp.MustCompile(`\[([0-9, ]+)\]`)
-
-// judgeAnswer asks the judge LLM to score a single (question, answer, sources)
-// triple. pre-filters sources to only those cited in the answer, so
-// the judge can't be confused by uncited tangential material in the list.
-// Returns a judgeScore or an error.
-func judgeAnswer(ctx context.Context, judge embed.ChatClient, question, answer string, sources []struct {
-	ID    int    `json:"id"`
-	URL   string `json:"url"`
-	Title string `json:"title"`
-}) (judgeScore, error) {
-	cited := citedSourceIDs(answer)
-	var sb strings.Builder
-	for _, src := range sources {
-		if !cited[src.ID] {
-			continue // skip uncited sources — judge only evaluates citation accuracy
-		}
-		fmt.Fprintf(&sb, "[%d] %s (%s)\n", src.ID, src.Title, src.URL)
-	}
-	citedBlock := sb.String()
-	if citedBlock == "" {
-		citedBlock = "(the answer did not cite any sources via [N])\n"
-	}
-	userMsg := fmt.Sprintf("Question: %s\n\nSources actually cited by the answer:\n%s\nAnswer:\n%s\n\nScore this answer.", question, citedBlock, answer)
-	raw, err := judge.Chat(ctx, []embed.ChatMsg{
-		{Role: "system", Content: judgeSystemPrompt},
-		{Role: "user", Content: userMsg},
-	})
-	if err != nil {
-		return judgeScore{}, fmt.Errorf("judge call: %w", err)
-	}
-	raw = strings.TrimSpace(raw)
-	for _, fence := range []string{"```json", "```"} {
-		raw = strings.TrimPrefix(raw, fence)
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	raw = strings.TrimSpace(raw)
-	startIdx := strings.Index(raw, "{")
-	endIdx := strings.LastIndex(raw, "}")
-	if startIdx < 0 || endIdx <= startIdx {
-		return judgeScore{}, fmt.Errorf("judge: no JSON object in response: %q", raw)
-	}
-	var sc judgeScore
-	if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &sc); err != nil {
-		return judgeScore{}, fmt.Errorf("judge: parse JSON: %w (raw=%q)", err, raw)
-	}
-	if sc.Coverage < 1 || sc.Coverage > 5 || sc.Grounding < 1 || sc.Grounding > 5 {
-		return judgeScore{}, fmt.Errorf("judge: scores out of 1-5 range: %+v", sc)
-	}
-	return sc, nil
-}
-
-// savedAnswerEvalReport mirrors the structure runAnswerEval writes. Used by
-// runAnswerEvalCompare to load and diff two prior reports without re-running
-// the eval. The shape is the same map[string]any savedJSON shape; this struct
-// carries only the fields we need for comparison.
-type savedAnswerEvalReport struct {
-	SynthModel string `json:"synth_model"`
-	JudgeModel string `json:"judge_model"`
-	Queries    string `json:"queries"`
-	Rerank     bool   `json:"rerank"`
-	SynthK     int    `json:"synth_k"`
-	When       string `json:"when"`
-	Summary    map[string]struct {
-		N             int     `json:"n"`
-		MeanCoverage  float64 `json:"mean_coverage"`
-		MeanGrounding float64 `json:"mean_grounding"`
-		Combined      float64 `json:"combined"`
-	} `json:"summary"`
-	Reports []struct {
-		Query   string `json:"query"`
-		Reports []struct {
-			Strategy     string `json:"strategy"`
-			Coverage     int    `json:"coverage"`
-			Grounding    int    `json:"grounding"`
-			JudgeComment string `json:"judge_comment"`
-		} `json:"reports"`
-	} `json:"reports"`
-}
-
-func loadAnswerEvalReport(path string) (*savedAnswerEvalReport, error) {
-	b, err := os.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("read %s: %w", path, err)
-	}
-	var r savedAnswerEvalReport
-	if err := json.Unmarshal(b, &r); err != nil {
-		return nil, fmt.Errorf("parse %s: %w", path, err)
-	}
-	// Old reports had an empty summary because the agg struct used
-	// unexported fields. Recompute the summary on the fly for those.
-	if len(r.Summary) == 0 || r.Summary["planner"].N == 0 {
-		r.Summary = recomputeSummary(r)
-	}
-	return &r, nil
-}
-
-func recomputeSummary(r savedAnswerEvalReport) map[string]struct {
-	N             int     `json:"n"`
-	MeanCoverage  float64 `json:"mean_coverage"`
-	MeanGrounding float64 `json:"mean_grounding"`
-	Combined      float64 `json:"combined"`
-} {
-	out := make(map[string]struct {
-		N             int     `json:"n"`
-		MeanCoverage  float64 `json:"mean_coverage"`
-		MeanGrounding float64 `json:"mean_grounding"`
-		Combined      float64 `json:"combined"`
-	})
-	for _, strategy := range []string{"planner", "paraphrase"} {
-		var covSum, grdSum, n int
-		for _, q := range r.Reports {
-			for _, sr := range q.Reports {
-				if sr.Strategy == strategy {
-					covSum += sr.Coverage
-					grdSum += sr.Grounding
-					n++
-				}
-			}
-		}
-		s := out[strategy]
-		s.N = n
-		if n > 0 {
-			s.MeanCoverage = float64(covSum) / float64(n)
-			s.MeanGrounding = float64(grdSum) / float64(n)
-			s.Combined = s.MeanCoverage + s.MeanGrounding
-		}
-		out[strategy] = s
-	}
-	return out
-}
-
-// runAnswerEvalCompare loads two saved answer-eval JSON reports and prints
-// side-by-side coverage/grounding/combined deltas + per-query moves above a
-// configurable threshold. No LLM calls; pure file-to-file diff —
-// first-class replacement for the ad-hoc python one-liners operators
-// previously chained for the same report.
-func runAnswerEvalCompare(_ context.Context, args []string) error {
-	fs := flag.NewFlagSet("answer-eval-compare", flag.ExitOnError)
-	threshold := fs.Int("query-threshold", 2, "report per-query moves of |Δgrounding| or |Δcoverage| ≥ this value (1-5 scale)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	rest := fs.Args()
-	if len(rest) != 2 {
-		return errors.New("usage: cosift answer-eval-compare [-query-threshold N] BASELINE.json NEW.json")
-	}
-	a, err := loadAnswerEvalReport(rest[0])
-	if err != nil {
-		return fmt.Errorf("baseline: %w", err)
-	}
-	b, err := loadAnswerEvalReport(rest[1])
-	if err != nil {
-		return fmt.Errorf("new: %w", err)
-	}
-	// Header — record what we're comparing.
-	fmt.Printf("BASELINE: %s\n", rest[0])
-	fmt.Printf("  synth=%s judge=%s rerank=%v synth_k=%d queries=%s\n", a.SynthModel, a.JudgeModel, a.Rerank, a.SynthK, a.Queries)
-	fmt.Printf("  when: %s\n", a.When)
-	fmt.Printf("NEW:      %s\n", rest[1])
-	fmt.Printf("  synth=%s judge=%s rerank=%v synth_k=%d queries=%s\n", b.SynthModel, b.JudgeModel, b.Rerank, b.SynthK, b.Queries)
-	fmt.Printf("  when: %s\n", b.When)
-	fmt.Println()
-
-	// Summary deltas.
-	fmt.Printf("%-12s   N     cov     grnd    comb   |   Δcov    Δgrnd   Δcomb\n", "strategy")
-	fmt.Println("-----------------------------------------------------------------------")
-	for _, strategy := range []string{"planner", "paraphrase"} {
-		ab, bb := a.Summary[strategy], b.Summary[strategy]
-		if ab.N == 0 && bb.N == 0 {
-			fmt.Printf("%-12s   (not present in either report)\n", strategy)
-			continue
-		}
-		fmt.Printf("%-12s %3d→%-3d %.2f→%.2f %.2f→%.2f %.2f→%.2f | %+.2f  %+.2f  %+.2f\n",
-			strategy, ab.N, bb.N,
-			ab.MeanCoverage, bb.MeanCoverage,
-			ab.MeanGrounding, bb.MeanGrounding,
-			ab.Combined, bb.Combined,
-			bb.MeanCoverage-ab.MeanCoverage,
-			bb.MeanGrounding-ab.MeanGrounding,
-			bb.Combined-ab.Combined,
-		)
-	}
-
-	// Per-query moves.
-	if *threshold > 0 {
-		// Index by (query, strategy) for fast lookup.
-		aIdx := make(map[string]struct{ cov, grd int })
-		for _, q := range a.Reports {
-			for _, sr := range q.Reports {
-				aIdx[q.Query+"\x00"+sr.Strategy] = struct{ cov, grd int }{sr.Coverage, sr.Grounding}
-			}
-		}
-		type move struct {
-			query, strategy string
-			oldCov, oldGrd  int
-			newCov, newGrd  int
-		}
-		var moves []move
-		for _, q := range b.Reports {
-			for _, sr := range q.Reports {
-				ab, ok := aIdx[q.Query+"\x00"+sr.Strategy]
-				if !ok {
-					continue
-				}
-				dc := abs(sr.Coverage - ab.cov)
-				dg := abs(sr.Grounding - ab.grd)
-				if dc >= *threshold || dg >= *threshold {
-					moves = append(moves, move{q.Query, sr.Strategy, ab.cov, ab.grd, sr.Coverage, sr.Grounding})
-				}
-			}
-		}
-		if len(moves) > 0 {
-			fmt.Printf("\nPer-query moves with |Δ| ≥ %d:\n", *threshold)
-			fmt.Printf("  %-10s  %-50s  cov   grnd\n", "strategy", "query")
-			for _, m := range moves {
-				fmt.Printf("  %-10s  %-50s  %d→%d  %d→%d\n",
-					m.strategy, truncate(m.query, 50),
-					m.oldCov, m.newCov,
-					m.oldGrd, m.newGrd,
-				)
-			}
-		} else {
-			fmt.Printf("\nNo per-query moves with |Δ| ≥ %d.\n", *threshold)
-		}
-	}
-	return nil
-}
-
-func abs(n int) int {
-	if n < 0 {
-		return -n
-	}
-	return n
-}
-
-func truncate(s string, n int) string {
-	if len(s) <= n {
-		return s
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
 	}
 	return s[:n-1] + "…"
 }
 
-// peekWarnings extracts the warnings field from a server
-// response body without committing to a full typed decode. When the CLI hits
-// pebble-serve, the response carries a warnings[] slice naming silent no-ops
-// (unknown expand, rerank without reranker, malformed include_domains, etc).
-// Without surfacing these on stderr the human CLI mode swallows them.
-func peekWarnings(body []byte) []string {
-	var probe struct {
-		Warnings []string `json:"warnings"`
-	}
-	if err := json.Unmarshal(body, &probe); err != nil {
-		return nil
-	}
-	return probe.Warnings
-}
-
-// emitWarnings prints each warning to stderr with a 'cosift:' prefix.
-func emitWarnings(body []byte) {
-	for _, w := range peekWarnings(body) {
-		fmt.Fprintln(os.Stderr, "cosift: warning:", w)
-	}
-}
-
-func runStats(ctx context.Context, cfg *config.Config, args []string) error {
-	// -backend flag (-backend=pebble reads the Pebble subdir).
-	fs := flag.NewFlagSet("stats", flag.ExitOnError)
-	backend := fs.String("backend", "sqlite", "storage backend: sqlite (default) | pebble")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	switch *backend {
-	case "sqlite", "":
-		s, err := store.Open(cfg.DataDir)
-		if err != nil {
-			return err
-		}
-		defer s.Close()
-		stats, err := s.Stats(ctx)
-		if err != nil {
-			return err
-		}
-		fmt.Printf("documents: %d\nterms: %d\ndata_dir: %s\nbackend: sqlite\n",
-			stats.Documents, stats.Terms, cfg.DataDir)
-	case "pebble":
-		pebbleDir := filepath.Join(cfg.DataDir, "pebble")
-		ps, err := openPebbleOrFriendlyErr(pebbleDir)
-		if err != nil {
-			return err
-		}
-		defer ps.Close()
-		stats, err := ps.Stats(ctx)
-		if err != nil {
-			return err
-		}
-		fmt.Printf("documents: %d\ndata_dir: %s\nbackend: pebble\n",
-			stats.Documents, pebbleDir)
-	default:
-		return fmt.Errorf("stats: unknown -backend %q (want: sqlite | pebble)", *backend)
-	}
-	return nil
-}
-
-// runCrawlStatus prints an operator-friendly snapshot of an ongoing crawl:
-// counts, frontier breakdown, top hosts by indexed-doc count, recent error
-// classes, and rolling-window doc rates with a 1M-doc ETA. Safe to run
-// concurrently with a live `cosift crawl` — SQLite WAL mode allows readers
-// alongside the writer.
-// runStatusFile reads the crawl-status.json (the file the live
-// crawler writes every 10s) and pretty-prints it. Useful when an operator
-// can't run `cosift stats -backend=pebble` because Pebble's single-writer
-// lock is held by the crawl process.
-//
-// Default path: <cfg.DataDir>/crawl-status.json. -file flag overrides.
-func runStatusFile(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("status-file", flag.ExitOnError)
-	path := fs.String("file", "", "path to crawl-status.json (defaults to <data_dir>/crawl-status.json)")
-	asJSON := fs.Bool("json", false, "emit raw status JSON plus a derived 'age_seconds' field; suitable for jq / Prometheus")
-	target := fs.Int64("target", 0, "doc count target — show indexed/target progress percent (default 0 = no target line)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	p := *path
-	if p == "" {
-		p = filepath.Join(cfg.DataDir, "crawl-status.json")
-	}
-	buf, err := os.ReadFile(p)
-	if err != nil {
-		return fmt.Errorf("read %s: %w", p, err)
-	}
-	var d struct {
-		Queued             int64     `json:"frontier_queued"`
-		InFlight           int64     `json:"frontier_in_flight"`
-		Done               int64     `json:"frontier_done"`
-		Errored            int64     `json:"frontier_errored"`
-		IndexedDocs        int64     `json:"indexed_docs,omitempty"`
-		IndexedDocsAtStart int64     `json:"indexed_docs_at_start,omitempty"`
-		AvgDocLen          float64   `json:"avg_doc_len,omitempty"`
-		StartedAt          time.Time `json:"started_at,omitempty"`
-		WrittenAt          time.Time `json:"written_at"`
-	}
-	if err := json.Unmarshal(buf, &d); err != nil {
-		return fmt.Errorf("decode %s: %w", p, err)
-	}
-	age := time.Since(d.WrittenAt).Round(time.Second)
-	if *asJSON {
-		out := map[string]any{
-			"path":                  p,
-			"frontier_queued":       d.Queued,
-			"frontier_in_flight":    d.InFlight,
-			"frontier_done":         d.Done,
-			"frontier_errored":      d.Errored,
-			"indexed_docs":          d.IndexedDocs,
-			"indexed_docs_at_start": d.IndexedDocsAtStart,
-			"avg_doc_len":           d.AvgDocLen,
-			"started_at":            d.StartedAt,
-			"written_at":            d.WrittenAt,
-			"age_seconds":           int64(age.Seconds()),
-			"stale":                 age > 30*time.Second,
-		}
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "  ")
-		return enc.Encode(out)
-	}
-	fmt.Printf("status file: %s  (written %s ago)\n\n", p, age)
-	fmt.Printf("  queued:     %d\n", d.Queued)
-	fmt.Printf("  in_flight:  %d\n", d.InFlight)
-	fmt.Printf("  done:       %d\n", d.Done)
-	fmt.Printf("  errored:    %d\n", d.Errored)
-	if d.IndexedDocs > 0 {
-		fmt.Printf("  indexed:    %d (avg doc length: %.0f tokens)\n", d.IndexedDocs, d.AvgDocLen)
-	}
-	total := d.Queued + d.InFlight + d.Done + d.Errored
-	if total > 0 {
-		processed := d.Done + d.Errored
-		fmt.Printf("  processed:  %d / %d (%.1f%%)\n",
-			processed, total, float64(processed)/float64(total)*100)
-	}
-	// ?-target N → 'indexed/target (pct)' line for long crawls toward
-	// a known doc-count goal. No-op when target is unset or already met.
-	// ETA from started_at + indexed_docs_at_start fields.
-	// Rate is averaged since the dumper's first poll, not instantaneous.
-	if *target > 0 && d.IndexedDocs > 0 {
-		pct := float64(d.IndexedDocs) / float64(*target) * 100
-		// cap pct display at 100% and append a 'reached' marker when
-		// the crawl has met or exceeded the goal — operators watching a long
-		// crawl see the win line instead of '237.4%' growing confusingly.
-		switch {
-		case d.IndexedDocs >= *target:
-			fmt.Printf("  target:     %d / %d (100.0%%, reached)\n", d.IndexedDocs, *target)
-		default:
-			fmt.Printf("  target:     %d / %d (%.1f%%)\n", d.IndexedDocs, *target, pct)
-		}
-		gained := d.IndexedDocs - d.IndexedDocsAtStart
-		elapsed := d.WrittenAt.Sub(d.StartedAt)
-		if gained > 0 && elapsed > time.Second && d.IndexedDocs < *target {
-			rate := float64(gained) / elapsed.Seconds()
-			remaining := float64(*target - d.IndexedDocs)
-			eta := time.Duration(remaining/rate) * time.Second
-			fmt.Printf("  rate:       %.1f docs/sec  eta: %s\n", rate, eta.Round(time.Second))
-		}
-	}
-	if age > 30*time.Second {
-		fmt.Printf("\n  WARNING: status is %s old; crawler may have stopped\n", age)
-	}
-	return nil
-}
-
-func runCrawlStatus(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("crawl-status", flag.ExitOnError)
-	hostsN := fs.Int("hosts", 10, "show top N hosts by indexed-doc count")
-	errsN := fs.Int("errors", 8, "show top N distinct error classes")
-	target := fs.Int64("target", 1_000_000, "doc count target for ETA projection")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	s, err := store.Open(cfg.DataDir)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	r, err := s.CrawlStatus(ctx, *hostsN, *errsN)
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("crawl status — %s\n\n", cfg.DataDir)
-	fmt.Printf("  documents:        %d\n", r.Documents)
-	fmt.Printf("  terms:            %d\n", r.Terms)
-	fmt.Printf("  unique hosts:     %d\n", r.UniqueHosts)
-	fmt.Println()
-	fmt.Println("  frontier:")
-	for _, fs := range r.FrontierByStatus {
-		fmt.Printf("    %-12s %d\n", fs.Status, fs.Count)
-	}
-	fmt.Println()
-	fmt.Println("  doc rate (rolling, by fetched_at):")
-	for _, win := range r.RateWindows {
-		ratePerMin := float64(win.Count) / float64(win.WindowSec) * 60.0
-		fmt.Printf("    last %2dmin: %5d docs (%7.0f /min)\n", win.WindowSec/60, win.Count, ratePerMin)
-	}
-	if len(r.RateWindows) > 0 {
-		// Use the longest available rolling window for ETA — most stable signal.
-		win := r.RateWindows[len(r.RateWindows)-1]
-		ratePerMin := float64(win.Count) / float64(win.WindowSec) * 60.0
-		if ratePerMin > 0 {
-			remaining := *target - r.Documents
-			if remaining < 0 {
-				remaining = 0
-			}
-			etaHours := float64(remaining) / ratePerMin / 60.0
-			fmt.Printf("    → ETA to %d docs: %.1f hours = %.2f days (at the %dmin rate)\n",
-				*target, etaHours, etaHours/24.0, win.WindowSec/60)
-		}
-	}
-	if len(r.TopHosts) > 0 {
-		fmt.Printf("\n  top %d hosts (by indexed doc count):\n", len(r.TopHosts))
-		for _, h := range r.TopHosts {
-			fmt.Printf("    %-50s %d\n", truncStr(h.Host, 50), h.Count)
-		}
-	}
-	if len(r.ErrorClasses) > 0 {
-		fmt.Printf("\n  top %d error classes:\n", len(r.ErrorClasses))
-		for _, e := range r.ErrorClasses {
-			fmt.Printf("    %-60s %d\n", truncStr(e.LastError, 60), e.Count)
-		}
-	}
-	return nil
-}
-
 func truncStr(s string, n int) string {
 	if len(s) <= n {
 		return s
diff --git a/cmd/cosift/pebble_serve.go b/cmd/cosift/pebble_serve.go
deleted file mode 100644
index 6d231bd..0000000
--- a/cmd/cosift/pebble_serve.go
+++ /dev/null
@@ -1,7572 +0,0 @@
-// minimal HTTP server backed by PebbleStore + PebbleBM25.
-//
-// Parallel to the SQLite-backed `cosift serve`. Read-only endpoints only:
-// /healthz, /stats, /search, /contents. No crawler, no admin, no /answer
-// or /research yet — those need the SQLite-side server's chat-client +
-// admin-token plumbing, which is the work.
-//
-// Purpose: proves the path-2 storage rework works end-to-end through HTTP,
-// and gives a clean benchmark surface against the existing SQLite server.
-// Operators evaluating cosift's billion-scale capability run this against
-// a Pebble store and compare /search latency + index size to the SQLite
-// equivalent.
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	_ "embed"
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"math"
-	"net"
-	"net/http"
-	"net/http/pprof"
-	"net/url"
-	"os"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"crypto/sha256"
-	"encoding/hex"
-
-	"github.com/pilot-protocol/cosift/internal/crawler"
-
-	"github.com/pilot-protocol/cosift/internal/answercache"
-	"github.com/pilot-protocol/cosift/internal/authority"
-	"github.com/pilot-protocol/cosift/internal/chatgate"
-	"github.com/pilot-protocol/cosift/internal/config"
-	"github.com/pilot-protocol/cosift/internal/embed"
-	"github.com/pilot-protocol/cosift/internal/index"
-	"github.com/pilot-protocol/cosift/internal/judge"
-	"github.com/pilot-protocol/cosift/internal/qexpand"
-	"github.com/pilot-protocol/cosift/internal/rerank"
-	"github.com/pilot-protocol/cosift/internal/sla"
-	"github.com/pilot-protocol/cosift/internal/store"
-)
-
-func runPebbleServe(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("pebble-serve", flag.ExitOnError)
-	dir := fs.String("dir", "", "PebbleStore directory (required; the SQLite cfg.DataDir is ignored)")
-	addr := fs.String("addr", cfg.Server.Addr, "listen address (defaults to server.addr from cosift.json)")
-	// -crawl-seeds-file lets pebble-serve run an in-process crawler
-	// against the same PebbleStore. Eliminates the writer-lock dance between
-	// cosift-serve and cosift-crawl — search + crawl + index growth all in one
-	// process, one binary, one lock. The crawler runs as a goroutine for the
-	// lifetime of the server.
-	crawlSeeds := fs.String("crawl-seeds-file", "", "if set, run a continuous in-process crawler against these seeds while serving")
-	crawlCheckpoint := fs.Duration("crawl-checkpoint", 60*time.Second, "HNSW persist cadence for the in-serve crawler")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *dir == "" {
-		return errors.New("pebble-serve: -dir is required")
-	}
-	if *addr == "" {
-		*addr = "127.0.0.1:7777"
-	}
-
-	ps, err := store.OpenPebble(*dir)
-	if err != nil {
-		return fmt.Errorf("open pebble: %w", err)
-	}
-	defer ps.Close()
-
-	// log corpus size at startup so operators see whether the
-	// store opened with the expected doc count (silent open would force them
-	// to curl /stats just to confirm a restart loaded the right data dir).
-	if _, indexedDocs, err := ps.CorpusStats(ctx); err == nil && indexedDocs > 0 {
-		log.Printf("pebble-serve: opened store with %d indexed docs", indexedDocs)
-	}
-
-	// Meta is 20 bytes (dim+nodeCount); first-entry probe is the fallback
-	// when meta is absent but vector entries exist (partial persist).
-	// We never load the full graph here — gigabytes of RAM at 10M-vector scale.
-	hasVectors := false
-	var vectorDim, vectorNodes int
-	if meta, ok, err := index.LoadHNSWMeta(ctx, ps); err == nil && ok {
-		hasVectors = true
-		vectorDim = meta.Dim
-		vectorNodes = meta.NodeCount
-	} else {
-		_ = ps.IterateVectorNodes(ctx, func(_ uint64, _ []byte) bool {
-			hasVectors = true
-			return false
-		})
-	}
-	// Loading is
-	// gigabytes of RAM at production scale (10M vectors × 1536 dim ≈ 60GB),
-	// so it's opt-in. When the env var isn't set we just keep the cheap meta
-	// snapshot from and operators see has_vectors=true.
-	var hnswGraph *index.HNSW
-	if hasVectors && os.Getenv("COSIFT_LOAD_HNSW") == "true" {
-		g, ok, err := index.LoadHNSW(ctx, ps)
-		switch {
-		case err != nil:
-			log.Printf("pebble-serve: COSIFT_LOAD_HNSW=true but LoadHNSW failed: %v", err)
-		case !ok:
-			log.Printf("pebble-serve: COSIFT_LOAD_HNSW=true but no HNSW meta on store")
-		default:
-			hnswGraph = g
-			log.Printf("pebble-serve: HNSW graph loaded into memory: %d nodes, dim=%d", g.Len(), vectorDim)
-			// Default 50 from NewHNSW
-			// underfits big graphs grown via AddPassage; raising to ~200
-			// restored Recall@10 from 0.47 to ~0.85 on the 800K-vector
-			// production corpus without a graph rebuild.
-			if v := os.Getenv("COSIFT_HNSW_EF_SEARCH"); v != "" {
-				if n, err := strconv.Atoi(v); err == nil && n > 0 {
-					g.SetEfSearch(n)
-					log.Printf("pebble-serve: HNSW efSearch override = %d", n)
-				}
-			}
-			// if a PQ codebook + codes exist in this store, wire
-			// them so /search uses asymmetric PQ distance (much faster on
-			// large graphs). When absent, search falls back to raw vectors.
-			//
-			// bench-pq exposed that PQ on this dim=768 corpus drops
-			// Recall@10 from ~0.89 to ~0.60 — the 32× compression has a
-			// recall cost that may exceed the speed benefit. Operators can
-			// keep the codes on disk but disable the runtime path by setting
-			// COSIFT_DISABLE_PQ=true.
-			if os.Getenv("COSIFT_DISABLE_PQ") == "true" {
-				log.Printf("pebble-serve: PQ disabled via COSIFT_DISABLE_PQ — using raw vectors for search")
-			} else if cb, cbOK, _ := index.LoadPQCodebook(ctx, ps); cbOK {
-				codes := make([][]uint16, g.Len())
-				loaded := 0
-				if err := ps.IteratePQCodes(ctx, func(nodeID uint64, blob []byte) bool {
-					if int(nodeID) >= len(codes) {
-						return true
-					}
-					// codebook-aware decode handles both byte-packed
-					// (new, K≤256) and uint16 LE (legacy) blob shapes.
-					code, err := cb.DecodeCodeBlob(blob)
-					if err != nil || len(code) != cb.M {
-						return true
-					}
-					codes[int(nodeID)] = code
-					loaded++
-					return true
-				}); err != nil {
-					log.Printf("pebble-serve: PQ codes iterate failed: %v — falling back to raw vectors", err)
-				} else {
-					g.UsePQ(cb, codes)
-					log.Printf("pebble-serve: PQ search enabled (codebook: dim=%d M=%d K=%d, codes=%d/%d)",
-						cb.Dim, cb.M, cb.K, loaded, g.Len())
-				}
-			}
-		}
-	}
-	if hasVectors {
-		if vectorNodes > 0 {
-			suffix := "(graph not loaded into memory; set COSIFT_LOAD_HNSW=true to load)"
-			if hnswGraph != nil {
-				suffix = "(graph loaded)"
-			}
-			log.Printf("pebble-serve: HNSW index present: %d nodes, dim=%d %s", vectorNodes, vectorDim, suffix)
-		} else {
-			log.Printf("pebble-serve: HNSW vector entries present (no meta blob — partial persist?)")
-		}
-	}
-
-	// configurable HyDE + paraphrase cache caps (defaults 256).
-	hydeCap := 256
-	if v := os.Getenv("COSIFT_HYDE_CACHE_SIZE"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			hydeCap = n
-			log.Printf("pebble-serve: HyDE cache size override = %d", n)
-		} else {
-			// surface unparseable env vars as warnings at startup
-			// instead of silently keeping defaults. Catches typos like
-			// COSIFT_HYDE_CACHE_SIZE="512m" or negative values.
-			log.Printf("pebble-serve: WARN COSIFT_HYDE_CACHE_SIZE=%q is not a positive integer — using default %d", v, hydeCap)
-		}
-	}
-	paraCap := 256
-	if v := os.Getenv("COSIFT_PARA_CACHE_SIZE"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			paraCap = n
-			log.Printf("pebble-serve: paraphrase cache size override = %d", n)
-		} else {
-			log.Printf("pebble-serve: WARN COSIFT_PARA_CACHE_SIZE=%q is not a positive integer — using default %d", v, paraCap)
-		}
-	}
-	idx := index.NewPebbleBM25(ps)
-	// Wikipedia / kernel.org / arxiv float
-	// above spam directories in the long tail. Optional Tranco /
-	// Majestic CSVs at COSIFT_TRANCO_CSV / COSIFT_MAJESTIC_CSV improve
-	// the score; with no CSVs the embedded whitelist + TLD heuristics
-	// still produce useful signals.
-	scorer := authority.New()
-	if v := os.Getenv("COSIFT_AUTHORITY_ALPHA"); v != "" {
-		if a, err := strconv.ParseFloat(v, 64); err == nil && a >= 0 {
-			scorer = scorer.WithAlpha(a)
-		}
-	}
-	if path := os.Getenv("COSIFT_TRANCO_CSV"); path != "" {
-		if f, err := os.Open(path); err == nil {
-			n, _ := scorer.LoadTranco(f)
-			_ = f.Close()
-			log.Printf("pebble-serve: authority Tranco loaded (%d entries from %s)", n, path)
-		} else {
-			log.Printf("pebble-serve: WARN COSIFT_TRANCO_CSV=%q: %v", path, err)
-		}
-	}
-	if path := os.Getenv("COSIFT_MAJESTIC_CSV"); path != "" {
-		if f, err := os.Open(path); err == nil {
-			n, _ := scorer.LoadMajestic(f)
-			_ = f.Close()
-			log.Printf("pebble-serve: authority Majestic loaded (%d entries from %s)", n, path)
-		} else {
-			log.Printf("pebble-serve: WARN COSIFT_MAJESTIC_CSV=%q: %v", path, err)
-		}
-	}
-	idx = idx.WithAuthority(scorer)
-	log.Printf("pebble-serve: authority scoring enabled (alpha=%.2f, trusted=%d, tranco=%d, majestic=%d)",
-		scorer.Alpha(), scorer.Stats().TrustedHosts, scorer.Stats().TrancoEntries, scorer.Stats().MajesticEntries)
-	// COSIFT_BM25_K1 / COSIFT_BM25_B override per instance.
-	// Shared with runQuery via applyBM25EnvOverrides so any PebbleBM25 built
-	// from CLI or server honors the same env.
-	o := applyBM25EnvOverrides(idx)
-	if o.k1Set {
-		log.Printf("pebble-serve: BM25 k1 override = %.2f", o.k1Val)
-	}
-	if o.bSet {
-		log.Printf("pebble-serve: BM25 b override = %.2f", o.bVal)
-	}
-	if o.k1Bad != "" {
-		log.Printf("pebble-serve: WARN COSIFT_BM25_K1=%q is not a positive float — using default %.2f", o.k1Bad, idx.K1())
-	}
-	if o.bBad != "" {
-		log.Printf("pebble-serve: WARN COSIFT_BM25_B=%q is not a positive float — using default %.2f", o.bBad, idx.B())
-	}
-	srv := &pebbleHTTP{
-		store:        ps,
-		idx:          idx,
-		hydeCache:    make(map[string]string, hydeCap),
-		hydeCacheCap: hydeCap,
-		paraCache:    make(map[string][]string, paraCap),
-		paraCacheCap: paraCap,
-		hasVectors:   hasVectors,
-		vectorDim:    vectorDim,
-		vectorNodes:  vectorNodes,
-		hnsw:         hnswGraph,
-		started:      time.Now(),
-		authority:    scorer,
-		// capture doc count at startup so /stats can compute
-		// per-minute crawl rate without long-term counters.
-		startupDocs: vectorNodes, // placeholder; overwritten below
-	}
-	// Read actual doc count at startup for crawl-rate baseline. Uses
-	// CorpusStats (the atomic-mirror indexed_docs counter) — NOT
-	// Stats.Documents which scans the raw 'd' family. The two diverge
-	// in two ways:
-	//
-	//   - Stats.Documents may include partially-upserted rows that the
-	//     indexed_docs counter hasn't seen yet, leaving startupDocs >
-	//     indexedDocs and freezing docs_added_since_start at 0 forever.
-	//   - On a cold corpus, Stats does a 12s scan that returns the
-	//     count AT END-OF-SCAN, by which time the crawler has already
-	//     added 50+ more docs, pushing startupDocs above the post-boot
-	//     indexed_docs atomic.
-	//
-	// CorpusStats reads the atomic directly (or bootstraps it from the
-	// indexed_docs meta key on first call), giving an exact match with
-	// the indexedDocs value the rate calc uses below.
-	if _, count, err := ps.CorpusStats(ctx); err == nil && count > 0 {
-		srv.startupDocs = int(count)
-	} else {
-		srv.startupDocs = 0
-	}
-	// stamp cluster config onto srv early so /search's
-	// gateway-mode check fires even on nodes that don't run a crawler.
-	if err := cfg.Cluster.Validate(); err != nil {
-		log.Printf("pebble-serve: cluster config invalid (%v) — running single-node", err)
-	} else {
-		srv.cluster = cfg.Cluster
-		if cfg.Cluster.IsClustered() {
-			role := "leaf"
-			if cfg.Cluster.GatewayMode {
-				role = "gateway+leaf"
-			}
-			log.Printf("pebble-serve: cluster mode (role=%s, shard=%d/%d, peers=%d)",
-				role, cfg.Cluster.MyShardID, cfg.Cluster.NumShards, len(cfg.Cluster.Peers))
-		}
-	}
-	// Uses the same OpenAI-compatible chat
-	// client the SQLite-side server uses; works against OpenAI, Together,
-	// Azure, llama.cpp, vLLM, Ollama, anything speaking /v1/chat/completions.
-	// anonymous auth when cfg.Chat.URL is set (self-hosted, no key
-	// needed). Required only for hosted defaults (empty URL = api.openai.com).
-	// Shared between answer + rerank pools so an
-	// /answer burst can't also exhaust the rerank pool's goroutines.
-	srv.chatGate = chatgate.New(
-		envIntDefault("COSIFT_LLM_GATE_RERANK", 8),
-		envIntDefault("COSIFT_LLM_GATE_ANSWER", 4),
-	)
-	// /answer response cache. Default TTL 60s + 1024-entry cap is a
-	// reasonable starting point for a workload with a few hundred
-	// recurring queries; raise via COSIFT_ANSWER_CACHE_TTL_SEC for
-	// stable corpora, disable with =0 if every query is unique.
-	srv.answerCache = answercache.New(
-		time.Duration(envIntDefault("COSIFT_ANSWER_CACHE_TTL_SEC", 60))*time.Second,
-		envIntDefault("COSIFT_ANSWER_CACHE_CAP", 1024),
-	)
-	// Tries the vLLM /metrics endpoint at the chat origin (works for
-	// self-hosted vLLM, no-op against OpenAI). Threshold default 8 — that
-	// matches the answer-pool cap so when the chat backend is already
-	// saturated we stop trying to add more LLM work.
-	if cfg.Chat.URL != "" {
-		probeURL := chatgate.VLLMMetricsURLFrom(cfg.Chat.URL)
-		threshold := envIntDefault("COSIFT_LLM_DEGRADE_QUEUE", 8)
-		interval := envDurationMsDefault("COSIFT_LLM_PROBE_MS", 2*time.Second)
-		srv.llmProbe = chatgate.NewLoadProbe(probeURL, threshold, interval)
-		go srv.llmProbe.Run(ctx)
-		log.Printf("pebble-serve: vLLM load probe started (url=%s, threshold=%d, interval=%s)", probeURL, threshold, interval)
-	}
-	// Defaults match the
-	// June-3 baseline measurements; ops can override per env. Violation
-	// log defaults under cfg.DataDir so it co-locates with the rest of
-	// the operational state.
-	slaLogPath := os.Getenv("COSIFT_SLA_LOG_PATH")
-	if slaLogPath == "" && cfg.DataDir != "" {
-		slaLogPath = filepath.Join(cfg.DataDir, "sla-violations.jsonl")
-	}
-	slaTargets := []sla.Target{
-		{Endpoint: "/search", P95: envDurationMsDefault("COSIFT_SLA_SEARCH_P95_MS", 1500*time.Millisecond), P99: envDurationMsDefault("COSIFT_SLA_SEARCH_P99_MS", 4*time.Second), MaxErrorRate: 0.02},
-		{Endpoint: "/answer", P95: envDurationMsDefault("COSIFT_SLA_ANSWER_P95_MS", 8*time.Second), P99: envDurationMsDefault("COSIFT_SLA_ANSWER_P99_MS", 20*time.Second), MaxErrorRate: 0.05},
-		{Endpoint: "/research", P95: envDurationMsDefault("COSIFT_SLA_RESEARCH_P95_MS", 30*time.Second), P99: envDurationMsDefault("COSIFT_SLA_RESEARCH_P99_MS", 60*time.Second), MaxErrorRate: 0.05},
-		{Endpoint: "/healthz", P95: 50 * time.Millisecond, P99: 200 * time.Millisecond, MaxErrorRate: 0.01},
-		{Endpoint: "/stats", P95: 200 * time.Millisecond, P99: time.Second, MaxErrorRate: 0.01},
-	}
-	slaWindow := envDurationMsDefault("COSIFT_SLA_WINDOW_MS", 5*time.Minute)
-	slaEvalInterval := envDurationMsDefault("COSIFT_SLA_EVAL_MS", 30*time.Second)
-	if m, err := sla.New(slaTargets, slaWindow, slaLogPath); err == nil {
-		srv.sla = m
-		go srv.sla.Run(ctx.Done(), slaEvalInterval)
-		log.Printf("pebble-serve: SLA monitor started (window=%s, eval=%s, log=%s)", slaWindow, slaEvalInterval, slaLogPath)
-	} else {
-		log.Printf("pebble-serve: SLA monitor disabled: %v", err)
-	}
-	// Holds the unwrapped OpenAIChatClient so the rerank
-	// wrapper below can sit directly on top of the inner client (single
-	// gate acquisition per call) instead of stacking on srv.chatSafe.
-	var rawChatForRerank embed.ChatClient
-	if cfg.Chat.Model != "" {
-		apiKey := resolveAPIKey("chat")
-		rawChat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Chat.Model)
-		rawChatForRerank = rawChat
-		srv.chatSafe = chatgate.NewSafeChat(rawChat, chatgate.Options{
-			Gate:          srv.chatGate,
-			Kind:          chatgate.KindAnswer,
-			StageDeadline: envDurationMsDefault("COSIFT_LLM_DEADLINE_ANSWER_MS", 30*time.Second),
-			MaxRetries:    envIntDefault("COSIFT_LLM_RETRIES", 1),
-		})
-		srv.chat = srv.chatSafe
-		auth := "anonymous"
-		if apiKey != "" {
-			auth = "bearer-token"
-		}
-		st := srv.chatGate.Stats()
-		log.Printf("pebble-serve: /answer enabled (chat model=%s, auth=%s, gate=answer:%d/rerank:%d)",
-			cfg.Chat.Model, auth, st.AnswerCap, st.RerankCap)
-	}
-	// Same anonymous-when-URL
-	// semantics as chat above.
-	if cfg.Embeddings.Model != "" {
-		apiKey := resolveAPIKey("embed")
-		// when cfg.Embeddings.URLs is set, build one client per
-		// URL and wrap them in a RoundRobinEmbedder so cosift fans embed
-		// requests across multiple backends. Falls back to single-URL
-		// behavior when URLs is empty.
-		urls := cfg.Embeddings.URLs
-		if len(urls) == 0 {
-			urls = []string{cfg.Embeddings.URL}
-		}
-		clients := make([]embed.Embedder, 0, len(urls))
-		for _, u := range urls {
-			clients = append(clients, embed.NewOpenAIClient(apiKey, u, cfg.Embeddings.Model, cfg.Embeddings.Dim))
-		}
-		var base embed.Embedder = embed.NewRoundRobinEmbedder(clients)
-		// Both search and
-		// crawler embed paths share the cache layer — same text returns
-		// instantly on re-fetch / re-query, no ollama call.
-		cacheNote := ""
-		if cfg.Embeddings.CacheDir != "" {
-			base = embed.NewCachedEmbedder(base, cfg.Embeddings.CacheDir)
-			cacheNote = fmt.Sprintf(", cache=%s", cfg.Embeddings.CacheDir)
-		}
-		srv.embedder = base
-		auth := "anonymous"
-		if apiKey != "" {
-			auth = "bearer-token"
-		}
-		log.Printf("pebble-serve: embedder configured (model=%s, dim=%d, auth=%s, backends=%d%s)",
-			cfg.Embeddings.Model, cfg.Embeddings.Dim, auth, len(clients), cacheNote)
-	}
-	// wire rerank.Reranker when cfg.Rerank is configured. Two paths
-	// (matching the SQLite-side): cfg.Rerank.URL → HTTPReranker (Cohere/Voyage/
-	// Jina/TEI wire shape); otherwise cfg.Chat.Model present → LLMReranker
-	// (RankGPT-style listwise via chat). /search?rerank=true gates the wrapper.
-	if cfg.Rerank.URL != "" {
-		apiKey := cfg.Rerank.APIKey
-		if apiKey == "" {
-			apiKey = os.Getenv("COHERE_API_KEY")
-			if apiKey == "" {
-				apiKey = os.Getenv("VOYAGE_API_KEY")
-			}
-		}
-		srv.reranker = rerank.NewHTTPReranker(cfg.Rerank.URL, apiKey, cfg.Rerank.Model)
-		log.Printf("pebble-serve: rerank enabled (http: %s, model=%s)", cfg.Rerank.URL, cfg.Rerank.Model)
-	} else if cfg.Rerank.Enabled && srv.chat != nil {
-		// when cfg.Rerank.Model is set and differs from cfg.Chat.Model,
-		// build a separate chat client for rerank pointing at the same /v1 URL
-		// but with the alternate model. Lets operators dedicate a small/fast
-		// model (e.g. qwen3.5:0.8b) to rerank so it doesn't queue behind chat
-		// generations on the same model. Falls back to the main chat client
-		// when Rerank.Model is empty or equal to Chat.Model.
-		var rerankInner embed.ChatClient = rawChatForRerank
-		if cfg.Rerank.Model != "" && cfg.Rerank.Model != cfg.Chat.Model {
-			apiKey := resolveAPIKey("chat")
-			rerankInner = embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Rerank.Model)
-			log.Printf("pebble-serve: rerank using dedicated chat (model=%s, url=%s)", cfg.Rerank.Model, cfg.Chat.URL)
-		}
-		srv.rerankSafe = chatgate.NewSafeChat(rerankInner, chatgate.Options{
-			Gate:          srv.chatGate,
-			Kind:          chatgate.KindRerank,
-			StageDeadline: envDurationMsDefault("COSIFT_LLM_DEADLINE_RERANK_MS", 5*time.Second),
-			MaxRetries:    envIntDefault("COSIFT_LLM_RETRIES", 1),
-		})
-		srv.reranker = rerank.NewLLMReranker(srv.rerankSafe)
-		log.Printf("pebble-serve: rerank enabled (llm: %s, gated)", srv.rerankSafe.Model())
-	}
-	srv.rerankCandK = cfg.Rerank.CandidateK
-	if srv.rerankCandK <= 0 {
-		srv.rerankCandK = 20
-	}
-	srv.hostBoosts = cfg.Defaults.HostBoosts
-
-	mux := http.NewServeMux()
-	// Built once from env;
-	// nil when disabled (COSIFT_RATELIMIT_RPM unset or 0). Wraps every route
-	// below — including /healthz so monitoring hits are budgeted too;
-	// operators wanting unlimited probes should set
-	// COSIFT_RATELIMIT_WHITELIST to include their monitoring source.
-	srv.rl = newRateLimiterFromEnv()
-	if srv.rl != nil {
-		log.Printf("pebble-serve: rate limit active (rpm=%.0f burst=%.0f whitelist=%v)", srv.rl.rpm, srv.rl.burst, srv.rl.whitelistList())
-	}
-	wrap := func(h http.HandlerFunc) http.HandlerFunc { return srv.count(srv.rateLimit(h)) }
-	// landing page at / and OpenAPI 3.1 spec at /openapi.json.
-	// Both embedded into the binary at build time — operators get a single
-	// self-contained executable, no separate static-asset deployment.
-	mux.HandleFunc("GET /", wrap(srv.handleLanding))
-	// Streams /answer or /research SSE
-	// into a multi-turn conversation view with citation rendering.
-	mux.HandleFunc("GET /chat", wrap(srv.handleChat))
-	mux.HandleFunc("GET /openapi.json", wrap(srv.handleOpenAPI))
-	// Swagger UI bundled into the binary. /docs renders the
-	// interactive spec; /docs/swagger-ui.css and /docs/swagger-ui-bundle.js
-	// serve the dist assets locally so the page works air-gapped (no CDN).
-	mux.HandleFunc("GET /docs", wrap(srv.handleSwaggerUI))
-	mux.HandleFunc("GET /docs/{file...}", wrap(srv.handleSwaggerAsset))
-	mux.HandleFunc("GET /healthz", wrap(srv.handleHealthz))
-	mux.HandleFunc("GET /stats", wrap(srv.handleStats))
-	mux.HandleFunc("GET /domains", wrap(srv.handleDomains))
-	// frontier queue visibility — counts by status + top-N
-	// hosts in queue. Distinct from /domains which is over INDEXED docs.
-	mux.HandleFunc("GET /queue", wrap(srv.handleQueue))
-	// Single-node deployments still expose
-	// this; it's just unused. Authenticated by cfg.Cluster.PeerAuthToken
-	// (Bearer); when token is empty, requests from any source are accepted.
-	mux.HandleFunc("POST /admin/crawl-enqueue", wrap(srv.handleCrawlEnqueue))
-	mux.HandleFunc("POST /admin/frontier-purge-host", wrap(srv.handleFrontierPurgeHost))
-	mux.HandleFunc("POST /admin/frontier-clear", wrap(srv.handleFrontierClear))
-	mux.HandleFunc("POST /admin/frontier-demote-host", wrap(srv.handleFrontierDemoteHost))
-	mux.HandleFunc("POST /admin/frontier-purge-stale-inflight", wrap(srv.handleFrontierPurgeStaleInFlight))
-	mux.HandleFunc("POST /admin/rss-import", wrap(srv.handleRSSImport))
-	mux.HandleFunc("POST /admin/crawl-now", wrap(srv.handleCrawlNow))
-	mux.HandleFunc("POST /admin/wet-import", wrap(srv.handleWETImport))
-	mux.HandleFunc("POST /admin/wet-import-bulk", wrap(srv.handleWETImportBulk))
-	mux.HandleFunc("POST /admin/site-pack", wrap(srv.handleSitePack))
-	mux.HandleFunc("POST /admin/site-submit", wrap(srv.handleSiteSubmit))
-	mux.HandleFunc("POST /admin/embed-backfill", wrap(srv.handleEmbedBackfill))
-	mux.HandleFunc("GET /admin/eval-quick", wrap(srv.handleEvalQuick))
-	mux.HandleFunc("POST /admin/hnsw-compact", wrap(srv.handleHNSWCompact))
-	mux.HandleFunc("GET /query", wrap(srv.handleQuery))
-	mux.HandleFunc("POST /query", wrap(srv.handleQuery))
-	// import a sitemap.xml (or sitemap-index) and push every
-	// listed URL into the live frontier.
-	mux.HandleFunc("POST /admin/sitemap-import", wrap(srv.handleSitemapImport))
-	// PQ training admin endpoint. Same auth as crawl-enqueue
-	// (Bearer cfg.Cluster.PeerAuthToken). Runs synchronously — for the
-	// 224K-vec corpus we have today it takes ~minutes; operator-only.
-	mux.HandleFunc("POST /admin/pq-train", wrap(srv.handlePQTrain))
-	// backfill-only — re-encode every node that doesn't have a
-	// code yet against the existing codebook. No retrain. Fast.
-	mux.HandleFunc("POST /admin/pq-encode", wrap(srv.handlePQEncode))
-	// Creates a hard-linked, consistent
-	// snapshot dir that's safe to tar without racing background compactions.
-	mux.HandleFunc("POST /admin/checkpoint", wrap(srv.handleCheckpoint))
-	mux.HandleFunc("GET /search", wrap(srv.handleSearch))
-	mux.HandleFunc("POST /search", wrap(srv.handleSearchPOST))
-	mux.HandleFunc("GET /contents", wrap(srv.handleContents))
-	mux.HandleFunc("POST /contents", wrap(srv.handleContentsBatch))
-	mux.HandleFunc("GET /verify", wrap(srv.handleVerify))
-	mux.HandleFunc("GET /metrics", wrap(srv.handleMetrics))
-	mux.HandleFunc("GET /sla", wrap(srv.handleSLA))
-	mux.HandleFunc("GET /admin/domains-audit", wrap(srv.handleDomainsAudit))
-	mux.HandleFunc("GET /find_similar", wrap(srv.handleFindSimilar))
-	mux.HandleFunc("POST /find_similar", wrap(srv.handleFindSimilarPOST))
-	mux.HandleFunc("GET /answer", wrap(srv.handleAnswer))
-	mux.HandleFunc("POST /answer", wrap(srv.handleAnswerPOST))
-	mux.HandleFunc("GET /research", wrap(srv.handleResearch))
-	mux.HandleFunc("POST /research", wrap(srv.handleResearchPOST))
-
-	httpSrv := &http.Server{
-		Addr:              *addr,
-		Handler:           mux,
-		ReadHeaderTimeout: 10 * time.Second,
-		ReadTimeout:       30 * time.Second,
-		WriteTimeout:      60 * time.Second,
-		IdleTimeout:       120 * time.Second,
-		MaxHeaderBytes:    1 << 20, // 1 MB
-	}
-
-	log.Printf("pebble-serve: listening on %s (PebbleStore at %s)", *addr, *dir)
-	// Production state observed on GH200: after a restart series that
-	// healed HNSW zombies, the meta counters showed 0 while the 'l'
-	// family had 10.87M entries — degraded BM25 length normalization.
-	// Off the hot path so HNSW load + first request aren't blocked.
-	go func() {
-		// Brief settle so an active IndexDocument can populate counters
-		// the cheap way before we kick the O(N) scan.
-		time.Sleep(10 * time.Second)
-		sumLen, count, recomputed, err := ps.BootstrapCorpusStats(ctx)
-		if err != nil {
-			log.Printf("pebble-serve: corpus stats bootstrap failed: %v", err)
-			return
-		}
-		if recomputed {
-			log.Printf("pebble-serve: corpus stats bootstrapped (indexed_docs=%d, sum_doc_len=%d)", count, sumLen)
-		}
-	}()
-	// Without this the
-	// subdomain-density penalty in authority.Score never fires — we
-	// saw 6.68M .cfd/.sbs hosts on GH200 sitting at score 0.20 instead
-	// of dropping to "block." Single-pass iteration is ~18s at 9M hosts;
-	// runs off the hot path so HNSW load isn't blocked. Re-runs every
-	// 6h to track ongoing crawl growth without operator intervention.
-	go func() {
-		// Initial settle so the rest of startup completes first.
-		time.Sleep(15 * time.Second)
-		refresh := func() {
-			counts := map[string]int{}
-			err := ps.IterateDomains(ctx, func(host string, _ int) bool {
-				// Use the Scorer's eTLD+1 logic so the bucket keys we
-				// produce here match what Score() looks up — without this
-				// the publicsuffix-aware Scorer keys on "bbc.co.uk" while
-				// the bootstrap keys on "co.uk" and the lookup misses.
-				e := authority.ETLD1(host)
-				if e == "" {
-					return true
-				}
-				counts[e]++
-				return true
-			})
-			if err != nil {
-				log.Printf("pebble-serve: subdomain-count refresh: %v", err)
-				return
-			}
-			srv.authority.SetSubdomainCounts(counts)
-			log.Printf("pebble-serve: authority subdomain counts refreshed (%d eTLD+1 buckets)", len(counts))
-		}
-		refresh()
-		// Re-scan periodically to absorb ongoing crawl growth.
-		t := time.NewTicker(6 * time.Hour)
-		defer t.Stop()
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-t.C:
-				refresh()
-			}
-		}
-	}()
-	// Only
-	// listens on loopback so it can never be exposed publicly through the
-	// reverse proxy. Set COSIFT_PPROF_ADDR=127.0.0.1:6060 to enable.
-	if pprofAddr := os.Getenv("COSIFT_PPROF_ADDR"); pprofAddr != "" {
-		// optional mutex / block sampling so the
-		// dense-search-latency investigation has data. Off by default
-		// because each sample is a couple-µs overhead per contended
-		// lock event. Set rate ≥1 to enable; 1000 means "sample 1 in
-		// 1000 lock waits" which is plenty for diagnosis.
-		if v := os.Getenv("COSIFT_MUTEX_PROFILE_FRACTION"); v != "" {
-			if n, err := strconv.Atoi(v); err == nil && n > 0 {
-				runtime.SetMutexProfileFraction(n)
-				log.Printf("pebble-serve: mutex profile fraction = %d", n)
-			}
-		}
-		if v := os.Getenv("COSIFT_BLOCK_PROFILE_RATE"); v != "" {
-			if n, err := strconv.Atoi(v); err == nil && n > 0 {
-				runtime.SetBlockProfileRate(n)
-				log.Printf("pebble-serve: block profile rate = %d", n)
-			}
-		}
-		pprofMux := http.NewServeMux()
-		pprofMux.HandleFunc("/debug/pprof/", pprof.Index)
-		pprofMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		pprofMux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		go func() {
-			log.Printf("pebble-serve: pprof listening on %s", pprofAddr)
-			// Bare ListenAndServe uses zero-value Server with no timeouts —
-			// a slow-reader on the pprof port can pin goroutines indefinitely.
-			// WriteTimeout is generous because CPU profile and trace endpoints
-			// stream for 30s+ by design.
-			srv := &http.Server{
-				Addr:              pprofAddr,
-				Handler:           pprofMux,
-				ReadHeaderTimeout: 10 * time.Second,
-				ReadTimeout:       30 * time.Second,
-				WriteTimeout:      5 * time.Minute,
-				IdleTimeout:       120 * time.Second,
-				MaxHeaderBytes:    1 << 20,
-			}
-			if err := srv.ListenAndServe(); err != nil {
-				log.Printf("pebble-serve: pprof exited: %v", err)
-			}
-		}()
-	}
-	go func() {
-		<-ctx.Done()
-		log.Printf("pebble-serve: shutting down")
-		shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer cancel()
-		_ = httpSrv.Shutdown(shutCtx)
-	}()
-
-	// Reuses ps + srv.hnsw + srv.embedder. The
-	// crawler.Crawler uses concurrency from cfg.Crawler.MaxConcurrent. Vectors
-	// land into the SAME HNSW pointer that /search reads from, so freshly
-	// crawled content is searchable as soon as AddPassage returns.
-	// Checkpoint goroutine persists the graph every crawlCheckpoint.
-	// collect a WaitGroup of crawler goroutines so the final
-	// HNSW persist runs BEFORE ps.Close() — fixes a 'pebble: closed' panic.
-	var crawlWG sync.WaitGroup
-	if *crawlSeeds != "" {
-		if err := srv.startInProcessCrawl(ctx, ps, *crawlSeeds, *crawlCheckpoint, cfg, &crawlWG); err != nil {
-			log.Printf("pebble-serve: in-process crawler not started: %v", err)
-		}
-	}
-
-	servErr := httpSrv.ListenAndServe()
-	// wait for crawler + checkpoint goroutines to finish their
-	// final persist before the deferred ps.Close() runs. Otherwise Persist
-	// can race ps.Close() and panic.
-	crawlWG.Wait()
-	if servErr != nil && servErr != http.ErrServerClosed {
-		return servErr
-	}
-	return nil
-}
-
-// startInProcessCrawl wires the crawler-inside-serve flow. Bumps
-// srv.hnsw to a fresh empty graph if none was loaded (so dense retrieval is
-// live as soon as the first passage lands), then runs the crawler in a
-// goroutine for the server's lifetime.
-func (s *pebbleHTTP) startInProcessCrawl(ctx context.Context, ps *store.PebbleStore, seedsFile string, ckpEvery time.Duration, cfg *config.Config, wg *sync.WaitGroup) error {
-	if s.embedder == nil {
-		return errors.New("crawl requires embedder configuration (cfg.Embeddings.Model)")
-	}
-	buf, err := os.ReadFile(seedsFile)
-	if err != nil {
-		return fmt.Errorf("read seeds file: %w", err)
-	}
-	var seeds []string
-	for _, line := range strings.Split(string(buf), "\n") {
-		t := strings.TrimSpace(line)
-		if t == "" || strings.HasPrefix(t, "#") {
-			continue
-		}
-		seeds = append(seeds, t)
-	}
-	if len(seeds) == 0 {
-		return errors.New("seeds file empty after parsing")
-	}
-
-	// Ensure HNSW is non-nil so the bridge can call AddPassage. /search reads
-	// the same pointer, so growth is immediately searchable.
-	if s.hnsw == nil {
-		s.hnsw = index.NewHNSW(s.embedder.Dim())
-		log.Printf("in-serve crawler: created fresh HNSW (dim=%d)", s.embedder.Dim())
-	}
-
-	c := crawler.NewWithBackend(cfg.Crawler, ps, index.NewPebbleBM25(ps))
-	// crawler embedder is wrapped in a semaphore-throttled
-	// wrapper so the bulk crawl can never queue enough requests at
-	// ollama to starve interactive /search?retriever=dense calls.
-	// Default cap = 8 (well under OLLAMA_NUM_PARALLEL=32); operator
-	// override via COSIFT_CRAWL_EMBED_CONCURRENCY.
-	crawlEmbedCap := 8
-	if v := os.Getenv("COSIFT_CRAWL_EMBED_CONCURRENCY"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			crawlEmbedCap = n
-		}
-	}
-	// Coalesces many
-	// per-doc embed calls into one larger inner.Embed per ~20 ms window
-	// (up to maxBatch=128 texts). Search bypasses the batcher (uses
-	// s.embedder directly) so interactive latency stays sub-100 ms.
-	// Operator overrides:
-	//   COSIFT_EMBED_BATCH=128       — texts per inner call
-	//   COSIFT_EMBED_BATCH_WAIT_MS=20 — drain timer in ms
-	batchMax := 128
-	if v := os.Getenv("COSIFT_EMBED_BATCH"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			batchMax = n
-		}
-	}
-	batchWait := 20 * time.Millisecond
-	if v := os.Getenv("COSIFT_EMBED_BATCH_WAIT_MS"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			batchWait = time.Duration(n) * time.Millisecond
-		}
-	}
-	batchedEmbedder := embed.NewBatchingEmbedder(s.embedder, batchMax, batchWait)
-	crawlEmbedder := embed.NewThrottledEmbedder(batchedEmbedder, crawlEmbedCap)
-	log.Printf("pebble-serve: crawler embed = throttle(%d) → batch(max=%d, wait=%s) → round-robin → backends; search bypasses batcher",
-		crawlEmbedCap, batchMax, batchWait)
-	c = c.WithEmbedder(crawlEmbedder)
-	c = c.WithPassageWriter(&hnswPassageWriter{ps: ps, hnsw: s.hnsw})
-	// Single-node config
-	// (NumShards <= 1) makes route fn a no-op (every URL ownsLocally).
-	// s.cluster was stamped at server-init time so search-only nodes also
-	// see it; we just need the router here.
-	if cfg.Cluster.IsClustered() {
-		c = c.WithRouter(
-			func(url string) (bool, string) {
-				if cfg.Cluster.OwnsURL(url) {
-					return true, ""
-				}
-				return false, cfg.Cluster.PeerForURL(url)
-			},
-			s.forwardURLToPeer,
-		)
-		log.Printf("in-serve crawler: cluster mode (shard=%d/%d, peers=%d)",
-			cfg.Cluster.MyShardID, cfg.Cluster.NumShards, len(cfg.Cluster.Peers))
-	}
-	// Expose Seed so /admin/crawl-enqueue can hand off forwarded URLs.
-	s.crawlSeed = c.Seed
-	// expose SeedSitemap so /admin/sitemap-import can push
-	// sitemap-discovered URLs into the live frontier.
-	s.crawlSeedSitemap = c.SeedSitemap
-	// expose SeedSitemapLane so /admin/site-submit can push a whole site's
-	// URLs into a chosen priority lane (default: submitted/priority).
-	s.crawlSeedSitemapLane = c.SeedSitemapLane
-	s.crawlSeedRSS = c.SeedRSS
-	s.crawlFetchNow = c.FetchAndIndexNow
-	s.crawlSeedWET = c.SeedWET
-	for _, u := range seeds {
-		// Only seed locally-owned URLs in cluster mode; the rest get forwarded.
-		if cfg.Cluster.IsClustered() && !cfg.Cluster.OwnsURL(u) {
-			if err := s.forwardURLToPeer(ctx, u, cfg.Cluster.PeerForURL(u)); err != nil {
-				log.Printf("in-serve crawler: forward initial seed %s: %v", u, err)
-			}
-			continue
-		}
-		_ = c.Seed(u)
-	}
-	log.Printf("in-serve crawler: %d seeds queued (concurrency=%d, depth=%d, checkpoint=%s)",
-		len(seeds), cfg.Crawler.MaxConcurrent, cfg.Crawler.MaxDepth, ckpEvery)
-	s.crawlActive = true
-
-	// Checkpoint goroutine: incremental persist via PersistFrom — each
-	// tick only writes nodes [lastN, n). Shutdown does a full persist
-	// from 0 so any backlinks added to older nodes get refreshed.
-	// Seeding lastN from the loaded graph means the first checkpoint
-	// after restart only writes nodes added during this run; the prior
-	// N are already on disk.
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		t := time.NewTicker(ckpEvery)
-		defer t.Stop()
-		lastN := s.hnsw.Len()
-		if lastN > 0 {
-			log.Printf("in-serve crawler: checkpoint baseline = %d nodes (loaded from disk)", lastN)
-		}
-		for {
-			select {
-			case <-ctx.Done():
-				n := s.hnsw.Len()
-				if n > 0 {
-					t0 := time.Now()
-					log.Printf("in-serve crawler: final HNSW persist at shutdown (%d nodes, full)", n)
-					if err := s.hnsw.Persist(context.Background(), ps); err != nil {
-						log.Printf("in-serve crawler: final HNSW persist failed: %v", err)
-					} else {
-						log.Printf("in-serve crawler: final HNSW persist complete in %s", time.Since(t0))
-					}
-				}
-				return
-			case <-t.C:
-				n := s.hnsw.Len()
-				if n == 0 || n == lastN {
-					continue
-				}
-				// graph can shrink (e.g., /admin/hnsw-compact rewrites
-				// indices and writes a smaller meta). When that happens, lastN
-				// from before the compaction is stale and > n; PersistFrom(lastN)
-				// would be a no-op forever, stranding any new AddPassages until
-				// shutdown. The compact handler does its own full Persist so disk
-				// is already in sync; we just need to resync lastN here.
-				if n < lastN {
-					lastN = n
-					continue
-				}
-				t0 := time.Now()
-				if err := s.hnsw.PersistFrom(context.Background(), ps, lastN); err != nil {
-					log.Printf("in-serve crawler: HNSW persist (incremental from %d) failed: %v", lastN, err)
-					continue
-				}
-				// alongside HNSW node writes, persist any newly-
-				// encoded PQ codes for nodes [lastN, n). Skipped silently
-				// when no codebook is loaded.
-				pqWritten := 0
-				if s.hnsw.HasPQ() {
-					if w, err := s.hnsw.PersistPQCodesFrom(context.Background(), ps, lastN); err != nil {
-						log.Printf("in-serve crawler: PQ codes persist failed: %v", err)
-					} else {
-						pqWritten = w
-					}
-				}
-				if pqWritten > 0 {
-					log.Printf("in-serve crawler: HNSW checkpoint at %d nodes (+%d incremental, +%d PQ codes, took %s)",
-						n, n-lastN, pqWritten, time.Since(t0))
-				} else {
-					log.Printf("in-serve crawler: HNSW checkpoint at %d nodes (+%d incremental, took %s)",
-						n, n-lastN, time.Since(t0))
-				}
-				lastN = n
-			}
-		}
-	}()
-
-	// Crawler goroutine.
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		log.Printf("in-serve crawler: starting")
-		if err := c.Run(ctx); err != nil {
-			log.Printf("in-serve crawler: exited with error: %v", err)
-			return
-		}
-		log.Printf("in-serve crawler: frontier drained")
-	}()
-	return nil
-}
-
-// pebbleHTTP holds the read-side handles. Stays small on purpose — the
-// SQLite-side Server struct accumulated a lot of config knobs over many iters;
-// the Pebble surface starts minimal and grows feature-by-feature.
-type pebbleHTTP struct {
-	store       *store.PebbleStore
-	idx         *index.PebbleBM25
-	chat        embed.ChatClient // nil when cfg.Chat.Model is unset; /answer returns 501
-	reranker    rerank.Reranker  // nil when no rerank is configured; ?rerank=true is a no-op then
-	rerankCandK int              // candidates pulled for rerank; default 20
-
-	// Bounded LLM concurrency + circuit breaker. chatGate is shared
-	// between the answer and rerank pools so a burst on one side can't
-	// also pin the goroutines of the other (the auto-sitemap-style leak
-	// that took the box down). chatSafe wraps the answer-class client
-	// (used for /answer, planner, HyDE, paraphrase) with KindAnswer;
-	// rerankSafe wraps the per-passage scoring client with KindRerank.
-	// Nil = uninitialized (tests / mocks).
-	chatGate   *chatgate.Gate
-	chatSafe   *chatgate.SafeChat
-	rerankSafe *chatgate.SafeChat
-
-	// TTL'd response cache + singleflight for /answer. A popular query
-	// during a load spike costs one hybrid + rerank + LLM call; the
-	// next 100 callers hit the cache in microseconds OR join the
-	// in-flight call (sub-second). Disabled when ttl=0.
-	answerCache *answercache.Cache
-
-	// Polls vLLM /metrics. When num_requests_waiting exceeds
-	// COSIFT_LLM_DEGRADE_QUEUE, optional LLM stages (rerank, HyDE,
-	// paraphrase) are silently skipped — graceful degradation rather
-	// than queue-then-timeout.
-	llmProbe *chatgate.LoadProbe
-
-	// Per-host authority Scorer; multiplies BM25 / hybrid scores by
-	// (1 + alpha * Score(host)). Wikipedia / kernel.org / arxiv float
-	// over the spam directories in the long tail of the crawl.
-	authority *authority.Scorer
-
-	// Per-endpoint latency + error-rate tracking against
-	// operator-configured targets. Violations are appended to disk
-	// (COSIFT_SLA_LOG_PATH) and surfaced on /sla.
-	sla *sla.Monitor
-
-	// hostBoosts is the operator-configured host-suffix →
-	// multiplier map, applied to fused retrieval scores by /query.
-	// Empty / nil = no boosts (fast path).
-	hostBoosts map[string]float64
-
-	// Nil = disabled.
-	rl *rateLimiter
-
-	// Empty cluster cfg = single-node, no-ops below.
-	cluster config.Cluster
-	// crawlSeed is set after startInProcessCrawl runs so /admin/crawl-enqueue
-	// can hand off forwarded URLs into the in-process frontier. Nil when no
-	// in-serve crawler is wired.
-	crawlSeed func(url string) error
-	// crawlSeedSitemap wraps Crawler.SeedSitemap so the /admin/
-	// sitemap-import endpoint can push sitemap URLs into the live frontier.
-	crawlSeedSitemap func(ctx context.Context, url string) (int, error)
-	// crawlSeedSitemapLane is like crawlSeedSitemap but lets the caller pick
-	// the frontier lane — used by /admin/site-submit to land a site's URLs
-	// in the high-priority submitted lane.
-	crawlSeedSitemapLane func(ctx context.Context, url string, lane byte) (int, error)
-	crawlSeedRSS         func(ctx context.Context, url string) (int, error)
-	crawlFetchNow        func(ctx context.Context, url string) error
-	crawlSeedWET         func(ctx context.Context, url string, dedupeFresh, lexicalOnly bool) (int, error)
-
-	// doc count at startup so /stats can report crawl rate
-	// without persistent counter tables. docs_added = current - startup,
-	// rate = docs_added / uptime.
-	startupDocs int
-	// crawlActive = true when in-serve crawler is running.
-	crawlActive bool
-
-	// /research?expand=true issues a
-	// chat call PER sub-query, so a sticky workload (repeated queries, slow
-	// sub-query rephrasings from the planner) hits the chat provider many
-	// times for the same passage. Cap at 256 entries; on overflow drop one
-	// arbitrary entry (Go map iteration order is randomized, good enough).
-	hydeMu       sync.RWMutex
-	hydeCache    map[string]string
-	hydeCacheCap int // env-configurable via COSIFT_HYDE_CACHE_SIZE
-
-	// PQStatus's lock contended
-	// hard with the crawler's AddPassage writers — observed /stats
-	// latency varied 0.2 s to 5.5 s. 5 s TTL keeps the cache effective
-	// for the landing page (polls every ~30 s) without serving very
-	// stale numbers.
-	//
-	// After TTL, return stale body
-	// immediately and refresh async; only the first-ever call pays the
-	// cold-cache cost. statsRefreshing is a single-flight guard so we
-	// don't fan out N background refreshes when a burst arrives stale.
-	statsBodyMu     sync.Mutex
-	statsBodyBlob   []byte
-	statsBodyAt     time.Time
-	statsRefreshing atomic.Bool
-
-	// /research?expand=paraphrase fans
-	// out 3 paraphrases × N sub-queries — same hot path as HyDE but each
-	// miss is 3x larger by output volume. Keyed on q only (fixed n=3 today).
-	paraMu       sync.RWMutex
-	paraCache    map[string][]string
-	paraCacheCap int // env-configurable via COSIFT_PARA_CACHE_SIZE
-	paraHits     atomic.Int64
-	paraMisses   atomic.Int64
-
-	// atomic counters surfaced on /metrics so operators can size the
-	// HyDE cache against a real workload — if misses dominate, raise the cap
-	// or move to an L2 store; if hits dominate at a tight working set, the
-	// cap is sufficient and you can mostly forget about it.
-	hydeHits   atomic.Int64
-	hydeMisses atomic.Int64
-
-	// snapshot at startup of whether persisted HNSW vectors exist
-	// in the 'v' family. Cheap peek (first-hit short-circuit). Surfaced on
-	// /stats so operators know whether the store is ready for the future
-	// ?retriever=dense path without needing to grep pebble-info.
-	hasVectors bool
-	// when an HNSW meta blob is persisted (the normal case),
-	// surface dim + node count too. Cheap 20-byte read at startup.
-	vectorDim   int
-	vectorNodes int
-	// Loaded
-	// only when COSIFT_LOAD_HNSW=true at startup (gigabytes RAM at scale).
-	// Nil = graph not loaded; /search?retriever=dense returns a warning.
-	hnsw *index.HNSW
-	// Required by
-	// /search?retriever=dense alongside s.hnsw. Built at startup when
-	// cfg.Embeddings.Model is set. Nil → ?retriever=dense warns + falls back.
-	embedder embed.Embedder
-
-	// Rerank failures fall back
-	// to BM25 order silently — that's the right reliability move, but without
-	// a counter operators can't tell whether their LLM/HTTP reranker is
-	// healthy or quietly broken.
-	rerankAttempts atomic.Int64
-	rerankFailures atomic.Int64
-
-	// chat call attempt + failure counters across HyDE expansion,
-	// /answer synth (sync + SSE), and /research plan + synth (sync + SSE).
-	// A spike in failures (provider 429s, network blips) is the clearest
-	// early signal that synth endpoints are degraded.
-	chatAttempts atomic.Int64
-	chatFailures atomic.Int64
-
-	// count responses that carried at least one warning.
-	// Operators alerting on this catch 'a deploy started sending malformed
-	// requests' without having to parse response bodies.
-	warningsEmitted atomic.Int64
-
-	// /metrics divides this by
-	// chatAttempts to give mean chat latency, separated from the
-	// per-endpoint duration. Diagnoses 'where did the seconds go' on a
-	// slow /research stream: chat-side or retrieval-side?
-	chatDurationNanos atomic.Int64
-
-	// per-endpoint request counters + duration sums via a
-	// counting middleware wrapping every mux entry. sync.Map keeps the hot
-	// path lock-free; /metrics reads via Range. Path is the label so a
-	// misrouted call (404) doesn't get counted under an existing endpoint.
-	// rate(sum)/rate(count) over the duration sum gives mean latency in PromQL.
-	requestCounts sync.Map // map[string]*endpointMetrics
-
-	started time.Time
-}
-
-type endpointMetrics struct {
-	count    atomic.Int64
-	sumNanos atomic.Int64
-}
-
-// count is the request-counting middleware. Bumps a per-path
-// {count, sumNanos} struct, lazily created on first request to a path. Hot
-// path is sync.Map.Load + two atomic Adds — no contention even under high
-// RPS. Duration is sampled after the handler returns so streaming endpoints
-// account for their full open connection time.
-// envIntDefault reads name as an int; falls back to def when unset or
-// unparseable. Used by gate / deadline knobs so operators can tune via
-// env without a config-file roundtrip.
-func envIntDefault(name string, def int) int {
-	if v := os.Getenv(name); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			return n
-		}
-	}
-	return def
-}
-
-// envDurationMsDefault reads name as a millisecond integer; returns def
-// on missing/invalid. Chose ms over Go's "2s" parse to keep operator
-// muscle memory consistent with the other COSIFT_*_MS knobs.
-func envDurationMsDefault(name string, def time.Duration) time.Duration {
-	if v := os.Getenv(name); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			return time.Duration(n) * time.Millisecond
-		}
-	}
-	return def
-}
-
-// rateLimiter is a per-IP token-bucket limiter. Active when
-// COSIFT_RATELIMIT_RPM > 0; nil = disabled. Whitelisted IPs bypass entirely.
-type rateLimiter struct {
-	rpm       float64
-	burst     float64
-	whitelist map[string]bool
-	buckets   sync.Map // string IP → *rateLimitBucket
-}
-
-type rateLimitBucket struct {
-	mu     sync.Mutex
-	tokens float64
-	last   time.Time
-}
-
-// newRateLimiterFromEnv reads COSIFT_RATELIMIT_RPM / _BURST / _WHITELIST.
-// Returns nil when RPM is unset or non-positive — limiting is off by default
-// so the no-config self-host story stays simple.
-func newRateLimiterFromEnv() *rateLimiter {
-	rpmStr := os.Getenv("COSIFT_RATELIMIT_RPM")
-	if rpmStr == "" {
-		return nil
-	}
-	rpm, err := strconv.ParseFloat(rpmStr, 64)
-	if err != nil || rpm <= 0 {
-		return nil
-	}
-	burst := 10.0
-	if v := os.Getenv("COSIFT_RATELIMIT_BURST"); v != "" {
-		if b, err := strconv.ParseFloat(v, 64); err == nil && b > 0 {
-			burst = b
-		}
-	}
-	wl := map[string]bool{}
-	for _, ip := range strings.Split(os.Getenv("COSIFT_RATELIMIT_WHITELIST"), ",") {
-		ip = strings.TrimSpace(ip)
-		if ip != "" {
-			wl[ip] = true
-		}
-	}
-	return &rateLimiter{rpm: rpm, burst: burst, whitelist: wl}
-}
-
-// whitelistList returns the whitelisted IPs as a slice (for logging only).
-func (rl *rateLimiter) whitelistList() []string {
-	if rl == nil {
-		return nil
-	}
-	out := make([]string, 0, len(rl.whitelist))
-	for ip := range rl.whitelist {
-		out = append(out, ip)
-	}
-	sort.Strings(out)
-	return out
-}
-
-// allow returns whether the request from ip may proceed. Side-effects: drains
-// one token from the IP's bucket on success.
-func (rl *rateLimiter) allow(ip string) bool {
-	if rl == nil {
-		return true
-	}
-	if rl.whitelist[ip] {
-		return true
-	}
-	bv, _ := rl.buckets.LoadOrStore(ip, &rateLimitBucket{tokens: rl.burst, last: time.Now()})
-	b := bv.(*rateLimitBucket)
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	now := time.Now()
-	elapsed := now.Sub(b.last).Seconds()
-	b.tokens += elapsed * rl.rpm / 60.0
-	if b.tokens > rl.burst {
-		b.tokens = rl.burst
-	}
-	b.last = now
-	if b.tokens < 1 {
-		return false
-	}
-	b.tokens--
-	return true
-}
-
-// stripPort drops ":port" from a "host:port" or "[v6]:port" RemoteAddr. Falls
-// back to the input on parse failure (so we still get SOME per-client key).
-func stripPort(remoteAddr string) string {
-	host, _, err := net.SplitHostPort(remoteAddr)
-	if err != nil {
-		return remoteAddr
-	}
-	return host
-}
-
-// rateLimit is the HTTP middleware that gates each request through the per-IP
-// limiter. Returns 429 with a JSON problem doc + Retry-After hint when the
-// bucket is empty. No-op when s.rl is nil.
-//
-// X-Forwarded-For is honored ONLY when the request came from a configured
-// trusted proxy; otherwise clients could spoof their IP by setting the header.
-// For self-host with cosift directly on the public network, leave
-// cfg.Server.TrustedProxies empty (default) and the RemoteAddr is used.
-func (s *pebbleHTTP) rateLimit(h http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ip := stripPort(r.RemoteAddr)
-		if !s.rl.allow(ip) {
-			w.Header().Set("Retry-After", "60")
-			writeProblem(w, http.StatusTooManyRequests, "rate limit exceeded for ip="+ip)
-			return
-		}
-		h(w, r)
-	}
-}
-
-func (s *pebbleHTTP) count(h http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		key := r.URL.Path
-		v, ok := s.requestCounts.Load(key)
-		if !ok {
-			v, _ = s.requestCounts.LoadOrStore(key, &endpointMetrics{})
-		}
-		m := v.(*endpointMetrics)
-		m.count.Add(1)
-		start := time.Now()
-		// Wrap the writer so the SLA monitor knows whether the call
-		// succeeded; default status to 200 because handlers that just
-		// w.Write() never call WriteHeader explicitly.
-		sw := &statusCapturingWriter{ResponseWriter: w, status: 200}
-		h(sw, r)
-		dur := time.Since(start)
-		m.sumNanos.Add(dur.Nanoseconds())
-		if s.sla != nil {
-			s.sla.Observe(key, dur, sw.status < 500)
-		}
-	}
-}
-
-// statusCapturingWriter lets the count middleware record the response
-// status for the SLA observer. The middleware treats 5xx as failures
-// for SLA purposes; 4xx are client errors and don't count against us.
-type statusCapturingWriter struct {
-	http.ResponseWriter
-	status      int
-	wroteHeader bool
-}
-
-func (s *statusCapturingWriter) WriteHeader(code int) {
-	if !s.wroteHeader {
-		s.status = code
-		s.wroteHeader = true
-	}
-	s.ResponseWriter.WriteHeader(code)
-}
-
-func (s *statusCapturingWriter) Write(p []byte) (int, error) {
-	if !s.wroteHeader {
-		s.wroteHeader = true
-	}
-	return s.ResponseWriter.Write(p)
-}
-
-// Flush passes through to support SSE / streaming handlers.
-func (s *statusCapturingWriter) Flush() {
-	if f, ok := s.ResponseWriter.(http.Flusher); ok {
-		f.Flush()
-	}
-}
-
-//go:embed assets/landing.html
-var landingHTML []byte
-
-//go:embed assets/chat.html
-var chatHTML []byte
-
-//go:embed assets/openapi.json
-var openapiJSON []byte
-
-//go:embed assets/swagger/index.html
-var swaggerIndexHTML []byte
-
-//go:embed assets/swagger/swagger-ui.css
-var swaggerUICSS []byte
-
-//go:embed assets/swagger/swagger-ui-bundle.js
-var swaggerUIJS []byte
-
-// handleLanding serves the self-host dashboard. Only matches exact
-// "/" — Go 1.22's ServeMux routes sub-paths to the longest match, so this
-// won't accidentally swallow /search etc.
-func (s *pebbleHTTP) handleLanding(w http.ResponseWriter, r *http.Request) {
-	if r.URL.Path != "/" {
-		http.NotFound(w, r)
-		return
-	}
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	w.Header().Set("Cache-Control", "public, max-age=60")
-	w.Write(landingHTML)
-}
-
-// handleChat serves the multi-turn chat UI. JS calls /answer
-// or /research with stream=true and renders SSE events incrementally.
-func (s *pebbleHTTP) handleChat(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	w.Header().Set("Cache-Control", "public, max-age=60")
-	_, _ = w.Write(chatHTML)
-}
-
-func (s *pebbleHTTP) handleOpenAPI(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Cache-Control", "public, max-age=3600")
-	w.Write(openapiJSON)
-}
-
-func (s *pebbleHTTP) handleSwaggerUI(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	w.Header().Set("Cache-Control", "public, max-age=300")
-	w.Write(swaggerIndexHTML)
-}
-
-func (s *pebbleHTTP) handleSwaggerAsset(w http.ResponseWriter, r *http.Request) {
-	file := r.PathValue("file")
-	switch file {
-	case "swagger-ui.css":
-		w.Header().Set("Content-Type", "text/css")
-		w.Header().Set("Cache-Control", "public, max-age=86400")
-		w.Write(swaggerUICSS)
-	case "swagger-ui-bundle.js":
-		w.Header().Set("Content-Type", "application/javascript")
-		w.Header().Set("Cache-Control", "public, max-age=86400")
-		w.Write(swaggerUIJS)
-	default:
-		http.NotFound(w, r)
-	}
-}
-
-func (s *pebbleHTTP) handleHealthz(w http.ResponseWriter, r *http.Request) {
-	writeJSON(w, http.StatusOK, map[string]any{"status": "ok"})
-}
-
-// forwardURLToPeer POSTs a URL to peer's /admin/crawl-enqueue. Best-effort:
-// caller logs failures and the URL is dropped — no retry or persistent queue.
-var forwardHTTP = &http.Client{Timeout: 10 * time.Second}
-
-func (s *pebbleHTTP) forwardURLToPeer(ctx context.Context, rawURL, peerAddr string) error {
-	body, _ := json.Marshal(crawlEnqueueReq{URL: rawURL})
-	// peerAddr is host:port; assume http inside the cluster (mTLS / VPN
-	// would be a wrapper concern). Switch to https://... if peers expose TLS.
-	endpoint := "http://" + peerAddr + "/admin/crawl-enqueue"
-	if strings.HasPrefix(peerAddr, "http://") || strings.HasPrefix(peerAddr, "https://") {
-		endpoint = strings.TrimRight(peerAddr, "/") + "/admin/crawl-enqueue"
-	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if t := s.cluster.PeerAuthToken; t != "" {
-		req.Header.Set("Authorization", "Bearer "+t)
-	}
-	resp, err := forwardHTTP.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 300 {
-		b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
-		return fmt.Errorf("peer %s returned %d: %s", peerAddr, resp.StatusCode, b)
-	}
-	return nil
-}
-
-// scatterSearch fans /search out to every peer with the given params,
-// collects per-peer hit lists, RRF-merges them, and returns the merged
-// top-k. Used by /search, /answer, /research gateway modes.
-// includeText forces text inlining on the scatter so callers that need
-// passages (synth endpoints) get them in one round trip.
-func (s *pebbleHTTP) scatterSearch(ctx context.Context, q string, k int, perPeerK int, params url.Values, includeText bool) (hits []searchHit, warns []string, totalCandidates int) {
-	srcQ := url.Values{}
-	for kk, vv := range params {
-		srcQ[kk] = vv
-	}
-	srcQ.Set("q", q)
-	srcQ.Set("k", strconv.Itoa(perPeerK))
-	srcQ.Set("cluster_local", "1")
-	if includeText {
-		srcQ.Set("include_text", "true")
-	}
-	queryStr := srcQ.Encode()
-
-	type peerResult struct {
-		peerIdx int
-		hits    []searchHit
-		err     error
-	}
-	resCh := make(chan peerResult, len(s.cluster.Peers))
-	httpClient := &http.Client{Timeout: 15 * time.Second}
-	for i, peer := range s.cluster.Peers {
-		if peer == "" {
-			continue
-		}
-		i, peer := i, peer
-		go func() {
-			endpoint := "http://" + peer + "/search?" + queryStr
-			if strings.HasPrefix(peer, "http://") || strings.HasPrefix(peer, "https://") {
-				endpoint = strings.TrimRight(peer, "/") + "/search?" + queryStr
-			}
-			req, _ := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
-			if t := s.cluster.PeerAuthToken; t != "" {
-				req.Header.Set("Authorization", "Bearer "+t)
-			}
-			resp, err := httpClient.Do(req)
-			if err != nil {
-				resCh <- peerResult{peerIdx: i, err: err}
-				return
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode >= 300 {
-				resCh <- peerResult{peerIdx: i, err: fmt.Errorf("peer %s returned %d", peer, resp.StatusCode)}
-				return
-			}
-			var sr struct {
-				Hits []searchHit `json:"hits"`
-			}
-			body, _ := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
-			if err := json.Unmarshal(body, &sr); err != nil {
-				resCh <- peerResult{peerIdx: i, err: err}
-				return
-			}
-			resCh <- peerResult{peerIdx: i, hits: sr.Hits}
-		}()
-	}
-	lists := [][]index.Hit{}
-	hitFull := map[string]searchHit{}
-gather:
-	for i := 0; i < len(s.cluster.Peers); i++ {
-		if s.cluster.Peers[i] == "" {
-			continue
-		}
-		select {
-		case res := <-resCh:
-			if res.err != nil {
-				warns = append(warns, fmt.Sprintf("peer %d (%s) failed: %v", res.peerIdx, s.cluster.Peers[res.peerIdx], res.err))
-				continue
-			}
-			totalCandidates += len(res.hits)
-			peerList := make([]index.Hit, len(res.hits))
-			for j, h := range res.hits {
-				peerList[j] = index.Hit{URL: h.URL, Title: h.Title, Score: h.Score}
-				if _, ok := hitFull[h.URL]; !ok {
-					hitFull[h.URL] = h
-				}
-			}
-			lists = append(lists, peerList)
-		case <-ctx.Done():
-			warns = append(warns, "client context cancelled before all peers responded")
-			break gather
-		}
-	}
-	fused := rrfFuse(lists)
-	if len(fused) > k {
-		fused = fused[:k]
-	}
-	hits = make([]searchHit, 0, len(fused))
-	for _, f := range fused {
-		full, ok := hitFull[f.URL]
-		if !ok {
-			full = searchHit{URL: f.URL, Title: f.Title}
-		}
-		full.Score = f.Score
-		hits = append(hits, full)
-	}
-	return hits, warns, totalCandidates
-}
-
-// handleSearchGateway is the scatter-gather entry. It fans out the
-// search to every peer's /search (including its own shard via the peers[]
-// table), each peer over-fetches k*2 candidates locally, gateway RRF-merges
-// the per-peer lists, and returns the top-k. Slow / failing peers are
-// included in a warnings[] entry but don't block the response.
-func (s *pebbleHTTP) handleSearchGateway(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	k := 10
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
-			k = n
-		}
-	}
-	perPeerK := k * 2
-	if perPeerK < 20 {
-		perPeerK = 20
-	}
-	// delegated to shared scatterSearch helper.
-	hits, warns, total := s.scatterSearch(r.Context(), q, k, perPeerK, r.URL.Query(), r.URL.Query().Get("include_text") == "true")
-	resp := searchResponse{
-		Query:           q,
-		Retriever:       fmt.Sprintf("gateway:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
-		Hits:            hits,
-		TotalCandidates: total,
-		Warnings:        append(warns, s.warningsFor(r)...),
-		Took:            time.Since(start).String(),
-	}
-	writeJSON(w, http.StatusOK, resp)
-}
-
-// numNonEmpty counts non-empty entries in a peer list (peers[i]="" means
-// "skip", typically MyShardID).
-func numNonEmpty(ps []string) int {
-	n := 0
-	for _, p := range ps {
-		if p != "" {
-			n++
-		}
-	}
-	return n
-}
-
-// handleFindSimilarGateway scatters /find_similar to every shard. The
-// owning shard (URL-mode) or every shard (text-mode) produces real
-// neighbors; others return empty. Gateway RRF-merges.
-func (s *pebbleHTTP) handleFindSimilarGateway(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	k := 10
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
-			k = n
-		}
-	}
-	perPeerK := k * 2
-	if perPeerK < 20 {
-		perPeerK = 20
-	}
-	srcQ := url.Values{}
-	for kk, vv := range r.URL.Query() {
-		srcQ[kk] = vv
-	}
-	srcQ.Set("k", strconv.Itoa(perPeerK))
-	srcQ.Set("cluster_local", "1")
-	queryStr := srcQ.Encode()
-
-	type peerResult struct {
-		peerIdx int
-		hits    []searchHit
-		err     error
-	}
-	resCh := make(chan peerResult, len(s.cluster.Peers))
-	httpClient := &http.Client{Timeout: 15 * time.Second}
-	for i, peer := range s.cluster.Peers {
-		if peer == "" {
-			continue
-		}
-		i, peer := i, peer
-		go func() {
-			endpoint := "http://" + peer + "/find_similar?" + queryStr
-			if strings.HasPrefix(peer, "http://") || strings.HasPrefix(peer, "https://") {
-				endpoint = strings.TrimRight(peer, "/") + "/find_similar?" + queryStr
-			}
-			req, _ := http.NewRequestWithContext(r.Context(), http.MethodGet, endpoint, http.NoBody)
-			if t := s.cluster.PeerAuthToken; t != "" {
-				req.Header.Set("Authorization", "Bearer "+t)
-			}
-			resp, err := httpClient.Do(req)
-			if err != nil {
-				resCh <- peerResult{peerIdx: i, err: err}
-				return
-			}
-			defer resp.Body.Close()
-			if resp.StatusCode >= 300 {
-				resCh <- peerResult{peerIdx: i, err: fmt.Errorf("peer %s returned %d", peer, resp.StatusCode)}
-				return
-			}
-			var sr struct {
-				Hits []searchHit `json:"hits"`
-			}
-			body, _ := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
-			if err := json.Unmarshal(body, &sr); err != nil {
-				resCh <- peerResult{peerIdx: i, err: err}
-				return
-			}
-			resCh <- peerResult{peerIdx: i, hits: sr.Hits}
-		}()
-	}
-	lists := [][]index.Hit{}
-	hitFull := map[string]searchHit{}
-	warns := []string{}
-	total := 0
-gather:
-	for i := 0; i < len(s.cluster.Peers); i++ {
-		if s.cluster.Peers[i] == "" {
-			continue
-		}
-		select {
-		case res := <-resCh:
-			if res.err != nil {
-				warns = append(warns, fmt.Sprintf("peer %d (%s) failed: %v", res.peerIdx, s.cluster.Peers[res.peerIdx], res.err))
-				continue
-			}
-			total += len(res.hits)
-			peerList := make([]index.Hit, len(res.hits))
-			for j, h := range res.hits {
-				peerList[j] = index.Hit{URL: h.URL, Title: h.Title, Score: h.Score}
-				if _, ok := hitFull[h.URL]; !ok {
-					hitFull[h.URL] = h
-				}
-			}
-			lists = append(lists, peerList)
-		case <-r.Context().Done():
-			warns = append(warns, "client context cancelled")
-			break gather
-		}
-	}
-	fused := rrfFuse(lists)
-	if len(fused) > k {
-		fused = fused[:k]
-	}
-	out := make([]searchHit, 0, len(fused))
-	for _, f := range fused {
-		full, ok := hitFull[f.URL]
-		if !ok {
-			full = searchHit{URL: f.URL, Title: f.Title}
-		}
-		full.Score = f.Score
-		out = append(out, full)
-	}
-	writeJSON(w, http.StatusOK, searchResponse{
-		Query:           r.URL.Query().Get("url"),
-		Retriever:       fmt.Sprintf("gateway:find_similar:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
-		Hits:            out,
-		TotalCandidates: total,
-		Warnings:        append(warns, s.warningsFor(r)...),
-		Took:            time.Since(start).String(),
-	})
-}
-
-// handleResearchGateway runs the planner on the gateway, scatters
-// each sub-query to peers (collecting RRF-merged hits with text), then
-// runs a single research synth.
-func (s *pebbleHTTP) handleResearchGateway(w http.ResponseWriter, r *http.Request) {
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented, "/research requires cfg.Chat.Model on the gateway")
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	k := 8
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
-			k = n
-		}
-	}
-	// 1. Plan via chat.
-	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: researchPlanPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
-		return
-	}
-	subs := parseSubQueries(planRaw, q)
-	if len(subs) > 5 {
-		subs = subs[:5]
-	}
-	// 2. Scatter per sub-query. Dedup by URL, keep best fused score.
-	type ranked struct {
-		score float64
-		hit   searchHit
-	}
-	best := make(map[string]ranked, k*len(subs))
-	allWarns := []string{}
-	perSub := k * 2
-	for _, sq := range subs {
-		hits, warns, _ := s.scatterSearch(r.Context(), sq, perSub, perSub, r.URL.Query(), true)
-		allWarns = append(allWarns, warns...)
-		for _, h := range hits {
-			if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
-				best[h.URL] = ranked{score: h.Score, hit: h}
-			}
-		}
-	}
-	pooled := make([]ranked, 0, len(best))
-	for _, v := range best {
-		pooled = append(pooled, v)
-	}
-	sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
-	if len(pooled) > k {
-		pooled = pooled[:k]
-	}
-	// 3. Build prompt.
-	sources := make([]answerSource, 0, len(pooled))
-	var promptSources strings.Builder
-	for i, p := range pooled {
-		src := answerSource{ID: i + 1, URL: p.hit.URL, Title: p.hit.Title, Excerpt: p.hit.Excerpt, Author: p.hit.Author, PublishedAt: p.hit.PublishedAt}
-		text := p.hit.Text
-		if text == "" {
-			text = p.hit.Excerpt
-		}
-		sources = append(sources, src)
-		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, p.hit.Title, p.hit.URL, text)
-	}
-	if len(sources) == 0 {
-		writeJSON(w, http.StatusOK, researchResponse{
-			Query: q, Plan: subs, Answer: "No matching sources across the cluster.",
-			Sources: sources, Model: s.chat.Model(),
-			Warnings: allWarns, Took: time.Since(start).String(),
-		})
-		return
-	}
-	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: researchSynthPrompt},
-		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Original question: " + q},
-	})
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, researchResponse{
-		Query: q, Plan: subs, Answer: answer, Sources: sources,
-		Model:     s.chat.Model(),
-		Retriever: fmt.Sprintf("gateway:rrf(%d-shard,%d-sub)", numNonEmpty(s.cluster.Peers), len(subs)),
-		Warnings:  allWarns, Took: time.Since(start).String(),
-	})
-}
-
-// handleAnswerGateway scatters /search across peers with include_text, runs
-// a SINGLE chat synth on the gateway.
-func (s *pebbleHTTP) handleAnswerGateway(w http.ResponseWriter, r *http.Request) {
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented, "/answer requires cfg.Chat.Model on the gateway")
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	k := 5
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
-			k = n
-		}
-	}
-	hits, warns, total := s.scatterSearch(r.Context(), q, k, k*2, r.URL.Query(), true)
-	if len(hits) == 0 {
-		writeJSON(w, http.StatusOK, answerResponse{
-			Query: q, Answer: "No matching sources across the cluster.",
-			Sources: []answerSource{}, Model: s.chat.Model(),
-			TotalCandidates: total, Warnings: warns, Took: time.Since(start).String(),
-		})
-		return
-	}
-	sources := make([]answerSource, 0, len(hits))
-	var promptSources strings.Builder
-	for i, h := range hits {
-		src := answerSource{ID: i + 1, URL: h.URL, Title: h.Title, Excerpt: h.Excerpt, Author: h.Author, PublishedAt: h.PublishedAt}
-		text := h.Text
-		if text == "" {
-			text = h.Excerpt
-		}
-		sources = append(sources, src)
-		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, h.Title, h.URL, text)
-	}
-	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: answerSystemPrompt},
-		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Question: " + q},
-	})
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, answerResponse{
-		Query: q, Answer: answer, Sources: sources, Model: s.chat.Model(),
-		Retriever:       fmt.Sprintf("gateway:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
-		TotalCandidates: total, Warnings: warns, Took: time.Since(start).String(),
-	})
-}
-
-// handlePQEncode backfills PQ codes for every node currently missing one,
-// against the codebook already loaded on this serve. Order-of-magnitude
-// faster than handlePQTrain because it skips k-means; just encode loop.
-func (s *pebbleHTTP) handlePQEncode(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.hnsw == nil {
-		writeProblem(w, http.StatusBadRequest, "pq-encode: no HNSW loaded")
-		return
-	}
-	if !s.hnsw.HasPQ() {
-		writeProblem(w, http.StatusBadRequest, "pq-encode: no codebook loaded — run /admin/pq-train first")
-		return
-	}
-	t0 := time.Now()
-	ids, codes, err := s.hnsw.EncodeMissing()
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, "encode: "+err.Error())
-		return
-	}
-	encodeElapsed := time.Since(t0)
-	if len(ids) == 0 {
-		writeJSON(w, http.StatusOK, map[string]any{
-			"encoded":       0,
-			"total":         s.hnsw.Len(),
-			"already_full":  true,
-			"total_elapsed": encodeElapsed.String(),
-		})
-		return
-	}
-	// Persist freshly-encoded codes in a single batch.
-	pq := s.hnsw.PQStatus()
-	cb := &index.PQCodebook{Dim: pq.Dim, M: pq.M, K: pq.K, SubDim: pq.Dim / pq.M}
-	entries := make([]store.PQCodeEntry, len(ids))
-	for i := range ids {
-		entries[i] = store.PQCodeEntry{ID: ids[i], Blob: cb.EncodeCodeBlob(codes[i])}
-	}
-	persistT0 := time.Now()
-	if err := s.store.PutPQCodesBatch(r.Context(), entries); err != nil {
-		writeProblem(w, http.StatusInternalServerError, "persist codes: "+err.Error())
-		return
-	}
-	persistElapsed := time.Since(persistT0)
-	total := s.hnsw.Len()
-	coverage := 100.0 * float64(pq.NodesWithCode+len(ids)) / float64(total)
-	totalElapsed := time.Since(t0)
-	log.Printf("pq-encode: backfilled %d codes in %s (persist %s, total %s) → coverage %.1f%%",
-		len(ids), encodeElapsed-persistElapsed, persistElapsed, totalElapsed, coverage)
-	writeJSON(w, http.StatusOK, map[string]any{
-		"encoded":         len(ids),
-		"total":           total,
-		"coverage_pct":    coverage,
-		"encode_elapsed":  (encodeElapsed - persistElapsed).String(),
-		"persist_elapsed": persistElapsed.String(),
-		"total_elapsed":   totalElapsed.String(),
-	})
-}
-
-// handleCheckpoint creates a Pebble checkpoint (hard-linked, point-in-time)
-// at a server-chosen path under COSIFT_CHECKPOINT_DIR (default /tmp). The
-// returned path is safe to tar — Pebble's compactor cannot mutate hard-linked
-// SSTs. Caller is responsible for deleting the dir after consuming it.
-func (s *pebbleHTTP) handleCheckpoint(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	base := os.Getenv("COSIFT_CHECKPOINT_DIR")
-	if base == "" {
-		base = "/tmp"
-	}
-	dest := filepath.Join(base, fmt.Sprintf("cosift-ckpt-%d", time.Now().UnixNano()))
-	t0 := time.Now()
-	if err := s.store.Checkpoint(dest); err != nil {
-		writeProblem(w, http.StatusInternalServerError, "checkpoint: "+err.Error())
-		return
-	}
-	elapsed := time.Since(t0)
-	log.Printf("checkpoint: created %q in %s", dest, elapsed)
-	writeJSON(w, http.StatusOK, map[string]any{
-		"path":    dest,
-		"elapsed": elapsed.String(),
-	})
-}
-
-// handlePQTrain trains a PQ codebook on a sample of the current HNSW
-// graph, then encodes ALL nodes and persists codes + codebook to Pebble.
-// Synchronous; returns when training completes.
-type pqTrainReq struct {
-	SampleSize int `json:"sample_size"` // default 50000
-	M          int `json:"m"`           // default dim/8 (subspaces of size 8)
-	K          int `json:"k"`           // default 256
-	Iters      int `json:"iters"`       // default 15
-	Parallel   int `json:"parallel"`    // default GOMAXPROCS
-}
-
-func (s *pebbleHTTP) handlePQTrain(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.hnsw == nil || s.hnsw.Len() == 0 {
-		writeProblem(w, http.StatusBadRequest, "pq-train requires an in-memory HNSW with nodes")
-		return
-	}
-	var req pqTrainReq
-	if r.ContentLength > 0 {
-		body, _ := io.ReadAll(io.LimitReader(r.Body, 4<<10))
-		_ = json.Unmarshal(body, &req)
-	}
-	if req.SampleSize <= 0 {
-		req.SampleSize = 50000
-	}
-	if req.K <= 0 {
-		req.K = 256
-	}
-	if req.Iters <= 0 {
-		req.Iters = 15
-	}
-	if req.Parallel <= 0 {
-		req.Parallel = 8
-	}
-	if s.embedder == nil {
-		writeProblem(w, http.StatusBadRequest, "pq-train needs an embedder to know the vector dim")
-		return
-	}
-	dim := s.embedder.Dim()
-	M := req.M
-	if M <= 0 {
-		// Default: 1 subspace per 8 dims. For 768d → 96 subspaces.
-		M = dim / 8
-		if M <= 0 {
-			M = 1
-		}
-	}
-	if dim%M != 0 {
-		writeProblem(w, http.StatusBadRequest, fmt.Sprintf("dim %d not divisible by M %d", dim, M))
-		return
-	}
-	t0 := time.Now()
-	log.Printf("pq-train: sampling %d / %d vectors (dim=%d, M=%d, K=%d, iters=%d, parallel=%d)",
-		req.SampleSize, s.hnsw.Len(), dim, M, req.K, req.Iters, req.Parallel)
-	sample := s.hnsw.SampleVectors(req.SampleSize, time.Now().UnixNano())
-	sampledN := len(sample)
-	log.Printf("pq-train: training codebook on %d samples...", sampledN)
-	cb, err := index.TrainPQCodebookParallel(sample, dim, M, req.K, req.Iters, req.Parallel, time.Now().UnixNano())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, "train: "+err.Error())
-		return
-	}
-	trainElapsed := time.Since(t0)
-	log.Printf("pq-train: codebook trained in %s; persisting...", trainElapsed)
-	if err := cb.Persist(r.Context(), s.store); err != nil {
-		writeProblem(w, http.StatusInternalServerError, "persist codebook: "+err.Error())
-		return
-	}
-	encodeT0 := time.Now()
-	log.Printf("pq-train: encoding %d nodes...", s.hnsw.Len())
-	ids, codes, err := s.hnsw.EncodeAll(cb)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, "encode: "+err.Error())
-		return
-	}
-	entries := make([]store.PQCodeEntry, len(ids))
-	for i := range ids {
-		// K≤256 → byte-packed for 2x disk savings.
-		entries[i] = store.PQCodeEntry{ID: ids[i], Blob: cb.EncodeCodeBlob(codes[i])}
-	}
-	if err := s.store.PutPQCodesBatch(r.Context(), entries); err != nil {
-		writeProblem(w, http.StatusInternalServerError, "put codes: "+err.Error())
-		return
-	}
-	encodeElapsed := time.Since(encodeT0)
-	totalElapsed := time.Since(t0)
-	log.Printf("pq-train: done in %s (train=%s, encode+persist=%s, %d codes written)",
-		totalElapsed, trainElapsed, encodeElapsed, len(ids))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"sample_size":    sampledN,
-		"dim":            dim,
-		"M":              M,
-		"K":              req.K,
-		"iters":          req.Iters,
-		"nodes_encoded":  len(ids),
-		"train_elapsed":  trainElapsed.String(),
-		"encode_elapsed": encodeElapsed.String(),
-		"total_elapsed":  totalElapsed.String(),
-	})
-}
-
-// handleCrawlEnqueue accepts a single URL forwarded from a peer shard and
-// pushes it onto the local crawler's frontier. Auth via cfg.Cluster.
-// PeerAuthToken Bearer header.
-type crawlEnqueueReq struct {
-	URL string `json:"url"`
-}
-
-func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request) {
-	// Auth
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
-			return
-		}
-	}
-	if s.crawlSeed == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req crawlEnqueueReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"...\"}")
-		return
-	}
-	if err := s.crawlSeed(req.URL); err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]any{"queued": req.URL})
-}
-
-// handleFrontierClear wipes the entire frontier in a single Pebble
-// DeleteRange — every queued URL, primary + secondary index. Used when
-// the frontier is so polluted by spam-discovery crawls that
-// claim-time exclude rules can't drain it in reasonable time. Operator
-// re-seeds via seeds-file restart or /admin/sitemap-import calls.
-//
-// Sees the same PeerAuthToken as the other admin endpoints. Returns the
-// approximate count of deleted entries (DeleteRange is O(tombstones),
-// not O(N), so the count is reported as -1 to signal "swept range").
-func (s *pebbleHTTP) handleFrontierClear(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
-			return
-		}
-	}
-	if err := s.store.ClearFrontier(r.Context()); err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]any{"cleared": true})
-}
-
-// handleFrontierPurgeHost deletes every queued frontier entry whose host
-// matches the supplied value. Used to drain blacklisted hosts that the
-// round-robin cursor would take days to chew through one-by-one (the
-// cursor visits each host once per cycle; a 5M-entry host needs 5M
-// cycles to drain, days of cycle time at typical claim rates).
-type frontierPurgeReq struct {
-	Host string `json:"host"`
-}
-
-func (s *pebbleHTTP) handleFrontierPurgeHost(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
-			return
-		}
-	}
-	var req frontierPurgeReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"host\": \"...\"}")
-		return
-	}
-	n, err := s.store.PurgeFrontierByHost(r.Context(), req.Host)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]any{"purged": n, "host": req.Host})
-}
-
-// handleSitemapImport fetches a sitemap.xml (or sitemap-index, one level
-// of recursion) and pushes every <loc> entry to the live frontier. Same
-// auth as crawl-enqueue. Synchronous — returns when all URLs are queued.
-// For very large sitemaps (>50K URLs) the call may take seconds.
-type sitemapImportReq struct {
-	URL string `json:"url"`
-}
-
-func (s *pebbleHTTP) handleSitemapImport(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedSitemap == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req sitemapImportReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"https://.../sitemap.xml\"}")
-		return
-	}
-	t0 := time.Now()
-	n, err := s.crawlSeedSitemap(r.Context(), req.URL)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	log.Printf("sitemap-import: queued %d URLs from %s in %s", n, req.URL, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"sitemap": req.URL,
-		"queued":  n,
-		"elapsed": time.Since(t0).String(),
-	})
-}
-
-// handleCrawlNow runs the fetch-parse-index pipeline synchronously for one
-// or more URLs, bypassing the persistent frontier entirely. Use this when
-// the round-robin cursor in a large frontier (10M+) would take hours to
-// reach an explicitly-seeded URL — direct fetch + UpsertDocument + chunk +
-// embed all happen in this request. Returns per-URL outcomes.
-type crawlNowReq struct {
-	URLs []string `json:"urls"`
-	URL  string   `json:"url,omitempty"`
-}
-
-func (s *pebbleHTTP) handleCrawlNow(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlFetchNow == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req crawlNowReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20))
-	if err := json.Unmarshal(body, &req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "expected {\"urls\":[...]} or {\"url\":\"...\"}")
-		return
-	}
-	urls := req.URLs
-	if req.URL != "" {
-		urls = append(urls, req.URL)
-	}
-	if len(urls) == 0 {
-		writeProblem(w, http.StatusBadRequest, "no URLs in body")
-		return
-	}
-	results := make([]map[string]any, 0, len(urls))
-	t0 := time.Now()
-	for _, u := range urls {
-		err := s.crawlFetchNow(r.Context(), u)
-		out := map[string]any{"url": u}
-		if err != nil {
-			out["error"] = err.Error()
-		} else {
-			out["ok"] = true
-		}
-		results = append(results, out)
-	}
-	writeJSON(w, http.StatusOK, map[string]any{
-		"results": results,
-		"elapsed": time.Since(t0).String(),
-	})
-}
-
-// evalQuickQueries is the canned smoke-eval set. Chosen for breadth:
-// 3 definition + 3 technical-howto + 2 comparison + 2 findability. Tracks
-// the same answered/no-info/error verdicts the external 60-Q harness uses,
-// so a regression here predicts a regression in the bigger eval. Hardcoded
-// (no new deps) so any operator can hit /admin/eval-quick and get an
-// answer rate without setting up Python or external test infra.
-var evalQuickQueries = []string{
-	"what is BM25 ranking function",
-	"what is HNSW algorithm",
-	"what is reciprocal rank fusion",
-	"how does retrieval augmented generation work",
-	"how does Go goroutines scheduling work",
-	"explain CAP theorem in distributed systems",
-	"compare BM25 vs dense vector retrieval",
-	"compare REST vs GraphQL APIs",
-	"what is Pilot Protocol overlay network",
-	"what is sentence embedding",
-}
-
-// handleEvalQuick runs the 10-query smoke eval against the running cosift
-// and returns answered-rate + per-query verdicts. Each query hits /answer
-// in-process (re-uses the same chat/retrieval stack as a real /answer call),
-// so the rate that comes back matches what users see.
-func (s *pebbleHTTP) handleEvalQuick(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented, "eval-quick requires cfg.Chat.Model")
-		return
-	}
-	// the server-wide WriteTimeout (60s) was killing this handler
-	// before 10 sequential /answer calls finished — connection closed mid-write,
-	// curl saw "empty reply from server" (exit 52). Disable the deadline for
-	// this long-running admin endpoint via ResponseController. Pairs with
-	// bounded-parallel dispatch below so the 10-query batch finishes in
-	// ~one chat-LLM round-trip instead of ten.
-	if rc := http.NewResponseController(w); rc != nil {
-		_ = rc.SetWriteDeadline(time.Time{})
-	}
-	type queryResult struct {
-		Query     string `json:"query"`
-		Verdict   string `json:"verdict"` // answered | no_info | empty | error
-		LatencyMs int    `json:"latency_ms"`
-		Sources   int    `json:"sources"`
-		Suggest   string `json:"suggest_escalation,omitempty"`
-	}
-	results := make([]queryResult, len(evalQuickQueries))
-	t0 := time.Now()
-
-	// Bounded-parallel dispatch. vLLM batches concurrent requests so 4-wide
-	// fan-out finishes in roughly one chat round-trip rather than 10.
-	sem := make(chan struct{}, 4)
-	var wg sync.WaitGroup
-	for i, q := range evalQuickQueries {
-		wg.Add(1)
-		go func(i int, q string) {
-			defer wg.Done()
-			sem <- struct{}{}
-			defer func() { <-sem }()
-			qStart := time.Now()
-			subReq, _ := http.NewRequestWithContext(r.Context(), http.MethodGet,
-				"/answer?q="+url.QueryEscape(q)+"&stream=false&retriever=hybrid", nil)
-			rec := newResponseRecorder()
-			s.handleAnswer(rec, subReq)
-			latencyMs := int(time.Since(qStart) / time.Millisecond)
-
-			var ar answerResponse
-			if jerr := json.Unmarshal(rec.body.Bytes(), &ar); jerr != nil || rec.code >= 400 {
-				results[i] = queryResult{Query: q, Verdict: "error", LatencyMs: latencyMs}
-				return
-			}
-			var verdict string
-			switch {
-			case len(ar.Answer) < 30:
-				verdict = "empty"
-			case answerLooksLikeNoInfo(ar.Answer):
-				verdict = "no_info"
-			default:
-				verdict = "answered"
-			}
-			results[i] = queryResult{
-				Query:     q,
-				Verdict:   verdict,
-				LatencyMs: latencyMs,
-				Sources:   len(ar.Sources),
-				Suggest:   ar.SuggestEscalation,
-			}
-		}(i, q)
-	}
-	wg.Wait()
-
-	answered, noInfo, errCount, empty, totalMs := 0, 0, 0, 0, 0
-	for _, r := range results {
-		totalMs += r.LatencyMs
-		switch r.Verdict {
-		case "answered":
-			answered++
-		case "no_info":
-			noInfo++
-		case "empty":
-			empty++
-		default:
-			errCount++
-		}
-	}
-	n := len(evalQuickQueries)
-	writeJSON(w, http.StatusOK, map[string]any{
-		"queries":         n,
-		"answered":        answered,
-		"no_info":         noInfo,
-		"empty":           empty,
-		"errors":          errCount,
-		"answer_rate_pct": 100 * answered / n,
-		"avg_latency_ms":  totalMs / n,
-		"total_elapsed":   time.Since(t0).String(),
-		"chat_model":      s.chat.Model(),
-		"results":         results,
-	})
-}
-
-// handleHNSWCompact runs HNSW.Compact() in-place, then clears the persisted
-// 'v' family and writes a fresh full snapshot so disk matches the compacted
-// in-memory graph. Cheaper than the offline hnsw-rebuild subcommand: Compact
-// keeps the existing topology among surviving nodes (O(N + edges)), whereas
-// Rebuild re-inserts every node via HNSW search (multiple minutes per million
-// passages). Operators run this when stats.zombie_nodes climbs above ~30% of
-// nodes_total.
-//
-// Synchronous; holds the HNSW write lock during the compact step and the
-// read lock during the persist step. Dense retrieval and AddPassage calls
-// queue for the duration. The server-wide WriteTimeout is disabled here via
-// ResponseController because compacting a multi-million-node graph routinely
-// runs past 60s. Returns counters so operators can confirm progress.
-func (s *pebbleHTTP) handleHNSWCompact(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.hnsw == nil {
-		writeProblem(w, http.StatusNotImplemented, "hnsw-compact requires a loaded HNSW graph")
-		return
-	}
-	if rc := http.NewResponseController(w); rc != nil {
-		_ = rc.SetWriteDeadline(time.Time{})
-	}
-	skipPersist := r.URL.Query().Get("skip_persist") == "1"
-
-	before := s.hnsw.Len()
-	t0 := time.Now()
-	removed := s.hnsw.Compact()
-	compactDur := time.Since(t0)
-	after := s.hnsw.Len()
-
-	resp := map[string]any{
-		"nodes_before": before,
-		"nodes_after":  after,
-		"removed":      removed,
-		"compact_ms":   compactDur.Milliseconds(),
-		"persisted":    false,
-	}
-
-	if removed == 0 || skipPersist {
-		writeJSON(w, http.StatusOK, resp)
-		return
-	}
-
-	// Compact remapped node indices; the persisted 'v' family now points at
-	// stale slots. Wipe and full-rewrite. PQ codes follow node indices too,
-	// so clear 'q' as well — operators must re-run /admin/pq-train if PQ was
-	// in use.
-	persistT0 := time.Now()
-	ctx := r.Context()
-	if err := s.store.ClearVectorFamily(ctx); err != nil {
-		resp["persist_error"] = "clear vector family: " + err.Error()
-		writeJSON(w, http.StatusInternalServerError, resp)
-		return
-	}
-	if err := s.store.ClearPQFamily(ctx); err != nil {
-		resp["persist_error"] = "clear pq family: " + err.Error()
-		writeJSON(w, http.StatusInternalServerError, resp)
-		return
-	}
-	if err := s.hnsw.Persist(ctx, s.store); err != nil {
-		resp["persist_error"] = "persist: " + err.Error()
-		writeJSON(w, http.StatusInternalServerError, resp)
-		return
-	}
-	resp["persisted"] = true
-	resp["persist_ms"] = time.Since(persistT0).Milliseconds()
-	log.Printf("hnsw-compact: removed=%d (%.1f%% zombies) compact=%s persist=%s nodes %d→%d",
-		removed, 100*float64(removed)/float64(before),
-		compactDur.Round(time.Millisecond), time.Since(persistT0).Round(time.Millisecond),
-		before, after)
-	writeJSON(w, http.StatusOK, resp)
-}
-
-// responseRecorder captures an http.Handler's output for in-process
-// dispatch. Minimal — only what handleEvalQuick needs.
-type responseRecorder struct {
-	hdr  http.Header
-	body bytes.Buffer
-	code int
-}
-
-func newResponseRecorder() *responseRecorder {
-	return &responseRecorder{hdr: make(http.Header), code: 200}
-}
-
-func (r *responseRecorder) Header() http.Header         { return r.hdr }
-func (r *responseRecorder) Write(b []byte) (int, error) { return r.body.Write(b) }
-func (r *responseRecorder) WriteHeader(c int)           { r.code = c }
-
-// handleEmbedBackfill walks famDocMeta, finds docs whose URL has zero
-// HNSW vector nodes (lexical-only ingest from WET imports, or any docs
-// indexed before an embedder was wired), and embeds them. Pairs with
-// /admin/wet-import?lexical_only=true so bulk-loaded content gets dense
-// retrieval after the fast lexical pass.
-//
-// Body: {"limit": 10000, "workers": 4}
-//   - limit:   cap on docs processed this call (0 = unlimited)
-//   - workers: concurrent embed pipelines (default 4)
-//
-// Idempotent against concurrent re-runs — re-embedding a doc that
-// already has vectors is a no-op because UpsertPassage just adds more
-// nodes pointing at the same URL (which the dedup-by-URL in
-// HNSW.Search picks the best of anyway). Cleaner: zombie-reclaim kicks
-// in to invalidate prior, then fresh vectors take over.
-type embedBackfillReq struct {
-	Limit   int `json:"limit,omitempty"`
-	Workers int `json:"workers,omitempty"`
-}
-
-func (s *pebbleHTTP) handleEmbedBackfill(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.embedder == nil || s.hnsw == nil {
-		writeProblem(w, http.StatusNotImplemented, "embed backfill requires both an embedder and HNSW graph to be configured")
-		return
-	}
-	var req embedBackfillReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	_ = json.Unmarshal(body, &req)
-	if req.Workers <= 0 {
-		req.Workers = 4
-	}
-	if req.Workers > 32 {
-		req.Workers = 32
-	}
-
-	// Pipeline: iter docs → filter (no-vector) → chunker → embed → write.
-	// Bounded channel so we don't materialize the candidate list in memory.
-	type cand struct {
-		docID int64
-		url   string
-	}
-	candCh := make(chan cand, req.Workers*4)
-
-	t0 := time.Now()
-	var scanned, missing, embedded atomic.Int64
-
-	// Find candidates: docs with zero HNSW vectors. Producer.
-	go func() {
-		defer close(candCh)
-		_ = s.store.IterDocsLite(r.Context(), func(docID int64, url string) error {
-			scanned.Add(1)
-			if req.Limit > 0 && missing.Load() >= int64(req.Limit) {
-				return errors.New("limit reached") // sentinel — stops IterDocsLite
-			}
-			if _, ok := s.hnsw.LookupVectorByURL(url); ok {
-				return nil // already has vectors
-			}
-			missing.Add(1)
-			select {
-			case candCh <- cand{docID: docID, url: url}:
-			case <-r.Context().Done():
-				return r.Context().Err()
-			}
-			return nil
-		})
-	}()
-
-	// Worker pool: chunk + embed + write passages per doc.
-	var wg sync.WaitGroup
-	for i := 0; i < req.Workers; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for c := range candCh {
-				doc, err := s.store.GetDocByID(r.Context(), c.docID)
-				if err != nil || doc == nil || doc.Text == "" {
-					continue
-				}
-				// Use index defaults (NewChunker passes 0,0 = default 320/64).
-				chunker := index.NewChunker()
-				chunks := chunker.Chunk(doc.Text)
-				if len(chunks) == 0 {
-					continue
-				}
-				const tokenCap = 1500
-				texts := make([]string, len(chunks))
-				for j, ch := range chunks {
-					texts[j] = truncateForEmbedLite(ch.Text, tokenCap)
-				}
-				vecs, eErr := s.embedder.Embed(r.Context(), texts)
-				if eErr != nil || len(vecs) != len(chunks) {
-					continue
-				}
-				for j, ch := range chunks {
-					s.hnsw.AddPassage(doc.URL, doc.Title, ch.Offset, ch.Length, vecs[j])
-					_ = j
-				}
-				embedded.Add(1)
-			}
-		}()
-	}
-	wg.Wait()
-
-	log.Printf("embed-backfill: scanned=%d missing=%d embedded=%d in %s",
-		scanned.Load(), missing.Load(), embedded.Load(), time.Since(t0).Round(time.Second))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"scanned":  scanned.Load(),
-		"missing":  missing.Load(),
-		"embedded": embedded.Load(),
-		"elapsed":  time.Since(t0).String(),
-	})
-}
-
-// truncateForEmbedLite mirrors the crawler's helper without pulling the
-// crawler package in. Same heuristic: cap by approximate token count.
-func truncateForEmbedLite(s string, tokenCap int) string {
-	// Rough: 1 token ≈ 4 chars for ASCII; cap at 4×tokenCap bytes as a
-	// fast upper bound. Real chunker is at ~320 words = ~1200 tokens, so
-	// 1500-token cap rarely triggers.
-	maxBytes := tokenCap * 4
-	if len(s) <= maxBytes {
-		return s
-	}
-	return s[:maxBytes]
-}
-
-// handleSitePack discovers a site's sitemaps + RSS feeds and bulk-enqueues
-// everything found. Targets the "I want to index THIS site" operator flow
-// without editing cosift.json + restarting + waiting for the cursor.
-// Discovery order:
-//  1. GET https://<host>/robots.txt — look for `Sitemap:` directives
-//  2. Fallback to canonical /sitemap.xml + /sitemap_index.xml
-//  3. RSS at common paths: /feed, /feed.xml, /rss, /rss.xml, /atom.xml, /feed/atom
-//
-// Each found resource is run through the existing SeedSitemap / SeedRSS
-// primitives. Returns per-resource counts.
-type sitePackReq struct {
-	Host string `json:"host"` // e.g. "example.com" or "blog.example.com" — no scheme
-}
-
-func (s *pebbleHTTP) handleSitePack(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedSitemap == nil || s.crawlSeedRSS == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req sitePackReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"example.com\"}")
-		return
-	}
-	host, ok := normalizeBareHost(req.Host)
-	if !ok {
-		writeProblem(w, http.StatusBadRequest, "host must be a bare hostname like example.com")
-		return
-	}
-	base := "https://" + host
-	hc := &http.Client{Timeout: 20 * time.Second}
-
-	type result struct {
-		Source  string `json:"source"` // "robots-sitemap" | "fallback-sitemap" | "rss"
-		URL     string `json:"url"`
-		Indexed int    `json:"indexed"`
-		Error   string `json:"error,omitempty"`
-	}
-	results := make([]result, 0, 8)
-	t0 := time.Now()
-
-	candidateSitemaps, fromRobots := discoverSitemaps(r.Context(), hc, base)
-	for _, su := range candidateSitemaps {
-		n, err := s.crawlSeedSitemap(r.Context(), su)
-		res := result{URL: su, Indexed: n}
-		if fromRobots {
-			res.Source = "robots-sitemap"
-		} else {
-			res.Source = "fallback-sitemap"
-		}
-		if err != nil {
-			res.Error = err.Error()
-		}
-		results = append(results, res)
-	}
-	// Step 3: common RSS paths.
-	for _, p := range []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/feed/atom"} {
-		fu := base + p
-		n, err := s.crawlSeedRSS(r.Context(), fu)
-		// Silently skip RSS paths that 404 / don't parse — most sites have
-		// only one of the listed paths.
-		if err != nil && n == 0 {
-			continue
-		}
-		results = append(results, result{Source: "rss", URL: fu, Indexed: n})
-	}
-
-	total := 0
-	for _, r := range results {
-		total += r.Indexed
-	}
-	log.Printf("site-pack: %s discovered %d resources, enqueued %d URLs in %s",
-		host, len(results), total, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"host":          host,
-		"resources":     len(results),
-		"total_indexed": total,
-		"elapsed":       time.Since(t0).String(),
-		"results":       results,
-	})
-}
-
-// normalizeBareHost strips scheme/trailing-slash from a host or URL and
-// returns the bare hostname. ok is false when the result still contains a
-// path segment (so callers can reject "example.com/foo" as a host).
-func normalizeBareHost(s string) (host string, ok bool) {
-	host = strings.TrimSpace(s)
-	host = strings.TrimPrefix(host, "https://")
-	host = strings.TrimPrefix(host, "http://")
-	host = strings.TrimSuffix(host, "/")
-	host = strings.ToLower(host)
-	if host == "" || strings.Contains(host, "/") {
-		return "", false
-	}
-	return host, true
-}
-
-// discoverSitemaps returns candidate sitemap URLs for a site, given its base
-// origin (e.g. "https://example.com"). It prefers Sitemap: directives in
-// robots.txt; when robots.txt yields none it falls back to a small ordered
-// list of canonical/CMS paths. fromRobots reports which source was used.
-func discoverSitemaps(ctx context.Context, hc *http.Client, base string) (sitemaps []string, fromRobots bool) {
-	if hc == nil {
-		hc = &http.Client{Timeout: 20 * time.Second}
-	}
-	if req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/robots.txt", nil); err == nil {
-		if rresp, err := hc.Do(req); err == nil {
-			if rresp.StatusCode < 400 {
-				rbody, _ := io.ReadAll(io.LimitReader(rresp.Body, 2<<20))
-				for _, line := range strings.Split(string(rbody), "\n") {
-					line = strings.TrimSpace(line)
-					if strings.HasPrefix(strings.ToLower(line), "sitemap:") {
-						if val := strings.TrimSpace(line[len("sitemap:"):]); val != "" {
-							sitemaps = append(sitemaps, val)
-						}
-					}
-				}
-			}
-			rresp.Body.Close()
-		}
-	}
-	if len(sitemaps) > 0 {
-		return sitemaps, true
-	}
-	// /sitemap.xml is the canonical spec but many CMSes (WordPress, Yoast,
-	// Ghost, Hugo themes) ship at non-canonical paths. Try a small ordered
-	// list before giving up: /sitemap.xml (most common), then WordPress's
-	// /wp-sitemap.xml + Yoast's per-content-type splits, then index variants.
-	for _, p := range []string{
-		"/sitemap.xml",
-		"/wp-sitemap.xml",    // WordPress 5.5+
-		"/sitemap_index.xml", // Yoast SEO
-		"/post-sitemap.xml",  // Yoast posts
-		"/page-sitemap.xml",  // Yoast pages
-		"/sitemap-index.xml", // some CMSes hyphenate
-		"/sitemap.xml.gz",    // gzipped variant (sitemap.go handles .gz)
-	} {
-		sitemaps = append(sitemaps, base+p)
-	}
-	return sitemaps, false
-}
-
-// parseLaneName maps a friendly lane name to a frontier lane byte. The empty
-// string and unknown values default to the high-priority submitted lane,
-// which is the point of /admin/site-submit: jump a site to the front.
-func parseLaneName(s string) byte {
-	switch strings.ToLower(strings.TrimSpace(s)) {
-	case "refresh":
-		return store.LaneRefresh
-	case "discovered":
-		return store.LaneDiscovered
-	case "bulk":
-		return store.LaneBulk
-	default: // "", "priority", "submitted", or anything unrecognized
-		return store.LaneSubmitted
-	}
-}
-
-func laneName(lane byte) string {
-	switch lane {
-	case store.LaneSubmitted:
-		return "submitted"
-	case store.LaneRefresh:
-		return "refresh"
-	case store.LaneDiscovered:
-		return "discovered"
-	case store.LaneBulk:
-		return "bulk"
-	default:
-		return "submitted"
-	}
-}
-
-// handleSiteSubmit discovers every URL of a website (via robots.txt sitemaps,
-// then canonical fallbacks) and pushes them all onto the live crawl frontier
-// in a chosen priority lane — by default the high-priority "submitted" lane,
-// so the whole site jumps ahead of the generic discovery backlog. Same auth
-// as the other admin endpoints. Synchronous; large sitemaps take seconds.
-//
-// Body: {"host":"pilotprotocol.network", "lane":"priority"}
-//
-//	lane: "priority" (default) | "refresh" | "discovered" | "bulk"
-type siteSubmitReq struct {
-	Host string `json:"host"`
-	Lane string `json:"lane,omitempty"`
-}
-
-func (s *pebbleHTTP) handleSiteSubmit(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedSitemapLane == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req siteSubmitReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"example.com\", \"lane\":\"priority\"}")
-		return
-	}
-	host, ok := normalizeBareHost(req.Host)
-	if !ok {
-		writeProblem(w, http.StatusBadRequest, "host must be a bare hostname like example.com")
-		return
-	}
-	lane := parseLaneName(req.Lane)
-	base := "https://" + host
-	hc := &http.Client{Timeout: 20 * time.Second}
-	t0 := time.Now()
-
-	type result struct {
-		Source string `json:"source"` // "robots-sitemap" | "fallback-sitemap"
-		URL    string `json:"url"`
-		Queued int    `json:"queued"`
-		Error  string `json:"error,omitempty"`
-	}
-	candidateSitemaps, fromRobots := discoverSitemaps(r.Context(), hc, base)
-	source := "fallback-sitemap"
-	if fromRobots {
-		source = "robots-sitemap"
-	}
-	results := make([]result, 0, len(candidateSitemaps))
-	total := 0
-	for _, su := range candidateSitemaps {
-		n, err := s.crawlSeedSitemapLane(r.Context(), su, lane)
-		res := result{Source: source, URL: su, Queued: n}
-		if err != nil {
-			res.Error = err.Error()
-		}
-		total += n
-		results = append(results, res)
-	}
-	log.Printf("site-submit: %s → lane=%s discovered %d sitemap(s), queued %d URLs in %s",
-		host, laneName(lane), len(results), total, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"host":         host,
-		"lane":         laneName(lane),
-		"sitemaps":     len(results),
-		"total_queued": total,
-		"elapsed":      time.Since(t0).String(),
-		"results":      results,
-	})
-}
-
-// handleWETImportBulk fetches a CommonCrawl `wet.paths.gz` manifest,
-// takes the first N entries (or skip+take), and runs `/admin/wet-import`
-// against each one in parallel. Lets operators bulk-ingest a release with
-// one call instead of repeatedly POSTing per file. Synchronous — blocks
-// until all N finish, returns total docs indexed.
-//
-// Example body:
-//
-//	{"manifest_url":"https://data.commoncrawl.org/crawl-data/CC-MAIN-2024-51/wet.paths.gz",
-//	 "count":4, "skip":0, "concurrency":2, "lexical_only":true}
-type wetImportBulkReq struct {
-	ManifestURL string `json:"manifest_url"`
-	Count       int    `json:"count"`
-	Skip        int    `json:"skip,omitempty"`
-	Concurrency int    `json:"concurrency,omitempty"`
-	LexicalOnly bool   `json:"lexical_only,omitempty"`
-	DedupeFresh bool   `json:"dedupe_fresh,omitempty"`
-}
-
-func (s *pebbleHTTP) handleWETImportBulk(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedWET == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req wetImportBulkReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.ManifestURL == "" || req.Count <= 0 {
-		writeProblem(w, http.StatusBadRequest, "expected {\"manifest_url\":\"https://.../wet.paths.gz\",\"count\":N}")
-		return
-	}
-	if req.Concurrency <= 0 {
-		req.Concurrency = 2
-	}
-	if req.Concurrency > 8 {
-		req.Concurrency = 8 // beyond this the pebble write lock dominates anyway
-	}
-
-	// Fetch the manifest. wet.paths.gz is a gzipped newline-delimited list
-	// of relative paths under data.commoncrawl.org.
-	mreq, err := http.NewRequestWithContext(r.Context(), http.MethodGet, req.ManifestURL, http.NoBody)
-	if err != nil {
-		writeProblem(w, http.StatusBadRequest, "build manifest request: "+err.Error())
-		return
-	}
-	mreq.Header.Set("User-Agent", "cosift-bulk-import")
-	mresp, err := (&http.Client{Timeout: 60 * time.Second}).Do(mreq)
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "fetch manifest: "+err.Error())
-		return
-	}
-	defer mresp.Body.Close()
-	if mresp.StatusCode >= 400 {
-		writeProblem(w, http.StatusBadGateway, fmt.Sprintf("manifest http %d", mresp.StatusCode))
-		return
-	}
-	zr, err := gzip.NewReader(mresp.Body)
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "gunzip manifest: "+err.Error())
-		return
-	}
-	defer zr.Close()
-	mbody, err := io.ReadAll(io.LimitReader(zr, 32<<20))
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "read manifest: "+err.Error())
-		return
-	}
-	// Each line is a path like "crawl-data/CC-MAIN-.../wet/...warc.wet.gz".
-	// Prepend the data.commoncrawl.org base.
-	all := strings.Split(strings.TrimSpace(string(mbody)), "\n")
-	if req.Skip >= len(all) {
-		writeProblem(w, http.StatusBadRequest, fmt.Sprintf("skip=%d exceeds manifest size %d", req.Skip, len(all)))
-		return
-	}
-	end := req.Skip + req.Count
-	if end > len(all) {
-		end = len(all)
-	}
-	paths := all[req.Skip:end]
-
-	// Run imports in parallel, bounded by concurrency.
-	type result struct {
-		URL     string `json:"url"`
-		Indexed int    `json:"indexed"`
-		Elapsed string `json:"elapsed"`
-		Error   string `json:"error,omitempty"`
-	}
-	results := make([]result, len(paths))
-	sem := make(chan struct{}, req.Concurrency)
-	var wg sync.WaitGroup
-	t0 := time.Now()
-	for i, p := range paths {
-		i, p := i, strings.TrimSpace(p)
-		if p == "" {
-			continue
-		}
-		wg.Add(1)
-		sem <- struct{}{}
-		go func() {
-			defer wg.Done()
-			defer func() { <-sem }()
-			full := "https://data.commoncrawl.org/" + p
-			start := time.Now()
-			n, err := s.crawlSeedWET(r.Context(), full, req.DedupeFresh, req.LexicalOnly)
-			results[i].URL = full
-			results[i].Indexed = n
-			results[i].Elapsed = time.Since(start).Round(time.Second).String()
-			if err != nil {
-				results[i].Error = err.Error()
-			}
-		}()
-	}
-	wg.Wait()
-
-	total := 0
-	for _, r := range results {
-		total += r.Indexed
-	}
-	log.Printf("wet-import-bulk: indexed %d docs from %d WET files in %s", total, len(paths), time.Since(t0).Round(time.Second))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"manifest_url":  req.ManifestURL,
-		"files":         len(paths),
-		"total_indexed": total,
-		"elapsed":       time.Since(t0).String(),
-		"results":       results,
-	})
-}
-
-// handleWETImport streams a CommonCrawl WET file (gzipped, pre-extracted
-// plain text per URL) and runs each record through UpsertDocument + BM25
-// indexing + chunk + embed. Bypasses the fetch-and-parse pipeline entirely
-// because WET bodies are already extracted text — typically 50-100× faster
-// than open-web crawling.
-//
-// Example: POST {"url":"https://data.commoncrawl.org/crawl-data/CC-MAIN-2025-09/segments/.../wet/CC-MAIN-...wet.gz"}
-type wetImportReq struct {
-	URL         string `json:"url"`
-	DedupeFresh bool   `json:"dedupe_fresh,omitempty"` // skip URLs already-fresh in famDoc
-	LexicalOnly bool   `json:"lexical_only,omitempty"` // BM25-only ingest, defer dense embed to /admin/embed-backfill
-}
-
-func (s *pebbleHTTP) handleWETImport(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedWET == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req wetImportReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"url\":\"https://.../...warc.wet.gz\"}")
-		return
-	}
-	t0 := time.Now()
-	n, err := s.crawlSeedWET(r.Context(), req.URL, req.DedupeFresh, req.LexicalOnly)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	log.Printf("wet-import: indexed %d docs from %s in %s", n, req.URL, time.Since(t0).Round(time.Second))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"wet":     req.URL,
-		"indexed": n,
-		"elapsed": time.Since(t0).String(),
-	})
-}
-
-// handleFrontierDemoteHost re-keys every queued URL for a host into a
-// different lane. The escape hatch for the cloud.google.com problem:
-// 2.8M queued URLs on one host (65% of the queue) starve host-fair
-// claim slots from fresher lanes. Demote to lane 3 (bulk, 5% weight)
-// and lane 1/2 actually get the work.
-//
-// POST body: {"host": "cloud.google.com", "lane": 3}
-type frontierDemoteHostReq struct {
-	Host string `json:"host"`
-	Lane int    `json:"lane"`
-}
-
-func (s *pebbleHTTP) handleFrontierDemoteHost(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	ps, ok := any(s.store).(*store.PebbleStore)
-	if !ok {
-		writeProblem(w, http.StatusNotImplemented, "lanes are PebbleStore-only")
-		return
-	}
-	var req frontierDemoteHostReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"foo.com\",\"lane\":0..3}")
-		return
-	}
-	if req.Lane < 0 || req.Lane > 3 {
-		writeProblem(w, http.StatusBadRequest, "lane must be 0..3")
-		return
-	}
-	t0 := time.Now()
-	n, err := ps.DemoteHostToLane(r.Context(), req.Host, byte(req.Lane))
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	log.Printf("frontier-demote-host: moved %d URLs (%s -> lane %d) in %s", n, req.Host, req.Lane, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"host":    req.Host,
-		"lane":    req.Lane,
-		"moved":   n,
-		"elapsed": time.Since(t0).String(),
-	})
-}
-
-// handleFrontierPurgeStaleInFlight clears the stale 'i' secondary keys
-// left over from the pre-fix RecoverInFlight bug. Pre-fix, every restart
-// re-queued in-flight URLs via the LEGACY 'q' index only and skipped the
-// lane-aware 'i' delete, so each restart leaked the URL's lane-aware 'i'
-// key. GetLaneStats then reported impossibly-high in_flight counts
-// (>max_concurrent). Idempotent — re-running is a no-op once clean.
-func (s *pebbleHTTP) handleFrontierPurgeStaleInFlight(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	ps, ok := any(s.store).(*store.PebbleStore)
-	if !ok {
-		writeProblem(w, http.StatusNotImplemented, "PebbleStore-only")
-		return
-	}
-	t0 := time.Now()
-	n, err := ps.PurgeStaleInFlight(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	log.Printf("frontier-purge-stale-inflight: purged %d keys in %s", n, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{"purged": n, "elapsed": time.Since(t0).String()})
-}
-
-// handleRSSImport fetches an RSS 2.0 or Atom feed and pushes every <item>/
-// <entry> link to the live frontier. Same auth shape as sitemap-import.
-// Designed to be cron-friendly: idempotent against the frontier (re-seeding
-// the same feed only adds newly-listed items).
-type rssImportReq struct {
-	URL string `json:"url"`
-}
-
-func (s *pebbleHTTP) handleRSSImport(w http.ResponseWriter, r *http.Request) {
-	if want := s.cluster.PeerAuthToken; want != "" {
-		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
-		if got != want {
-			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
-			return
-		}
-	}
-	if s.crawlSeedRSS == nil {
-		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
-		return
-	}
-	var req rssImportReq
-	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
-	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
-		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"https://.../feed.xml\"}")
-		return
-	}
-	t0 := time.Now()
-	n, err := s.crawlSeedRSS(r.Context(), req.URL)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	log.Printf("rss-import: queued %d URLs from %s in %s", n, req.URL, time.Since(t0).Round(time.Millisecond))
-	writeJSON(w, http.StatusOK, map[string]any{
-		"feed":    req.URL,
-		"queued":  n,
-		"elapsed": time.Since(t0).String(),
-	})
-}
-
-// handleDomains returns indexed hosts by doc count.
-//
-// Query params:
-//   - top:    legacy alias for limit (capped at 500). Preserved for the
-//     landing page contract.
-//   - q:      substring filter (case-insensitive) on the host name.
-//   - offset: pagination offset (default 0).
-//   - limit:  page size (default 50, capped at 500).
-//
-// Response:
-//
-//	{"total": N, "offset": M, "limit": L, "domains": [...]}
-func (s *pebbleHTTP) handleDomains(w http.ResponseWriter, r *http.Request) {
-	q := r.URL.Query().Get("q")
-	offset := 0
-	if v := r.URL.Query().Get("offset"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n >= 0 {
-			offset = n
-		}
-	}
-	limit := 50
-	if v := r.URL.Query().Get("limit"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
-			limit = n
-		}
-	}
-	// Legacy 'top' alias from When set, behaves like limit + no
-	// search/offset; lands as the first page.
-	if v := r.URL.Query().Get("top"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
-			limit = n
-			offset = 0
-		}
-	}
-	domains, total, err := s.store.ListDomains(r.Context(), q, offset, limit)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]any{
-		"total":   total,
-		"offset":  offset,
-		"limit":   limit,
-		"q":       q,
-		"domains": domains,
-	})
-}
-
-// handleQueue surfaces frontier queue depth + top-N hosts currently queued
-// for crawl. Fills the gap left by /domains, which only shows
-// already-indexed hosts.
-func (s *pebbleHTTP) handleQueue(w http.ResponseWriter, r *http.Request) {
-	topN := 25
-	if v := r.URL.Query().Get("top"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
-			topN = n
-		}
-	}
-	fs, err := s.store.GetFrontierStats(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	hosts, err := s.store.TopQueuedHosts(r.Context(), topN)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	body := map[string]any{
-		"queued":    fs.Queued,
-		"in_flight": fs.InFlight,
-		"done":      fs.Done,
-		"errored":   fs.Errored,
-		"top_hosts": hosts,
-	}
-	// Lane breakdown: PebbleStore-only (SQLite Store has no lanes). When the
-	// store is a PebbleStore, surface the per-lane queued/in_flight counts so
-	// operators can see whether the weighted RR is actually draining RSS
-	// (lane 1) ahead of bulk (lane 3).
-	if ps, ok := any(s.store).(*store.PebbleStore); ok {
-		if ls, lerr := ps.GetLaneStats(r.Context()); lerr == nil {
-			laneNames := [4]string{"submitted", "refresh", "discovered", "bulk"}
-			lanesOut := make([]map[string]any, 0, 4)
-			for i, n := range laneNames {
-				lanesOut = append(lanesOut, map[string]any{
-					"lane":      i,
-					"name":      n,
-					"queued":    ls.Lanes[i].Queued,
-					"in_flight": ls.Lanes[i].InFlight,
-				})
-			}
-			body["lanes"] = lanesOut
-			body["legacy_queued"] = ls.LegacyQueued
-			body["legacy_in_flight"] = ls.LegacyInFlight
-		}
-	}
-	writeJSON(w, http.StatusOK, body)
-}
-
-// handleDomainsAudit streams the entire 'h' family as JSONL, one
-// {host, count, authority, etld1, tld} record per line. Single pass
-// over the index — at 9M hosts on the GH200 this is ~30 seconds of
-// streaming, vs the 18000 full scans that brute-paginating /domains
-// would force. Read-only; safe to run live.
-//
-// Query params:
-//
-//	min_count=N — emit only hosts with count >= N (default 1)
-//	limit=N     — cap the number of emitted rows (default unlimited)
-//	below=F     — emit only hosts with authority score < F (filter
-//	              for the blocklist proposal stream; default 1.0)
-//	above=F     — emit only hosts with score >= F (filter for the
-//	              authority-list inspection; default 0.0)
-func (s *pebbleHTTP) handleDomainsAudit(w http.ResponseWriter, r *http.Request) {
-	minCount := 1
-	if v := r.URL.Query().Get("min_count"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			minCount = n
-		}
-	}
-	limit := -1
-	if v := r.URL.Query().Get("limit"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 {
-			limit = n
-		}
-	}
-	below := 1.0
-	if v := r.URL.Query().Get("below"); v != "" {
-		if f, err := strconv.ParseFloat(v, 64); err == nil {
-			below = f
-		}
-	}
-	above := 0.0
-	if v := r.URL.Query().Get("above"); v != "" {
-		if f, err := strconv.ParseFloat(v, 64); err == nil {
-			above = f
-		}
-	}
-	w.Header().Set("Content-Type", "application/x-ndjson")
-	w.Header().Set("Cache-Control", "no-store")
-	enc := json.NewEncoder(w)
-	emitted := 0
-	flusher, _ := w.(http.Flusher)
-	flushEvery := 256
-	err := s.store.IterateDomains(r.Context(), func(host string, count int) bool {
-		if count < minCount {
-			return true
-		}
-		var score float64 = 0.5
-		if s.authority != nil {
-			score = s.authority.Score(host)
-		}
-		if score >= below || score < above {
-			return true
-		}
-		rec := map[string]any{
-			"host":      host,
-			"count":     count,
-			"authority": score,
-			"recommend": classify(score),
-		}
-		if err := enc.Encode(rec); err != nil {
-			return false
-		}
-		emitted++
-		if limit > 0 && emitted >= limit {
-			return false
-		}
-		if flusher != nil && emitted%flushEvery == 0 {
-			flusher.Flush()
-		}
-		return true
-	})
-	if err != nil {
-		// Best-effort: log and stop; we may have already flushed
-		// partial output, so don't try to writeProblem.
-		log.Printf("handleDomainsAudit: %v", err)
-	}
-	if flusher != nil {
-		flusher.Flush()
-	}
-}
-
-// handleSLA returns the current SLA evaluation snapshot. Read-only.
-// Surfaces per-endpoint p50/p95/p99 + error rate, configured targets,
-// and the most recent violations. Operators consume this in dashboards
-// + on-call paging.
-func (s *pebbleHTTP) handleSLA(w http.ResponseWriter, _ *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Cache-Control", "no-store")
-	if s.sla == nil {
-		_, _ = w.Write([]byte(`{"enabled":false}`))
-		return
-	}
-	enc := json.NewEncoder(w)
-	enc.SetIndent("", "  ")
-	_ = enc.Encode(s.sla.Snapshot())
-}
-
-func (s *pebbleHTTP) handleStats(w http.ResponseWriter, r *http.Request) {
-	// stale-while-revalidate cache.
-	//   - Fresh hit (< TTL): return cached, X-Cache: HIT.
-	//   - Stale (have cache, past TTL but under maxStale): return cached,
-	//     X-Cache: STALE, kick a background refresh (single-flight).
-	//   - Very stale (past maxStale): fall through to synchronous compute
-	//     — protects against the case where a refresh goroutine
-	//     hangs under HNSW lock contention and the cache shows ancient
-	//     values (we saw a startup-time vec=654 served for ~15 min).
-	//   - Cold (no cache): compute synchronously, X-Cache: MISS.
-	const statsBodyTTL = 5 * time.Second
-	const statsBodyMaxStale = 60 * time.Second
-	s.statsBodyMu.Lock()
-	body := s.statsBodyBlob
-	age := time.Since(s.statsBodyAt)
-	cached := body != nil && !s.statsBodyAt.IsZero()
-	s.statsBodyMu.Unlock()
-
-	if cached && age < statsBodyTTL {
-		w.Header().Set("Content-Type", "application/json")
-		w.Header().Set("X-Cache", "HIT")
-		_, _ = w.Write(body)
-		return
-	}
-	if cached && age < statsBodyMaxStale {
-		if s.statsRefreshing.CompareAndSwap(false, true) {
-			go func() {
-				defer s.statsRefreshing.Store(false)
-				ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-				defer cancel()
-				if newBody, err := s.buildStatsBody(ctx); err == nil {
-					s.statsBodyMu.Lock()
-					s.statsBodyBlob = newBody
-					s.statsBodyAt = time.Now()
-					s.statsBodyMu.Unlock()
-				}
-			}()
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.Header().Set("X-Cache", "STALE")
-		_, _ = w.Write(body)
-		return
-	}
-
-	// Cold path: compute synchronously and populate the cache.
-	newBody, err := s.buildStatsBody(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	s.statsBodyMu.Lock()
-	s.statsBodyBlob = newBody
-	s.statsBodyAt = time.Now()
-	s.statsBodyMu.Unlock()
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("X-Cache", "MISS")
-	_, _ = w.Write(newBody)
-}
-
-// buildStatsBody collects every signal /stats surfaces and marshals it.
-// Heavy paths: PebbleStore.Stats (d-family scan, partially counter-cached
-// since), HNSW.PQStatus (O(N) under h.mu read lock that
-// contends with crawler writers). Called by handleStats both
-// synchronously (cold) and from a background goroutine (SWR refresh).
-func (s *pebbleHTTP) buildStatsBody(ctx context.Context) ([]byte, error) {
-	// /stats was observed taking 31s under load
-	// when the SWR cache exceeded maxStale: store.Stats() iterates the
-	// entire 'd' family (10.87M entries) every cold compute. The
-	// CorpusStats counter is an O(1) atomic read mirror of the persisted
-	// indexed_docs meta — equivalent doc count for any operator-facing
-	// purpose, microseconds instead of seconds. Terms isn't populated in
-	// the Pebble path (always 0) so dropping the scan loses nothing.
-	sumLen, indexedDocs, err := s.store.CorpusStats(ctx)
-	if err != nil {
-		return nil, err
-	}
-	var avg float64
-	if indexedDocs > 0 {
-		avg = float64(sumLen) / float64(indexedDocs)
-	}
-	// surface config knobs that affect retrieval/synth quality so
-	// operators can verify env overrides took effect from one endpoint.
-	out := map[string]any{
-		"documents":    indexedDocs,
-		"terms":        int64(0),
-		"indexed_docs": indexedDocs,
-		"sum_doc_len":  sumLen,
-		"avg_doc_len":  avg,
-		"uptime":       time.Since(s.started).String(),
-		"backend":      "pebble",
-		"bm25_k1":      s.idx.K1(),
-		"bm25_b":       s.idx.B(),
-	}
-	if s.reranker != nil {
-		out["reranker"] = s.reranker.Name()
-		out["rerank_candidate_k"] = s.rerankCandK
-	}
-	if s.chat != nil {
-		out["chat_model"] = s.chat.Model()
-		// surface the cache caps so operators can verify
-		// COSIFT_HYDE_CACHE_SIZE / _PARA_CACHE_SIZE overrides took effect.
-		out["hyde_cache_size"] = s.hydeCacheCap
-		out["paraphrase_cache_size"] = s.paraCacheCap
-	}
-	// Bounded LLM concurrency + circuit-breaker state. Surfaced so
-	// operators can see whether the gate is shedding load (rejected > 0)
-	// or holding requests (in_flight = cap) and whether the breaker has
-	// tripped (state != "closed").
-	if s.chatGate != nil {
-		out["llm_gate"] = s.chatGate.Stats()
-	}
-	if s.chatSafe != nil {
-		out["llm_answer_circuit"] = s.chatSafe.CircuitState()
-	}
-	if s.rerankSafe != nil {
-		out["llm_rerank_circuit"] = s.rerankSafe.CircuitState()
-	}
-	if s.answerCache != nil {
-		out["answer_cache"] = s.answerCache.Stats()
-	}
-	if s.llmProbe != nil {
-		out["llm_load"] = s.llmProbe.Stats()
-	}
-	if s.sla != nil {
-		out["sla"] = s.sla.Snapshot()
-	}
-	if s.authority != nil {
-		out["authority"] = s.authority.Stats()
-	}
-	// embed cache hit/miss counters so operators can see how
-	// often re-fetches are skipping ollama. The cache wraps the round-
-	// robin embedder when cfg.Embeddings.CacheDir is set; type-assert
-	// down to extract the counters.
-	if ce, ok := s.embedder.(*embed.CachedEmbedder); ok {
-		hits := ce.Hits()
-		misses := ce.Misses()
-		total := hits + misses
-		hitRate := 0.0
-		if total > 0 {
-			hitRate = 100 * float64(hits) / float64(total)
-		}
-		out["embed_cache"] = map[string]any{
-			"hits":         hits,
-			"misses":       misses,
-			"hit_rate_pct": hitRate,
-		}
-	}
-	// signal whether the store has HNSW vectors persisted, and
-	// (when meta is available) surface dim + node count.
-	// when the graph is loaded in memory, report s.hnsw.Len()
-	// instead of the startup-cached vectorNodes count — otherwise /stats
-	// shows the corpus frozen in time while the in-serve crawler keeps
-	// growing the graph.
-	// "% of the indexed web" novelty stat. The web is too big to
-	// give a meaningful number, but Common Crawl's monthly snapshot is
-	// ~3.5 B unique pages — a recognizable reference point. Operators can
-	// override with COSIFT_WEB_DENOMINATOR if they want a different scale
-	// (e.g. their own intranet, a TLD slice, etc.).
-	webDenom := int64(3_500_000_000)
-	if v := os.Getenv("COSIFT_WEB_DENOMINATOR"); v != "" {
-		if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
-			webDenom = n
-		}
-	}
-	out["web_pct"] = 100.0 * float64(indexedDocs) / float64(webDenom)
-	out["web_denominator"] = webDenom
-	out["has_vectors"] = s.hasVectors
-	switch {
-	case s.hnsw != nil:
-		out["vector_nodes"] = s.hnsw.Len()
-		out["vector_dim"] = s.vectorDim
-	case s.vectorNodes > 0:
-		out["vector_nodes"] = s.vectorNodes
-		out["vector_dim"] = s.vectorDim
-	}
-	// whether the graph is loaded into memory for dense retrieval.
-	out["hnsw_loaded"] = s.hnsw != nil
-	// PQ status — operator-facing visibility into compression
-	// state. Only present when the graph is loaded; nil otherwise.
-	if s.hnsw != nil {
-		pq := s.hnsw.PQStatus()
-		// coverage is over VALID nodes (vec != nil), not raw total —
-		// zombie slots from pre partial persists inflate the total
-		// without being searchable. NodesTotal still surfaced for context.
-		denom := pq.NodesValid
-		if denom == 0 {
-			denom = pq.NodesTotal // avoid div-by-zero on a fresh load
-		}
-		coverage := 0.0
-		if denom > 0 {
-			coverage = 100 * float64(pq.NodesWithCode) / float64(denom)
-		}
-		zombies := pq.NodesTotal - pq.NodesValid
-		pqInfo := map[string]any{
-			"enabled":         pq.Enabled,
-			"nodes_with_code": pq.NodesWithCode,
-			"nodes_valid":     pq.NodesValid,
-			"nodes_total":     pq.NodesTotal,
-			"zombie_nodes":    zombies,
-			"coverage_pct":    coverage,
-		}
-		if pq.Enabled {
-			pqInfo["dim"] = pq.Dim
-			pqInfo["m"] = pq.M
-			pqInfo["k"] = pq.K
-		}
-		out["pq"] = pqInfo
-	}
-	// Clients can read
-	// this once instead of probing ?retriever=dense + parsing the warning.
-	// bm25 always — index is always available. dense/hybrid require a
-	// loaded graph; the embedder lets dense/hybrid handle text-mode and
-	// new queries. /find_similar URL-mode dense works without embedder, but
-	// that's an endpoint-specific carve-out — keep this list general.
-	retrievers := []string{"bm25", "bm25-mlt"}
-	if s.hnsw != nil && s.embedder != nil {
-		retrievers = append(retrievers, "dense", "hybrid")
-	} else if s.hnsw != nil {
-		// Graph but no embedder — only URL-mode /find_similar can use dense.
-		retrievers = append(retrievers, "dense:find_similar_url_only")
-	}
-	out["retrievers"] = retrievers
-	// crawl-rate surface for the in-serve crawler.
-	// Computes docs added since process start and per-minute rate.
-	if s.crawlActive {
-		uptimeMin := time.Since(s.started).Minutes()
-		added := int(indexedDocs) - s.startupDocs
-		if added < 0 {
-			added = 0
-		}
-		rate := 0.0
-		if uptimeMin > 0 {
-			rate = float64(added) / uptimeMin
-		}
-		out["crawl_active"] = true
-		out["docs_added_since_start"] = added
-		out["docs_per_minute"] = rate
-	}
-	return json.Marshal(out)
-}
-
-// Prometheus-format scrape endpoint. Hand-written plain text
-// (no client_golang dep) — exposition format is simple enough that
-// pulling in a dep just to print four gauges isn't justified.
-// Quantities chosen to be O(1) reads (CorpusStats counters + uptime);
-// frontier scans deliberately excluded so scrape latency stays flat
-// regardless of corpus size.
-func (s *pebbleHTTP) handleMetrics(w http.ResponseWriter, r *http.Request) {
-	sumLen, count, err := s.store.CorpusStats(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	var avg float64
-	if count > 0 {
-		avg = float64(sumLen) / float64(count)
-	}
-	uptime := time.Since(s.started).Seconds()
-	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	fmt.Fprintf(w, "# HELP cosift_indexed_docs Number of documents passed through IndexDocument.\n")
-	fmt.Fprintf(w, "# TYPE cosift_indexed_docs gauge\n")
-	fmt.Fprintf(w, "cosift_indexed_docs %d\n", count)
-	fmt.Fprintf(w, "# HELP cosift_sum_doc_len_total Sum of indexed document lengths in tokens.\n")
-	fmt.Fprintf(w, "# TYPE cosift_sum_doc_len_total counter\n")
-	fmt.Fprintf(w, "cosift_sum_doc_len_total %d\n", sumLen)
-	fmt.Fprintf(w, "# HELP cosift_avg_doc_len Average document length in tokens (derived).\n")
-	fmt.Fprintf(w, "# TYPE cosift_avg_doc_len gauge\n")
-	fmt.Fprintf(w, "cosift_avg_doc_len %.2f\n", avg)
-	fmt.Fprintf(w, "# HELP cosift_uptime_seconds Seconds since pebble-serve started.\n")
-	fmt.Fprintf(w, "# TYPE cosift_uptime_seconds counter\n")
-	fmt.Fprintf(w, "cosift_uptime_seconds %.0f\n", uptime)
-	// crawl_active gauges whether
-	// the crawler goroutine is running; docs_added is monotonic
-	// since process start; docs_per_minute is the rolling rate.
-	crawlActiveGauge := 0
-	if s.crawlActive {
-		crawlActiveGauge = 1
-	}
-	fmt.Fprintf(w, "# HELP cosift_crawl_active 1 if in-serve crawler is running, 0 otherwise.\n")
-	fmt.Fprintf(w, "# TYPE cosift_crawl_active gauge\n")
-	fmt.Fprintf(w, "cosift_crawl_active %d\n", crawlActiveGauge)
-	if s.crawlActive {
-		added := count - int64(s.startupDocs)
-		if added < 0 {
-			added = 0
-		}
-		fmt.Fprintf(w, "# HELP cosift_crawl_docs_added_total Documents added by the in-serve crawler since process start.\n")
-		fmt.Fprintf(w, "# TYPE cosift_crawl_docs_added_total counter\n")
-		fmt.Fprintf(w, "cosift_crawl_docs_added_total %d\n", added)
-		rate := 0.0
-		if uptime > 0 {
-			rate = float64(added) / (uptime / 60)
-		}
-		fmt.Fprintf(w, "# HELP cosift_crawl_docs_per_minute Recent crawl rate (docs added per minute, averaged since process start).\n")
-		fmt.Fprintf(w, "# TYPE cosift_crawl_docs_per_minute gauge\n")
-		fmt.Fprintf(w, "cosift_crawl_docs_per_minute %.2f\n", rate)
-	}
-	// HyDE cache effectiveness. Hits/misses both monotonic so
-	// Prometheus rate() over these gives cache pressure under load.
-	fmt.Fprintf(w, "# HELP cosift_hyde_cache_hits_total HyDE cache hits (expandQuery served from memory).\n")
-	fmt.Fprintf(w, "# TYPE cosift_hyde_cache_hits_total counter\n")
-	fmt.Fprintf(w, "cosift_hyde_cache_hits_total %d\n", s.hydeHits.Load())
-	fmt.Fprintf(w, "# HELP cosift_hyde_cache_misses_total HyDE cache misses (expandQuery called the LLM).\n")
-	fmt.Fprintf(w, "# TYPE cosift_hyde_cache_misses_total counter\n")
-	fmt.Fprintf(w, "cosift_hyde_cache_misses_total %d\n", s.hydeMisses.Load())
-	fmt.Fprintf(w, "# HELP cosift_paraphrase_cache_hits_total Paraphrase cache hits (paraphraseQuery served from memory).\n")
-	fmt.Fprintf(w, "# TYPE cosift_paraphrase_cache_hits_total counter\n")
-	fmt.Fprintf(w, "cosift_paraphrase_cache_hits_total %d\n", s.paraHits.Load())
-	fmt.Fprintf(w, "# HELP cosift_paraphrase_cache_misses_total Paraphrase cache misses (paraphraseQuery called the LLM).\n")
-	fmt.Fprintf(w, "# TYPE cosift_paraphrase_cache_misses_total counter\n")
-	fmt.Fprintf(w, "cosift_paraphrase_cache_misses_total %d\n", s.paraMisses.Load())
-	fmt.Fprintf(w, "# HELP cosift_rerank_attempts_total Rerank calls invoked (any endpoint with ?rerank=true).\n")
-	fmt.Fprintf(w, "# TYPE cosift_rerank_attempts_total counter\n")
-	fmt.Fprintf(w, "cosift_rerank_attempts_total %d\n", s.rerankAttempts.Load())
-	fmt.Fprintf(w, "# HELP cosift_rerank_failures_total Rerank calls that returned an error (silently fell back to BM25 order).\n")
-	fmt.Fprintf(w, "# TYPE cosift_rerank_failures_total counter\n")
-	fmt.Fprintf(w, "cosift_rerank_failures_total %d\n", s.rerankFailures.Load())
-	fmt.Fprintf(w, "# HELP cosift_chat_attempts_total Chat-client calls invoked (HyDE, /answer synth, /research plan + synth).\n")
-	fmt.Fprintf(w, "# TYPE cosift_chat_attempts_total counter\n")
-	fmt.Fprintf(w, "cosift_chat_attempts_total %d\n", s.chatAttempts.Load())
-	fmt.Fprintf(w, "# HELP cosift_chat_failures_total Chat-client calls that returned an error.\n")
-	fmt.Fprintf(w, "# TYPE cosift_chat_failures_total counter\n")
-	fmt.Fprintf(w, "cosift_chat_failures_total %d\n", s.chatFailures.Load())
-	fmt.Fprintf(w, "# HELP cosift_chat_duration_seconds_sum Cumulative wall-clock spent inside chat-client calls.\n")
-	fmt.Fprintf(w, "# TYPE cosift_chat_duration_seconds_sum counter\n")
-	fmt.Fprintf(w, "cosift_chat_duration_seconds_sum %.6f\n", float64(s.chatDurationNanos.Load())/1e9)
-	fmt.Fprintf(w, "# HELP cosift_warnings_emitted_total Responses that carried at least one warning (misconfigured request).\n")
-	fmt.Fprintf(w, "# TYPE cosift_warnings_emitted_total counter\n")
-	fmt.Fprintf(w, "cosift_warnings_emitted_total %d\n", s.warningsEmitted.Load())
-	// HNSW vector index shape. Live count when graph is
-	// loaded; startup-cached otherwise.
-	vectorNodesLive := s.vectorNodes
-	if s.hnsw != nil {
-		vectorNodesLive = s.hnsw.Len()
-	}
-	if vectorNodesLive > 0 {
-		fmt.Fprintf(w, "# HELP cosift_vector_nodes Number of HNSW vector nodes in memory (or persisted when not loaded).\n")
-		fmt.Fprintf(w, "# TYPE cosift_vector_nodes gauge\n")
-		fmt.Fprintf(w, "cosift_vector_nodes %d\n", vectorNodesLive)
-		fmt.Fprintf(w, "# HELP cosift_vector_dim Embedding dimension of the persisted HNSW index.\n")
-		fmt.Fprintf(w, "# TYPE cosift_vector_dim gauge\n")
-		fmt.Fprintf(w, "cosift_vector_dim %d\n", s.vectorDim)
-	}
-	// PromQL
-	// rate(cosift_request_duration_seconds_sum) / rate(cosift_requests_total)
-	// gives mean latency in any window. Labels = path; misrouted calls (404)
-	// don't share a label with any handled path.
-	fmt.Fprintf(w, "# HELP cosift_requests_total HTTP requests served, by endpoint.\n")
-	fmt.Fprintf(w, "# TYPE cosift_requests_total counter\n")
-	fmt.Fprintf(w, "# HELP cosift_request_duration_seconds_sum Cumulative request duration, by endpoint.\n")
-	fmt.Fprintf(w, "# TYPE cosift_request_duration_seconds_sum counter\n")
-	s.requestCounts.Range(func(k, v any) bool {
-		path := k.(string)
-		m := v.(*endpointMetrics)
-		fmt.Fprintf(w, "cosift_requests_total{endpoint=%q} %d\n", path, m.count.Load())
-		fmt.Fprintf(w, "cosift_request_duration_seconds_sum{endpoint=%q} %.6f\n", path, float64(m.sumNanos.Load())/1e9)
-		return true
-	})
-}
-
-// HTTP form of `cosift verify`. Same comparison (running
-// counters vs authoritative 'l' family scan) on the already-open store, so
-// monitoring can poll without contending for the single-writer lock or
-// shelling into the container. 503 on drift makes this composable with
-// Kubernetes liveness / load-balancer health checks.
-func (s *pebbleHTTP) handleVerify(w http.ResponseWriter, r *http.Request) {
-	counterSum, counterCount, err := s.store.CorpusStats(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	scanSum, scanCount, err := s.store.SumDocLengths(r.Context())
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	driftCount := counterCount - scanCount
-	driftSum := counterSum - scanSum
-	ok := driftCount == 0 && driftSum == 0
-	body := map[string]any{
-		"ok":                   ok,
-		"indexed_docs_counter": counterCount,
-		"indexed_docs_scan":    scanCount,
-		"indexed_docs_drift":   driftCount,
-		"sum_doc_len_counter":  counterSum,
-		"sum_doc_len_scan":     scanSum,
-		"sum_doc_len_drift":    driftSum,
-	}
-	status := http.StatusOK
-	if !ok {
-		status = http.StatusServiceUnavailable
-	}
-	writeJSON(w, status, body)
-}
-
-// searchHit is the minimal hit shape returned by pebble-serve's /search.
-// Intentionally narrower than the SQLite server's SearchHit — feature
-// parity (highlight, excerpt, calibration, paragraph filters) grows as
-// follow-up iters port each one through the Pebble side.
-type searchHit struct {
-	URL         string     `json:"url"`
-	Title       string     `json:"title"`
-	Score       float64    `json:"score"`
-	Excerpt     string     `json:"excerpt,omitempty"`
-	PublishedAt *time.Time `json:"published_at,omitempty"`
-	Author      string     `json:"author,omitempty"`
-	Text        string     `json:"text,omitempty"`
-}
-
-type searchResponse struct {
-	Query           string      `json:"query"`
-	EffectiveQuery  string      `json:"effective_query,omitempty"`
-	Expand          string      `json:"expand,omitempty"`
-	Retriever       string      `json:"retriever"`
-	Hits            []searchHit `json:"hits"`
-	TotalCandidates int         `json:"total_candidates,omitempty"`
-	Warnings        []string    `json:"warnings,omitempty"`
-	Took            string      `json:"took"`
-}
-
-// POST /search with JSON body — for callers whose query lists,
-// quoted phrases, or filter CSVs are awkward to URL-encode. Decodes into a
-// searchRequest, re-encodes as r.URL.RawQuery, then calls handleSearch so
-// every param the GET form supports works identically here. The hand-off
-// shape (URL.Values) is the cosift contract — easier to keep one parser
-// than to fork it across method handlers.
-type searchRequest struct {
-	Q              string `json:"q"`
-	K              int    `json:"k,omitempty"`
-	IncludeDomains string `json:"include_domains,omitempty"`
-	ExcludeDomains string `json:"exclude_domains,omitempty"`
-	Site           string `json:"site,omitempty"`
-	Since          string `json:"since,omitempty"`
-	Until          string `json:"until,omitempty"`
-	Sort           string `json:"sort,omitempty"`
-	Enrich         *bool  `json:"enrich,omitempty"`
-	IncludeText    bool   `json:"include_text,omitempty"`
-	Rerank         bool   `json:"rerank,omitempty"`
-	Expand         string `json:"expand,omitempty"`
-}
-
-func (s *pebbleHTTP) handleSearchPOST(w http.ResponseWriter, r *http.Request) {
-	var req searchRequest
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
-		return
-	}
-	v := url.Values{}
-	if req.Q != "" {
-		v.Set("q", req.Q)
-	}
-	if req.K > 0 {
-		v.Set("k", strconv.Itoa(req.K))
-	}
-	if req.IncludeDomains != "" {
-		v.Set("include_domains", req.IncludeDomains)
-	}
-	if req.ExcludeDomains != "" {
-		v.Set("exclude_domains", req.ExcludeDomains)
-	}
-	if req.Site != "" {
-		v.Set("site", req.Site)
-	}
-	if req.Since != "" {
-		v.Set("since", req.Since)
-	}
-	if req.Until != "" {
-		v.Set("until", req.Until)
-	}
-	if req.Sort != "" {
-		v.Set("sort", req.Sort)
-	}
-	if req.Enrich != nil && !*req.Enrich {
-		v.Set("enrich", "false")
-	}
-	if req.IncludeText {
-		v.Set("include_text", "true")
-	}
-	if req.Rerank {
-		v.Set("rerank", "true")
-	}
-	if req.Expand != "" {
-		v.Set("expand", req.Expand)
-	}
-	r.URL.RawQuery = v.Encode()
-	s.handleSearch(w, r)
-}
-
-// POST variants of /find_similar, /answer, /research. Same pattern
-// as POST /search — re-encode JSON body as URL.Values, hand off to the GET
-// handler. The GET handlers own the param semantics; POST is a wire-level
-// alternative for callers whose payloads don't fit cleanly into a query string.
-
-type findSimilarRequest struct {
-	URL            string `json:"url,omitempty"`
-	Text           string `json:"text,omitempty"`  // content-based MLT, no source URL needed
-	Title          string `json:"title,omitempty"` // optional title-boost when using text mode
-	K              int    `json:"k,omitempty"`
-	Q              string `json:"q,omitempty"`
-	IncludeDomains string `json:"include_domains,omitempty"`
-	ExcludeDomains string `json:"exclude_domains,omitempty"`
-	Since          string `json:"since,omitempty"`
-	Until          string `json:"until,omitempty"`
-	IncludeText    bool   `json:"include_text,omitempty"`
-	Rerank         bool   `json:"rerank,omitempty"`
-}
-
-func (s *pebbleHTTP) handleFindSimilarPOST(w http.ResponseWriter, r *http.Request) {
-	var req findSimilarRequest
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
-		return
-	}
-	v := url.Values{}
-	if req.URL != "" {
-		v.Set("url", req.URL)
-	}
-	if req.Text != "" {
-		v.Set("text", req.Text)
-	}
-	if req.Title != "" {
-		v.Set("title", req.Title)
-	}
-	if req.K > 0 {
-		v.Set("k", strconv.Itoa(req.K))
-	}
-	if req.Q != "" {
-		v.Set("q", req.Q)
-	}
-	if req.IncludeDomains != "" {
-		v.Set("include_domains", req.IncludeDomains)
-	}
-	if req.ExcludeDomains != "" {
-		v.Set("exclude_domains", req.ExcludeDomains)
-	}
-	if req.Since != "" {
-		v.Set("since", req.Since)
-	}
-	if req.Until != "" {
-		v.Set("until", req.Until)
-	}
-	if req.IncludeText {
-		v.Set("include_text", "true")
-	}
-	if req.Rerank {
-		v.Set("rerank", "true")
-	}
-	r.URL.RawQuery = v.Encode()
-	s.handleFindSimilar(w, r)
-}
-
-type synthRequest struct {
-	Q              string `json:"q"`
-	K              int    `json:"k,omitempty"`
-	IncludeDomains string `json:"include_domains,omitempty"`
-	ExcludeDomains string `json:"exclude_domains,omitempty"`
-	Site           string `json:"site,omitempty"`
-	Since          string `json:"since,omitempty"`
-	Until          string `json:"until,omitempty"`
-	IncludeText    bool   `json:"include_text,omitempty"`
-	Rerank         bool   `json:"rerank,omitempty"`
-	Expand         string `json:"expand,omitempty"`
-	Stream         bool   `json:"stream,omitempty"`
-}
-
-func (req synthRequest) toValues() url.Values {
-	v := url.Values{}
-	if req.Q != "" {
-		v.Set("q", req.Q)
-	}
-	if req.K > 0 {
-		v.Set("k", strconv.Itoa(req.K))
-	}
-	if req.IncludeDomains != "" {
-		v.Set("include_domains", req.IncludeDomains)
-	}
-	if req.ExcludeDomains != "" {
-		v.Set("exclude_domains", req.ExcludeDomains)
-	}
-	if req.Site != "" {
-		v.Set("site", req.Site)
-	}
-	if req.Since != "" {
-		v.Set("since", req.Since)
-	}
-	if req.Until != "" {
-		v.Set("until", req.Until)
-	}
-	if req.IncludeText {
-		v.Set("include_text", "true")
-	}
-	if req.Rerank {
-		v.Set("rerank", "true")
-	}
-	if req.Expand != "" {
-		v.Set("expand", req.Expand)
-	}
-	if req.Stream {
-		v.Set("stream", "true")
-	}
-	return v
-}
-
-func (s *pebbleHTTP) handleAnswerPOST(w http.ResponseWriter, r *http.Request) {
-	var req synthRequest
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
-		return
-	}
-	r.URL.RawQuery = req.toValues().Encode()
-	s.handleAnswer(w, r)
-}
-
-func (s *pebbleHTTP) handleResearchPOST(w http.ResponseWriter, r *http.Request) {
-	var req synthRequest
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
-		return
-	}
-	r.URL.RawQuery = req.toValues().Encode()
-	s.handleResearch(w, r)
-}
-
-func (s *pebbleHTTP) handleSearch(w http.ResponseWriter, r *http.Request) {
-	// When this process is in cluster
-	// gateway-mode AND the caller hasn't already set ?cluster_local=1
-	// (which is how the gateway tells peers "just do your local search,
-	// don't fan out again"), fan out to peers and RRF-merge their results.
-	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
-		s.handleSearchGateway(w, r)
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	k := 10
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
-			k = n
-		}
-	}
-	// include_domains / exclude_domains — mirrors the
-	// SQLite-side server semantics (CSV, dot-boundary suffix match).
-	// since / until — ISO-date filters on doc.PublishedAt.
-	// When any filter is active we over-fetch so the post-filter has
-	// enough candidates to fill k; same brute-force shape SQLite used
-	// before it grew a proper index, fine at pebble-serve's scale.
-	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
-	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
-	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
-	if sinceErr != nil {
-		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
-		return
-	}
-	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
-	if untilErr != nil {
-		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
-		return
-	}
-	dateFilter := !since.IsZero() || !until.IsZero()
-	// site — scope results to one or more host[/path] sections, e.g.
-	// ?site=pilotprotocol.network/docs. Host-suffix + path-prefix match,
-	// ANDed with include/exclude. Applied post-retrieval like the domain
-	// filters, so it widens the over-fetch below.
-	sites := parseSiteScopes(r.URL.Query().Get("site"))
-	siteFilter := len(sites) > 0
-	// rerank widens both the fetch and the keep-cap before filtering,
-	// so the reranker sees a healthy candidate pool even with restrictive filters.
-	wantRerank := r.URL.Query().Get("rerank") == "true" && s.reranker != nil
-	keepCap := k
-	if wantRerank {
-		keepCap = s.rerankCandK
-		if keepCap < k {
-			keepCap = k
-		}
-	}
-	fetchK := keepCap
-	if len(include) > 0 || len(exclude) > 0 || dateFilter || siteFilter {
-		mult := 5
-		if dateFilter {
-			mult = 10
-		}
-		fetchK = keepCap * mult
-		if fetchK > 500 {
-			fetchK = 500
-		}
-	} else if wantRerank {
-		fetchK = keepCap * 2
-	}
-	// expansion dispatch (bare / HyDE / paraphrase+RRF).
-	// Shared
-	// helper used by /answer + /research so all three get the same matrix.
-	// Reranker still scores against the original q regardless of strategy.
-	expandMode := r.URL.Query().Get("expand")
-	retrieverParam := r.URL.Query().Get("retriever")
-	denseReady := s.hnsw != nil && s.embedder != nil
-	hits, effectiveQuery, err := s.retrieve(r.Context(), q, fetchK, retrieverParam, expandMode)
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	// enrich each surviving hit with Excerpt + PublishedAt + Author
-	// via a single GetDocByURL per hit. Cost: k extra Gets — block-cache hot,
-	// ~ms-scale at the k≤100 we accept. Opt out with ?enrich=false for callers
-	// that only need scoring. The date filter forces a doc fetch
-	// even when enrich=false, since PublishedAt lives on the gob.
-	// ?include_text=true inlines doc.Text on each hit so research
-	// pipelines avoid the N+1 round trip to /contents. Off by default —
-	// payload size grows linearly with k and average doc length.
-	enrich := r.URL.Query().Get("enrich") != "false"
-	if wantRerank {
-		enrich = true // rerank needs per-doc text — overrides enrich opt-out
-	}
-	// time-decay multiplier needs PublishedAt per hit, which lives
-	// behind the enrich flag. Force enrich when decay is requested so the
-	// signal is available downstream.
-	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
-	if decaySet {
-		enrich = true
-	}
-	includeText := r.URL.Query().Get("include_text") == "true"
-	out := make([]searchHit, 0, keepCap)
-	var rerankTexts []string
-	if wantRerank {
-		rerankTexts = make([]string, 0, keepCap)
-	}
-	for _, h := range hits {
-		if len(include) > 0 || len(exclude) > 0 {
-			host := hostOf(h.URL)
-			if len(include) > 0 && !matchesAnyDomain(host, include) {
-				continue
-			}
-			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
-				continue
-			}
-		}
-		if siteFilter && !matchesAnySite(h.URL, sites) {
-			continue
-		}
-		hit := searchHit{URL: h.URL, Title: h.Title, Score: h.Score}
-		if enrich || dateFilter || includeText {
-			doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
-			if derr != nil || doc == nil {
-				continue
-			}
-			if dateFilter {
-				if doc.PublishedAt.IsZero() {
-					continue // zero PublishedAt = unknown; drop under any date filter
-				}
-				if !since.IsZero() && doc.PublishedAt.Before(since) {
-					continue
-				}
-				if !until.IsZero() && doc.PublishedAt.After(until) {
-					continue
-				}
-			}
-			if enrich {
-				hit.Excerpt = textExcerpt(doc.Text, 320)
-				if !doc.PublishedAt.IsZero() {
-					t := doc.PublishedAt
-					hit.PublishedAt = &t
-				}
-				hit.Author = doc.Author
-			}
-			if includeText {
-				hit.Text = doc.Text
-			}
-			if wantRerank {
-				rerankTexts = append(rerankTexts, doc.Title+"\n"+doc.Text)
-			}
-		}
-		out = append(out, hit)
-		if len(out) >= keepCap {
-			break
-		}
-	}
-	// Runs BEFORE rerank so the rerank pool
-	// reflects both quality and recency — when fetchK is over-fetched, the
-	// reranker still sees the freshest-AND-most-relevant top-N. Hits without
-	// PublishedAt are left alone (no signal, no penalty). The pool re-sort
-	// re-aligns rerankTexts via a URL→text map so both knobs compose.
-	if decaySet {
-		var textByURL map[string]string
-		if wantRerank && len(rerankTexts) == len(out) {
-			textByURL = make(map[string]string, len(out))
-			for i, h := range out {
-				textByURL[h.URL] = rerankTexts[i]
-			}
-		}
-		applyTimeDecay(out, decayHalfLife, time.Now())
-		if textByURL != nil {
-			rerankTexts = rerankTexts[:0]
-			for _, h := range out {
-				rerankTexts = append(rerankTexts, textByURL[h.URL])
-			}
-		}
-	}
-	// rerank now that we have keepCap candidates with text.
-	if wantRerank && len(out) > 1 && len(rerankTexts) == len(out) {
-		cands := make([]rerank.Candidate, len(out))
-		for i := range out {
-			cands[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: rerankTexts[i]}
-		}
-		if order, rerr := s.doRerank(r.Context(), q, cands); rerr == nil && len(order) > 0 {
-			reordered := make([]searchHit, 0, len(out))
-			seen := make(map[int]bool, len(out))
-			for _, id := range order {
-				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(out) && !seen[n] {
-					reordered = append(reordered, out[n])
-					seen[n] = true
-				}
-			}
-			// Drop nothing: if the reranker omitted any IDs, append in original order.
-			for i, h := range out {
-				if !seen[i] {
-					reordered = append(reordered, h)
-				}
-			}
-			out = reordered
-		}
-	}
-	// MMR diversification. ?mmr=<lambda> (0..1) reorders the
-	// candidate pool to balance query relevance against diversity from
-	// previously-selected hits. Requires HNSW (per-hit vectors) and an
-	// embedder when the query vector wasn't already computed by the
-	// dense/hybrid path. Composes after rerank — rerank gives quality
-	// ordering, MMR diversifies it. warningsFor() handles the silent
-	// fall-through when MMR was requested but couldn't fire.
-	if mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr")); mmrSet && len(out) > 1 && s.hnsw != nil {
-		var qVec []float32
-		// Hybrid/dense path already embedded the query — try to reuse it.
-		// For BM25-only path we need to embed q if an embedder is configured.
-		if s.embedder != nil {
-			if vecs, embErr := s.embedder.Embed(r.Context(), []string{q}); embErr == nil && len(vecs) > 0 {
-				qVec = vecs[0]
-			}
-		}
-		if len(qVec) > 0 {
-			hitVecs := make([][]float32, len(out))
-			for i := range out {
-				if v, ok := s.hnsw.LookupVectorByURL(out[i].URL); ok {
-					hitVecs[i] = v
-				}
-			}
-			out = mmrSelect(qVec, out, hitVecs, mmrLambda)
-		}
-	}
-	if len(out) > k {
-		out = out[:k]
-	}
-	// Applies to
-	// the already-collected top-k pool; raise k to widen the pool before
-	// re-sorting if you need more candidates for date ordering.
-	switch r.URL.Query().Get("sort") {
-	case "date_desc":
-		sortHitsByDate(out, false)
-	case "date_asc":
-		sortHitsByDate(out, true)
-	}
-	// label centralized in buildRetrieverLabel so /answer
-	// and /research report the same vocabulary as /search.
-	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, effectiveQuery != q, wantRerank)
-	// mmr suffix when diversification actually fired (HNSW loaded
-	// + embedder available). Same conditional as the apply site above —
-	// the label tracks the real pipeline.
-	if mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr")); mmrSet && s.hnsw != nil && s.embedder != nil && mmrLambda < 1.0 {
-		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
-	}
-	// decay suffix when time-decay was applied.
-	if decaySet {
-		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
-	}
-	resp := searchResponse{
-		Query:     q,
-		Expand:    normalizeExpandMode(expandMode),
-		Retriever: retrieverLabel,
-		Hits:      out,
-		// total_candidates = BM25 candidates considered before
-		// filter (capped at fetchK). Operators tuning over-fetch can see
-		// whether their filter is dropping a lot — when out=k but
-		// total_candidates is close to fetchK, the filter is restrictive
-		// enough that you may want to raise k or relax filters.
-		TotalCandidates: len(hits),
-		Took:            time.Since(start).String(),
-	}
-	// surface the post-HyDE query when it actually changed, so callers
-	// can debug whether expand=true contributed any extra terms or returned q
-	// unchanged (chat down, empty passage, etc).
-	if effectiveQuery != q {
-		resp.EffectiveQuery = effectiveQuery
-	}
-	resp.Warnings = s.warningsFor(r)
-	writeJSON(w, http.StatusOK, resp)
-}
-
-// sortHitsByDate orders hits by PublishedAt. Hits with no PublishedAt sink to
-// the end regardless of direction — they have no comparable value. asc=true
-// gives oldest-first; asc=false gives newest-first.
-func sortHitsByDate(hits []searchHit, asc bool) {
-	sort.SliceStable(hits, func(i, j int) bool {
-		ai := hits[i].PublishedAt != nil
-		aj := hits[j].PublishedAt != nil
-		if ai != aj {
-			return ai // hits with dates come before hits without
-		}
-		if !ai {
-			return false
-		}
-		if asc {
-			return hits[i].PublishedAt.Before(*hits[j].PublishedAt)
-		}
-		return hits[i].PublishedAt.After(*hits[j].PublishedAt)
-	})
-}
-
-// BM25-only "more like this". Mirrors EXA's /findSimilar shape but
-// stays dependency-free: no embeddings required. Algorithm is Lucene's MLT —
-// pick the source doc's top-N terms by tf·idf, build a query from them, run
-// the existing BM25 search, drop the source URL itself. With dense vectors
-// off the table on the Pebble path (HNSW indexing during crawl is the iter
-// follow-up), this is the cheapest credible /find_similar we can ship.
-func (s *pebbleHTTP) handleFindSimilar(w http.ResponseWriter, r *http.Request) {
-	// /find_similar runs locally with a
-	// known URL+vec, but in a cluster the URL's shard owns its vector and
-	// only one shard has it. We fan-out to all shards anyway: the owning
-	// shard does the real work; others either fall back to text-mode (if
-	// text supplied) or return empty.
-	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
-		s.handleFindSimilarGateway(w, r)
-		return
-	}
-	start := time.Now()
-	// accept either ?url= (existing behavior) or ?text= (content-
-	// based similarity for unindexed drafts). text path skips the source-URL
-	// exclusion since there's no source URL to exclude.
-	rawURL := r.URL.Query().Get("url")
-	rawText := r.URL.Query().Get("text")
-	if rawURL == "" && rawText == "" {
-		writeProblem(w, http.StatusBadRequest, "missing url or text parameter")
-		return
-	}
-	var srcTitle, srcText, srcURL string
-	if rawURL != "" {
-		decoded, err := url.QueryUnescape(rawURL)
-		if err != nil {
-			writeProblem(w, http.StatusBadRequest, "url parameter is not URL-encoded")
-			return
-		}
-		src, err := s.store.GetDocByURL(r.Context(), decoded)
-		if errors.Is(err, store.ErrNotFound) {
-			writeProblem(w, http.StatusNotFound, "url not in index")
-			return
-		}
-		if err != nil {
-			writeProblem(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		srcTitle, srcText, srcURL = src.Title, src.Text, src.URL
-	} else {
-		srcText = rawText
-		srcTitle = r.URL.Query().Get("title") // optional title boost when text mode
-	}
-	k := 10
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
-			k = n
-		}
-	}
-
-	// Tokenize title (×3) and body; mirror IndexDocument's title boost so
-	// title terms dominate the similarity query.
-	tf := make(map[string]int, 256)
-	for _, t := range index.Tokenize(srcTitle) {
-		tf[t] += 3
-	}
-	for _, t := range index.Tokenize(srcText) {
-		tf[t]++
-	}
-
-	_, n, _ := s.store.CorpusStats(r.Context())
-	if n <= 0 {
-		n = 1
-	}
-	logN := math.Log(float64(n))
-
-	type termScore struct {
-		term  string
-		score float64
-	}
-	scored := make([]termScore, 0, len(tf))
-	for term, freq := range tf {
-		info, ok, err := s.store.GetTermInfo(r.Context(), term)
-		if err != nil || !ok || info.DocFreq <= 0 {
-			continue
-		}
-		idf := logN - math.Log(float64(info.DocFreq))
-		if idf <= 0 {
-			continue // term appears in every doc; useless signal
-		}
-		scored = append(scored, termScore{term: term, score: float64(freq) * idf})
-	}
-	sort.Slice(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
-
-	topN := 10
-	if len(scored) < topN {
-		topN = len(scored)
-	}
-	if topN == 0 {
-		// previously referenced `decoded` (only in scope in the
-		// url-mode branch) — broken since's text-mode addition.
-		// Use srcURL when present, else echo the bare text/title as the
-		// source identifier so the empty response still tells the caller
-		// what was searched.
-		emptyQuery := srcURL
-		if emptyQuery == "" {
-			emptyQuery = srcTitle
-			if emptyQuery == "" {
-				emptyQuery = srcText
-			}
-		}
-		writeJSON(w, http.StatusOK, searchResponse{
-			Query: emptyQuery, Retriever: "bm25-mlt",
-			Hits: []searchHit{}, Took: time.Since(start).String(),
-		})
-		return
-	}
-	terms := make([]string, topN)
-	for i := 0; i < topN; i++ {
-		terms[i] = scored[i].term
-	}
-	queryStr := strings.Join(terms, " ")
-	// optional ?q= augments the auto-derived MLT query so callers
-	// can constrain "more like this URL" with an extra concept (e.g.
-	// /find_similar?url=...&q=pricing). Appended verbatim — supports the
-	// same quoted-phrase shape /search accepts.
-	if extra := strings.TrimSpace(r.URL.Query().Get("q")); extra != "" {
-		queryStr = queryStr + " " + extra
-	}
-
-	// scope MLT with the same retrieval filters /search and /answer
-	// accept. 'find pages similar to X but only on docs.example.com' is the
-	// archetype EXA findSimilar shape. Over-fetch enough that the source
-	// exclusion + filter still fills k.
-	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
-	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
-	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
-	if sinceErr != nil {
-		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
-		return
-	}
-	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
-	if untilErr != nil {
-		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
-		return
-	}
-	dateFilter := !since.IsZero() || !until.IsZero()
-	includeText := r.URL.Query().Get("include_text") == "true"
-	// ?rerank=true closes /find_similar's parity gap with the other
-	// retrieval endpoints. The MLT query is auto-derived from the source doc,
-	// so reranking the candidate neighbors against THAT query is exactly what
-	// EXA's findSimilar quality boost is doing under the hood.
-	wantRerank := r.URL.Query().Get("rerank") == "true" && s.reranker != nil
-	keepCap := k
-	if wantRerank {
-		keepCap = s.rerankCandK
-		if keepCap < k {
-			keepCap = k
-		}
-	}
-	fetchK := keepCap + 1 // +1 to absorb the source-URL exclusion
-	if len(include) > 0 || len(exclude) > 0 || dateFilter {
-		mult := 5
-		if dateFilter {
-			mult = 10
-		}
-		fetchK = keepCap * mult
-		if fetchK > 500 {
-			fetchK = 500
-		}
-	} else if wantRerank {
-		fetchK = keepCap * 2
-	}
-
-	// /find_similar?retriever=dense reuses the source's
-	// persisted vector (URL-mode) or embeds the user's text (text-mode) and
-	// runs an HNSW cosine search instead of BM25-MLT. ?retriever=hybrid
-	// runs BOTH BM25-MLT and dense, then RRF-fuses — the
-	// strongest "find similar" signal: lexical precision + semantic recall.
-	//
-	// Requires COSIFT_LOAD_HNSW=true at server start (for the graph);
-	// text-mode additionally needs a configured embedder. URL-mode dense
-	// works without an embedder — the source vector is already in the graph
-	// from indexing. Missing requirements fall through to BM25-MLT;
-	// warningsFor() flags it (carves out the URL-mode-no-embedder
-	// case so the warning isn't misleading).
-	retrieverParam := r.URL.Query().Get("retriever")
-	useDense := retrieverParam == "dense" && s.hnsw != nil
-	useHybrid := retrieverParam == "hybrid" && s.hnsw != nil
-	var (
-		hits       []index.Hit
-		denseFired bool
-		bm25Fired  bool
-	)
-	// Helper: fetch the query vector (URL-mode lookup, falling back to
-	// text-mode embed when allowed). Returns ok=false when neither path
-	// produced a usable vector.
-	getQueryVec := func() ([]float32, bool) {
-		if srcURL != "" {
-			if v, ok := s.hnsw.LookupVectorByURL(srcURL); ok {
-				return v, true
-			}
-		}
-		if s.embedder != nil && (srcText != "" || srcTitle != "") {
-			seed := strings.TrimSpace(srcTitle + " " + srcText)
-			if seed != "" {
-				vecs, embErr := s.embedder.Embed(r.Context(), []string{seed})
-				if embErr == nil && len(vecs) > 0 {
-					return vecs[0], true
-				}
-			}
-		}
-		return nil, false
-	}
-	if useDense {
-		if queryVec, ok := getQueryVec(); ok {
-			vhits := s.hnsw.Search(r.Context(), queryVec, fetchK)
-			hits = make([]index.Hit, len(vhits))
-			for i, vh := range vhits {
-				hits[i] = index.Hit{URL: vh.URL, Title: vh.Title, Score: vh.Score}
-			}
-			denseFired = true
-		}
-	} else if useHybrid {
-		// Run BM25-MLT and dense in series; RRF-fuse the two ranked lists.
-		// Each retriever votes fetchK candidates so the fused top-fetchK has
-		// room to balance both signals before filter/enrich/rerank.
-		bm25Hits, bm25Err := s.idx.Search(r.Context(), queryStr, fetchK)
-		if bm25Err != nil {
-			writeProblem(w, http.StatusInternalServerError, bm25Err.Error())
-			return
-		}
-		bm25Fired = true
-		if queryVec, ok := getQueryVec(); ok {
-			vhits := s.hnsw.Search(r.Context(), queryVec, fetchK)
-			denseHits := s.applyAuthorityToDense(vhits)
-			hits = rrfFuse([][]index.Hit{bm25Hits, denseHits})
-			if len(hits) > fetchK {
-				hits = hits[:fetchK]
-			}
-			denseFired = true
-		} else {
-			// Hybrid fell through to BM25-only (e.g. text-mode + no embedder).
-			// Keep the BM25 hits we already paid for.
-			hits = bm25Hits
-		}
-	}
-	if !denseFired && !bm25Fired {
-		var err error
-		hits, err = s.idx.Search(r.Context(), queryStr, fetchK)
-		if err != nil {
-			writeProblem(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-	}
-	type fsCand struct {
-		hit        searchHit
-		rerankText string
-	}
-	cands := make([]fsCand, 0, keepCap)
-	for _, h := range hits {
-		if srcURL != "" && h.URL == srcURL {
-			continue
-		}
-		if len(include) > 0 || len(exclude) > 0 {
-			host := hostOf(h.URL)
-			if len(include) > 0 && !matchesAnyDomain(host, include) {
-				continue
-			}
-			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
-				continue
-			}
-		}
-		hit := searchHit{URL: h.URL, Title: h.Title, Score: h.Score}
-		doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
-		if derr != nil || doc == nil {
-			continue
-		}
-		if dateFilter {
-			if doc.PublishedAt.IsZero() {
-				continue
-			}
-			if !since.IsZero() && doc.PublishedAt.Before(since) {
-				continue
-			}
-			if !until.IsZero() && doc.PublishedAt.After(until) {
-				continue
-			}
-		}
-		hit.Excerpt = textExcerpt(doc.Text, 320)
-		if !doc.PublishedAt.IsZero() {
-			t := doc.PublishedAt
-			hit.PublishedAt = &t
-		}
-		hit.Author = doc.Author
-		if includeText {
-			hit.Text = doc.Text
-		}
-		c := fsCand{hit: hit}
-		if wantRerank {
-			c.rerankText = doc.Title + "\n" + doc.Text
-		}
-		cands = append(cands, c)
-		if len(cands) >= keepCap {
-			break
-		}
-	}
-	// Same pre-rerank placement as
-	// /search — re-sort cands by adjusted score so rerank sees a freshness-
-	// aware pool. Re-aligns rerankText too via URL→text map.
-	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
-	if decaySet && len(cands) > 0 {
-		now := time.Now()
-		for i := range cands {
-			cands[i].hit.Score *= decayMultiplier(cands[i].hit.PublishedAt, now, decayHalfLife)
-		}
-		sort.SliceStable(cands, func(i, j int) bool { return cands[i].hit.Score > cands[j].hit.Score })
-	}
-	if wantRerank && len(cands) > 1 {
-		rc := make([]rerank.Candidate, len(cands))
-		for i := range cands {
-			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
-		}
-		if order, rerr := s.doRerank(r.Context(), queryStr, rc); rerr == nil && len(order) > 0 {
-			reordered := make([]fsCand, 0, len(cands))
-			seen := make(map[int]bool, len(cands))
-			for _, id := range order {
-				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
-					reordered = append(reordered, cands[n])
-					seen[n] = true
-				}
-			}
-			for i, c := range cands {
-				if !seen[i] {
-					reordered = append(reordered, c)
-				}
-			}
-			cands = reordered
-		}
-	}
-	// MMR diversification on /find_similar. Anchored at the
-	// source vector — URL-mode reuses the persisted vec, text-mode embeds
-	// the seed via getQueryVec (defined above for dense/hybrid dispatch).
-	// Without an explicit user query, the source IS the query — this is
-	// the right anchor for "find similar but diverse".
-	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
-	mmrFired := false
-	if mmrSet && len(cands) > 1 && s.hnsw != nil {
-		if qVec, ok := getQueryVec(); ok {
-			hitVecs := make([][]float32, len(cands))
-			for i, c := range cands {
-				if v, vok := s.hnsw.LookupVectorByURL(c.hit.URL); vok {
-					hitVecs[i] = v
-				}
-			}
-			order := mmrOrder(qVec, hitVecs, mmrLambda)
-			if order != nil {
-				reordered := make([]fsCand, len(order))
-				for i, idx := range order {
-					reordered[i] = cands[idx]
-				}
-				cands = reordered
-				mmrFired = true
-			}
-		}
-	}
-	if len(cands) > k {
-		cands = cands[:k]
-	}
-	out := make([]searchHit, 0, len(cands))
-	for _, c := range cands {
-		out = append(out, c.hit)
-	}
-	// label tracks which retrievers actually fired.
-	//   bm25-mlt              — default, BM25 only
-	//   dense                 — ?retriever=dense fired (graph + vec found)
-	//   bm25-mlt+dense:rrf    — ?retriever=hybrid, both BM25 and dense fired
-	// If dense was requested but fell through (no graph, no vec), the label
-	// stays bm25-mlt and warningsFor() carries the reason.
-	retrieverLabel := "bm25-mlt"
-	switch {
-	case bm25Fired && denseFired:
-		retrieverLabel = "bm25-mlt+dense:rrf"
-	case denseFired:
-		retrieverLabel = "dense"
-	}
-	if wantRerank {
-		retrieverLabel += "+rerank:" + s.reranker.Name()
-	}
-	if mmrFired {
-		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
-	}
-	if decaySet {
-		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
-	}
-	writeJSON(w, http.StatusOK, searchResponse{
-		Query:           queryStr,
-		Retriever:       retrieverLabel,
-		Hits:            out,
-		TotalCandidates: len(hits),
-		Warnings:        s.warningsFor(r),
-		Took:            time.Since(start).String(),
-	})
-}
-
-// /answer — synthesis over BM25 retrieval. Mirrors the SQLite-side
-// /answer in spirit (top-k sources → cited synthesis) but stays minimal: no
-// streaming, no rerank, no query expansion. Those land in follow-up iters
-// once this surface is exercised. Returns 501 when no chat model is
-// configured so the absent capability fails loud instead of silent.
-const answerSystemPrompt = `You are a research assistant. Answer the user's question using the provided sources.
-- Cite sources by their numeric id in square brackets, e.g. [1] or [2,3]. Every factual claim needs a citation.
-- Synthesize the answer from what the sources state, including reasonable inferences from adjacent context — sources rarely state every answer verbatim.
-- Only fall back to "the sources do not cover X" if NONE of the sources have any relevant material at all. Do not refuse just because the exact formula, number, or phrase isn't quoted verbatim — extract whatever IS there and say so.
-- IGNORE sources that are irrelevant to the question — login forms, page-not-found stubs, wiki edit/diff/admin pages, navigation menus, raw image directories, or content about a different topic. Do not cite them. Do not mention them in your answer. Pretend they weren't in the source list.
-- Do not invent facts not supported by the sources.
-- Keep the answer focused on what the sources actually say; do not pad.`
-
-// HyDE-style query expansion prompt. Borrowed verbatim from
-// internal/server/hyde.go so the SQLite and Pebble paths produce comparable
-// expansions and operators don't have to learn two prompt shapes.
-const hydeSystemPrompt = `Write a brief, factual passage (2-4 sentences) that would directly answer the user's question. Output ONLY the passage — no preamble, no commentary, no apology if you're uncertain. If the question is ambiguous, pick the most plausible interpretation and answer that. The passage doesn't need to be true; it needs to be the SHAPE of what a relevant document would say. Embedding this passage and searching by its vector will find documents that look like real answers, even if the user's original query was just a few keywords.`
-
-// doChat wraps ChatClient.Chat with attempt/failure counters.
-// Takes the client as a parameter so it works for both s.chat and the
-// StreamingChatClient passed into streamResearch/streamAnswer.
-func (s *pebbleHTTP) doChat(ctx context.Context, c embed.ChatClient, msgs []embed.ChatMsg) (string, error) {
-	s.chatAttempts.Add(1)
-	start := time.Now()
-	out, err := c.Chat(ctx, msgs)
-	s.chatDurationNanos.Add(time.Since(start).Nanoseconds())
-	if err != nil {
-		s.chatFailures.Add(1)
-	}
-	return out, err
-}
-
-// doChatStream wraps StreamingChatClient.ChatStream with the same counters.
-func (s *pebbleHTTP) doChatStream(ctx context.Context, c embed.StreamingChatClient, msgs []embed.ChatMsg, onChunk func(string)) (string, error) {
-	s.chatAttempts.Add(1)
-	start := time.Now()
-	out, err := c.ChatStream(ctx, msgs, onChunk)
-	s.chatDurationNanos.Add(time.Since(start).Nanoseconds())
-	if err != nil {
-		s.chatFailures.Add(1)
-	}
-	return out, err
-}
-
-// doRerank wraps s.reranker.Rerank with attempt/failure counters.
-// Returning the original error unchanged so callers keep their existing
-// silent-fallback behavior; only the operator-side visibility changed.
-func (s *pebbleHTTP) doRerank(ctx context.Context, q string, cands []rerank.Candidate) ([]string, error) {
-	s.rerankAttempts.Add(1)
-	order, err := s.reranker.Rerank(ctx, q, cands)
-	if err != nil {
-		s.rerankFailures.Add(1)
-	}
-	return order, err
-}
-
-// applyBM25EnvOverrides reads COSIFT_BM25_K1 / _B and applies them
-// to idx. Shared by runPebbleServe and runQuery so both honor the same env.
-// Returns which knobs landed so callers can log selectively.
-type bm25EnvResult struct {
-	k1Set, bSet bool
-	k1Val, bVal float64
-	k1Bad, bBad string // non-empty when env was set but unparseable
-}
-
-func applyBM25EnvOverrides(idx *index.PebbleBM25) bm25EnvResult {
-	var out bm25EnvResult
-	if v := os.Getenv("COSIFT_BM25_K1"); v != "" {
-		if k1, err := strconv.ParseFloat(v, 64); err == nil && k1 > 0 {
-			idx.WithBM25Params(k1, 0)
-			out.k1Set, out.k1Val = true, k1
-		} else {
-			out.k1Bad = v
-		}
-	}
-	if v := os.Getenv("COSIFT_BM25_B"); v != "" {
-		if b, err := strconv.ParseFloat(v, 64); err == nil && b > 0 {
-			idx.WithBM25Params(0, b)
-			out.bSet, out.bVal = true, b
-		} else {
-			out.bBad = v
-		}
-	}
-	return out
-}
-
-// paraphraseQuery returns up to n paraphrases of q via the chat
-// client, parses the JSON array shape the SQLite-side paraphraser already
-// uses, and falls back to nil on any failure (no chat client, empty / malformed
-// reply, parse error). Caller decides what to do with an empty list — the
-// downstream RRF strategy treats it as 'no expansion, fall back to single
-// query'. No cache yet — keyed on (q, n) would need a different shape than
-// the HyDE cache; revisit when workload justifies.
-func (s *pebbleHTTP) paraphraseQuery(ctx context.Context, q string, n int) []string {
-	if s.chat == nil || n <= 0 {
-		return nil
-	}
-	s.paraMu.RLock()
-	if cached, ok := s.paraCache[q]; ok {
-		s.paraMu.RUnlock()
-		s.paraHits.Add(1)
-		return cached
-	}
-	s.paraMu.RUnlock()
-	s.paraMisses.Add(1)
-	const sys = `Generate paraphrases of a search query. Each paraphrase preserves the semantic intent but uses different vocabulary — different keywords that a target document might also use. Output ONLY a JSON array of strings.
-Example output for "go programming language": ["golang concurrent compiled language", "Google's systems programming language with goroutines"]`
-	resp, err := s.doChat(ctx, s.chat, []embed.ChatMsg{
-		{Role: "system", Content: sys},
-		{Role: "user", Content: fmt.Sprintf("Generate %d paraphrases of: %s", n, q)},
-	})
-	if err != nil {
-		return nil
-	}
-	raw := strings.TrimSpace(resp)
-	for _, fence := range []string{"```json", "```"} {
-		raw = strings.TrimPrefix(raw, fence)
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	raw = strings.TrimSpace(raw)
-	startIdx := strings.Index(raw, "[")
-	endIdx := strings.LastIndex(raw, "]")
-	if startIdx < 0 || endIdx <= startIdx {
-		return nil
-	}
-	var arr []string
-	if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &arr); err != nil {
-		return nil
-	}
-	out := make([]string, 0, len(arr))
-	for _, p := range arr {
-		t := strings.TrimSpace(p)
-		if t != "" {
-			out = append(out, t)
-		}
-	}
-	if len(out) > 0 {
-		s.paraMu.Lock()
-		if len(s.paraCache) >= s.paraCacheCap {
-			for k := range s.paraCache {
-				delete(s.paraCache, k)
-				break
-			}
-		}
-		s.paraCache[q] = out
-		s.paraMu.Unlock()
-	}
-	return out
-}
-
-// rrfFuse implements Reciprocal Rank Fusion across N ranked lists.
-// k=60 is the standard Cormack et al. constant; tweaking rarely changes top-k
-// ordering meaningfully. Each list contributes 1/(k + rank+1) to a URL's
-// fused score; URLs appearing in more lists at higher ranks rise. Returns
-// synthesized Hits ordered by fused score (URL/Title from first encounter,
-// Score = fused RRF score).
-func rrfFuse(lists [][]index.Hit) []index.Hit {
-	const fuseK = 60
-	type fused struct {
-		hit   index.Hit
-		score float64
-	}
-	scored := make(map[string]*fused, 64)
-	for _, list := range lists {
-		for rank, h := range list {
-			if existing, ok := scored[h.URL]; ok {
-				existing.score += 1.0 / float64(fuseK+rank+1)
-			} else {
-				scored[h.URL] = &fused{hit: h, score: 1.0 / float64(fuseK+rank+1)}
-			}
-		}
-	}
-	flat := make([]*fused, 0, len(scored))
-	for _, f := range scored {
-		flat = append(flat, f)
-	}
-	sort.Slice(flat, func(i, j int) bool { return flat[i].score > flat[j].score })
-	out := make([]index.Hit, len(flat))
-	for i, f := range flat {
-		out[i] = f.hit
-		out[i].Score = f.score
-	}
-	return out
-}
-
-// warningsFor surfaces silent no-ops that callers used to have to
-// derive from absent effective_query / retriever fields. Each warning is one
-// human-readable sentence; consumers programmatically inspect the slice.
-func (s *pebbleHTTP) warningsFor(r *http.Request) []string {
-	var w []string
-	if mode := r.URL.Query().Get("expand"); mode != "" {
-		if s.chat == nil {
-			w = append(w, "expand="+mode+" requested but no chat client configured (set cfg.Chat.Model)")
-		} else if normalizeExpandMode(mode) == "" {
-			// unknown expand value used to be silently ignored.
-			w = append(w, "expand="+mode+" is not a known strategy (try: hyde, paraphrase) — treated as no expansion")
-		}
-	}
-	if r.URL.Query().Get("rerank") == "true" && s.reranker == nil {
-		w = append(w, "rerank=true requested but no reranker configured (set cfg.Rerank.URL or cfg.Rerank.Enabled)")
-	}
-	// dense + hybrid retrievers need both a loaded HNSW graph
-	// and an embedder. Missing either falls through to BM25 with a warning.
-	// /find_similar?url=X URL-mode-dense reads the source vector
-	// directly from the graph via LookupVectorByURL — no embed RPC. Skip the
-	// embedder warning in that case (would otherwise be misleading: the
-	// request succeeded without an embedder).
-	// /find_similar falls back to bm25-mlt (not bm25), so the
-	// warning says so — caller's retriever label matches the warning text.
-	if rv := r.URL.Query().Get("retriever"); rv == "dense" || rv == "hybrid" {
-		isFindSimilar := strings.HasSuffix(r.URL.Path, "/find_similar")
-		isFindSimilarURLMode := isFindSimilar && r.URL.Query().Get("url") != ""
-		fallback := "BM25"
-		if isFindSimilar {
-			fallback = "BM25-MLT"
-		}
-		switch {
-		case s.hnsw == nil:
-			w = append(w, "retriever="+rv+" requested but HNSW graph not loaded (set COSIFT_LOAD_HNSW=true at server start) — fell back to "+fallback)
-		case s.embedder == nil && !isFindSimilarURLMode:
-			w = append(w, "retriever="+rv+" requested but no embedder configured (set cfg.Embeddings.Model) — fell back to "+fallback)
-		}
-	}
-	// MMR diversification needs HNSW + embedder. Bad values (non-
-	// float, out of [0,1]) fall through silently — flag them. Missing
-	// requirements also warn.
-	// /find_similar?url=X reuses the source vector from the graph
-	// for the MMR anchor — no embedder needed. Same carve-out as
-	// for the retriever warning.
-	if raw := r.URL.Query().Get("mmr"); raw != "" {
-		isFindSimilarURLMode := strings.HasSuffix(r.URL.Path, "/find_similar") && r.URL.Query().Get("url") != ""
-		if _, ok := parseMMRLambda(raw); !ok {
-			w = append(w, "mmr="+raw+" is not a float in [0,1] — diversification skipped")
-		} else if s.hnsw == nil {
-			w = append(w, "mmr requires HNSW graph (set COSIFT_LOAD_HNSW=true at server start) — diversification skipped")
-		} else if s.embedder == nil && !isFindSimilarURLMode {
-			w = append(w, "mmr requires an embedder to vectorize the query (set cfg.Embeddings.Model) — diversification skipped")
-		}
-	}
-	// invalid decay half-life flags loudly instead of silently
-	// being ignored. Empty value is fine (no decay requested).
-	if raw := r.URL.Query().Get("decay"); raw != "" {
-		if _, ok := parseDecayHalfLife(raw); !ok {
-			w = append(w, "decay="+raw+" is not a positive half-life in days (≤ 36500) — time-decay skipped")
-		}
-	}
-	// catch unknown ?sort= values (silently treated as relevance).
-	if sortVal := r.URL.Query().Get("sort"); sortVal != "" {
-		switch sortVal {
-		case "relevance", "date_desc", "date_asc":
-			// valid
-		default:
-			w = append(w, "sort="+sortVal+" is not a known mode (try: relevance, date_desc, date_asc) — treated as relevance")
-		}
-	}
-	// catch obviously-bad ?k= values that silently fell back to the
-	// per-endpoint default. Upper-bound clamping varies per endpoint so we
-	// flag only the universally-invalid cases (non-integer, zero, negative).
-	if kVal := r.URL.Query().Get("k"); kVal != "" {
-		if n, err := strconv.Atoi(kVal); err != nil || n <= 0 {
-			w = append(w, "k="+kVal+" is not a positive integer — using server default")
-		}
-	}
-	// Users sometimes
-	// pass 'https://example.com/foo' when 'example.com' was wanted; the
-	// dot-boundary matcher silently drops every result.
-	for _, key := range []string{"include_domains", "exclude_domains"} {
-		if v := r.URL.Query().Get(key); v != "" {
-			for _, d := range strings.Split(v, ",") {
-				d = strings.TrimSpace(d)
-				if strings.Contains(d, "://") || strings.Contains(d, "/") {
-					w = append(w, key+" entry "+d+" looks like a URL — pass a hostname (e.g., example.com), not a URL")
-					break // one warning per filter list is enough
-				}
-			}
-		}
-	}
-	if len(w) > 0 {
-		s.warningsEmitted.Add(1)
-	}
-	return w
-}
-
-// normalizeExpandMode returns the canonical strategy name for an `expand`
-// param value: "true" → "hyde" (alias), "hyde"/"paraphrase" pass through,
-// anything else → "" (so the response field doesn't echo a value that had
-// no effect).
-func normalizeExpandMode(raw string) string {
-	switch raw {
-	case "true", "hyde":
-		return "hyde"
-	case "paraphrase":
-		return "paraphrase"
-	default:
-		return ""
-	}
-}
-
-// parseDecayHalfLife parses ?decay= as a positive half-life in days. Returns
-// (halfLife, true) when valid. Empty / non-positive → (0, false) so caller
-// short-circuits. Bounded at 36500 (100 years) to avoid pathological values.
-func parseDecayHalfLife(raw string) (float64, bool) {
-	if raw == "" {
-		// Half-life from COSIFT_DEFAULT_DECAY_DAYS
-		// (default 180 days = 6 months). Docs older than 6 months get exponentially
-		// less weight in the score; recent docs surface naturally without keyword
-		// hacks like the JS regex. Set the env to 0 to disable globally.
-		// Explicit ?decay=N still wins.
-		def := 180.0
-		if v := os.Getenv("COSIFT_DEFAULT_DECAY_DAYS"); v != "" {
-			if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 {
-				def = f
-			}
-		}
-		if def <= 0 {
-			return 0, false
-		}
-		return def, true
-	}
-	// explicit decay=0 disables (even if env default is on).
-	if raw == "0" {
-		return 0, false
-	}
-	v, err := strconv.ParseFloat(raw, 64)
-	if err != nil {
-		return 0, false
-	}
-	if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 36500 {
-		return 0, false
-	}
-	return v, true
-}
-
-// decayMultiplier computes the time-decay weight for a single PublishedAt.
-// Half-life H: doc from H days ago → 0.5x, 2H → 0.25x. Missing date or
-// non-positive halfLife → 1.0 (no decay).
-func decayMultiplier(publishedAt *time.Time, now time.Time, halfLifeDays float64) float64 {
-	if halfLifeDays <= 0 || publishedAt == nil || publishedAt.IsZero() {
-		return 1
-	}
-	ageDays := now.Sub(*publishedAt).Hours() / 24.0
-	if ageDays < 0 {
-		ageDays = 0 // future-dated → treated as fresh, never boosted
-	}
-	return math.Exp(-math.Ln2 * ageDays / halfLifeDays)
-}
-
-// applyTimeDecay multiplies each hit's Score by decayMultiplier and resorts.
-func applyTimeDecay(hits []searchHit, halfLifeDays float64, now time.Time) {
-	if halfLifeDays <= 0 || len(hits) == 0 {
-		return
-	}
-	for i := range hits {
-		hits[i].Score *= decayMultiplier(hits[i].PublishedAt, now, halfLifeDays)
-	}
-	sort.SliceStable(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
-}
-
-// parseMMRLambda parses ?mmr= as a float in [0, 1]. Returns (lambda, true)
-// when valid. Empty value or out-of-range returns (0, false) so the caller
-// can short-circuit without firing MMR.
-func parseMMRLambda(raw string) (float64, bool) {
-	if raw == "" {
-		return 0, false
-	}
-	v, err := strconv.ParseFloat(raw, 64)
-	if err != nil {
-		return 0, false
-	}
-	if v < 0 || v > 1 {
-		return 0, false
-	}
-	return v, true
-}
-
-// cosineUnit returns the dot product of two unit-normalized float32 slices.
-// HNSW stores vectors in unit form, so this is equivalent to cosine similarity.
-// Returns 0 on length mismatch — caller is responsible for filtering missing
-// vectors before calling.
-func cosineUnit(a, b []float32) float32 {
-	if len(a) != len(b) {
-		return 0
-	}
-	var s float32
-	i := 0
-	for ; i <= len(a)-4; i += 4 {
-		s += a[i]*b[i] + a[i+1]*b[i+1] + a[i+2]*b[i+2] + a[i+3]*b[i+3]
-	}
-	for ; i < len(a); i++ {
-		s += a[i] * b[i]
-	}
-	return s
-}
-
-// mmrOrder computes a permutation of [0..n-1] via Maximal Marginal Relevance
-// (Carbonell & Goldstein '98). At each step picks the candidate maximizing
-//
-//	score = λ · sim(q, c) - (1-λ) · max_{s∈selected} sim(c, s)
-//
-// Candidates whose vectors are missing (zero-length) score 0 on relevance
-// and 0 against any other missing-vec candidate — MMR can only diversify
-// what it can compare. lambda=1 → pure relevance (no diversification);
-// lambda=0 → pure diversity. 0.5–0.7 is a typical starting point.
-//
-// Returns the permutation as a []int; never drops anything. The caller
-// applies it to whatever URL-keyed slice it has.
-func mmrOrder(qVec []float32, hitVecs [][]float32, lambda float64) []int {
-	n := len(hitVecs)
-	if n == 0 {
-		return nil
-	}
-	selected := make([]bool, n)
-	order := make([]int, 0, n)
-	rel := make([]float64, n)
-	for i := range hitVecs {
-		if len(hitVecs[i]) > 0 {
-			rel[i] = float64(cosineUnit(qVec, hitVecs[i]))
-		}
-	}
-	for len(order) < n {
-		bestI := -1
-		bestScore := -math.MaxFloat64
-		for i := 0; i < n; i++ {
-			if selected[i] {
-				continue
-			}
-			var maxSim float64
-			for _, j := range order {
-				if len(hitVecs[i]) == 0 || len(hitVecs[j]) == 0 {
-					continue
-				}
-				sim := float64(cosineUnit(hitVecs[i], hitVecs[j]))
-				if sim > maxSim {
-					maxSim = sim
-				}
-			}
-			score := lambda*rel[i] - (1.0-lambda)*maxSim
-			if score > bestScore {
-				bestScore = score
-				bestI = i
-			}
-		}
-		if bestI < 0 {
-			break
-		}
-		selected[bestI] = true
-		order = append(order, bestI)
-	}
-	return order
-}
-
-// mmrSelect — thin /search wrapper around mmrOrder that operates on []searchHit
-// directly.
-func mmrSelect(qVec []float32, hits []searchHit, hitVecs [][]float32, lambda float64) []searchHit {
-	if len(hits) <= 1 || lambda >= 1.0 {
-		return hits
-	}
-	order := mmrOrder(qVec, hitVecs, lambda)
-	out := make([]searchHit, 0, len(hits))
-	for _, i := range order {
-		out = append(out, hits[i])
-	}
-	return out
-}
-
-// applyMMRPermutation embeds q, looks up per-URL vectors from HNSW, and
-// returns an MMR permutation. Returns nil when MMR can't fire (no graph,
-// no embedder, embed failed, λ≥1, single-element pool). Shared by /answer
-// and /research synth endpoints.
-func (s *pebbleHTTP) applyMMRPermutation(ctx context.Context, urls []string, q string, lambda float64) []int {
-	if s.hnsw == nil || s.embedder == nil || len(urls) <= 1 || lambda >= 1.0 {
-		return nil
-	}
-	vecs, err := s.embedder.Embed(ctx, []string{q})
-	if err != nil || len(vecs) == 0 {
-		return nil
-	}
-	hitVecs := make([][]float32, len(urls))
-	for i, u := range urls {
-		if v, ok := s.hnsw.LookupVectorByURL(u); ok {
-			hitVecs[i] = v
-		}
-	}
-	return mmrOrder(vecs[0], hitVecs, lambda)
-}
-
-// buildRetrieverLabel produces the human-readable retriever string
-// surfaced on /search, /answer, /research responses. Mirrors the
-// /search inline switch so all three endpoints report the same vocabulary.
-// expansionFired is the post-hoc "did expand=hyde/paraphrase actually run"
-// signal (effectiveQuery != q for /search & /answer; chat-present for
-// /research where sub-queries are individual). wantRerank appends the
-// "+rerank:<name>" suffix when rerank fired.
-func (s *pebbleHTTP) buildRetrieverLabel(retrieverParam, expandMode string, denseReady, expansionFired, wantRerank bool) string {
-	label := "bm25"
-	switch {
-	case retrieverParam == "dense" && denseReady:
-		label = "dense"
-	case retrieverParam == "hybrid" && denseReady:
-		label = "bm25+dense:rrf"
-	default:
-		switch expandMode {
-		case "paraphrase":
-			if expansionFired {
-				label = "bm25+paraphrase"
-			}
-		case "true", "hyde":
-			if expansionFired {
-				label = "bm25+hyde"
-			}
-		}
-	}
-	if wantRerank && s.reranker != nil {
-		label += "+rerank:" + s.reranker.Name()
-	}
-	return label
-}
-
-// applyLLMEndpointDefaults sets the retriever-choice and rerank defaults
-// the LLM-synthesized endpoints (/answer, /research) want. BM25 alone is
-// too slow at multi-million-doc corpora for synchronous LLM use, and the
-// reranker is the proven quality recovery layer — so /answer and /research
-// default to hybrid + rerank whenever the components are wired, while
-// /search keeps its plain-BM25 default for low-latency operator use.
-//
-// Operators retain full control: ?retriever=bm25|dense|hybrid and
-// ?rerank=true|false|1|0 override either default.
-func (s *pebbleHTTP) applyLLMEndpointDefaults(retrieverParam, rerankParam string) (string, bool) {
-	denseReady := s.hnsw != nil && s.embedder != nil
-	if retrieverParam == "" && denseReady {
-		retrieverParam = "hybrid"
-	}
-	wantRerank := s.reranker != nil
-	switch rerankParam {
-	case "false", "0":
-		wantRerank = false
-	case "true", "1":
-		wantRerank = s.reranker != nil
-	}
-	// When the vLLM probe says the backend is queue-deep, silently
-	// disable rerank. Operators can override by passing ?rerank=true
-	// explicitly — but with COSIFT_LLM_DEGRADE_QUEUE breached, the
-	// rerank call would just queue and time out anyway. Better to ship
-	// hybrid scores fast than block on a doomed LLM call.
-	if wantRerank && rerankParam == "" && s.llmProbe != nil && s.llmProbe.Loaded() {
-		wantRerank = false
-	}
-	return retrieverParam, wantRerank
-}
-
-// retrieve dispatches retriever choice (bm25 / dense / hybrid) and
-// then expansion (bare / HyDE / paraphrase+RRF) for BM25 paths. Shared by
-// /search, /answer, /research so all three endpoints get the same retriever
-// matrix. Dense / hybrid require both the loaded HNSW graph and an
-// embedder; missing either falls through to BM25 — warningsFor()
-// surfaces that to the client.
-// applyAuthorityToDense converts HNSW VectorHits into index.Hits with
-// the per-host authority multiplier applied to the score, AND re-sorts
-// the result by the new score so the rank order reflects authority.
-// RRF fusion downstream uses rank position, not raw score, so without
-// the re-sort the multiplier was a no-op (observed: 'What is the C10K
-// problem?' still cited 3 luanjiao.cfd subdomains after applying the
-// multiplier alone). PebbleBM25.Search already multiplies AND sorts
-// internally; this helper mirrors that contract for the dense path.
-func (s *pebbleHTTP) applyAuthorityToDense(vhits []index.VectorHit) []index.Hit {
-	out := make([]index.Hit, len(vhits))
-	for i, vh := range vhits {
-		score := vh.Score
-		if s.authority != nil {
-			score *= s.authority.Multiplier(hostFromURL(vh.URL))
-		}
-		out[i] = index.Hit{URL: vh.URL, Title: vh.Title, Score: score}
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score })
-	return out
-}
-
-func (s *pebbleHTTP) retrieve(ctx context.Context, q string, fetchK int, retrieverParam, expandMode string) ([]index.Hit, string, error) {
-	denseReady := s.hnsw != nil && s.embedder != nil
-	switch {
-	case retrieverParam == "dense" && denseReady:
-		vecs, err := s.embedder.Embed(ctx, []string{q})
-		if err != nil {
-			return nil, "", fmt.Errorf("embedder: %w", err)
-		}
-		vhits := s.hnsw.Search(ctx, vecs[0], fetchK)
-		hits := s.applyAuthorityToDense(vhits)
-		return hits, q, nil
-	case retrieverParam == "hybrid" && denseReady:
-		bm25Hits, bm25Eff, bm25Err := s.retrieveWithExpansion(ctx, q, fetchK, expandMode)
-		if bm25Err != nil {
-			return nil, "", bm25Err
-		}
-		vecs, embErr := s.embedder.Embed(ctx, []string{q})
-		if embErr != nil {
-			return nil, "", fmt.Errorf("embedder: %w", embErr)
-		}
-		denseV := s.hnsw.Search(ctx, vecs[0], fetchK)
-		denseHits := s.applyAuthorityToDense(denseV)
-		hits := rrfFuse([][]index.Hit{bm25Hits, denseHits})
-		if len(hits) > fetchK {
-			hits = hits[:fetchK]
-		}
-		return hits, bm25Eff, nil
-	default:
-		return s.retrieveWithExpansion(ctx, q, fetchK, expandMode)
-	}
-}
-
-// retrieveWithExpansion dispatches the BM25 call across the three
-// expansion strategies /search and /answer share — bare, HyDE, paraphrase+RRF.
-// Returns (hits, effectiveQuery, err). effectiveQuery == q when no expansion
-// fired (bare path, or expansion no-op'd because chat is down).
-func (s *pebbleHTTP) retrieveWithExpansion(ctx context.Context, q string, fetchK int, expandMode string) ([]index.Hit, string, error) {
-	switch expandMode {
-	case "paraphrase":
-		paras := s.paraphraseQuery(ctx, q, 3)
-		if len(paras) == 0 {
-			hits, err := s.idx.Search(ctx, q, fetchK)
-			return hits, q, err
-		}
-		queries := append([]string{q}, paras...)
-		lists := make([][]index.Hit, 0, len(queries))
-		for _, qq := range queries {
-			h, lerr := s.idx.Search(ctx, qq, fetchK)
-			if lerr != nil {
-				continue
-			}
-			lists = append(lists, h)
-		}
-		hits := rrfFuse(lists)
-		if len(hits) > fetchK {
-			hits = hits[:fetchK]
-		}
-		return hits, q + " | " + strings.Join(paras, " | "), nil
-	case "true", "hyde":
-		eq := s.expandQuery(ctx, q)
-		hits, err := s.idx.Search(ctx, eq, fetchK)
-		return hits, eq, err
-	case "entity":
-		// Rule-based query rewrite for question-form entity lookups
-		// ("who created X", "how tall is X", "what is the capital of
-		// X"). Strips the question form, appends canonical-attribute
-		// keywords to the original, single BM25 call. The earlier
-		// RRF-fusion variant regressed legitimate passes because
-		// equal-weight fusion across 4 queries pushed the original's
-		// top hit out of the reranker's top-30. Concat preserves the
-		// original signal while letting the new keywords add positive
-		// scoring contributions where biographical pages exist.
-		rewrites := qexpand.RewriteEntity(q)
-		if len(rewrites) == 0 {
-			hits, err := s.idx.Search(ctx, q, fetchK)
-			return hits, q, err
-		}
-		expanded := q + " " + strings.Join(rewrites, " ")
-		hits, err := s.idx.Search(ctx, expanded, fetchK)
-		return hits, expanded, err
-	default:
-		// Bare path runs entity-expansion when the query matches a
-		// question-form pattern. Earlier RRF-fuse experiment regressed
-		// 'who created the World Wide Web' (passing → bail) because
-		// equal-weight fusion across 4 queries pushed the
-		// previously-#1 Berners-Lee bio out of the reranker's top-30.
-		// Concat-into-single-query avoids the fusion-weight problem:
-		// the rewrites only add positive scoring contributions for
-		// pages that contain the canonical-attribute words ("creator,"
-		// "inventor"), without diluting the original query's ranking
-		// signal. Operators can disable with COSIFT_DISABLE_ENTITY_EXPAND=1.
-		if os.Getenv("COSIFT_DISABLE_ENTITY_EXPAND") == "" {
-			if rewrites := qexpand.RewriteEntity(q); len(rewrites) > 0 {
-				expanded := q + " " + strings.Join(rewrites, " ")
-				hits, err := s.idx.Search(ctx, expanded, fetchK)
-				return hits, expanded, err
-			}
-		}
-		hits, err := s.idx.Search(ctx, q, fetchK)
-		return hits, q, err
-	}
-}
-
-// expandQuery returns q + " " + a HyDE-generated passage when a chat client is
-// configured. On any error (no chat client, chat call fails, empty passage),
-// returns q unchanged so callers can compose safely without explicit guards.
-// added a bounded in-memory cache to skip the chat call on
-// repeat queries — important for /research?expand=true where the same
-// sub-query rephrasing can fire across many similar research requests.
-func (s *pebbleHTTP) expandQuery(ctx context.Context, q string) string {
-	if s.chat == nil {
-		return q
-	}
-	s.hydeMu.RLock()
-	cached, hit := s.hydeCache[q]
-	s.hydeMu.RUnlock()
-	if hit {
-		s.hydeHits.Add(1)
-		return q + " " + cached
-	}
-	s.hydeMisses.Add(1)
-	passage, err := s.doChat(ctx, s.chat, []embed.ChatMsg{
-		{Role: "system", Content: hydeSystemPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		return q
-	}
-	passage = strings.TrimSpace(passage)
-	if passage == "" {
-		return q
-	}
-	s.hydeMu.Lock()
-	if len(s.hydeCache) >= s.hydeCacheCap {
-		// Bounded: drop one arbitrary entry (Go map range is randomized).
-		for k := range s.hydeCache {
-			delete(s.hydeCache, k)
-			break
-		}
-	}
-	s.hydeCache[q] = passage
-	s.hydeMu.Unlock()
-	return q + " " + passage
-}
-
-type answerSource struct {
-	// Matches the [N] tokens the synth prompt produces in the answer
-	// text. SQLite-side AnswerSource has had this; pebble's was missing,
-	// which caused the CLI to render every source as '[0]' before the
-	// i+1 fallback landed.
-	ID          int        `json:"id"`
-	URL         string     `json:"url"`
-	Title       string     `json:"title"`
-	Excerpt     string     `json:"excerpt,omitempty"`
-	PublishedAt *time.Time `json:"published_at,omitempty"`
-	Author      string     `json:"author,omitempty"`
-	Text        string     `json:"text,omitempty"`
-}
-
-type answerResponse struct {
-	Query          string `json:"query"`
-	EffectiveQuery string `json:"effective_query,omitempty"`
-	Expand         string `json:"expand,omitempty"`
-	// retriever label — same vocabulary as /search ("bm25",
-	// "dense", "bm25+dense:rrf", "+hyde"/"+paraphrase", "+rerank:<name>").
-	Retriever       string         `json:"retriever,omitempty"`
-	Answer          string         `json:"answer"`
-	Sources         []answerSource `json:"sources"`
-	Model           string         `json:"model"`
-	Warnings        []string       `json:"warnings,omitempty"`
-	TotalCandidates int            `json:"total_candidates,omitempty"`
-	Took            string         `json:"took"`
-	// when the answer matches a "sources do not contain" pattern,
-	// surface a hint URL pointing at /research?strategy=planner. The 60-Q
-	// eval showed multi-hop decomposition rescues ~40% of findability +
-	// factual-lookup misses. Purely additive — clients decide whether to
-	// render a "Try research mode" affordance.
-	SuggestEscalation string `json:"suggest_escalation,omitempty"`
-}
-
-// answerLooksLikeNoInfo returns true when the answer reads as "sources don't
-// cover this" — the canonical /research-escalation trigger. Same phrase set
-// the eval uses; kept in sync so eval results predict escalation rate.
-func answerLooksLikeNoInfo(s string) bool {
-	if len(s) > 800 {
-		// Long answers with one disclaimer line aren't bails — only treat
-		// short, mostly-disclaimer responses as escalation candidates.
-		return false
-	}
-	low := strings.ToLower(s)
-	for _, p := range []string{
-		"do not contain", "do not provide", "sources don",
-		"not contain the", "no information", "do not include",
-		"do not have", "do not cover", "not mention", "no mention",
-	} {
-		if strings.Contains(low, p) {
-			return true
-		}
-	}
-	return false
-}
-
-func (s *pebbleHTTP) handleAnswer(w http.ResponseWriter, r *http.Request) {
-	// When clustered + gateway mode +
-	// not already serving as a leaf for someone else, fan-out retrieval
-	// to peers (with include_text), then run a SINGLE synth here.
-	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
-		s.handleAnswerGateway(w, r)
-		return
-	}
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented,
-			"/answer requires cfg.Chat.Model to be set (point cosift.json's chat.url at any OpenAI-compatible endpoint)")
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	// Cache + singleflight on /answer. Key is the full query-string
-	// (sorted) so different filter combos don't collide. We capture the
-	// rendered body into a buffer; on success the buffer is the cache
-	// value. Skip the cache for explicit no-cache requests and for
-	// streaming clients — bufferedResponse can't flush, so SSE would
-	// silently fall through to the sync JSON path.
-	if s.answerCache != nil && r.Header.Get("Cache-Control") != "no-cache" && !wantsSSE(r) {
-		key := answerCacheKey(r)
-		body, err, shared := s.answerCache.Do(key, func() ([]byte, error) {
-			rec := &bufferedResponse{header: http.Header{}, status: 200}
-			s.handleAnswerInner(rec, r, start)
-			if rec.status != http.StatusOK {
-				return nil, errAnswerNotOK
-			}
-			return rec.buf.Bytes(), nil
-		})
-		if err == nil {
-			if shared {
-				w.Header().Set("X-Cache", "hit")
-			} else {
-				w.Header().Set("X-Cache", "miss")
-			}
-			w.Header().Set("Content-Type", "application/json")
-			_, _ = w.Write(body)
-			return
-		}
-		// On non-OK from the inner handler, fall through to the
-		// uncached path so the real status code reaches the client.
-	}
-	s.handleAnswerInner(w, r, start)
-}
-
-// errAnswerNotOK signals that the inner handler wrote a non-2xx
-// response — don't cache it, and let the caller re-run uncached so
-// the proper status reaches the client.
-var errAnswerNotOK = errors.New("answer: non-OK response, not cached")
-
-// answerCacheKey is the deterministic cache key for a given /answer
-// request. Sorts query parameters so URL-order doesn't fragment the
-// cache; drops parameters that don't affect the response.
-func answerCacheKey(r *http.Request) string {
-	q := r.URL.Query()
-	// Don't key on transport-only or debug-only flags.
-	q.Del("cluster_local")
-	q.Del("trace")
-	keys := make([]string, 0, len(q))
-	for k := range q {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	var sb strings.Builder
-	for _, k := range keys {
-		for _, v := range q[k] {
-			sb.WriteString(k)
-			sb.WriteByte('=')
-			sb.WriteString(v)
-			sb.WriteByte('\x00')
-		}
-	}
-	sum := sha256.Sum256([]byte(sb.String()))
-	return hex.EncodeToString(sum[:16])
-}
-
-// bufferedResponse captures a handler's writes into a buffer so we
-// can store the rendered JSON in answerCache before flushing to the
-// real response writer.
-type bufferedResponse struct {
-	buf    bytes.Buffer
-	header http.Header
-	status int
-}
-
-func (b *bufferedResponse) Header() http.Header        { return b.header }
-func (b *bufferedResponse) WriteHeader(statusCode int) { b.status = statusCode }
-func (b *bufferedResponse) Write(p []byte) (int, error) {
-	if b.status == 0 {
-		b.status = http.StatusOK
-	}
-	return b.buf.Write(p)
-}
-
-// handleAnswerInner is the uncached body. Called both via the cache
-// path (with a bufferedResponse) and directly when the cache is
-// disabled or a non-OK response needs the real writer.
-func (s *pebbleHTTP) handleAnswerInner(w http.ResponseWriter, r *http.Request, start time.Time) {
-	q := r.URL.Query().Get("q")
-	k := 5
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
-			k = n
-		}
-	}
-	// /answer respects the same retrieval filters /search does —
-	// scoping research to a domain or date window is the common EXA shape.
-	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
-	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
-	sites := parseSiteScopes(r.URL.Query().Get("site"))
-	siteFilter := len(sites) > 0
-	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
-	if sinceErr != nil {
-		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
-		return
-	}
-	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
-	if untilErr != nil {
-		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
-		return
-	}
-	dateFilter := !since.IsZero() || !until.IsZero()
-	includeText := r.URL.Query().Get("include_text") == "true"
-	// Default to hybrid + rerank on /answer — the reranker is the proven
-	// quality recovery layer; ?retriever=bm25 and ?rerank=false opt out.
-	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
-		r.URL.Query().Get("retriever"),
-		r.URL.Query().Get("rerank"),
-	)
-	keepCap := k
-	if wantRerank {
-		keepCap = s.rerankCandK
-		if keepCap < k {
-			keepCap = k
-		}
-	}
-	fetchK := keepCap
-	if len(include) > 0 || len(exclude) > 0 || dateFilter || siteFilter {
-		mult := 5
-		if dateFilter {
-			mult = 10
-		}
-		fetchK = keepCap * mult
-		if fetchK > 200 {
-			fetchK = 200 // /answer caps tighter than /search — full doc text per source is expensive
-		}
-	} else if wantRerank {
-		fetchK = keepCap * 2
-	}
-	// SSE opt-in. Open the stream BEFORE retrieve() so phase events can
-	// surface backend pipeline progress (retrieve / rerank / judge / mmr /
-	// synth_start) instead of just the synth-chunk tail. Opt in via
-	// ?stream=true or Accept: text/event-stream. Caps requirement: chat
-	// client must implement StreamingChatClient AND the ResponseWriter
-	// must support http.Flusher; if either is missing, fall through to
-	// sync (today's behavior).
-	wantStream := wantsSSE(r)
-	var sse *answerSSE
-	var streamChat embed.StreamingChatClient
-	if wantStream {
-		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
-			if a := newAnswerSSE(w, start); a != nil {
-				sse, streamChat = a, sc
-				stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
-				defer stopKA()
-				sse.warnings(s.warningsFor(r))
-				sse.phase("parsed", map[string]any{
-					"q": q, "k": k, "fetch_k": fetchK,
-					"retriever": retrieverParam, "rerank": wantRerank,
-				})
-			}
-		}
-	}
-
-	// expansion dispatch (bare / HyDE / paraphrase+RRF).
-	// retriever dispatch (bm25 / dense / hybrid) via shared helper.
-	expandMode := r.URL.Query().Get("expand")
-	hits, effectiveQuery, err := s.retrieve(r.Context(), q, fetchK, retrieverParam, expandMode)
-	if err != nil {
-		if sse != nil {
-			sse.errorEvt(err.Error())
-			return
-		}
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	if sse != nil {
-		sse.phase("retrieve", map[string]any{
-			"hits": len(hits), "effective_query": effectiveQuery,
-		})
-	}
-	type cand struct {
-		src        answerSource
-		excerpt    string
-		rerankText string
-		score      float64 // retrieval score, used by time-decay
-	}
-	cands := make([]cand, 0, keepCap)
-	for _, h := range hits {
-		if len(include) > 0 || len(exclude) > 0 {
-			host := hostOf(h.URL)
-			if len(include) > 0 && !matchesAnyDomain(host, include) {
-				continue
-			}
-			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
-				continue
-			}
-		}
-		if siteFilter && !matchesAnySite(h.URL, sites) {
-			continue
-		}
-		doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
-		if derr != nil || doc == nil {
-			continue
-		}
-		if dateFilter {
-			if doc.PublishedAt.IsZero() {
-				continue
-			}
-			if !since.IsZero() && doc.PublishedAt.Before(since) {
-				continue
-			}
-			if !until.IsZero() && doc.PublishedAt.After(until) {
-				continue
-			}
-		}
-		excerpt := textExcerpt(doc.Text, 1200)
-		src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
-		if !doc.PublishedAt.IsZero() {
-			t := doc.PublishedAt
-			src.PublishedAt = &t
-		}
-		if includeText {
-			src.Text = doc.Text
-		}
-		c := cand{src: src, excerpt: excerpt, score: h.Score}
-		if wantRerank {
-			c.rerankText = doc.Title + "\n" + doc.Text
-		}
-		cands = append(cands, c)
-		if len(cands) >= keepCap {
-			break
-		}
-	}
-	if sse != nil {
-		sse.phase("filter", map[string]any{"candidates": len(cands)})
-	}
-	// time-decay re-weights then re-sorts BEFORE rerank, mirroring
-	// the /search pre-rerank placement so the reranker sees a freshness-
-	// aware pool.
-	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
-	if decaySet && len(cands) > 0 {
-		now := time.Now()
-		for i := range cands {
-			cands[i].score *= decayMultiplier(cands[i].src.PublishedAt, now, decayHalfLife)
-		}
-		sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
-		if sse != nil {
-			sse.phase("decay", map[string]any{"half_life_days": decayHalfLife})
-		}
-	}
-	// Optional rerank, then truncate to k. Done in this order so citation
-	// numbers in the prompt match the final rank-ordered sources list.
-	if wantRerank && len(cands) > 1 {
-		rc := make([]rerank.Candidate, len(cands))
-		for i := range cands {
-			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
-		}
-		if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
-			reordered := make([]cand, 0, len(cands))
-			seen := make(map[int]bool, len(cands))
-			for _, id := range order {
-				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
-					reordered = append(reordered, cands[n])
-					seen[n] = true
-				}
-			}
-			for i, c := range cands {
-				if !seen[i] {
-					reordered = append(reordered, c)
-				}
-			}
-			cands = reordered
-		}
-		if sse != nil {
-			sse.phase("rerank", map[string]any{"reranker": s.reranker.Name(), "candidates": len(cands)})
-		}
-	}
-	// LLM relevance judge — STRICT gate. The reranker re-orders but
-	// doesn't drop; without this gate, marginal content (Wikipedia
-	// diff/admin pages, dead .edu PDFs, JS-only landing pages with
-	// stub HTML) leaks into the synth context and produces "based on
-	// the sources, here is some irrelevant trivia" answers.
-	//
-	// Strict means: if a candidate fails the relevance threshold, it
-	// is DROPPED — we do not top up from the rejected pool to reach k.
-	// Better to return 2 truly relevant sources than 5 sources where 3
-	// are noise. If 0 candidates pass, downstream returns the "no
-	// matching sources" response (already handled below).
-	//
-	// Skip when ?judge=false or no chat client (correctness-preserving).
-	if r.URL.Query().Get("judge") != "false" && s.chat != nil && len(cands) > 1 {
-		before := len(cands)
-		jCands := make([]judge.Candidate, len(cands))
-		for i, c := range cands {
-			jCands[i] = judge.Candidate{ID: strconv.Itoa(i), Excerpt: c.rerankText}
-		}
-		verdicts := judge.Judge(r.Context(), s.chat, q, jCands, judge.Options{MinScore: 0.3})
-		keep := make([]cand, 0, len(cands))
-		for i, c := range cands {
-			if i < len(verdicts) && verdicts[i].Keep {
-				keep = append(keep, c)
-			}
-		}
-		cands = keep
-		if sse != nil {
-			sse.phase("judge", map[string]any{
-				"before": before, "kept": len(cands), "dropped": before - len(cands),
-			})
-		}
-	}
-	// MMR diversification on /answer — same pattern as /search.
-	// Synth quality benefits when the top-k sources cover different
-	// angles instead of being 5 paraphrases of the same paper.
-	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
-	mmrFired := false
-	if mmrSet && len(cands) > 1 {
-		urls := make([]string, len(cands))
-		for i := range cands {
-			urls[i] = cands[i].src.URL
-		}
-		if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
-			reordered := make([]cand, len(order))
-			for i, idx := range order {
-				reordered[i] = cands[idx]
-			}
-			cands = reordered
-			mmrFired = true
-		}
-		if sse != nil && mmrFired {
-			sse.phase("mmr", map[string]any{"lambda": mmrLambda, "candidates": len(cands)})
-		}
-	}
-	if len(cands) > k {
-		cands = cands[:k]
-	}
-	sources := make([]answerSource, 0, len(cands))
-	var promptSources strings.Builder
-	for i, c := range cands {
-		// stamp ID = i+1 so the JSON response matches the [N]
-		// citation tokens we emit in the synth prompt below.
-		c.src.ID = i + 1
-		sources = append(sources, c.src)
-		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, c.excerpt)
-	}
-	// same retriever label vocabulary as /search.
-	denseReady := s.hnsw != nil && s.embedder != nil
-	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, effectiveQuery != q, wantRerank)
-	if mmrFired {
-		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
-	}
-	if decaySet {
-		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
-	}
-	if len(sources) == 0 {
-		// Distinguish the two failure modes so the client can tell
-		// "we have nothing on this topic" (retrieval returned 0 hits)
-		// from "we retrieved candidates but the relevance judge dropped
-		// every one of them" (the corpus has noise, not the answer).
-		msg := "No matching sources in the index."
-		if len(hits) > 0 {
-			msg = "Retrieval returned candidates but none were judged relevant to the question. The corpus may not cover this topic in usable detail."
-		}
-		if sse != nil {
-			sse.sources(q, sources, streamChat.Model(), retrieverLabel, len(hits))
-			sse.chunk(msg)
-			sse.done()
-			return
-		}
-		empty := answerResponse{
-			Query: q, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
-			Answer:  msg,
-			Sources: sources, Model: s.chat.Model(), TotalCandidates: len(hits), Took: time.Since(start).String(),
-		}
-		if effectiveQuery != q {
-			empty.EffectiveQuery = effectiveQuery
-		}
-		empty.Warnings = s.warningsFor(r)
-		writeJSON(w, http.StatusOK, empty)
-		return
-	}
-
-	msgs := []embed.ChatMsg{
-		{Role: "system", Content: answerSystemPrompt},
-		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Question: " + q},
-	}
-
-	if sse != nil {
-		sse.sources(q, sources, streamChat.Model(), retrieverLabel, len(hits))
-		sse.phase("synth_start", map[string]any{"sources": len(sources), "model": streamChat.Model()})
-		full, cerr := s.doChatStream(r.Context(), streamChat, msgs, sse.chunk)
-		if cerr != nil {
-			sse.errorEvt(cerr.Error())
-			return
-		}
-		if answerLooksLikeNoInfo(full) {
-			sse.suggestEscalation(q)
-		}
-		sse.done()
-		return
-	}
-
-	answer, err := s.doChat(r.Context(), s.chat, msgs)
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "chat: "+err.Error())
-		return
-	}
-	resp := answerResponse{
-		Query: q, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
-		Answer: answer, Sources: sources,
-		Model: s.chat.Model(), TotalCandidates: len(hits), Took: time.Since(start).String(),
-	}
-	if effectiveQuery != q {
-		resp.EffectiveQuery = effectiveQuery
-	}
-	resp.Warnings = s.warningsFor(r)
-	// suggest /research escalation when the model bailed.
-	if answerLooksLikeNoInfo(answer) {
-		resp.SuggestEscalation = "/research?q=" + url.QueryEscape(q) + "&strategy=planner"
-	}
-	writeJSON(w, http.StatusOK, resp)
-}
-
-// wantsSSE reports whether the request opts into Server-Sent Events,
-// either via ?stream=true or an Accept: text/event-stream header. Both
-// /answer, /query, and /research check the same envelope.
-func wantsSSE(r *http.Request) bool {
-	return r.URL.Query().Get("stream") == "true" ||
-		strings.Contains(r.Header.Get("Accept"), "text/event-stream")
-}
-
-// answerSSE wraps the SSE writer for /answer streaming. Methods emit the
-// well-known event types (phase, sources, answer_chunk, warnings, error,
-// suggest_escalation, done). Phase events surface pipeline progress
-// (retrieve / rerank / judge / mmr / synth_start) so clients can render a
-// timeline of what the backend is doing.
-//
-// Writes are mu-protected because the keepalive ticker can race with
-// the main handler goroutine. The handler does the substantive event
-// writes; the ticker emits SSE comment frames (": ka\n\n") to keep
-// the HTTP/2 stream alive across long retrieval / rerank / chat calls
-// — without them, browsers (Safari especially) close the stream when
-// no bytes flow for 10–30 s and the request fails mid-pipeline.
-type answerSSE struct {
-	w       http.ResponseWriter
-	flusher http.Flusher
-	start   time.Time
-	last    time.Time // last phase emit; used to compute per-phase duration
-	mu      sync.Mutex
-}
-
-// newAnswerSSE opens an SSE stream. Returns nil if the writer can't flush.
-// Caller is responsible for not calling writeProblem after this point —
-// the response status (200) and headers are committed here.
-//
-// Disables the server-wide WriteTimeout (60s) via ResponseController.
-// Multi-pass /research can run 90-180s with 3-5 passes against a slow
-// chat model; the deadline would kill the connection mid-stream and
-// the browser fetch would surface a generic "Load failed". SSE handlers
-// flush their own progress, so a stuck handler still gets killed by
-// ctx cancellation when the client disconnects.
-func newAnswerSSE(w http.ResponseWriter, start time.Time) *answerSSE {
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		return nil
-	}
-	w.Header().Set("Content-Type", "text/event-stream")
-	w.Header().Set("Cache-Control", "no-cache")
-	w.Header().Set("X-Accel-Buffering", "no")
-	if rc := http.NewResponseController(w); rc != nil {
-		_ = rc.SetWriteDeadline(time.Time{})
-	}
-	w.WriteHeader(http.StatusOK)
-	return &answerSSE{w: w, flusher: flusher, start: start, last: start}
-}
-
-func (a *answerSSE) send(payload any) {
-	buf, _ := json.Marshal(payload)
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	fmt.Fprintf(a.w, "data: %s\n\n", buf)
-	a.flusher.Flush()
-}
-
-// startKeepalive launches a goroutine that emits SSE data frames every
-// interval, keeping the HTTP/2 stream and any intermediate proxy alive
-// across long retrieval / chat calls that emit no real data. Returns a
-// stop function the caller defers; stopping is idempotent.
-//
-// We deliberately emit `data: {"type":"ka", ...}` frames rather than the
-// spec's `: comment` form. Safari's CFNetwork stream stack tracks
-// "no payload data received" separately from "no bytes received" — a
-// long-running stream of nothing but comment frames still trips its
-// internal timeout and the browser surfaces it as "Error: Load failed".
-// JSON ka frames count as real data; the chat UI ignores `type:"ka"`.
-func (a *answerSSE) startKeepalive(ctx context.Context, interval time.Duration) (stop func()) {
-	done := make(chan struct{})
-	go func() {
-		t := time.NewTicker(interval)
-		defer t.Stop()
-		for {
-			select {
-			case <-done:
-				return
-			case <-ctx.Done():
-				return
-			case <-t.C:
-				a.send(map[string]any{
-					"type":       "ka",
-					"elapsed_ms": time.Since(a.start).Milliseconds(),
-				})
-			}
-		}
-	}()
-	var once sync.Once
-	return func() { once.Do(func() { close(done) }) }
-}
-
-// phase emits a pipeline-step event with elapsed-since-start and
-// duration-since-last-phase, plus any caller-supplied detail fields.
-// Holds a.mu for the whole call so the read+write of a.last and the
-// write-to-the-wire are all atomic with respect to concurrent phase
-// calls and the keepalive ticker.
-func (a *answerSSE) phase(name string, details map[string]any) {
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	now := time.Now()
-	evt := map[string]any{
-		"type":        "phase",
-		"name":        name,
-		"elapsed_ms":  now.Sub(a.start).Milliseconds(),
-		"duration_ms": now.Sub(a.last).Milliseconds(),
-	}
-	for k, v := range details {
-		evt[k] = v
-	}
-	a.last = now
-	buf, _ := json.Marshal(evt)
-	fmt.Fprintf(a.w, "data: %s\n\n", buf)
-	a.flusher.Flush()
-}
-
-func (a *answerSSE) warnings(warns []string) {
-	if len(warns) == 0 {
-		return
-	}
-	a.send(map[string]any{"type": "warnings", "warnings": warns})
-}
-
-func (a *answerSSE) sources(q string, src []answerSource, model, retrieverLabel string, totalCandidates int) {
-	evt := map[string]any{
-		"type":             "sources",
-		"query":            q,
-		"sources":          src,
-		"model":            model,
-		"total_candidates": totalCandidates,
-	}
-	if retrieverLabel != "" {
-		evt["retriever"] = retrieverLabel
-	}
-	a.send(evt)
-}
-
-func (a *answerSSE) chunk(delta string) {
-	a.send(map[string]any{"type": "answer_chunk", "text": delta})
-}
-
-func (a *answerSSE) errorEvt(msg string) {
-	a.send(map[string]any{"type": "error", "error": msg})
-}
-
-func (a *answerSSE) suggestEscalation(q string) {
-	a.send(map[string]any{
-		"type":               "suggest_escalation",
-		"suggest_escalation": "/research?q=" + url.QueryEscape(q) + "&strategy=planner",
-	})
-}
-
-func (a *answerSSE) done() {
-	a.send(map[string]any{"type": "done", "took": time.Since(a.start).String()})
-}
-
-// /research — multi-step retrieval + synthesis. LLM decomposes the
-// question into 2-3 sub-queries, each sub-query runs BM25, results are deduped
-// by URL keeping the best score, top-k feed a cited synthesis. Mirrors the
-// SQLite-side /research planner strategy. No streaming, no rerank, no
-// paraphrase strategy yet — those follow once this surface is exercised.
-const researchPlanPrompt = `Decompose the user's research question into 2-3 focused sub-queries that, taken together, would cover the answer. Output ONLY a JSON array of strings — no prose, no markdown. Example: ["sub-query 1", "sub-query 2"]`
-
-// queryPlanPrompt is the LLM-orchestrator prompt. The model
-// analyzes the user's natural-language query, classifies intent, and
-// outputs a structured retrieval plan: expanded queries (3 variations
-// covering different angles), recency window, retriever choice, and
-// optionally a domain allowlist. The plan then drives a broader RRF-
-// fused retrieval. JSON schema is strict; sanity-checked on receive.
-const queryPlanPrompt = `You analyze a user's natural-language search query and output a JSON retrieval plan.
-
-Output ONLY a JSON object (no markdown, no prose, no commentary) with this exact shape:
-{
-  "intent": "current_news" | "factual_lookup" | "comparison" | "research" | "findability",
-  "queries": ["expanded query 1", "expanded query 2", "expanded query 3"],
-  "since_days": null | <positive integer, days lookback from today>,
-  "retriever": "bm25" | "dense" | "hybrid",
-  "decay_days": <positive integer half-life> | null
-}
-
-Guidance:
-- "queries": 3 paraphrases/expansions of the original. Include entity names, synonyms, related terms. e.g. "latest news from Romania" → ["Romania political news 2026", "Romanian government coalition PSD AUR", "Bucharest current events"]
-- "since_days": only set when the query implies recency (latest/current/recent/today/news). null otherwise.
-- "decay_days": half-life in days. For current-events queries use 60. For factual/research queries use null or 365.
-- "retriever": hybrid is the safe default. Use "bm25" only for exact-term lookups (model names, error codes). Use "dense" only for purely conceptual queries with no keyword overlap.
-- DO NOT invent include_domains. The corpus chooses sources, not you.
-
-Examples:
-Input: "latest news from Romania"
-{"intent":"current_news","queries":["Romania political news 2026","Romanian PSD AUR coalition government","Bucharest economy and politics current"],"since_days":90,"retriever":"hybrid","decay_days":60}
-
-Input: "what is BM25 ranking"
-{"intent":"factual_lookup","queries":["BM25 Okapi ranking function formula","BM25 information retrieval probabilistic","term frequency inverse document frequency BM25"],"since_days":null,"retriever":"hybrid","decay_days":null}
-
-Input: "compare HNSW vs IVF"
-{"intent":"comparison","queries":["HNSW vs IVF approximate nearest neighbor","Hierarchical Navigable Small World versus Inverted File","ANN index tradeoffs HNSW IVF performance"],"since_days":null,"retriever":"hybrid","decay_days":730}`
-
-// queryPlan captures the LLM's structured retrieval plan. Sanitized on
-// parse — out-of-range values clipped to safe defaults.
-type queryPlan struct {
-	Intent    string   `json:"intent"`
-	Queries   []string `json:"queries"`
-	SinceDays *int     `json:"since_days"`
-	Retriever string   `json:"retriever"`
-	DecayDays *int     `json:"decay_days"`
-}
-
-// parseQueryPlan extracts the planner's JSON output and applies guard-rails.
-// Returns a plan with sane defaults if the LLM produces garbage — never
-// errors at the caller, so /query always succeeds.
-func parseQueryPlan(raw, original string) queryPlan {
-	defaultPlan := queryPlan{
-		Intent:    "research",
-		Queries:   []string{original},
-		Retriever: "hybrid",
-	}
-	// Strip code fences if the LLM wrapped output despite the prompt.
-	raw = strings.TrimSpace(raw)
-	if strings.HasPrefix(raw, "```") {
-		if i := strings.Index(raw, "\n"); i > 0 {
-			raw = raw[i+1:]
-		}
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	var p queryPlan
-	if err := json.Unmarshal([]byte(raw), &p); err != nil {
-		return defaultPlan
-	}
-	if len(p.Queries) == 0 {
-		p.Queries = []string{original}
-	}
-	if len(p.Queries) > 5 {
-		p.Queries = p.Queries[:5]
-	}
-	switch p.Retriever {
-	case "bm25", "dense", "hybrid":
-		// ok
-	default:
-		p.Retriever = "hybrid"
-	}
-	if p.SinceDays != nil && (*p.SinceDays <= 0 || *p.SinceDays > 3650) {
-		p.SinceDays = nil
-	}
-	if p.DecayDays != nil && (*p.DecayDays <= 0 || *p.DecayDays > 3650) {
-		p.DecayDays = nil
-	}
-	if p.Intent == "" {
-		p.Intent = "research"
-	}
-	return p
-}
-
-// handleQuery is the LLM-orchestrated search/research endpoint.
-// Flow: (1) LLM planner analyzes intent + emits expanded queries, (2) each
-// expansion runs through /search internals (hybrid + decay), (3) RRF-fuse
-// the per-expansion result lists, (4) optional rerank, (5) synth with
-// citations. Returns the plan in the response so callers can see what was
-// done.
-//
-// Effectively: /research planner-style decomposition, but optimized for
-// QUERY EXPANSION not sub-question DECOMPOSITION. The two are different —
-// research splits multi-faceted questions ("compare X and Y" → "X facts",
-// "Y facts", "X vs Y tradeoffs"); query expands a single intent into
-// paraphrases that catch corpus phrasing variation.
-func (s *pebbleHTTP) handleQuery(w http.ResponseWriter, r *http.Request) {
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented, "/query requires cfg.Chat.Model")
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-
-	// SSE opt-in. Stream phase events through the planner → expand → fuse
-	// → synth pipeline so operators see the LLM's sub-queries, per-expansion
-	// hit counts, fusion result, and synth tokens as they're produced.
-	wantStream := wantsSSE(r)
-	var sse *answerSSE
-	var streamChat embed.StreamingChatClient
-	if wantStream {
-		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
-			if a := newAnswerSSE(w, start); a != nil {
-				sse, streamChat = a, sc
-				stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
-				defer stopKA()
-				sse.phase("plan_start", map[string]any{"q": q, "model": s.chat.Model()})
-			}
-		}
-	}
-
-	// Step 1: LLM planner.
-	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: queryPlanPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		if sse != nil {
-			sse.errorEvt("plan: " + err.Error())
-			return
-		}
-		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
-		return
-	}
-	plan := parseQueryPlan(planRaw, q)
-	if sse != nil {
-		details := map[string]any{
-			"intent":    plan.Intent,
-			"retriever": plan.Retriever,
-			"queries":   plan.Queries,
-		}
-		if plan.SinceDays != nil {
-			details["since_days"] = *plan.SinceDays
-		}
-		if plan.DecayDays != nil {
-			details["decay_days"] = *plan.DecayDays
-		}
-		sse.phase("plan", details)
-	}
-
-	// Step 2: run each expanded query through retrieval, dedupe by URL.
-	type hitInfo struct {
-		hit    index.Hit
-		bestRR float64 // best reciprocal rank across expansions
-	}
-	byURL := make(map[string]*hitInfo, 64)
-	fetchK := 30
-	for _, sub := range plan.Queries {
-		hits, _, err := s.retrieve(r.Context(), sub, fetchK, plan.Retriever, "")
-		if err != nil {
-			if sse != nil {
-				sse.phase("expand", map[string]any{"query": sub, "error": err.Error()})
-			}
-			continue
-		}
-		for rank, h := range hits {
-			rr := 1.0 / float64(60+rank) // RRF with k=60
-			if existing, ok := byURL[h.URL]; ok {
-				if rr > existing.bestRR {
-					existing.bestRR = rr
-				}
-			} else {
-				byURL[h.URL] = &hitInfo{hit: h, bestRR: rr}
-			}
-		}
-		if sse != nil {
-			sse.phase("expand", map[string]any{"query": sub, "hits": len(hits)})
-		}
-	}
-
-	// Step 3: sort by fused RRF score with optional per-host boosts.
-	fused := make([]index.Hit, 0, len(byURL))
-	for _, v := range byURL {
-		h := v.hit
-		h.Score = v.bestRR
-		if len(s.hostBoosts) > 0 {
-			h.Score *= index.HostBoostFor(h.URL, s.hostBoosts)
-		}
-		fused = append(fused, h)
-	}
-	sort.Slice(fused, func(i, j int) bool { return fused[i].Score > fused[j].Score })
-	if sse != nil {
-		sse.phase("fuse", map[string]any{"unique_urls": len(fused)})
-	}
-
-	// Step 4: optional since filter + decay (decay handled by /answer-style
-	// path when we materialize). Apply since filter here on PublishedAt.
-	keep := 10
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
-			keep = n
-		}
-	}
-	var since time.Time
-	if plan.SinceDays != nil {
-		since = time.Now().AddDate(0, 0, -(*plan.SinceDays))
-	}
-
-	// Step 5: enrich + filter by since.
-	type cand struct {
-		src     answerSource
-		excerpt string
-		text    string
-	}
-	cands := make([]cand, 0, keep)
-	for _, h := range fused {
-		if len(cands) >= keep {
-			break
-		}
-		doc, err := s.store.GetDocByURL(r.Context(), h.URL)
-		if err != nil || doc == nil {
-			continue
-		}
-		if !since.IsZero() && (doc.PublishedAt.IsZero() || doc.PublishedAt.Before(since)) {
-			continue
-		}
-		c := cand{
-			src: answerSource{
-				ID:    len(cands) + 1,
-				URL:   h.URL,
-				Title: h.Title,
-			},
-			excerpt: textExcerpt(doc.Text, 320),
-			text:    doc.Text,
-		}
-		if !doc.PublishedAt.IsZero() {
-			t := doc.PublishedAt
-			c.src.PublishedAt = &t
-		}
-		cands = append(cands, c)
-	}
-	if sse != nil {
-		sse.phase("materialize", map[string]any{"candidates": len(cands)})
-	}
-
-	// Step 6: synth via answerSystemPrompt.
-	var promptSrcs strings.Builder
-	sources := make([]answerSource, 0, len(cands))
-	for i, c := range cands {
-		c.src.ID = i + 1
-		c.src.Excerpt = c.excerpt
-		fmt.Fprintf(&promptSrcs, "[%d] %s\n%s\n\n", c.src.ID, c.src.URL, truncateForPromptLite(c.text, 1200))
-		sources = append(sources, c.src)
-	}
-
-	if sse != nil {
-		sse.sources(q, sources, s.chat.Model(), "query:planner+hybrid+rrf", len(fused))
-		if len(cands) == 0 {
-			sse.chunk("No sources matched the planner's expanded queries within the chosen time window.")
-			sse.done()
-			return
-		}
-		sse.phase("synth_start", map[string]any{"sources": len(sources), "model": streamChat.Model()})
-		_, cerr := s.doChatStream(r.Context(), streamChat, []embed.ChatMsg{
-			{Role: "system", Content: answerSystemPrompt},
-			{Role: "user", Content: "Sources:\n\n" + promptSrcs.String() + "Question: " + q},
-		}, sse.chunk)
-		if cerr != nil {
-			sse.errorEvt("synth: " + cerr.Error())
-			return
-		}
-		sse.done()
-		return
-	}
-
-	answerText := ""
-	if len(cands) > 0 {
-		answerText, err = s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-			{Role: "system", Content: answerSystemPrompt},
-			{Role: "user", Content: "Sources:\n\n" + promptSrcs.String() + "Question: " + q},
-		})
-		if err != nil {
-			writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
-			return
-		}
-	} else {
-		answerText = "No sources matched the planner's expanded queries within the chosen time window."
-	}
-
-	writeJSON(w, http.StatusOK, map[string]any{
-		"query":   q,
-		"plan":    plan,
-		"answer":  answerText,
-		"sources": sources,
-		"model":   s.chat.Model(),
-		"took":    time.Since(start).String(),
-	})
-}
-
-// truncateForPromptLite caps text by approximate char count for source
-// prompts. Same heuristic the eval-quick path uses.
-func truncateForPromptLite(s string, maxChars int) string {
-	if len(s) <= maxChars {
-		return s
-	}
-	return s[:maxChars] + "…"
-}
-
-const researchSynthPrompt = `You are a research assistant. Synthesize an answer to the original question using ONLY the provided sources.
-- Cite sources by their numeric id, e.g. [1] or [2,3]. Every factual claim needs a citation.
-- If the sources don't cover something, say so plainly — do not invent.
-- Keep the answer focused on what the sources actually say.`
-
-// researchRefineSynthPrompt is the system prompt for pass 2+ of a multi-pass
-// /research call. The user message includes the prior draft answer alongside
-// the expanded source pool. The instruction shape mirrors the standard synth
-// prompt but explicitly directs the model to revise rather than start fresh.
-const researchRefineSynthPrompt = `You are a research assistant. You have a draft answer from a prior research pass plus additional sources that were gathered to close gaps. Produce a REVISED answer using all the sources now available.
-- Cite sources by their numeric id, e.g. [1] or [2,3]. Every factual claim needs a citation.
-- Keep correct claims from the draft. Correct or extend where the new sources change the picture.
-- If sources still don't cover something, say so plainly — do not invent.
-- Output the FULL revised answer, not a delta.`
-
-// selfEvalPrompt asks the model to judge whether its own draft answer is
-// sufficient given the question and sources. JSON-only output keeps parsing
-// deterministic. The "missing" + "refine_queries" fields drive the next
-// research pass when sufficient=false.
-const selfEvalPrompt = `You wrote a research answer. Judge whether it's sufficient given the question and sources.
-
-Output ONLY a JSON object (no markdown, no prose):
-{"sufficient": true|false, "missing": "<one-line gap, empty if sufficient>", "refine_queries": ["<query>", ...]}
-
-Rules:
-- "sufficient": true ONLY if the answer fully addresses the question with current sources.
-- "missing": one short sentence describing what would strengthen the answer (empty string if sufficient).
-- "refine_queries": 1-3 specific search queries that would close the gap (empty array if sufficient). Make them concrete — entity + concept + qualifier — not vague paraphrases.
-- Be honest. Most first-pass answers benefit from one more pass on a specific gap.`
-
-// selfEval is the parsed shape of the JSON the self-eval LLM call emits.
-type selfEval struct {
-	Sufficient    bool     `json:"sufficient"`
-	Missing       string   `json:"missing"`
-	RefineQueries []string `json:"refine_queries"`
-}
-
-// parseSelfEval extracts the {sufficient, missing, refine_queries} JSON from
-// the LLM response. Tolerates leading/trailing prose and markdown fences. On
-// any parse failure returns ok=false so the caller can treat as "stop and
-// keep the current answer" instead of crashing the stream.
-func parseSelfEval(s string) (selfEval, bool) {
-	s = strings.TrimSpace(s)
-	s = jsonFenceRE.ReplaceAllString(s, "$1")
-	// Find the first '{' and last '}' to skip any preamble.
-	i := strings.IndexByte(s, '{')
-	j := strings.LastIndexByte(s, '}')
-	if i < 0 || j <= i {
-		return selfEval{}, false
-	}
-	var ev selfEval
-	if err := json.Unmarshal([]byte(s[i:j+1]), &ev); err != nil {
-		return selfEval{}, false
-	}
-	if len(ev.RefineQueries) > 3 {
-		ev.RefineQueries = ev.RefineQueries[:3]
-	}
-	return ev, true
-}
-
-// jsonFenceRE strips ```json fences and bare ``` fences from LLM output. The
-// judge package has its own copy with the same intent; not worth pulling into
-// a shared package for two callers.
-var jsonFenceRE = regexp.MustCompile("(?s)```(?:json)?\\s*(.*?)\\s*```")
-
-// researchMaxPasses parses ?max_passes from the request. Default 3, valid
-// range 1-10 (clamped). 1 disables the self-eval loop entirely (single-pass
-// /research, today's behavior). The upper cap exists to bound LLM cost +
-// wall-clock: each non-final pass costs 1 synth + 1 self-eval call on top
-// of the per-sub-query retrieval, and the model can in theory loop forever
-// emitting "not yet sufficient" verdicts.
-func researchMaxPasses(r *http.Request) int {
-	const defaultMax = 3
-	const hardCap = 10
-	v := r.URL.Query().Get("max_passes")
-	if v == "" {
-		return defaultMax
-	}
-	n, err := strconv.Atoi(v)
-	if err != nil || n < 1 {
-		return defaultMax
-	}
-	if n > hardCap {
-		return hardCap
-	}
-	return n
-}
-
-type researchResponse struct {
-	Query  string   `json:"query"`
-	Plan   []string `json:"plan"`
-	Expand string   `json:"expand,omitempty"`
-	// retriever label, mirroring /search/answer vocabulary.
-	Retriever       string         `json:"retriever,omitempty"`
-	Answer          string         `json:"answer"`
-	Sources         []answerSource `json:"sources"`
-	Model           string         `json:"model"`
-	Warnings        []string       `json:"warnings,omitempty"`
-	TotalCandidates int            `json:"total_candidates,omitempty"`
-	Took            string         `json:"took"`
-}
-
-func (s *pebbleHTTP) handleResearch(w http.ResponseWriter, r *http.Request) {
-	// Planner runs once on gateway,
-	// each sub-query scatters to peers, single synth here.
-	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
-		s.handleResearchGateway(w, r)
-		return
-	}
-	if s.chat == nil {
-		writeProblem(w, http.StatusNotImplemented,
-			"/research requires cfg.Chat.Model to be set")
-		return
-	}
-	start := time.Now()
-	q := r.URL.Query().Get("q")
-	if q == "" {
-		writeProblem(w, http.StatusBadRequest, "missing q parameter")
-		return
-	}
-	k := 8
-	if v := r.URL.Query().Get("k"); v != "" {
-		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
-			k = n
-		}
-	}
-	// same retrieval filters /search/answer/find_similar accept.
-	// Parsed once here so the streaming and sync paths share semantics.
-	filt, err := parseRetrievalFilters(r)
-	if err != nil {
-		writeProblem(w, http.StatusBadRequest, err.Error())
-		return
-	}
-	includeText := r.URL.Query().Get("include_text") == "true"
-	// SSE streaming for /research. Same trigger as /answer.
-	// Emits phase-aware events so the UI can render the plan and source list
-	// before the synth call completes — /research often runs 10–30s
-	// (2-3 plan→retrieve→synth chat rounds), so phase visibility matters.
-	wantStream := wantsSSE(r)
-	if wantStream {
-		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
-			s.streamResearch(w, r, sc, q, k, filt, start)
-			return
-		}
-	}
-
-	// Plan
-	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: researchPlanPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
-		return
-	}
-	subs := parseSubQueries(planRaw, q)
-	if len(subs) > 5 {
-		subs = subs[:5]
-	}
-
-	// Retrieve per sub-query, dedupe by URL keeping best score.
-	type ranked struct {
-		score float64
-		hit   index.Hit
-	}
-	best := make(map[string]ranked, k*len(subs))
-	perSub := k * 2
-	if perSub > 40 {
-		perSub = 40
-	}
-	// each sub-query goes through retrieveWithExpansion, so
-	// ?expand=hyde gives per-sub-query HyDE (was's behavior) and
-	// ?expand=paraphrase fans out 3 paraphrases × N sub-queries → RRF per
-	// sub-query → merge into best{}. The cross-sub-query merge stays
-	// score-keep-best (not a second RRF) — that's what specified.
-	// retriever dispatch (bm25 / dense / hybrid) applies per
-	// sub-query — every sub-query runs through the same retriever.
-	expandMode := r.URL.Query().Get("expand")
-	// Default to hybrid + rerank on /research — same rationale as /answer.
-	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
-		r.URL.Query().Get("retriever"),
-		r.URL.Query().Get("rerank"),
-	)
-	for _, sq := range subs {
-		hits, _, err := s.retrieve(r.Context(), sq, perSub, retrieverParam, expandMode)
-		if err != nil {
-			// log the specific sub-query so operators can diagnose
-			// 'why was this research thin on sources' — previously silent.
-			log.Printf("pebble-serve: /research sub-query %q failed: %v", sq, err)
-			continue
-		}
-		for _, h := range hits {
-			if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
-				best[h.URL] = ranked{score: h.Score, hit: h}
-			}
-		}
-	}
-	pooled := make([]ranked, 0, len(best))
-	for _, v := range best {
-		pooled = append(pooled, v)
-	}
-	sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
-	// same rerank wiring as /search and /answer — pool widens to
-	// rerankCandK before materialization, rerank reorders the pool, truncate
-	// to k after rerank so citation numbers track the final order.
-	keepCap := k
-	if wantRerank {
-		keepCap = s.rerankCandK
-		if keepCap < k {
-			keepCap = k
-		}
-	}
-	if len(pooled) > keepCap {
-		pooled = pooled[:keepCap]
-	}
-
-	// Materialize candidates; defer sources + prompt build until after rerank.
-	type cand struct {
-		src        answerSource
-		excerpt    string
-		rerankText string
-		score      float64 // pooled score, used by time-decay
-	}
-	cands := make([]cand, 0, len(pooled))
-	for _, p := range pooled {
-		doc, derr := s.store.GetDocByURL(r.Context(), p.hit.URL)
-		if derr != nil || doc == nil {
-			continue
-		}
-		if !filt.allow(doc.URL, doc.PublishedAt) {
-			continue
-		}
-		excerpt := textExcerpt(doc.Text, 1200)
-		src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
-		if !doc.PublishedAt.IsZero() {
-			t := doc.PublishedAt
-			src.PublishedAt = &t
-		}
-		if includeText {
-			src.Text = doc.Text
-		}
-		c := cand{src: src, excerpt: excerpt, score: p.score}
-		if wantRerank {
-			c.rerankText = doc.Title + "\n" + doc.Text
-		}
-		cands = append(cands, c)
-	}
-	// time-decay on /research sync — re-weight pooled cands before
-	// rerank. Sub-queries that hit recent docs bubble up first.
-	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
-	if decaySet && len(cands) > 0 {
-		now := time.Now()
-		for i := range cands {
-			cands[i].score *= decayMultiplier(cands[i].src.PublishedAt, now, decayHalfLife)
-		}
-		sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
-	}
-	if wantRerank && len(cands) > 1 {
-		rc := make([]rerank.Candidate, len(cands))
-		for i := range cands {
-			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
-		}
-		if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
-			reordered := make([]cand, 0, len(cands))
-			seen := make(map[int]bool, len(cands))
-			for _, id := range order {
-				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
-					reordered = append(reordered, cands[n])
-					seen[n] = true
-				}
-			}
-			for i, c := range cands {
-				if !seen[i] {
-					reordered = append(reordered, c)
-				}
-			}
-			cands = reordered
-		}
-	}
-	// MMR diversification on /research sync. /research aggregates
-	// across sub-queries — duplicate-ish docs accumulate fast. MMR after
-	// rerank, before truncation.
-	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
-	mmrFired := false
-	if mmrSet && len(cands) > 1 {
-		urls := make([]string, len(cands))
-		for i := range cands {
-			urls[i] = cands[i].src.URL
-		}
-		if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
-			reordered := make([]cand, len(order))
-			for i, idx := range order {
-				reordered[i] = cands[idx]
-			}
-			cands = reordered
-			mmrFired = true
-		}
-	}
-	if len(cands) > k {
-		cands = cands[:k]
-	}
-	sources := make([]answerSource, 0, len(cands))
-	var promptSources strings.Builder
-	for i, c := range cands {
-		// stamp ID = i+1 so the JSON response matches the [N]
-		// citation tokens we emit in the synth prompt below.
-		c.src.ID = i + 1
-		sources = append(sources, c.src)
-		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, c.excerpt)
-	}
-	// Sub-queries don't yield a
-	// single effectiveQuery, so "expansion fired" is approximated by intent:
-	// chat is up and expandMode requested it. Matches what warningsFor() uses
-	// to decide whether to flag a silent no-op.
-	denseReady := s.hnsw != nil && s.embedder != nil
-	expandFired := expandMode != "" && s.chat != nil
-	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, expandFired, wantRerank)
-	if mmrFired {
-		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
-	}
-	if decaySet {
-		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
-	}
-	if len(sources) == 0 {
-		writeJSON(w, http.StatusOK, researchResponse{
-			Query: q, Plan: subs, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
-			Answer:  "No matching sources for any sub-query.",
-			Sources: sources, Model: s.chat.Model(), Warnings: s.warningsFor(r),
-			TotalCandidates: len(best), Took: time.Since(start).String(),
-		})
-		return
-	}
-
-	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
-		{Role: "system", Content: researchSynthPrompt},
-		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Original question: " + q},
-	})
-	if err != nil {
-		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, researchResponse{
-		Query: q, Plan: subs, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
-		Answer: answer, Sources: sources,
-		Model: s.chat.Model(), Warnings: s.warningsFor(r),
-		TotalCandidates: len(best), Took: time.Since(start).String(),
-	})
-}
-
-// streamResearch is the SSE form of handleResearch. Event sequence:
-//
-//	plan    — sub-queries returned by the planner
-//	sources — deduped + ranked pool fed to the synthesizer
-//	chunk   — per chat delta during synth
-//	done    — final {"took": "..."}
-//	error   — terminal; either plan, retrieval, or synth failed
-func (s *pebbleHTTP) streamResearch(w http.ResponseWriter, r *http.Request, sc embed.StreamingChatClient, q string, k int, filt retrievalFilters, start time.Time) {
-	sse := newAnswerSSE(w, start)
-	if sse == nil {
-		writeProblem(w, http.StatusInternalServerError, "streaming requires http.Flusher")
-		return
-	}
-	// /research can sit on a single slow retrieve or chat call for
-	// 10–30 s. Without a keepalive, the browser's HTTP/2 stream times
-	// out as inactive and the request fails mid-pipeline. 7 s gives
-	// generous headroom under Safari's ~30 s idle limit.
-	stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
-	defer stopKA()
-	includeText := r.URL.Query().Get("include_text") == "true"
-
-	// surface silent no-ops upfront so SSE clients can render them
-	// while the plan call is still in flight. Fires before plan to give the
-	// fastest visual feedback when a misconfigured request arrives.
-	sse.warnings(s.warningsFor(r))
-	sse.phase("plan_start", map[string]any{"q": q, "model": sc.Model()})
-
-	planRaw, err := s.doChat(r.Context(), sc, []embed.ChatMsg{
-		{Role: "system", Content: researchPlanPrompt},
-		{Role: "user", Content: q},
-	})
-	if err != nil {
-		sse.send(map[string]any{"type": "error", "phase": "plan", "error": err.Error()})
-		return
-	}
-	subs := parseSubQueries(planRaw, q)
-	if len(subs) > 5 {
-		subs = subs[:5]
-	}
-	// per-sub-query expansion via retrieveWithExpansion.
-	// per-sub-query retriever dispatch (bm25 / dense / hybrid).
-	// read retriever + wantRerank early so the plan event can
-	// surface the full retriever label, not just expand.
-	// Default to hybrid + rerank on /research streaming — same rationale as /answer.
-	expandMode := r.URL.Query().Get("expand")
-	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
-		r.URL.Query().Get("retriever"),
-		r.URL.Query().Get("rerank"),
-	)
-	denseReady := s.hnsw != nil && s.embedder != nil
-	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, expandMode != "", wantRerank)
-
-	// surface the active expansion strategy on the plan event so
-	// UIs can render 'using paraphrase' or 'using HyDE' as soon as the plan
-	// arrives, before per-sub-query expansion fires.
-	planEvent := map[string]any{"type": "plan", "query": q, "plan": subs, "model": sc.Model()}
-	if mode := normalizeExpandMode(expandMode); mode != "" {
-		planEvent["expand"] = mode
-	}
-	if retrieverLabel != "" {
-		planEvent["retriever"] = retrieverLabel
-	}
-	sse.send(planEvent)
-	// Phase event mirrors /query's shape so the chat UI's timeline strip
-	// renders identical rows across all three synth endpoints. Co-exists
-	// with the legacy `type: plan` event above, which the CLI client at
-	// cmd/cosift/main.go keys on.
-	planPhase := map[string]any{"queries": subs, "retriever": retrieverLabel}
-	if mode := normalizeExpandMode(expandMode); mode != "" {
-		planPhase["expand"] = mode
-	}
-	sse.phase("plan", planPhase)
-
-	type ranked struct {
-		score float64
-		hit   index.Hit
-	}
-	type cand struct {
-		src        answerSource
-		excerpt    string
-		rerankText string
-		score      float64
-	}
-
-	// Multi-pass research loop. Each pass: retrieve for the current subs →
-	// fuse → materialize → rerank → mmr → synth → self-eval. Cumulative URL
-	// set across passes; new passes skip URLs already seen so retrieval
-	// widens rather than re-fetching the same docs. After synth, the model
-	// judges whether its draft is sufficient. If not, we use its
-	// refine_queries to drive the next pass. Cap at maxPasses to bound
-	// latency + LLM cost.
-	maxPasses := researchMaxPasses(r)
-	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
-	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
-	perSub := k * 2
-	if perSub > 40 {
-		perSub = 40
-	}
-	keepCap := k
-	if wantRerank {
-		keepCap = s.rerankCandK
-		if keepCap < k {
-			keepCap = k
-		}
-	}
-
-	seenURLs := make(map[string]bool)
-	allCands := make([]cand, 0, k*maxPasses)
-	totalCandidates := 0 // counts URL contributions across passes for the sources event
-	var lastAnswer string
-	var finalRetrieverLabel string
-
-	for pass := 1; pass <= maxPasses; pass++ {
-		sse.phase("pass_start", map[string]any{"pass": pass, "queries": subs})
-
-		// Retrieval round for this pass. Sub-queries run concurrently with
-		// a bounded fan-out — they're independent calls to s.retrieve and
-		// the per-call cost dominates total pass latency. Serial execution
-		// of N sub-queries at ~12 s each pushed pass-1+pass-2 wall-clock
-		// past 60 s on this corpus, blowing the browser's hardcoded fetch
-		// timeout on long-streaming responses. With fan-out the pass
-		// shrinks to ~max(sub-query latency) plus the per-goroutine
-		// bookkeeping. sse.send is mutex-protected; bestMu protects best.
-		// seenURLs is read-only during this loop (last written at the end
-		// of the prior pass), so no synchronization is needed for it.
-		best := make(map[string]ranked, k*len(subs))
-		var bestMu sync.Mutex
-		var wg sync.WaitGroup
-		const subFanOut = 3
-		sem := make(chan struct{}, subFanOut)
-		for _, sq := range subs {
-			sq := sq
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				sem <- struct{}{}
-				defer func() { <-sem }()
-				sse.phase("retrieving", map[string]any{"pass": pass, "query": sq})
-				hits, _, rerr := s.retrieve(r.Context(), sq, perSub, retrieverParam, expandMode)
-				if rerr != nil {
-					log.Printf("pebble-serve: /research sub-query %q failed: %v", sq, rerr)
-					sse.phase("expand", map[string]any{"pass": pass, "query": sq, "error": rerr.Error()})
-					return
-				}
-				bestMu.Lock()
-				for _, h := range hits {
-					if seenURLs[h.URL] {
-						continue
-					}
-					if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
-						best[h.URL] = ranked{score: h.Score, hit: h}
-					}
-				}
-				bestMu.Unlock()
-				sse.phase("expand", map[string]any{"pass": pass, "query": sq, "hits": len(hits)})
-			}()
-		}
-		wg.Wait()
-		totalCandidates += len(best)
-		pooled := make([]ranked, 0, len(best))
-		for _, v := range best {
-			pooled = append(pooled, v)
-		}
-		sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
-		sse.phase("fuse", map[string]any{"pass": pass, "unique_urls": len(pooled)})
-		if len(pooled) > keepCap {
-			pooled = pooled[:keepCap]
-		}
-
-		// Materialize this pass's new candidates (filt-aware).
-		newCands := make([]cand, 0, len(pooled))
-		for _, p := range pooled {
-			doc, derr := s.store.GetDocByURL(r.Context(), p.hit.URL)
-			if derr != nil || doc == nil {
-				continue
-			}
-			if !filt.allow(doc.URL, doc.PublishedAt) {
-				continue
-			}
-			excerpt := textExcerpt(doc.Text, 1200)
-			src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
-			if !doc.PublishedAt.IsZero() {
-				t := doc.PublishedAt
-				src.PublishedAt = &t
-			}
-			if includeText {
-				src.Text = doc.Text
-			}
-			c := cand{src: src, excerpt: excerpt, score: p.score}
-			if wantRerank {
-				c.rerankText = doc.Title + "\n" + doc.Text
-			}
-			newCands = append(newCands, c)
-		}
-		sse.phase("materialize", map[string]any{"pass": pass, "candidates": len(newCands)})
-
-		// Optional time-decay on this pass's new candidates.
-		passLabel := retrieverLabel
-		if decaySet && len(newCands) > 0 {
-			now := time.Now()
-			for i := range newCands {
-				newCands[i].score *= decayMultiplier(newCands[i].src.PublishedAt, now, decayHalfLife)
-			}
-			sort.SliceStable(newCands, func(i, j int) bool { return newCands[i].score > newCands[j].score })
-			passLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
-		}
-		if wantRerank && len(newCands) > 1 {
-			rc := make([]rerank.Candidate, len(newCands))
-			for i := range newCands {
-				rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: newCands[i].rerankText}
-			}
-			if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
-				reordered := make([]cand, 0, len(newCands))
-				seenIdx := make(map[int]bool, len(newCands))
-				for _, id := range order {
-					if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(newCands) && !seenIdx[n] {
-						reordered = append(reordered, newCands[n])
-						seenIdx[n] = true
-					}
-				}
-				for i, c := range newCands {
-					if !seenIdx[i] {
-						reordered = append(reordered, c)
-					}
-				}
-				newCands = reordered
-			}
-			sse.phase("rerank", map[string]any{"pass": pass, "reranker": s.reranker.Name(), "candidates": len(newCands)})
-		}
-		if mmrSet && len(newCands) > 1 {
-			urls := make([]string, len(newCands))
-			for i := range newCands {
-				urls[i] = newCands[i].src.URL
-			}
-			if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
-				reordered := make([]cand, len(order))
-				for i, idx := range order {
-					reordered[i] = newCands[idx]
-				}
-				newCands = reordered
-				passLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
-				sse.phase("mmr", map[string]any{"pass": pass, "lambda": mmrLambda, "candidates": len(newCands)})
-			}
-		}
-		if len(newCands) > k {
-			newCands = newCands[:k]
-		}
-		// Promote into the cumulative pool.
-		for _, c := range newCands {
-			seenURLs[c.src.URL] = true
-			allCands = append(allCands, c)
-		}
-		finalRetrieverLabel = passLabel
-
-		// First-pass-empty path mirrors today's behavior: if pass 1 finds
-		// no usable sources at all, bail with empty=true.
-		if pass == 1 && len(allCands) == 0 {
-			sse.send(map[string]any{"type": "done", "took": time.Since(start).String(), "empty": true})
-			return
-		}
-		// Later-pass empty (no new sources added): synth would just rewrite
-		// the prior answer with no new signal. Stop and keep lastAnswer.
-		if pass > 1 && len(newCands) == 0 {
-			break
-		}
-
-		// Build the cumulative sources slice + prompt block, stamping IDs
-		// 1..N in the order URLs were promoted.
-		cumulativeSources := make([]answerSource, 0, len(allCands))
-		var promptSources strings.Builder
-		for i, c := range allCands {
-			c.src.ID = i + 1
-			cumulativeSources = append(cumulativeSources, c.src)
-			fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, c.excerpt)
-		}
-		srcEvt := map[string]any{
-			"type": "sources", "sources": cumulativeSources,
-			"total_candidates": totalCandidates, "pass": pass,
-		}
-		if finalRetrieverLabel != "" {
-			srcEvt["retriever"] = finalRetrieverLabel
-		}
-		sse.send(srcEvt)
-
-		// Synth. Pass 1 uses the standard prompt; pass 2+ uses the refine
-		// prompt with the prior draft injected.
-		sse.phase("synth_start", map[string]any{
-			"pass": pass, "sources": len(cumulativeSources), "model": sc.Model(),
-		})
-		var synthMsgs []embed.ChatMsg
-		userMsg := "Sources:\n\n" + promptSources.String() + "Original question: " + q
-		if pass == 1 {
-			synthMsgs = []embed.ChatMsg{
-				{Role: "system", Content: researchSynthPrompt},
-				{Role: "user", Content: userMsg},
-			}
-		} else {
-			synthMsgs = []embed.ChatMsg{
-				{Role: "system", Content: researchRefineSynthPrompt},
-				{Role: "user", Content: userMsg + "\n\nYour prior draft answer:\n" + lastAnswer},
-			}
-		}
-		full, serr := s.doChatStream(r.Context(), sc, synthMsgs, sse.chunk)
-		if serr != nil {
-			sse.send(map[string]any{"type": "error", "phase": "synth", "error": serr.Error()})
-			return
-		}
-		lastAnswer = full
-
-		// On the final pass, skip self-eval — nothing to escalate to.
-		if pass == maxPasses {
-			break
-		}
-
-		// Self-evaluate. The model decides whether to escalate.
-		sse.phase("self_eval_start", map[string]any{"pass": pass})
-		evalUserMsg := fmt.Sprintf(
-			"Question: %s\n\nYour answer:\n%s\n\nSources used (id — title — url):\n%s",
-			q, lastAnswer, summarizeSourceList(cumulativeSources),
-		)
-		evalRaw, eerr := s.doChat(r.Context(), sc, []embed.ChatMsg{
-			{Role: "system", Content: selfEvalPrompt},
-			{Role: "user", Content: evalUserMsg},
-		})
-		if eerr != nil {
-			log.Printf("pebble-serve: /research self-eval pass %d failed: %v", pass, eerr)
-			sse.phase("self_eval", map[string]any{"pass": pass, "error": eerr.Error()})
-			break
-		}
-		ev, ok := parseSelfEval(evalRaw)
-		if !ok {
-			sse.phase("self_eval", map[string]any{"pass": pass, "parse_failed": true})
-			break
-		}
-		sse.phase("self_eval", map[string]any{
-			"pass": pass, "sufficient": ev.Sufficient,
-			"missing": ev.Missing, "refine_queries": ev.RefineQueries,
-		})
-		if ev.Sufficient {
-			break
-		}
-		// Blurb summarizing why we're escalating; UI renders inline.
-		blurbText := strings.TrimSpace(ev.Missing)
-		if blurbText == "" {
-			blurbText = "More depth needed"
-		}
-		blurbText += " — searching deeper."
-		sse.phase("blurb", map[string]any{"pass": pass, "text": blurbText})
-
-		// Drive the next pass with the refined queries the model proposed.
-		if len(ev.RefineQueries) == 0 {
-			break
-		}
-		subs = ev.RefineQueries
-		if len(subs) > 5 {
-			subs = subs[:5]
-		}
-	}
-
-	sse.done()
-}
-
-// summarizeSourceList formats sources into compact "[id] title — url" lines
-// for the self-eval user message. Avoids piping the full excerpts back into
-// the eval prompt — the eval is about coverage, not re-reading.
-func summarizeSourceList(srcs []answerSource) string {
-	var sb strings.Builder
-	for _, s := range srcs {
-		fmt.Fprintf(&sb, "[%d] %s — %s\n", s.ID, s.Title, s.URL)
-	}
-	return sb.String()
-}
-
-// retrievalFilters bundles the four post-retrieval predicates /search,
-// /answer, /find_similar, and /research all share. Centralized in
-// so /research's sync + stream paths stay in lockstep.
-type retrievalFilters struct {
-	include    []string
-	exclude    []string
-	sites      []siteScope
-	since      time.Time
-	until      time.Time
-	dateActive bool
-}
-
-func parseRetrievalFilters(r *http.Request) (retrievalFilters, error) {
-	f := retrievalFilters{
-		include: splitDomainsCSV(r.URL.Query().Get("include_domains")),
-		exclude: splitDomainsCSV(r.URL.Query().Get("exclude_domains")),
-		sites:   parseSiteScopes(r.URL.Query().Get("site")),
-	}
-	since, err := parseDateBound(r.URL.Query().Get("since"))
-	if err != nil {
-		return f, fmt.Errorf("since: %w", err)
-	}
-	until, err := parseDateBound(r.URL.Query().Get("until"))
-	if err != nil {
-		return f, fmt.Errorf("until: %w", err)
-	}
-	f.since = since
-	f.until = until
-	f.dateActive = !since.IsZero() || !until.IsZero()
-	return f, nil
-}
-
-// allow reports whether (url, publishedAt) clears the filter. publishedAt
-// can be the zero Time when the caller doesn't have it loaded; zero is
-// treated as "unknown" and dropped under any date filter.
-func (f retrievalFilters) allow(rawURL string, publishedAt time.Time) bool {
-	if len(f.include) > 0 || len(f.exclude) > 0 {
-		host := hostOf(rawURL)
-		if len(f.include) > 0 && !matchesAnyDomain(host, f.include) {
-			return false
-		}
-		if len(f.exclude) > 0 && matchesAnyDomain(host, f.exclude) {
-			return false
-		}
-	}
-	if len(f.sites) > 0 && !matchesAnySite(rawURL, f.sites) {
-		return false
-	}
-	if f.dateActive {
-		if publishedAt.IsZero() {
-			return false
-		}
-		if !f.since.IsZero() && publishedAt.Before(f.since) {
-			return false
-		}
-		if !f.until.IsZero() && publishedAt.After(f.until) {
-			return false
-		}
-	}
-	return true
-}
-
-func parseSubQueries(raw, fallback string) []string {
-	raw = strings.TrimSpace(raw)
-	for _, fence := range []string{"```json", "```"} {
-		raw = strings.TrimPrefix(raw, fence)
-		raw = strings.TrimSuffix(raw, "```")
-	}
-	raw = strings.TrimSpace(raw)
-	startIdx := strings.Index(raw, "[")
-	endIdx := strings.LastIndex(raw, "]")
-	if startIdx >= 0 && endIdx > startIdx {
-		var arr []string
-		if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &arr); err == nil && len(arr) > 0 {
-			return arr
-		}
-	}
-	return []string{fallback}
-}
-
-func splitDomainsCSV(s string) []string {
-	if s == "" {
-		return nil
-	}
-	parts := strings.Split(s, ",")
-	out := make([]string, 0, len(parts))
-	for _, p := range parts {
-		t := strings.ToLower(strings.TrimSpace(p))
-		if t != "" {
-			out = append(out, t)
-		}
-	}
-	return out
-}
-
-func matchesAnyDomain(host string, patterns []string) bool {
-	host = strings.ToLower(host)
-	for _, p := range patterns {
-		if host == p || strings.HasSuffix(host, "."+p) {
-			return true
-		}
-	}
-	return false
-}
-
-func hostOf(rawURL string) string {
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return ""
-	}
-	return u.Hostname()
-}
-
-// siteScope is one entry of the `site` search filter: a host (suffix-matched
-// on dot boundaries, same as include_domains) plus an optional URL path
-// prefix. A zero path matches the whole host; a non-empty path scopes results
-// to a section of the site (e.g. host "pilotprotocol.network" + path "/docs").
-type siteScope struct {
-	host string
-	path string // normalized, no trailing slash; "" = any path
-}
-
-// parseSiteScopes parses the `site` parameter — a CSV of host or host/path
-// (or full-URL) entries — into scopes. Examples of a single entry:
-//
-//	pilotprotocol.network              → whole host (and subdomains)
-//	pilotprotocol.network/docs         → only URLs under /docs
-//	https://pilotprotocol.network/docs → same (scheme tolerated)
-func parseSiteScopes(csv string) []siteScope {
-	if csv == "" {
-		return nil
-	}
-	var out []siteScope
-	for _, raw := range strings.Split(csv, ",") {
-		t := strings.TrimSpace(raw)
-		if t == "" {
-			continue
-		}
-		t = strings.TrimPrefix(t, "https://")
-		t = strings.TrimPrefix(t, "http://")
-		host, path := t, ""
-		if i := strings.IndexByte(t, '/'); i >= 0 {
-			host, path = t[:i], t[i:]
-		}
-		host = strings.ToLower(strings.TrimSpace(host))
-		path = strings.TrimRight(strings.TrimSpace(path), "/")
-		if host == "" {
-			continue
-		}
-		out = append(out, siteScope{host: host, path: path})
-	}
-	return out
-}
-
-// matchesAnySite reports whether rawURL falls within any of the scopes. Host
-// matching reuses include_domains semantics (exact or dot-boundary suffix);
-// path matching is a segment-boundary prefix so "/docs" matches "/docs" and
-// "/docs/x" but not "/docsearch".
-func matchesAnySite(rawURL string, scopes []siteScope) bool {
-	if len(scopes) == 0 {
-		return true
-	}
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return false
-	}
-	host := strings.ToLower(u.Hostname())
-	p := u.Path
-	for _, sc := range scopes {
-		if host != sc.host && !strings.HasSuffix(host, "."+sc.host) {
-			continue
-		}
-		if sc.path == "" || p == sc.path || strings.HasPrefix(p, sc.path+"/") {
-			return true
-		}
-	}
-	return false
-}
-
-// parseDateBound accepts the same forms as the SQLite-side server: empty
-// (zero time), RFC3339 ("2026-01-15T00:00:00Z"), or a bare date
-// ("2026-01-15", treated as UTC midnight).
-func parseDateBound(s string) (time.Time, error) {
-	if s == "" {
-		return time.Time{}, nil
-	}
-	if t, err := time.Parse(time.RFC3339, s); err == nil {
-		return t, nil
-	}
-	if t, err := time.Parse("2006-01-02", s); err == nil {
-		return t, nil
-	}
-	return time.Time{}, fmt.Errorf("expected YYYY-MM-DD or RFC3339, got %q", s)
-}
-
-// textExcerpt returns a body-prefix excerpt no longer than maxBytes, cut at a
-// word boundary if one is available within the trailing 16 bytes.
-// Static prefix (not query-aware) — matches the SQLite-side DocMeta.Excerpt
-// shape; a query-aware passage extractor is the path on the index
-// hot path and lives there, not in the HTTP handler.
-func textExcerpt(text string, maxBytes int) string {
-	if len(text) <= maxBytes {
-		return text
-	}
-	cut := text[:maxBytes]
-	for i := len(cut) - 1; i >= len(cut)-16 && i > 0; i-- {
-		if cut[i] == ' ' || cut[i] == '\n' || cut[i] == '\t' {
-			cut = cut[:i]
-			break
-		}
-	}
-	return strings.TrimSpace(cut) + "…"
-}
-
-func (s *pebbleHTTP) handleContents(w http.ResponseWriter, r *http.Request) {
-	rawURL := r.URL.Query().Get("url")
-	if rawURL == "" {
-		writeProblem(w, http.StatusBadRequest, "missing url parameter")
-		return
-	}
-	decoded, err := url.QueryUnescape(rawURL)
-	if err != nil {
-		writeProblem(w, http.StatusBadRequest, "url parameter is not URL-encoded")
-		return
-	}
-	doc, err := s.store.GetDocByURL(r.Context(), decoded)
-	if errors.Is(err, store.ErrNotFound) {
-		writeProblem(w, http.StatusNotFound, "url not in index")
-		return
-	}
-	if err != nil {
-		writeProblem(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-	writeJSON(w, http.StatusOK, map[string]any{
-		"url":          doc.URL,
-		"title":        doc.Title,
-		"text":         doc.Text,
-		"author":       doc.Author,
-		"published_at": doc.PublishedAt,
-		"fetched_at":   doc.FetchedAt,
-	})
-}
-
-// POST /contents — batch URL → document. Up to 100 URLs in
-// one round-trip. URLs not in the index emit a {url, found:false, error}
-// stub in the same slot so the caller can correlate positionally. Wire
-// shape (results+took, per-item found+cached+lang) mirrors the SQLite-side
-// /contents POST so `cosift contents <url> <url>` works against either
-// backend without code changes.
-type contentsBatchReq struct {
-	URLs []string `json:"urls"`
-}
-
-type contentsBatchItem struct {
-	URL         string    `json:"url"`
-	Found       bool      `json:"found"`
-	Title       string    `json:"title,omitempty"`
-	Text        string    `json:"text,omitempty"`
-	Lang        string    `json:"lang,omitempty"`
-	Cached      bool      `json:"cached"`
-	Author      string    `json:"author,omitempty"`
-	PublishedAt time.Time `json:"published_at,omitempty"`
-	FetchedAt   time.Time `json:"fetched_at,omitempty"`
-	Error       string    `json:"error,omitempty"`
-}
-
-func (s *pebbleHTTP) handleContentsBatch(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	var req contentsBatchReq
-	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
-		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
-		return
-	}
-	if len(req.URLs) == 0 {
-		writeProblem(w, http.StatusBadRequest, "urls: must be a non-empty array")
-		return
-	}
-	if len(req.URLs) > 100 {
-		writeProblem(w, http.StatusBadRequest, "urls: at most 100 per request")
-		return
-	}
-	results := make([]contentsBatchItem, 0, len(req.URLs))
-	for _, u := range req.URLs {
-		item := contentsBatchItem{URL: u}
-		doc, err := s.store.GetDocByURL(r.Context(), u)
-		switch {
-		case errors.Is(err, store.ErrNotFound):
-			item.Error = "not in index"
-		case err != nil:
-			item.Error = err.Error()
-		case doc == nil:
-			item.Error = "not in index"
-		default:
-			item.Found = true
-			item.Cached = true
-			item.Title = doc.Title
-			item.Text = doc.Text
-			item.Lang = doc.Lang
-			item.Author = doc.Author
-			item.PublishedAt = doc.PublishedAt
-			item.FetchedAt = doc.FetchedAt
-		}
-		results = append(results, item)
-	}
-	writeJSON(w, http.StatusOK, map[string]any{
-		"results": results,
-		"took":    time.Since(start).String(),
-	})
-}
-
-// runPebbleInfo prints Pebble's built-in metrics for the store at -dir.
-// openPebbleOrFriendlyErr wraps store.OpenPebble with a more
-// helpful error message when the underlying failure is lock contention (a
-// pebble-serve / live crawl is holding the single-writer lock). Callers
-// using offline CLI subcommands (pebble-info, verify) hit this when run
-// during an active deployment.
-func openPebbleOrFriendlyErr(d string) (*store.PebbleStore, error) {
-	ps, err := store.OpenPebble(d)
-	if err != nil {
-		msg := err.Error()
-		if strings.Contains(msg, "lock") || strings.Contains(msg, "resource temporarily unavailable") {
-			return nil, fmt.Errorf("open pebble at %s: writer lock is held by another process (pebble-serve / crawl in flight); stop the running service first", d)
-		}
-		return nil, fmt.Errorf("open pebble at %s: %w", d, err)
-	}
-	return ps, nil
-}
-
-// runVerifyViaServer GETs pebble-serve's /verify endpoint and
-// renders the JSON body in the same shape runVerifyPebble's local path emits.
-// Lets `cosift verify -server URL` work against a running pebble-serve while
-// the writer lock is held by the crawl / serve process.
-func runVerifyViaServer(ctx context.Context, serverURL string, asJSON bool) error {
-	endpoint := strings.TrimRight(serverURL, "/") + "/verify"
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
-	if err != nil {
-		return err
-	}
-	client := &http.Client{Timeout: 30 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("GET %s: %w", endpoint, err)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if asJSON {
-		fmt.Println(string(body))
-	} else {
-		var d struct {
-			OK                 bool  `json:"ok"`
-			IndexedDocsCounter int64 `json:"indexed_docs_counter"`
-			IndexedDocsScan    int64 `json:"indexed_docs_scan"`
-			IndexedDocsDrift   int64 `json:"indexed_docs_drift"`
-			SumDocLenCounter   int64 `json:"sum_doc_len_counter"`
-			SumDocLenScan      int64 `json:"sum_doc_len_scan"`
-			SumDocLenDrift     int64 `json:"sum_doc_len_drift"`
-		}
-		if err := json.Unmarshal(body, &d); err != nil {
-			return fmt.Errorf("decode %s: %w", endpoint, err)
-		}
-		fmt.Printf("pebble-serve: %s\n\n", serverURL)
-		fmt.Printf("  indexed_docs (counter): %d\n", d.IndexedDocsCounter)
-		fmt.Printf("  indexed_docs (scan):    %d\n", d.IndexedDocsScan)
-		fmt.Printf("  sum_doc_len  (counter): %d\n", d.SumDocLenCounter)
-		fmt.Printf("  sum_doc_len  (scan):    %d\n", d.SumDocLenScan)
-		if d.OK {
-			fmt.Println("\nOK: counters match the 'l' family scan.")
-		} else {
-			fmt.Printf("\nDRIFT: indexed_docs Δ=%+d, sum_doc_len Δ=%+d\n", d.IndexedDocsDrift, d.SumDocLenDrift)
-		}
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("counter drift detected (server returned %d)", resp.StatusCode)
-	}
-	return nil
-}
-
-// pebbleInfoJSON shapes the -json output of `cosift pebble-info`. Extracted
-// in so the shape can be unit-tested without spawning a subprocess.
-// Mirrors the /stats response — same jq filters compose against either source.
-func pebbleInfoJSON(path string, st store.Stats, sumLen, indexedDocs int64, meta index.HNSWMeta, hnswOK bool) map[string]any {
-	retrievers := []string{"bm25", "bm25-mlt"}
-	if hnswOK {
-		retrievers = append(retrievers, "dense", "hybrid")
-	}
-	out := map[string]any{
-		"path":         path,
-		"documents":    st.Documents,
-		"indexed_docs": indexedDocs,
-		"sum_doc_len":  sumLen,
-		"hnsw_loaded":  hnswOK,
-		"retrievers":   retrievers,
-	}
-	if indexedDocs > 0 {
-		out["avg_doc_len"] = float64(sumLen) / float64(indexedDocs)
-	}
-	if hnswOK {
-		out["vector_nodes"] = meta.NodeCount
-		out["vector_dim"] = meta.Dim
-	}
-	return out
-}
-
-// operator visibility into LSM levels, WAL state, on-disk
-// size, and compaction queue, surfaced via pebble.Metrics().String().
-func runPebbleInfo(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("pebble-info", flag.ExitOnError)
-	dir := fs.String("dir", "", "PebbleStore directory (defaults to <cfg.DataDir>/pebble)")
-	// Mirrors
-	// the shape /stats returns ( retrievers list,'s
-	// vector meta) so the same jq filters compose against either source.
-	asJSON := fs.Bool("json", false, "emit JSON instead of human-readable text (no pebble.Metrics dump)")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	d := *dir
-	if d == "" {
-		d = filepath.Join(cfg.DataDir, "pebble")
-	}
-	ps, err := openPebbleOrFriendlyErr(d)
-	if err != nil {
-		return err
-	}
-	defer ps.Close()
-
-	st, err := ps.Stats(ctx)
-	if err != nil {
-		return err
-	}
-	sumLen, indexedDocs, _ := ps.CorpusStats(ctx)
-	meta, hnswOK, _ := index.LoadHNSWMeta(ctx, ps)
-
-	if *asJSON {
-		out := pebbleInfoJSON(d, st, sumLen, indexedDocs, meta, hnswOK)
-		blob, _ := json.MarshalIndent(out, "", "  ")
-		fmt.Println(string(blob))
-		return nil
-	}
-
-	fmt.Printf("PebbleStore: %s\n\n", d)
-	fmt.Printf("  documents:    %d\n", st.Documents)
-	// pebble-info already needs
-	// to read the store; reading them costs O(1) and gives operators 'is this
-	// store healthy' without round-tripping through pebble-serve /stats.
-	if indexedDocs > 0 {
-		fmt.Printf("  indexed_docs: %d\n", indexedDocs)
-		fmt.Printf("  sum_doc_len:  %d\n", sumLen)
-		fmt.Printf("  avg_doc_len:  %.2f\n", float64(sumLen)/float64(indexedDocs))
-	}
-	// when an HNSW meta blob is persisted, surface dim+nodes here
-	// too. Same cheap 20-byte read pebble-serve does at startup.
-	// Operators running pebble-info to inspect an offline store get the dense
-	// shape without opening pebble-serve.
-	// bm25/bm25-mlt
-	// always. dense/hybrid need the graph (meta present) AND, for non-URL
-	// modes, a runtime embedder — offline we can only check the persisted
-	// shape, so we report 'available with embedder' rather than asserting.
-	if hnswOK {
-		fmt.Printf("  vector_nodes: %d\n", meta.NodeCount)
-		fmt.Printf("  vector_dim:   %d\n", meta.Dim)
-	}
-	fmt.Println()
-	fmt.Println("--- retrievers ---")
-	fmt.Println("  bm25, bm25-mlt: always (BM25 index)")
-	if hnswOK {
-		fmt.Println("  dense:          available — /find_similar?url= works without embedder;")
-		fmt.Println("                  /search /answer /research need an embedder configured at server start")
-		fmt.Println("  hybrid:         available (same requirements as dense)")
-	} else {
-		fmt.Println("  dense, hybrid:  unavailable — no HNSW graph persisted in this store")
-		fmt.Println("                  (build embeddings during crawl with cfg.Embeddings.Model set)")
-	}
-	fmt.Println()
-	fmt.Println("--- pebble.Metrics ---")
-	fmt.Println(ps.Metrics().String())
-	return nil
-}
-
-// verify the running counters ('m'+indexed_docs,
-// 'm'+sum_doc_len) against an authoritative scan of the 'l' family.
-// Drift would mean a crash or bug left the counter inconsistent —
-// flagged here so it can be re-derived from the scan.
-func runVerifyPebble(ctx context.Context, cfg *config.Config, args []string) error {
-	fs := flag.NewFlagSet("verify", flag.ExitOnError)
-	dir := fs.String("dir", "", "PebbleStore directory (defaults to <cfg.DataDir>/pebble)")
-	// machine-readable output for CI integration.
-	asJSON := fs.Bool("json", false, "emit JSON report instead of human text (suitable for jq / CI)")
-	// -server URL routes the check through a running pebble-serve's
-	// /verify endpoint instead of opening the store directly. Useful when a
-	// crawl or pebble-serve is holding the writer lock.
-	serverURL := fs.String("server", "", "pebble-serve URL — when set, GETs /verify instead of opening the store")
-	if err := fs.Parse(args); err != nil {
-		return err
-	}
-	if *serverURL != "" {
-		return runVerifyViaServer(ctx, *serverURL, *asJSON)
-	}
-	d := *dir
-	if d == "" {
-		d = filepath.Join(cfg.DataDir, "pebble")
-	}
-	ps, err := openPebbleOrFriendlyErr(d)
-	if err != nil {
-		return err
-	}
-	defer ps.Close()
-
-	counterSum, counterCount, err := ps.CorpusStats(ctx)
-	if err != nil {
-		return fmt.Errorf("read counters: %w", err)
-	}
-	scanSum, scanCount, err := ps.SumDocLengths(ctx)
-	if err != nil {
-		return fmt.Errorf("scan 'l' family: %w", err)
-	}
-
-	driftCount := counterCount - scanCount
-	driftSum := counterSum - scanSum
-	ok := driftCount == 0 && driftSum == 0
-
-	if *asJSON {
-		out := map[string]any{
-			"path":                 d,
-			"ok":                   ok,
-			"indexed_docs_counter": counterCount,
-			"indexed_docs_scan":    scanCount,
-			"indexed_docs_drift":   driftCount,
-			"sum_doc_len_counter":  counterSum,
-			"sum_doc_len_scan":     scanSum,
-			"sum_doc_len_drift":    driftSum,
-		}
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "  ")
-		_ = enc.Encode(out)
-		if !ok {
-			return fmt.Errorf("counter drift detected")
-		}
-		return nil
-	}
-
-	fmt.Printf("PebbleStore: %s\n\n", d)
-	fmt.Printf("  indexed_docs (counter): %d\n", counterCount)
-	fmt.Printf("  indexed_docs (scan):    %d\n", scanCount)
-	fmt.Printf("  sum_doc_len  (counter): %d\n", counterSum)
-	fmt.Printf("  sum_doc_len  (scan):    %d\n", scanSum)
-	if ok {
-		fmt.Println("\nOK: counters match the 'l' family scan.")
-		return nil
-	}
-	fmt.Printf("\nDRIFT: indexed_docs Δ=%+d, sum_doc_len Δ=%+d\n", driftCount, driftSum)
-	return fmt.Errorf("counter drift detected")
-}
-
-func writeJSON(w http.ResponseWriter, status int, body any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(status)
-	_ = json.NewEncoder(w).Encode(body)
-}
-
-func writeProblem(w http.ResponseWriter, status int, detail string) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(status)
-	_ = json.NewEncoder(w).Encode(map[string]any{
-		"error":  http.StatusText(status),
-		"status": status,
-		"detail": detail,
-	})
-}
diff --git a/cmd/cosift/querylog.go b/cmd/cosift/querylog.go
new file mode 100644
index 0000000..3f3c985
--- /dev/null
+++ b/cmd/cosift/querylog.go
@@ -0,0 +1,160 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Query logging — the observability substrate cosift lacked. Until now nothing
+// recorded what users ask, what returns empty, or how slow queries are, so
+// "is it useful?" was unanswerable except by eyeballing. qlog appends one JSON
+// line per query-endpoint request; /admin/query-log tails it.
+//
+// Privacy: this records query text for a public search service. Keep retention
+// conservative (rotate/truncate the file); don't hoard raw queries indefinitely.
+
+type queryLogRec struct {
+	Qid    string `json:"qid"` // stable id; returned to caller via X-Cosift-Query-Id so feedback can reference it
+	TS     string `json:"ts"`
+	Ep     string `json:"ep"`               // path, e.g. /search
+	Q      string `json:"q"`                // the q= param (empty for POST-body queries)
+	Status int    `json:"status"`           // HTTP status
+	MS     int64  `json:"ms"`               // server-side latency
+	Bytes  int64  `json:"bytes"`            // response body bytes (empty-result proxy)
+	Caller string `json:"caller,omitempty"` // X-Forwarded-For or RemoteAddr — separates self/test from organic
+}
+
+// newQueryID returns a short random hex id for correlating a query with later
+// feedback. crypto/rand so concurrent requests don't collide.
+func newQueryID() string {
+	var b [12]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		// Fallback: time-based (collision-tolerant; feedback is weak signal).
+		return "q" + strconv.FormatInt(time.Now().UnixNano(), 36)
+	}
+	return hex.EncodeToString(b[:])
+}
+
+// qlog wraps a query handler to append a structured log line after it runs.
+// No-op when COSIFT_QUERY_LOG is unset (s.qlogFile == nil).
+func (s *pebbleHTTP) qlog(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if s.qlogFile == nil {
+			h(w, r)
+			return
+		}
+		start := time.Now()
+		qid := newQueryID()
+		sw := &statusCapturingWriter{ResponseWriter: w, status: 200}
+		// Set the id header BEFORE the handler writes — the adapter surfaces it
+		// to the caller so feedback can reference this exact answer.
+		sw.Header().Set("X-Cosift-Query-Id", qid)
+		h(sw, r)
+		caller := r.Header.Get("X-Forwarded-For")
+		if caller == "" {
+			caller = r.RemoteAddr
+		}
+		s.writeQueryLog(queryLogRec{
+			Qid:    qid,
+			TS:     time.Now().UTC().Format(time.RFC3339),
+			Ep:     r.URL.Path,
+			Q:      r.URL.Query().Get("q"),
+			Status: sw.status,
+			MS:     time.Since(start).Milliseconds(),
+			Bytes:  sw.bytes,
+			Caller: caller,
+		})
+	}
+}
+
+func (s *pebbleHTTP) writeQueryLog(rec queryLogRec) {
+	b, err := json.Marshal(rec)
+	if err != nil {
+		return
+	}
+	s.qlogMu.Lock()
+	defer s.qlogMu.Unlock()
+	_, _ = s.qlogFile.Write(append(b, '\n'))
+}
+
+// handleQueryLog tails the query log. ?n=N (default 100, max 5000) returns the
+// last N JSON lines as a JSON array; ?raw=1 returns them as raw JSONL.
+func (s *pebbleHTTP) handleQueryLog(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := r.Header.Get("Authorization")
+		if got != "Bearer "+want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	path := os.Getenv("COSIFT_QUERY_LOG")
+	if path == "" {
+		writeProblem(w, http.StatusNotImplemented, "query log disabled (COSIFT_QUERY_LOG unset)")
+		return
+	}
+	n := 100
+	if v := r.URL.Query().Get("n"); v != "" {
+		if k, err := strconv.Atoi(v); err == nil && k > 0 && k <= 5000 {
+			n = k
+		}
+	}
+	lines := tailLines(path, n)
+	if r.URL.Query().Get("raw") == "1" {
+		w.Header().Set("Content-Type", "application/x-ndjson")
+		for _, ln := range lines {
+			_, _ = w.Write(append([]byte(ln), '\n'))
+		}
+		return
+	}
+	recs := make([]json.RawMessage, 0, len(lines))
+	for _, ln := range lines {
+		recs = append(recs, json.RawMessage(ln))
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"count": len(recs), "entries": recs})
+}
+
+// tailLines returns the last n newline-delimited lines of a file. Reads at most
+// the trailing window so a large log doesn't blow memory.
+func tailLines(path string, n int) []string {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil {
+		return nil
+	}
+	const window = 2 << 20 // 2 MB tail window
+	size := st.Size()
+	start := int64(0)
+	if size > window {
+		start = size - window
+	}
+	buf := make([]byte, size-start)
+	if _, err := f.ReadAt(buf, start); err != nil && err.Error() != "EOF" {
+		return nil
+	}
+	parts := strings.Split(string(buf), "\n")
+	// If we started mid-file, the first element is a partial line — drop it.
+	if start > 0 && len(parts) > 0 {
+		parts = parts[1:]
+	}
+	// Drop empty trailing element (file ends in newline) + any blanks.
+	lines := parts[:0]
+	for _, p := range parts {
+		if p != "" {
+			lines = append(lines, p)
+		}
+	}
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return lines
+}
diff --git a/cmd/cosift/querylog_test.go b/cmd/cosift/querylog_test.go
new file mode 100644
index 0000000..689b0dd
--- /dev/null
+++ b/cmd/cosift/querylog_test.go
@@ -0,0 +1,49 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestTailLines(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "q.jsonl")
+	var sb strings.Builder
+	for i := 0; i < 500; i++ {
+		sb.WriteString("line")
+		sb.WriteByte(byte('0' + i%10))
+		sb.WriteByte('\n')
+	}
+	if err := os.WriteFile(p, []byte(sb.String()), 0644); err != nil {
+		t.Fatal(err)
+	}
+	got := tailLines(p, 3)
+	if len(got) != 3 {
+		t.Fatalf("want 3 lines, got %d", len(got))
+	}
+	// Last three lines are i=497,498,499 -> line7,line8,line9
+	want := []string{"line7", "line8", "line9"}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("line %d: got %q want %q", i, got[i], want[i])
+		}
+	}
+	// n larger than file → all lines, no partials/blanks.
+	all := tailLines(p, 10000)
+	if len(all) != 500 {
+		t.Errorf("want 500 lines, got %d", len(all))
+	}
+	for _, l := range all {
+		if l == "" {
+			t.Error("blank line leaked into output")
+		}
+	}
+}
+
+func TestTailLinesMissingFile(t *testing.T) {
+	if got := tailLines("/nonexistent/path/q.jsonl", 5); got != nil {
+		t.Errorf("missing file should return nil, got %v", got)
+	}
+}
diff --git a/cmd/cosift/serve_admin.go b/cmd/cosift/serve_admin.go
new file mode 100644
index 0000000..3eabfa8
--- /dev/null
+++ b/cmd/cosift/serve_admin.go
@@ -0,0 +1,623 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// handlePQEncode backfills PQ codes for every node currently missing one,
+// against the codebook already loaded on this serve. Order-of-magnitude
+// faster than handlePQTrain because it skips k-means; just encode loop.
+func (s *pebbleHTTP) handlePQEncode(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.hnsw == nil {
+		writeProblem(w, http.StatusBadRequest, "pq-encode: no HNSW loaded")
+		return
+	}
+	if !s.hnsw.HasPQ() {
+		writeProblem(w, http.StatusBadRequest, "pq-encode: no codebook loaded — run /admin/pq-train first")
+		return
+	}
+	t0 := time.Now()
+	ids, codes, err := s.hnsw.EncodeMissing()
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, "encode: "+err.Error())
+		return
+	}
+	encodeElapsed := time.Since(t0)
+	if len(ids) == 0 {
+		writeJSON(w, http.StatusOK, map[string]any{
+			"encoded":       0,
+			"total":         s.hnsw.Len(),
+			"already_full":  true,
+			"total_elapsed": encodeElapsed.String(),
+		})
+		return
+	}
+	// Persist freshly-encoded codes in a single batch.
+	pq := s.hnsw.PQStatus()
+	cb := &index.PQCodebook{Dim: pq.Dim, M: pq.M, K: pq.K, SubDim: pq.Dim / pq.M}
+	entries := make([]store.PQCodeEntry, len(ids))
+	for i := range ids {
+		entries[i] = store.PQCodeEntry{ID: ids[i], Blob: cb.EncodeCodeBlob(codes[i])}
+	}
+	persistT0 := time.Now()
+	if err := s.store.PutPQCodesBatch(r.Context(), entries); err != nil {
+		writeProblem(w, http.StatusInternalServerError, "persist codes: "+err.Error())
+		return
+	}
+	persistElapsed := time.Since(persistT0)
+	total := s.hnsw.Len()
+	coverage := 100.0 * float64(pq.NodesWithCode+len(ids)) / float64(total)
+	totalElapsed := time.Since(t0)
+	log.Printf("pq-encode: backfilled %d codes in %s (persist %s, total %s) → coverage %.1f%%",
+		len(ids), encodeElapsed-persistElapsed, persistElapsed, totalElapsed, coverage)
+	writeJSON(w, http.StatusOK, map[string]any{
+		"encoded":         len(ids),
+		"total":           total,
+		"coverage_pct":    coverage,
+		"encode_elapsed":  (encodeElapsed - persistElapsed).String(),
+		"persist_elapsed": persistElapsed.String(),
+		"total_elapsed":   totalElapsed.String(),
+	})
+}
+
+// handleCheckpoint creates a Pebble checkpoint (hard-linked, point-in-time)
+// at a server-chosen path under COSIFT_CHECKPOINT_DIR (default /tmp). The
+// returned path is safe to tar — Pebble's compactor cannot mutate hard-linked
+// SSTs. Caller is responsible for deleting the dir after consuming it.
+func (s *pebbleHTTP) handleCheckpoint(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	base := os.Getenv("COSIFT_CHECKPOINT_DIR")
+	if base == "" {
+		base = "/tmp"
+	}
+	dest := filepath.Join(base, fmt.Sprintf("cosift-ckpt-%d", time.Now().UnixNano()))
+	t0 := time.Now()
+	if err := s.store.Checkpoint(dest); err != nil {
+		writeProblem(w, http.StatusInternalServerError, "checkpoint: "+err.Error())
+		return
+	}
+	elapsed := time.Since(t0)
+	log.Printf("checkpoint: created %q in %s", dest, elapsed)
+	writeJSON(w, http.StatusOK, map[string]any{
+		"path":    dest,
+		"elapsed": elapsed.String(),
+	})
+}
+
+// handlePQTrain trains a PQ codebook on a sample of the current HNSW
+// graph, then encodes ALL nodes and persists codes + codebook to Pebble.
+// Synchronous; returns when training completes.
+type pqTrainReq struct {
+	SampleSize int `json:"sample_size"` // default 50000
+	M          int `json:"m"`           // default dim/8 (subspaces of size 8)
+	K          int `json:"k"`           // default 256
+	Iters      int `json:"iters"`       // default 15
+	Parallel   int `json:"parallel"`    // default GOMAXPROCS
+}
+
+func (s *pebbleHTTP) handlePQTrain(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.hnsw == nil || s.hnsw.Len() == 0 {
+		writeProblem(w, http.StatusBadRequest, "pq-train requires an in-memory HNSW with nodes")
+		return
+	}
+	var req pqTrainReq
+	if r.ContentLength > 0 {
+		body, _ := io.ReadAll(io.LimitReader(r.Body, 4<<10))
+		_ = json.Unmarshal(body, &req)
+	}
+	if req.SampleSize <= 0 {
+		req.SampleSize = 50000
+	}
+	if req.K <= 0 {
+		req.K = 256
+	}
+	if req.Iters <= 0 {
+		req.Iters = 15
+	}
+	if req.Parallel <= 0 {
+		req.Parallel = 8
+	}
+	if s.embedder == nil {
+		writeProblem(w, http.StatusBadRequest, "pq-train needs an embedder to know the vector dim")
+		return
+	}
+	dim := s.embedder.Dim()
+	M := req.M
+	if M <= 0 {
+		// Default: 1 subspace per 8 dims. For 768d → 96 subspaces.
+		M = dim / 8
+		if M <= 0 {
+			M = 1
+		}
+	}
+	if dim%M != 0 {
+		writeProblem(w, http.StatusBadRequest, fmt.Sprintf("dim %d not divisible by M %d", dim, M))
+		return
+	}
+	t0 := time.Now()
+	log.Printf("pq-train: sampling %d / %d vectors (dim=%d, M=%d, K=%d, iters=%d, parallel=%d)",
+		req.SampleSize, s.hnsw.Len(), dim, M, req.K, req.Iters, req.Parallel)
+	sample := s.hnsw.SampleVectors(req.SampleSize, time.Now().UnixNano())
+	sampledN := len(sample)
+	log.Printf("pq-train: training codebook on %d samples...", sampledN)
+	cb, err := index.TrainPQCodebookParallel(sample, dim, M, req.K, req.Iters, req.Parallel, time.Now().UnixNano())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, "train: "+err.Error())
+		return
+	}
+	trainElapsed := time.Since(t0)
+	log.Printf("pq-train: codebook trained in %s; persisting...", trainElapsed)
+	if err := cb.Persist(r.Context(), s.store); err != nil {
+		writeProblem(w, http.StatusInternalServerError, "persist codebook: "+err.Error())
+		return
+	}
+	encodeT0 := time.Now()
+	log.Printf("pq-train: encoding %d nodes...", s.hnsw.Len())
+	ids, codes, err := s.hnsw.EncodeAll(cb)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, "encode: "+err.Error())
+		return
+	}
+	entries := make([]store.PQCodeEntry, len(ids))
+	for i := range ids {
+		// K≤256 → byte-packed for 2x disk savings.
+		entries[i] = store.PQCodeEntry{ID: ids[i], Blob: cb.EncodeCodeBlob(codes[i])}
+	}
+	if err := s.store.PutPQCodesBatch(r.Context(), entries); err != nil {
+		writeProblem(w, http.StatusInternalServerError, "put codes: "+err.Error())
+		return
+	}
+	encodeElapsed := time.Since(encodeT0)
+	totalElapsed := time.Since(t0)
+	log.Printf("pq-train: done in %s (train=%s, encode+persist=%s, %d codes written)",
+		totalElapsed, trainElapsed, encodeElapsed, len(ids))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"sample_size":    sampledN,
+		"dim":            dim,
+		"M":              M,
+		"K":              req.K,
+		"iters":          req.Iters,
+		"nodes_encoded":  len(ids),
+		"train_elapsed":  trainElapsed.String(),
+		"encode_elapsed": encodeElapsed.String(),
+		"total_elapsed":  totalElapsed.String(),
+	})
+}
+
+// evalQuickQueries is the canned smoke-eval set. Chosen for breadth:
+// 3 definition + 3 technical-howto + 2 comparison + 2 findability. Tracks
+// the same answered/no-info/error verdicts the external 60-Q harness uses,
+// so a regression here predicts a regression in the bigger eval. Hardcoded
+// (no new deps) so any operator can hit /admin/eval-quick and get an
+// answer rate without setting up Python or external test infra.
+var evalQuickQueries = []string{
+	"what is BM25 ranking function",
+	"what is HNSW algorithm",
+	"what is reciprocal rank fusion",
+	"how does retrieval augmented generation work",
+	"how does Go goroutines scheduling work",
+	"explain CAP theorem in distributed systems",
+	"compare BM25 vs dense vector retrieval",
+	"compare REST vs GraphQL APIs",
+	"what is Pilot Protocol overlay network",
+	"what is sentence embedding",
+}
+
+// handleEvalQuick runs the 10-query smoke eval against the running cosift
+// and returns answered-rate + per-query verdicts. Each query hits /answer
+// in-process (re-uses the same chat/retrieval stack as a real /answer call),
+// so the rate that comes back matches what users see.
+func (s *pebbleHTTP) handleEvalQuick(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented, "eval-quick requires cfg.Chat.Model")
+		return
+	}
+	// the server-wide WriteTimeout (60s) was killing this handler
+	// before 10 sequential /answer calls finished — connection closed mid-write,
+	// curl saw "empty reply from server" (exit 52). Disable the deadline for
+	// this long-running admin endpoint via ResponseController. Pairs with
+	// bounded-parallel dispatch below so the 10-query batch finishes in
+	// ~one chat-LLM round-trip instead of ten.
+	if rc := http.NewResponseController(w); rc != nil {
+		_ = rc.SetWriteDeadline(time.Time{})
+	}
+	type queryResult struct {
+		Query     string `json:"query"`
+		Verdict   string `json:"verdict"` // answered | no_info | empty | error
+		LatencyMs int    `json:"latency_ms"`
+		Sources   int    `json:"sources"`
+		Suggest   string `json:"suggest_escalation,omitempty"`
+	}
+	results := make([]queryResult, len(evalQuickQueries))
+	t0 := time.Now()
+
+	// Bounded-parallel dispatch. vLLM batches concurrent requests so 4-wide
+	// fan-out finishes in roughly one chat round-trip rather than 10.
+	sem := make(chan struct{}, 4)
+	var wg sync.WaitGroup
+	for i, q := range evalQuickQueries {
+		wg.Add(1)
+		go func(i int, q string) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			qStart := time.Now()
+			subReq, _ := http.NewRequestWithContext(r.Context(), http.MethodGet,
+				"/answer?q="+url.QueryEscape(q)+"&stream=false&retriever=hybrid", nil)
+			rec := newResponseRecorder()
+			s.handleAnswer(rec, subReq)
+			latencyMs := int(time.Since(qStart) / time.Millisecond)
+
+			var ar answerResponse
+			if jerr := json.Unmarshal(rec.body.Bytes(), &ar); jerr != nil || rec.code >= 400 {
+				results[i] = queryResult{Query: q, Verdict: "error", LatencyMs: latencyMs}
+				return
+			}
+			var verdict string
+			switch {
+			case len(ar.Answer) < 30:
+				verdict = "empty"
+			case answerLooksLikeNoInfo(ar.Answer):
+				verdict = "no_info"
+			default:
+				verdict = "answered"
+			}
+			results[i] = queryResult{
+				Query:     q,
+				Verdict:   verdict,
+				LatencyMs: latencyMs,
+				Sources:   len(ar.Sources),
+				Suggest:   ar.SuggestEscalation,
+			}
+		}(i, q)
+	}
+	wg.Wait()
+
+	answered, noInfo, errCount, empty, totalMs := 0, 0, 0, 0, 0
+	for _, r := range results {
+		totalMs += r.LatencyMs
+		switch r.Verdict {
+		case "answered":
+			answered++
+		case "no_info":
+			noInfo++
+		case "empty":
+			empty++
+		default:
+			errCount++
+		}
+	}
+	n := len(evalQuickQueries)
+	writeJSON(w, http.StatusOK, map[string]any{
+		"queries":         n,
+		"answered":        answered,
+		"no_info":         noInfo,
+		"empty":           empty,
+		"errors":          errCount,
+		"answer_rate_pct": 100 * answered / n,
+		"avg_latency_ms":  totalMs / n,
+		"total_elapsed":   time.Since(t0).String(),
+		"chat_model":      s.chat.Model(),
+		"results":         results,
+	})
+}
+
+// handleHNSWCompact runs HNSW.Compact() in-place, then clears the persisted
+// 'v' family and writes a fresh full snapshot so disk matches the compacted
+// in-memory graph. Cheaper than the offline hnsw-rebuild subcommand: Compact
+// keeps the existing topology among surviving nodes (O(N + edges)), whereas
+// Rebuild re-inserts every node via HNSW search (multiple minutes per million
+// passages). Operators run this when stats.zombie_nodes climbs above ~30% of
+// nodes_total.
+//
+// Synchronous; holds the HNSW write lock during the compact step and the
+// read lock during the persist step. Dense retrieval and AddPassage calls
+// queue for the duration. The server-wide WriteTimeout is disabled here via
+// ResponseController because compacting a multi-million-node graph routinely
+// runs past 60s. Returns counters so operators can confirm progress.
+func (s *pebbleHTTP) handleHNSWCompact(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.hnsw == nil {
+		writeProblem(w, http.StatusNotImplemented, "hnsw-compact requires a loaded HNSW graph")
+		return
+	}
+	if rc := http.NewResponseController(w); rc != nil {
+		_ = rc.SetWriteDeadline(time.Time{})
+	}
+	skipPersist := r.URL.Query().Get("skip_persist") == "1"
+
+	before := s.hnsw.Len()
+	t0 := time.Now()
+	removed := s.hnsw.Compact()
+	compactDur := time.Since(t0)
+	after := s.hnsw.Len()
+
+	resp := map[string]any{
+		"nodes_before": before,
+		"nodes_after":  after,
+		"removed":      removed,
+		"compact_ms":   compactDur.Milliseconds(),
+		"persisted":    false,
+	}
+
+	if removed == 0 || skipPersist {
+		writeJSON(w, http.StatusOK, resp)
+		return
+	}
+
+	// Compact remapped node indices; the persisted 'v' family now points at
+	// stale slots. Wipe and full-rewrite. PQ codes follow node indices too,
+	// so clear 'q' as well — operators must re-run /admin/pq-train if PQ was
+	// in use.
+	persistT0 := time.Now()
+	ctx := r.Context()
+	if err := s.store.ClearVectorFamily(ctx); err != nil {
+		resp["persist_error"] = "clear vector family: " + err.Error()
+		writeJSON(w, http.StatusInternalServerError, resp)
+		return
+	}
+	if err := s.store.ClearPQFamily(ctx); err != nil {
+		resp["persist_error"] = "clear pq family: " + err.Error()
+		writeJSON(w, http.StatusInternalServerError, resp)
+		return
+	}
+	if err := s.hnsw.Persist(ctx, s.store); err != nil {
+		resp["persist_error"] = "persist: " + err.Error()
+		writeJSON(w, http.StatusInternalServerError, resp)
+		return
+	}
+	resp["persisted"] = true
+	resp["persist_ms"] = time.Since(persistT0).Milliseconds()
+	log.Printf("hnsw-compact: removed=%d (%.1f%% zombies) compact=%s persist=%s nodes %d→%d",
+		removed, 100*float64(removed)/float64(before),
+		compactDur.Round(time.Millisecond), time.Since(persistT0).Round(time.Millisecond),
+		before, after)
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// responseRecorder captures an http.Handler's output for in-process
+// dispatch. Minimal — only what handleEvalQuick needs.
+type responseRecorder struct {
+	hdr  http.Header
+	body bytes.Buffer
+	code int
+}
+
+func newResponseRecorder() *responseRecorder {
+	return &responseRecorder{hdr: make(http.Header), code: 200}
+}
+
+func (r *responseRecorder) Header() http.Header { return r.hdr }
+
+func (r *responseRecorder) Write(b []byte) (int, error) { return r.body.Write(b) }
+
+func (r *responseRecorder) WriteHeader(c int) { r.code = c }
+
+// handleEmbedBackfill walks famDocMeta, finds docs whose URL has zero
+// HNSW vector nodes (lexical-only ingest from WET imports, or any docs
+// indexed before an embedder was wired), and embeds them. Pairs with
+// /admin/wet-import?lexical_only=true so bulk-loaded content gets dense
+// retrieval after the fast lexical pass.
+//
+// Body: {"limit": 10000, "workers": 4}
+//   - limit:   cap on docs processed this call (0 = unlimited)
+//   - workers: concurrent embed pipelines (default 4)
+//
+// Idempotent against concurrent re-runs — re-embedding a doc that
+// already has vectors is a no-op because UpsertPassage just adds more
+// nodes pointing at the same URL (which the dedup-by-URL in
+// HNSW.Search picks the best of anyway). Cleaner: zombie-reclaim kicks
+// in to invalidate prior, then fresh vectors take over.
+type embedBackfillReq struct {
+	Limit   int `json:"limit,omitempty"`
+	Workers int `json:"workers,omitempty"`
+}
+
+// handleHostBackfill triggers the 'P' host-partition backfill IN-PROCESS, in a
+// detached background goroutine (Pebble is single-writer, so this can't run as
+// a separate process against a live store). Returns immediately; progress is
+// logged. ?host=<host> runs the fast targeted path; omit for the full corpus.
+//
+// Run this with the COSIFT_HOST_PARTITION write flag OFF so the backfill is the
+// sole writer of the 'T'/'next_host_id' families (no race with crawl indexing).
+// After it logs "done", set COSIFT_HOST_PARTITION_READ=1 (and =1 for the write
+// flag to keep it fresh) and restart.
+func (s *pebbleHTTP) handleHostBackfill(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.store == nil {
+		writeProblem(w, http.StatusNotImplemented, "host-backfill requires a PebbleStore")
+		return
+	}
+	host := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("host")))
+	// Detached context so the long backfill survives the request returning.
+	go func() {
+		t0 := time.Now()
+		scope := "full corpus"
+		if host != "" {
+			scope = "host=" + host
+		}
+		log.Printf("host-backfill: starting (%s)", scope)
+		written, err := s.store.BackfillHostPostings(context.Background(), host, func(n int64) {
+			log.Printf("host-backfill: %d postings written (%s elapsed)", n, time.Since(t0).Round(time.Second))
+		})
+		if err != nil {
+			log.Printf("host-backfill: FAILED after %d postings: %v", written, err)
+			return
+		}
+		log.Printf("host-backfill: DONE — %d postings in %s. Set COSIFT_HOST_PARTITION_READ=1 and restart.",
+			written, time.Since(t0).Round(time.Second))
+	}()
+	writeJSON(w, http.StatusAccepted, map[string]any{
+		"status": "started", "scope": host, "note": "running in background; watch server logs for progress + 'DONE'",
+	})
+}
+
+func (s *pebbleHTTP) handleEmbedBackfill(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.embedder == nil || s.hnsw == nil {
+		writeProblem(w, http.StatusNotImplemented, "embed backfill requires both an embedder and HNSW graph to be configured")
+		return
+	}
+	var req embedBackfillReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	_ = json.Unmarshal(body, &req)
+	if req.Workers <= 0 {
+		req.Workers = 4
+	}
+	if req.Workers > 32 {
+		req.Workers = 32
+	}
+
+	// Pipeline: iter docs → filter (no-vector) → chunker → embed → write.
+	// Bounded channel so we don't materialize the candidate list in memory.
+	type cand struct {
+		docID int64
+		url   string
+	}
+	candCh := make(chan cand, req.Workers*4)
+
+	t0 := time.Now()
+	var scanned, missing, embedded atomic.Int64
+
+	// Find candidates: docs with zero HNSW vectors. Producer.
+	go func() {
+		defer close(candCh)
+		_ = s.store.IterDocsLite(r.Context(), func(docID int64, url string) error {
+			scanned.Add(1)
+			if req.Limit > 0 && missing.Load() >= int64(req.Limit) {
+				return errors.New("limit reached") // sentinel — stops IterDocsLite
+			}
+			if _, ok := s.hnsw.LookupVectorByURL(url); ok {
+				return nil // already has vectors
+			}
+			missing.Add(1)
+			select {
+			case candCh <- cand{docID: docID, url: url}:
+			case <-r.Context().Done():
+				return r.Context().Err()
+			}
+			return nil
+		})
+	}()
+
+	// Worker pool: chunk + embed + write passages per doc.
+	var wg sync.WaitGroup
+	for i := 0; i < req.Workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for c := range candCh {
+				doc, err := s.store.GetDocByID(r.Context(), c.docID)
+				if err != nil || doc == nil || doc.Text == "" {
+					continue
+				}
+				// Use index defaults (NewChunker passes 0,0 = default 320/64).
+				chunker := index.NewChunker()
+				chunks := chunker.Chunk(doc.Text)
+				if len(chunks) == 0 {
+					continue
+				}
+				const tokenCap = 1500
+				texts := make([]string, len(chunks))
+				for j, ch := range chunks {
+					texts[j] = truncateForEmbedLite(ch.Text, tokenCap)
+				}
+				vecs, eErr := s.embedder.Embed(r.Context(), texts)
+				if eErr != nil || len(vecs) != len(chunks) {
+					continue
+				}
+				for j, ch := range chunks {
+					s.hnsw.AddPassage(doc.URL, doc.Title, ch.Offset, ch.Length, vecs[j])
+					_ = j
+				}
+				embedded.Add(1)
+			}
+		}()
+	}
+	wg.Wait()
+
+	log.Printf("embed-backfill: scanned=%d missing=%d embedded=%d in %s",
+		scanned.Load(), missing.Load(), embedded.Load(), time.Since(t0).Round(time.Second))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"scanned":  scanned.Load(),
+		"missing":  missing.Load(),
+		"embedded": embedded.Load(),
+		"elapsed":  time.Since(t0).String(),
+	})
+}
+
+// truncateForEmbedLite mirrors the crawler's helper without pulling the
+// crawler package in. Same heuristic: cap by approximate token count.
+func truncateForEmbedLite(s string, tokenCap int) string {
+	// Rough: 1 token ≈ 4 chars for ASCII; cap at 4×tokenCap bytes as a
+	// fast upper bound. Real chunker is at ~320 words = ~1200 tokens, so
+	// 1500-token cap rarely triggers.
+	maxBytes := tokenCap * 4
+	if len(s) <= maxBytes {
+		return s
+	}
+	return s[:maxBytes]
+}
diff --git a/cmd/cosift/serve_answer.go b/cmd/cosift/serve_answer.go
new file mode 100644
index 0000000..ef312ed
--- /dev/null
+++ b/cmd/cosift/serve_answer.go
@@ -0,0 +1,1697 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"crypto/sha256"
+	"encoding/hex"
+
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/judge"
+	"github.com/pilot-protocol/cosift/internal/rerank"
+)
+
+type synthRequest struct {
+	Q              string `json:"q"`
+	K              int    `json:"k,omitempty"`
+	IncludeDomains string `json:"include_domains,omitempty"`
+	ExcludeDomains string `json:"exclude_domains,omitempty"`
+	Site           string `json:"site,omitempty"`
+	Since          string `json:"since,omitempty"`
+	Until          string `json:"until,omitempty"`
+	IncludeText    bool   `json:"include_text,omitempty"`
+	Rerank         bool   `json:"rerank,omitempty"`
+	Expand         string `json:"expand,omitempty"`
+	Stream         bool   `json:"stream,omitempty"`
+}
+
+func (req synthRequest) toValues() url.Values {
+	v := url.Values{}
+	if req.Q != "" {
+		v.Set("q", req.Q)
+	}
+	if req.K > 0 {
+		v.Set("k", strconv.Itoa(req.K))
+	}
+	if req.IncludeDomains != "" {
+		v.Set("include_domains", req.IncludeDomains)
+	}
+	if req.ExcludeDomains != "" {
+		v.Set("exclude_domains", req.ExcludeDomains)
+	}
+	if req.Site != "" {
+		v.Set("site", req.Site)
+	}
+	if req.Since != "" {
+		v.Set("since", req.Since)
+	}
+	if req.Until != "" {
+		v.Set("until", req.Until)
+	}
+	if req.IncludeText {
+		v.Set("include_text", "true")
+	}
+	if req.Rerank {
+		v.Set("rerank", "true")
+	}
+	if req.Expand != "" {
+		v.Set("expand", req.Expand)
+	}
+	if req.Stream {
+		v.Set("stream", "true")
+	}
+	return v
+}
+
+func (s *pebbleHTTP) handleAnswerPOST(w http.ResponseWriter, r *http.Request) {
+	var req synthRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+	r.URL.RawQuery = req.toValues().Encode()
+	s.handleAnswer(w, r)
+}
+
+func (s *pebbleHTTP) handleResearchPOST(w http.ResponseWriter, r *http.Request) {
+	var req synthRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+	r.URL.RawQuery = req.toValues().Encode()
+	s.handleResearch(w, r)
+}
+
+// /answer — synthesis over BM25 retrieval. Mirrors the SQLite-side
+// /answer in spirit (top-k sources → cited synthesis) but stays minimal: no
+// streaming, no rerank, no query expansion. Those land in follow-up iters
+// once this surface is exercised. Returns 501 when no chat model is
+// configured so the absent capability fails loud instead of silent.
+const answerSystemPrompt = `You are a research assistant. Answer the user's question using the provided sources.
+- Cite sources by their numeric id in square brackets, e.g. [1] or [2,3]. Every factual claim needs a citation.
+- Synthesize the answer from what the sources state, including reasonable inferences from adjacent context — sources rarely state every answer verbatim.
+- Only fall back to "the sources do not cover X" if NONE of the sources have any relevant material at all. Do not refuse just because the exact formula, number, or phrase isn't quoted verbatim — extract whatever IS there and say so.
+- IGNORE sources that are irrelevant to the question — login forms, page-not-found stubs, wiki edit/diff/admin pages, navigation menus, raw image directories, or content about a different topic. Do not cite them. Do not mention them in your answer. Pretend they weren't in the source list.
+- Do not invent facts not supported by the sources.
+- Keep the answer focused on what the sources actually say; do not pad.`
+
+// HyDE-style query expansion prompt. Borrowed verbatim from
+// internal/server/hyde.go so the SQLite and Pebble paths produce comparable
+// expansions and operators don't have to learn two prompt shapes.
+const hydeSystemPrompt = `Write a brief, factual passage (2-4 sentences) that would directly answer the user's question. Output ONLY the passage — no preamble, no commentary, no apology if you're uncertain. If the question is ambiguous, pick the most plausible interpretation and answer that. The passage doesn't need to be true; it needs to be the SHAPE of what a relevant document would say. Embedding this passage and searching by its vector will find documents that look like real answers, even if the user's original query was just a few keywords.`
+
+// doChat wraps ChatClient.Chat with attempt/failure counters.
+// Takes the client as a parameter so it works for both s.chat and the
+// StreamingChatClient passed into streamResearch/streamAnswer.
+func (s *pebbleHTTP) doChat(ctx context.Context, c embed.ChatClient, msgs []embed.ChatMsg) (string, error) {
+	s.chatAttempts.Add(1)
+	start := time.Now()
+	out, err := c.Chat(ctx, msgs)
+	s.chatDurationNanos.Add(time.Since(start).Nanoseconds())
+	if err != nil {
+		s.chatFailures.Add(1)
+	}
+	return out, err
+}
+
+// doChatStream wraps StreamingChatClient.ChatStream with the same counters.
+func (s *pebbleHTTP) doChatStream(ctx context.Context, c embed.StreamingChatClient, msgs []embed.ChatMsg, onChunk func(string)) (string, error) {
+	s.chatAttempts.Add(1)
+	start := time.Now()
+	out, err := c.ChatStream(ctx, msgs, onChunk)
+	s.chatDurationNanos.Add(time.Since(start).Nanoseconds())
+	if err != nil {
+		s.chatFailures.Add(1)
+	}
+	return out, err
+}
+
+type answerSource struct {
+	// Matches the [N] tokens the synth prompt produces in the answer
+	// text. SQLite-side AnswerSource has had this; pebble's was missing,
+	// which caused the CLI to render every source as '[0]' before the
+	// i+1 fallback landed.
+	ID          int        `json:"id"`
+	URL         string     `json:"url"`
+	Title       string     `json:"title"`
+	Excerpt     string     `json:"excerpt,omitempty"`
+	PublishedAt *time.Time `json:"published_at,omitempty"`
+	Author      string     `json:"author,omitempty"`
+	Text        string     `json:"text,omitempty"`
+}
+
+type answerResponse struct {
+	Query          string `json:"query"`
+	EffectiveQuery string `json:"effective_query,omitempty"`
+	Expand         string `json:"expand,omitempty"`
+	// retriever label — same vocabulary as /search ("bm25",
+	// "dense", "bm25+dense:rrf", "+hyde"/"+paraphrase", "+rerank:<name>").
+	Retriever       string         `json:"retriever,omitempty"`
+	Answer          string         `json:"answer"`
+	Sources         []answerSource `json:"sources"`
+	Model           string         `json:"model"`
+	Warnings        []string       `json:"warnings,omitempty"`
+	TotalCandidates int            `json:"total_candidates,omitempty"`
+	Took            string         `json:"took"`
+	// when the answer matches a "sources do not contain" pattern,
+	// surface a hint URL pointing at /research?strategy=planner. The 60-Q
+	// eval showed multi-hop decomposition rescues ~40% of findability +
+	// factual-lookup misses. Purely additive — clients decide whether to
+	// render a "Try research mode" affordance.
+	SuggestEscalation string `json:"suggest_escalation,omitempty"`
+}
+
+// answerLooksLikeNoInfo returns true when the answer reads as "sources don't
+// cover this" — the canonical /research-escalation trigger. Same phrase set
+// the eval uses; kept in sync so eval results predict escalation rate.
+func answerLooksLikeNoInfo(s string) bool {
+	if len(s) > 800 {
+		// Long answers with one disclaimer line aren't bails — only treat
+		// short, mostly-disclaimer responses as escalation candidates.
+		return false
+	}
+	low := strings.ToLower(s)
+	for _, p := range []string{
+		"do not contain", "do not provide", "sources don",
+		"not contain the", "no information", "do not include",
+		"do not have", "do not cover", "not mention", "no mention",
+	} {
+		if strings.Contains(low, p) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *pebbleHTTP) handleAnswer(w http.ResponseWriter, r *http.Request) {
+	// When clustered + gateway mode +
+	// not already serving as a leaf for someone else, fan-out retrieval
+	// to peers (with include_text), then run a SINGLE synth here.
+	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
+		s.handleAnswerGateway(w, r)
+		return
+	}
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented,
+			"/answer requires cfg.Chat.Model to be set (point cosift.json's chat.url at any OpenAI-compatible endpoint)")
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	// Cache + singleflight on /answer. Key is the full query-string
+	// (sorted) so different filter combos don't collide. We capture the
+	// rendered body into a buffer; on success the buffer is the cache
+	// value. Skip the cache for explicit no-cache requests and for
+	// streaming clients — bufferedResponse can't flush, so SSE would
+	// silently fall through to the sync JSON path.
+	if s.answerCache != nil && r.Header.Get("Cache-Control") != "no-cache" && !wantsSSE(r) {
+		key := answerCacheKey(r)
+		body, err, shared := s.answerCache.Do(key, func() ([]byte, error) {
+			rec := &bufferedResponse{header: http.Header{}, status: 200}
+			s.handleAnswerInner(rec, r, start)
+			if rec.status != http.StatusOK {
+				return nil, errAnswerNotOK
+			}
+			return rec.buf.Bytes(), nil
+		})
+		if err == nil {
+			if shared {
+				w.Header().Set("X-Cache", "hit")
+			} else {
+				w.Header().Set("X-Cache", "miss")
+			}
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write(body)
+			return
+		}
+		// On non-OK from the inner handler, fall through to the
+		// uncached path so the real status code reaches the client.
+	}
+	s.handleAnswerInner(w, r, start)
+}
+
+// errAnswerNotOK signals that the inner handler wrote a non-2xx
+// response — don't cache it, and let the caller re-run uncached so
+// the proper status reaches the client.
+var errAnswerNotOK = errors.New("answer: non-OK response, not cached")
+
+// answerCacheKey is the deterministic cache key for a given /answer
+// request. Sorts query parameters so URL-order doesn't fragment the
+// cache; drops parameters that don't affect the response.
+func answerCacheKey(r *http.Request) string {
+	q := r.URL.Query()
+	// Don't key on transport-only or debug-only flags.
+	q.Del("cluster_local")
+	q.Del("trace")
+	keys := make([]string, 0, len(q))
+	for k := range q {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var sb strings.Builder
+	for _, k := range keys {
+		for _, v := range q[k] {
+			sb.WriteString(k)
+			sb.WriteByte('=')
+			sb.WriteString(v)
+			sb.WriteByte('\x00')
+		}
+	}
+	sum := sha256.Sum256([]byte(sb.String()))
+	return hex.EncodeToString(sum[:16])
+}
+
+// bufferedResponse captures a handler's writes into a buffer so we
+// can store the rendered JSON in answerCache before flushing to the
+// real response writer.
+type bufferedResponse struct {
+	buf    bytes.Buffer
+	header http.Header
+	status int
+}
+
+func (b *bufferedResponse) Header() http.Header { return b.header }
+
+func (b *bufferedResponse) WriteHeader(statusCode int) { b.status = statusCode }
+
+func (b *bufferedResponse) Write(p []byte) (int, error) {
+	if b.status == 0 {
+		b.status = http.StatusOK
+	}
+	return b.buf.Write(p)
+}
+
+// handleAnswerInner is the uncached body. Called both via the cache
+// path (with a bufferedResponse) and directly when the cache is
+// disabled or a non-OK response needs the real writer.
+func (s *pebbleHTTP) handleAnswerInner(w http.ResponseWriter, r *http.Request, start time.Time) {
+	q := r.URL.Query().Get("q")
+	k := 5
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
+			k = n
+		}
+	}
+	// /answer respects the same retrieval filters /search does —
+	// scoping research to a domain or date window is the common EXA shape.
+	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
+	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
+	sites := parseSiteScopes(r.URL.Query().Get("site"))
+	siteFilter := len(sites) > 0
+	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
+	if sinceErr != nil {
+		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
+		return
+	}
+	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
+	if untilErr != nil {
+		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
+		return
+	}
+	dateFilter := !since.IsZero() || !until.IsZero()
+	includeText := r.URL.Query().Get("include_text") == "true"
+	// Default to hybrid + rerank on /answer — the reranker is the proven
+	// quality recovery layer; ?retriever=bm25 and ?rerank=false opt out.
+	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
+		r.URL.Query().Get("retriever"),
+		r.URL.Query().Get("rerank"),
+	)
+	keepCap := k
+	if wantRerank {
+		keepCap = s.rerankCandK
+		if keepCap < k {
+			keepCap = k
+		}
+	}
+	fetchK := keepCap
+	if len(include) > 0 || len(exclude) > 0 || dateFilter || siteFilter {
+		mult := 5
+		if dateFilter {
+			mult = 10
+		}
+		if siteFilter {
+			mult = 50 // site= over-fetches aggressively: small site in a large corpus
+		}
+		fetchK = keepCap * mult
+		cap := 200 // /answer caps tighter than /search — full doc text per source is expensive
+		if siteFilter && cap < 500 {
+			cap = 500
+		}
+		if fetchK > cap {
+			fetchK = cap
+		}
+	} else if wantRerank {
+		fetchK = keepCap * 2
+	}
+	// SSE opt-in. Open the stream BEFORE retrieve() so phase events can
+	// surface backend pipeline progress (retrieve / rerank / judge / mmr /
+	// synth_start) instead of just the synth-chunk tail. Opt in via
+	// ?stream=true or Accept: text/event-stream. Caps requirement: chat
+	// client must implement StreamingChatClient AND the ResponseWriter
+	// must support http.Flusher; if either is missing, fall through to
+	// sync (today's behavior).
+	wantStream := wantsSSE(r)
+	var sse *answerSSE
+	var streamChat embed.StreamingChatClient
+	if wantStream {
+		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
+			if a := newAnswerSSE(w, start); a != nil {
+				sse, streamChat = a, sc
+				stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
+				defer stopKA()
+				sse.warnings(s.warningsFor(r))
+				sse.phase("parsed", map[string]any{
+					"q": q, "k": k, "fetch_k": fetchK,
+					"retriever": retrieverParam, "rerank": wantRerank,
+				})
+			}
+		}
+	}
+
+	// expansion dispatch (bare / HyDE / paraphrase+RRF).
+	// retriever dispatch (bm25 / dense / hybrid) via shared helper.
+	expandMode := r.URL.Query().Get("expand")
+	hits, effectiveQuery, err := s.retrieve(r.Context(), q, fetchK, retrieverParam, expandMode)
+	if err != nil {
+		if sse != nil {
+			sse.errorEvt(err.Error())
+			return
+		}
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if sse != nil {
+		sse.phase("retrieve", map[string]any{
+			"hits": len(hits), "effective_query": effectiveQuery,
+		})
+	}
+	type cand struct {
+		src        answerSource
+		excerpt    string
+		rerankText string
+		score      float64 // retrieval score, used by time-decay
+	}
+	cands := make([]cand, 0, keepCap)
+	for _, h := range hits {
+		if len(include) > 0 || len(exclude) > 0 {
+			host := hostOf(h.URL)
+			if len(include) > 0 && !matchesAnyDomain(host, include) {
+				continue
+			}
+			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
+				continue
+			}
+		}
+		if siteFilter && !matchesAnySite(h.URL, sites) {
+			continue
+		}
+		doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
+		if derr != nil || doc == nil {
+			continue
+		}
+		if dateFilter {
+			if doc.PublishedAt.IsZero() {
+				continue
+			}
+			if !since.IsZero() && doc.PublishedAt.Before(since) {
+				continue
+			}
+			if !until.IsZero() && doc.PublishedAt.After(until) {
+				continue
+			}
+		}
+		excerpt := textExcerpt(doc.Text, 1200)
+		src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
+		if !doc.PublishedAt.IsZero() {
+			t := doc.PublishedAt
+			src.PublishedAt = &t
+		}
+		if includeText {
+			src.Text = doc.Text
+		}
+		c := cand{src: src, excerpt: excerpt, score: h.Score}
+		if wantRerank {
+			c.rerankText = doc.Title + "\n" + doc.Text
+		}
+		cands = append(cands, c)
+		if len(cands) >= keepCap {
+			break
+		}
+	}
+	if sse != nil {
+		sse.phase("filter", map[string]any{"candidates": len(cands)})
+	}
+	// time-decay re-weights then re-sorts BEFORE rerank, mirroring
+	// the /search pre-rerank placement so the reranker sees a freshness-
+	// aware pool.
+	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
+	if decaySet && len(cands) > 0 {
+		now := time.Now()
+		for i := range cands {
+			cands[i].score *= decayMultiplier(cands[i].src.PublishedAt, now, decayHalfLife)
+		}
+		sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
+		if sse != nil {
+			sse.phase("decay", map[string]any{"half_life_days": decayHalfLife})
+		}
+	}
+	// Optional rerank, then truncate to k. Done in this order so citation
+	// numbers in the prompt match the final rank-ordered sources list.
+	if wantRerank && len(cands) > 1 {
+		rc := make([]rerank.Candidate, len(cands))
+		for i := range cands {
+			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
+		}
+		if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
+			reordered := make([]cand, 0, len(cands))
+			seen := make(map[int]bool, len(cands))
+			for _, id := range order {
+				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
+					reordered = append(reordered, cands[n])
+					seen[n] = true
+				}
+			}
+			for i, c := range cands {
+				if !seen[i] {
+					reordered = append(reordered, c)
+				}
+			}
+			cands = reordered
+		}
+		if sse != nil {
+			sse.phase("rerank", map[string]any{"reranker": s.reranker.Name(), "candidates": len(cands)})
+		}
+	}
+	// LLM relevance judge — STRICT gate. The reranker re-orders but
+	// doesn't drop; without this gate, marginal content (Wikipedia
+	// diff/admin pages, dead .edu PDFs, JS-only landing pages with
+	// stub HTML) leaks into the synth context and produces "based on
+	// the sources, here is some irrelevant trivia" answers.
+	//
+	// Strict means: if a candidate fails the relevance threshold, it
+	// is DROPPED — we do not top up from the rejected pool to reach k.
+	// Better to return 2 truly relevant sources than 5 sources where 3
+	// are noise. If 0 candidates pass, downstream returns the "no
+	// matching sources" response (already handled below).
+	//
+	// Skip when ?judge=false or no chat client (correctness-preserving).
+	if r.URL.Query().Get("judge") != "false" && s.chat != nil && len(cands) > 1 {
+		before := len(cands)
+		jCands := make([]judge.Candidate, len(cands))
+		for i, c := range cands {
+			jCands[i] = judge.Candidate{ID: strconv.Itoa(i), Excerpt: c.rerankText}
+		}
+		verdicts := judge.Judge(r.Context(), s.chat, q, jCands, judge.Options{MinScore: 0.3})
+		keep := make([]cand, 0, len(cands))
+		for i, c := range cands {
+			if i < len(verdicts) && verdicts[i].Keep {
+				keep = append(keep, c)
+			}
+		}
+		cands = keep
+		if sse != nil {
+			sse.phase("judge", map[string]any{
+				"before": before, "kept": len(cands), "dropped": before - len(cands),
+			})
+		}
+	}
+	// MMR diversification on /answer — same pattern as /search.
+	// Synth quality benefits when the top-k sources cover different
+	// angles instead of being 5 paraphrases of the same paper.
+	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
+	mmrFired := false
+	if mmrSet && len(cands) > 1 {
+		urls := make([]string, len(cands))
+		for i := range cands {
+			urls[i] = cands[i].src.URL
+		}
+		if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
+			reordered := make([]cand, len(order))
+			for i, idx := range order {
+				reordered[i] = cands[idx]
+			}
+			cands = reordered
+			mmrFired = true
+		}
+		if sse != nil && mmrFired {
+			sse.phase("mmr", map[string]any{"lambda": mmrLambda, "candidates": len(cands)})
+		}
+	}
+	if len(cands) > k {
+		cands = cands[:k]
+	}
+	sources := make([]answerSource, 0, len(cands))
+	var promptSources strings.Builder
+	for i, c := range cands {
+		// stamp ID = i+1 so the JSON response matches the [N]
+		// citation tokens we emit in the synth prompt below.
+		c.src.ID = i + 1
+		sources = append(sources, c.src)
+		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, c.excerpt)
+	}
+	// same retriever label vocabulary as /search.
+	denseReady := s.hnsw != nil && s.embedder != nil
+	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, effectiveQuery != q, wantRerank)
+	if mmrFired {
+		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
+	}
+	if decaySet {
+		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
+	}
+	if len(sources) == 0 {
+		// Distinguish the two failure modes so the client can tell
+		// "we have nothing on this topic" (retrieval returned 0 hits)
+		// from "we retrieved candidates but the relevance judge dropped
+		// every one of them" (the corpus has noise, not the answer).
+		msg := "No matching sources in the index."
+		if len(hits) > 0 {
+			msg = "Retrieval returned candidates but none were judged relevant to the question. The corpus may not cover this topic in usable detail."
+		}
+		if sse != nil {
+			sse.sources(q, sources, streamChat.Model(), retrieverLabel, len(hits))
+			sse.chunk(msg)
+			sse.done()
+			return
+		}
+		empty := answerResponse{
+			Query: q, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
+			Answer:  msg,
+			Sources: sources, Model: s.chat.Model(), TotalCandidates: len(hits), Took: time.Since(start).String(),
+		}
+		if effectiveQuery != q {
+			empty.EffectiveQuery = effectiveQuery
+		}
+		empty.Warnings = s.warningsFor(r)
+		writeJSON(w, http.StatusOK, empty)
+		return
+	}
+
+	msgs := []embed.ChatMsg{
+		{Role: "system", Content: answerSystemPrompt},
+		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Question: " + q},
+	}
+
+	if sse != nil {
+		sse.sources(q, sources, streamChat.Model(), retrieverLabel, len(hits))
+		sse.phase("synth_start", map[string]any{"sources": len(sources), "model": streamChat.Model()})
+		full, cerr := s.doChatStream(r.Context(), streamChat, msgs, sse.chunk)
+		if cerr != nil {
+			sse.errorEvt(cerr.Error())
+			return
+		}
+		if answerLooksLikeNoInfo(full) {
+			sse.suggestEscalation(q)
+		}
+		sse.done()
+		return
+	}
+
+	answer, err := s.doChat(r.Context(), s.chat, msgs)
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "chat: "+err.Error())
+		return
+	}
+	resp := answerResponse{
+		Query: q, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
+		Answer: answer, Sources: sources,
+		Model: s.chat.Model(), TotalCandidates: len(hits), Took: time.Since(start).String(),
+	}
+	if effectiveQuery != q {
+		resp.EffectiveQuery = effectiveQuery
+	}
+	resp.Warnings = s.warningsFor(r)
+	// suggest /research escalation when the model bailed.
+	if answerLooksLikeNoInfo(answer) {
+		resp.SuggestEscalation = "/research?q=" + url.QueryEscape(q) + "&strategy=planner"
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// recordingWriter wraps an http.ResponseWriter and captures every byte
+// written to it. Used to record SSE streams for the research cache — the
+// captured bytes can later be replayed to a new client directly.
+type recordingWriter struct {
+	http.ResponseWriter
+	buf bytes.Buffer
+}
+
+func (r *recordingWriter) Write(b []byte) (int, error) {
+	r.buf.Write(b)
+	return r.ResponseWriter.Write(b)
+}
+
+func (r *recordingWriter) Flush() {
+	if f, ok := r.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+// wantsSSE reports whether the request opts into Server-Sent Events,
+// either via ?stream=true or an Accept: text/event-stream header. Both
+// /answer, /query, and /research check the same envelope.
+func wantsSSE(r *http.Request) bool {
+	return r.URL.Query().Get("stream") == "true" ||
+		strings.Contains(r.Header.Get("Accept"), "text/event-stream")
+}
+
+// answerSSE wraps the SSE writer for /answer streaming. Methods emit the
+// well-known event types (phase, sources, answer_chunk, warnings, error,
+// suggest_escalation, done). Phase events surface pipeline progress
+// (retrieve / rerank / judge / mmr / synth_start) so clients can render a
+// timeline of what the backend is doing.
+//
+// Writes are mu-protected because the keepalive ticker can race with
+// the main handler goroutine. The handler does the substantive event
+// writes; the ticker emits SSE comment frames (": ka\n\n") to keep
+// the HTTP/2 stream alive across long retrieval / rerank / chat calls
+// — without them, browsers (Safari especially) close the stream when
+// no bytes flow for 10–30 s and the request fails mid-pipeline.
+type answerSSE struct {
+	w       http.ResponseWriter
+	flusher http.Flusher
+	start   time.Time
+	last    time.Time // last phase emit; used to compute per-phase duration
+	mu      sync.Mutex
+}
+
+// newAnswerSSE opens an SSE stream. Returns nil if the writer can't flush.
+// Caller is responsible for not calling writeProblem after this point —
+// the response status (200) and headers are committed here.
+//
+// Disables the server-wide WriteTimeout (60s) via ResponseController.
+// Multi-pass /research can run 90-180s with 3-5 passes against a slow
+// chat model; the deadline would kill the connection mid-stream and
+// the browser fetch would surface a generic "Load failed". SSE handlers
+// flush their own progress, so a stuck handler still gets killed by
+// ctx cancellation when the client disconnects.
+func newAnswerSSE(w http.ResponseWriter, start time.Time) *answerSSE {
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		return nil
+	}
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("X-Accel-Buffering", "no")
+	if rc := http.NewResponseController(w); rc != nil {
+		_ = rc.SetWriteDeadline(time.Time{})
+	}
+	w.WriteHeader(http.StatusOK)
+	return &answerSSE{w: w, flusher: flusher, start: start, last: start}
+}
+
+func (a *answerSSE) send(payload any) {
+	buf, _ := json.Marshal(payload)
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	fmt.Fprintf(a.w, "data: %s\n\n", buf)
+	a.flusher.Flush()
+}
+
+// startKeepalive launches a goroutine that emits SSE data frames every
+// interval, keeping the HTTP/2 stream and any intermediate proxy alive
+// across long retrieval / chat calls that emit no real data. Returns a
+// stop function the caller defers; stopping is idempotent.
+//
+// We deliberately emit `data: {"type":"ka", ...}` frames rather than the
+// spec's `: comment` form. Safari's CFNetwork stream stack tracks
+// "no payload data received" separately from "no bytes received" — a
+// long-running stream of nothing but comment frames still trips its
+// internal timeout and the browser surfaces it as "Error: Load failed".
+// JSON ka frames count as real data; the chat UI ignores `type:"ka"`.
+func (a *answerSSE) startKeepalive(ctx context.Context, interval time.Duration) (stop func()) {
+	done := make(chan struct{})
+	go func() {
+		t := time.NewTicker(interval)
+		defer t.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				a.send(map[string]any{
+					"type":       "ka",
+					"elapsed_ms": time.Since(a.start).Milliseconds(),
+				})
+			}
+		}
+	}()
+	var once sync.Once
+	return func() { once.Do(func() { close(done) }) }
+}
+
+// phase emits a pipeline-step event with elapsed-since-start and
+// duration-since-last-phase, plus any caller-supplied detail fields.
+// Holds a.mu for the whole call so the read+write of a.last and the
+// write-to-the-wire are all atomic with respect to concurrent phase
+// calls and the keepalive ticker.
+func (a *answerSSE) phase(name string, details map[string]any) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	now := time.Now()
+	evt := map[string]any{
+		"type":        "phase",
+		"name":        name,
+		"elapsed_ms":  now.Sub(a.start).Milliseconds(),
+		"duration_ms": now.Sub(a.last).Milliseconds(),
+	}
+	for k, v := range details {
+		evt[k] = v
+	}
+	a.last = now
+	buf, _ := json.Marshal(evt)
+	fmt.Fprintf(a.w, "data: %s\n\n", buf)
+	a.flusher.Flush()
+}
+
+func (a *answerSSE) warnings(warns []string) {
+	if len(warns) == 0 {
+		return
+	}
+	a.send(map[string]any{"type": "warnings", "warnings": warns})
+}
+
+func (a *answerSSE) sources(q string, src []answerSource, model, retrieverLabel string, totalCandidates int) {
+	evt := map[string]any{
+		"type":             "sources",
+		"query":            q,
+		"sources":          src,
+		"model":            model,
+		"total_candidates": totalCandidates,
+	}
+	if retrieverLabel != "" {
+		evt["retriever"] = retrieverLabel
+	}
+	a.send(evt)
+}
+
+func (a *answerSSE) chunk(delta string) {
+	a.send(map[string]any{"type": "answer_chunk", "text": delta})
+}
+
+func (a *answerSSE) errorEvt(msg string) {
+	a.send(map[string]any{"type": "error", "error": msg})
+}
+
+func (a *answerSSE) suggestEscalation(q string) {
+	a.send(map[string]any{
+		"type":               "suggest_escalation",
+		"suggest_escalation": "/research?q=" + url.QueryEscape(q) + "&strategy=planner",
+	})
+}
+
+func (a *answerSSE) done() {
+	a.send(map[string]any{"type": "done", "took": time.Since(a.start).String()})
+}
+
+// /research — multi-step retrieval + synthesis. LLM decomposes the
+// question into 2-3 sub-queries, each sub-query runs BM25, results are deduped
+// by URL keeping the best score, top-k feed a cited synthesis. Mirrors the
+// SQLite-side /research planner strategy. No streaming, no rerank, no
+// paraphrase strategy yet — those follow once this surface is exercised.
+const researchPlanPrompt = `Decompose the user's research question into 2-3 short keyword search phrases (3-6 words each, NOT full sentences). Each phrase MUST target a DIFFERENT facet so the phrases do not overlap: one conceptual/definitional (what it is), one procedural/how-to (steps, configuration, usage), and one edge-case/troubleshooting (errors, limits, gotchas, comparisons). Skip a facet only if it does not apply. Do NOT output near-duplicate rephrasings of the same idea. Output ONLY a JSON array of strings — no prose, no markdown. Example: ["consensus algorithm overview", "configure validator node", "leader election failure recovery"]`
+
+// truncateForPromptLite caps text by approximate char count for source
+// prompts. Same heuristic the eval-quick path uses.
+func truncateForPromptLite(s string, maxChars int) string {
+	if len(s) <= maxChars {
+		return s
+	}
+	return s[:maxChars] + "…"
+}
+
+const researchSynthPrompt = `You are a research assistant. Synthesize an answer to the original question using ONLY the provided sources.
+- Cite sources by their numeric id, e.g. [1] or [2,3]. Every factual claim needs a citation.
+- If the sources don't cover something, say so plainly — do not invent.
+- Keep the answer focused on what the sources actually say.`
+
+// researchRefineSynthPrompt is the system prompt for pass 2+ of a multi-pass
+// /research call. The user message includes the prior draft answer alongside
+// the expanded source pool. The instruction shape mirrors the standard synth
+// prompt but explicitly directs the model to revise rather than start fresh.
+const researchRefineSynthPrompt = `You are a research assistant. You have a draft answer from a prior research pass plus additional sources that were gathered to close gaps. Produce a REVISED answer using all the sources now available.
+- Cite sources by their numeric id, e.g. [1] or [2,3]. Every factual claim needs a citation.
+- Keep correct claims from the draft. Correct or extend where the new sources change the picture.
+- If sources still don't cover something, say so plainly — do not invent.
+- Output the FULL revised answer, not a delta.`
+
+// selfEvalPrompt asks the model to judge whether its own draft answer is
+// sufficient given the question and sources. JSON-only output keeps parsing
+// deterministic. The "missing" + "refine_queries" fields drive the next
+// research pass when sufficient=false.
+const selfEvalPrompt = `You wrote a research answer. Judge whether it's sufficient given the question and sources.
+
+Output ONLY a JSON object (no markdown, no prose):
+{"sufficient": true|false, "missing": "<one-line gap, empty if sufficient>", "refine_queries": ["<query>", ...]}
+
+Rules:
+- "sufficient": true ONLY if the answer fully addresses the question with current sources.
+- "missing": one short sentence describing what would strengthen the answer (empty string if sufficient).
+- "refine_queries": 1-3 specific search queries that would close the gap (empty array if sufficient). Make them concrete — entity + concept + qualifier — not vague paraphrases.
+- Be honest. Most first-pass answers benefit from one more pass on a specific gap.`
+
+// selfEval is the parsed shape of the JSON the self-eval LLM call emits.
+type selfEval struct {
+	Sufficient    bool     `json:"sufficient"`
+	Missing       string   `json:"missing"`
+	RefineQueries []string `json:"refine_queries"`
+}
+
+// parseSelfEval extracts the {sufficient, missing, refine_queries} JSON from
+// the LLM response. Tolerates leading/trailing prose and markdown fences. On
+// any parse failure returns ok=false so the caller can treat as "stop and
+// keep the current answer" instead of crashing the stream.
+func parseSelfEval(s string) (selfEval, bool) {
+	s = strings.TrimSpace(s)
+	s = jsonFenceRE.ReplaceAllString(s, "$1")
+	// Find the first '{' and last '}' to skip any preamble.
+	i := strings.IndexByte(s, '{')
+	j := strings.LastIndexByte(s, '}')
+	if i < 0 || j <= i {
+		return selfEval{}, false
+	}
+	var ev selfEval
+	if err := json.Unmarshal([]byte(s[i:j+1]), &ev); err != nil {
+		return selfEval{}, false
+	}
+	if len(ev.RefineQueries) > 3 {
+		ev.RefineQueries = ev.RefineQueries[:3]
+	}
+	return ev, true
+}
+
+// jsonFenceRE strips ```json fences and bare ``` fences from LLM output. The
+// judge package has its own copy with the same intent; not worth pulling into
+// a shared package for two callers.
+var jsonFenceRE = regexp.MustCompile("(?s)```(?:json)?\\s*(.*?)\\s*```")
+
+// researchMaxPasses parses ?max_passes from the request. Default 3, valid
+// range 1-10 (clamped). 1 disables the self-eval loop entirely (single-pass
+// /research, today's behavior). The upper cap exists to bound LLM cost +
+// wall-clock: each non-final pass costs 1 synth + 1 self-eval call on top
+// of the per-sub-query retrieval, and the model can in theory loop forever
+// emitting "not yet sufficient" verdicts.
+func researchMaxPasses(r *http.Request) int {
+	const defaultMax = 3
+	const hardCap = 10
+	v := r.URL.Query().Get("max_passes")
+	if v == "" {
+		return defaultMax
+	}
+	n, err := strconv.Atoi(v)
+	if err != nil || n < 1 {
+		return defaultMax
+	}
+	if n > hardCap {
+		return hardCap
+	}
+	return n
+}
+
+type researchResponse struct {
+	Query  string   `json:"query"`
+	Plan   []string `json:"plan"`
+	Expand string   `json:"expand,omitempty"`
+	// retriever label, mirroring /search/answer vocabulary.
+	Retriever       string         `json:"retriever,omitempty"`
+	Answer          string         `json:"answer"`
+	Sources         []answerSource `json:"sources"`
+	Model           string         `json:"model"`
+	Warnings        []string       `json:"warnings,omitempty"`
+	TotalCandidates int            `json:"total_candidates,omitempty"`
+	Took            string         `json:"took"`
+}
+
+// researchSubFanOut bounds how many sub-queries retrieve concurrently within
+// a single /research pass. Shared by the sync (handleResearch) and streaming
+// (streamResearch) paths so their latency profiles stay identical. 3 keeps the
+// site= budget at 3 sub-queries × fan-out-3 = one concurrent batch.
+const researchSubFanOut = 3
+
+func (s *pebbleHTTP) handleResearch(w http.ResponseWriter, r *http.Request) {
+	// Planner runs once on gateway,
+	// each sub-query scatters to peers, single synth here.
+	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
+		s.handleResearchGateway(w, r)
+		return
+	}
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented,
+			"/research requires cfg.Chat.Model to be set")
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	k := 8
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
+			k = n
+		}
+	}
+	// same retrieval filters /search/answer/find_similar accept.
+	// Parsed once here so the streaming and sync paths share semantics.
+	filt, err := parseRetrievalFilters(r)
+	if err != nil {
+		writeProblem(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	includeText := r.URL.Query().Get("include_text") == "true"
+	// SSE streaming for /research. Same trigger as /answer.
+	// Emits phase-aware events so the UI can render the plan and source list
+	// before the synth call completes — /research often runs 10–30s
+	// (2-3 plan→retrieve→synth chat rounds), so phase visibility matters.
+	wantStream := wantsSSE(r)
+	if wantStream {
+		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
+			cacheKey := "research|" + q + "|" + r.URL.Query().Get("site") + "|" + strconv.Itoa(k)
+			if r.Header.Get("Cache-Control") != "no-cache" {
+				if cached, ok := s.researchCache.Get(cacheKey); ok {
+					w.Header().Set("Content-Type", "text/event-stream")
+					w.Header().Set("Cache-Control", "no-cache")
+					w.Header().Set("X-Accel-Buffering", "no")
+					w.WriteHeader(http.StatusOK)
+					_, _ = w.Write(cached)
+					if f, ok2 := w.(http.Flusher); ok2 {
+						f.Flush()
+					}
+					return
+				}
+			}
+			rec := &recordingWriter{ResponseWriter: w}
+			s.streamResearch(rec, r, sc, q, k, filt, start)
+			if rec.buf.Len() > 0 {
+				s.researchCache.Set(cacheKey, rec.buf.Bytes())
+			}
+			return
+		}
+	}
+
+	// Plan — include site domain so the LLM generates site-specific sub-queries
+	// rather than generic web queries that miss small-site content.
+	planQ := q
+	if len(filt.sites) > 0 {
+		siteHints := make([]string, len(filt.sites))
+		for si, ss := range filt.sites {
+			siteHints[si] = ss.host
+			if ss.path != "" {
+				siteHints[si] += ss.path
+			}
+		}
+		planQ = q + "\n\nSite filter: " + strings.Join(siteHints, ", ") + ". Generate sub-queries using specific terminology likely found on this site."
+		if titles := s.getSiteTitles(r.Context(), filt.sites); len(titles) > 0 {
+			planQ += "\n\nExample pages on this site: " + strings.Join(titles, ", ") + "."
+		}
+	}
+	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: researchPlanPrompt},
+		{Role: "user", Content: planQ},
+	})
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
+		return
+	}
+	subs := parseSubQueries(planRaw, q)
+	maxSubs := 5
+	// site= scope limits the search to a small domain. 3 sub-queries with
+	// fan-out 3 complete in one concurrent batch (~21s each), keeping total
+	// retrieval under 25s so LLM synthesis fits within the 60s budget.
+	if len(filt.sites) > 0 && maxSubs > 3 {
+		maxSubs = 3
+	}
+	if len(subs) > maxSubs {
+		subs = subs[:maxSubs]
+	}
+
+	// Retrieve per sub-query, dedupe by URL keeping best score.
+	type ranked struct {
+		score float64
+		hit   index.Hit
+	}
+	best := make(map[string]ranked, k*len(subs))
+	// Build the site boost map first: its length equals the site's doc count,
+	// which drives the adaptive perSub below.
+	syncBoostIDs := s.getSiteBoostIDs(r.Context(), filt.sites)
+	perSub := k * 2
+	if perSub > 40 {
+		perSub = 40
+	}
+	if len(filt.sites) > 0 {
+		// Fetch deep enough to cover the whole site (×1.2 headroom) so the 50×
+		// boost can reorder every site doc into top-k. Floor 200 preserves the
+		// MaxScore-firing window for tiny site-key terms (e.g. "pilotctl");
+		// ceiling 1000 bounds per-sub BM25 latency. Issue #2's MaxScore
+		// pre-check is what keeps the larger fetch affordable.
+		perSub = max(200, min(1000, int(math.Ceil(float64(len(syncBoostIDs))*1.2))))
+	}
+	// each sub-query goes through retrieveWithExpansion, so
+	// ?expand=hyde gives per-sub-query HyDE (was's behavior) and
+	// ?expand=paraphrase fans out 3 paraphrases × N sub-queries → RRF per
+	// sub-query → merge into best{}. The cross-sub-query merge stays
+	// score-keep-best (not a second RRF) — that's what specified.
+	// retriever dispatch (bm25 / dense / hybrid) applies per
+	// sub-query — every sub-query runs through the same retriever.
+	expandMode := r.URL.Query().Get("expand")
+	// Default to hybrid + rerank on /research — same rationale as /answer.
+	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
+		r.URL.Query().Get("retriever"),
+		r.URL.Query().Get("rerank"),
+	)
+	// Sub-queries are independent retrieve calls and the per-call cost dominates
+	// total latency, so fan them out with a bounded concurrency identical to the
+	// streaming path. bestMu guards the shared best map; hits and syncBoostIDs
+	// are goroutine-local / read-only respectively. There is no seenURLs set in
+	// the sync path (single pass), so the merge is a plain keep-best-score dedupe.
+	var bestMu sync.Mutex
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, researchSubFanOut)
+	for _, sq := range subs {
+		sq := sq
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			hits, _, err := s.retrieveForSites(r.Context(), sq, perSub, retrieverParam, expandMode, syncBoostIDs, filt.sites)
+			if err != nil {
+				// log the specific sub-query so operators can diagnose
+				// 'why was this research thin on sources' — previously silent.
+				log.Printf("pebble-serve: /research sub-query %q failed: %v", sq, err)
+				return
+			}
+			bestMu.Lock()
+			for _, h := range hits {
+				if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
+					best[h.URL] = ranked{score: h.Score, hit: h}
+				}
+			}
+			bestMu.Unlock()
+		}()
+	}
+	wg.Wait()
+	pooled := make([]ranked, 0, len(best))
+	for _, v := range best {
+		pooled = append(pooled, v)
+	}
+	sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
+	// same rerank wiring as /search and /answer — pool widens to
+	// rerankCandK before materialization, rerank reorders the pool, truncate
+	// to k after rerank so citation numbers track the final order.
+	keepCap := k
+	if wantRerank {
+		keepCap = s.rerankCandK
+		if keepCap < k {
+			keepCap = k
+		}
+	}
+	// Truncate the sorted pool to keepCap before materialization. For site=
+	// queries the 50x boost (applied in Search before its descending sort)
+	// guarantees the site's matched docs occupy the front of pooled, so this
+	// cut keeps exactly the boosted docs and bounds the rerank candidate count
+	// (was ~138 for site= queries; now keepCap, default rerankCandK=20).
+	if len(pooled) > keepCap {
+		pooled = pooled[:keepCap]
+	}
+
+	// Materialize candidates; defer sources + prompt build until after rerank.
+	type cand struct {
+		src        answerSource
+		excerpt    string
+		rerankText string
+		score      float64 // pooled score, used by time-decay
+	}
+	cands := make([]cand, 0, len(pooled))
+	for _, p := range pooled {
+		doc, derr := s.store.GetDocByURL(r.Context(), p.hit.URL)
+		if derr != nil || doc == nil {
+			continue
+		}
+		if !filt.allow(doc.URL, doc.PublishedAt) {
+			continue
+		}
+		excerpt := textExcerpt(doc.Text, 1200)
+		src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
+		if !doc.PublishedAt.IsZero() {
+			t := doc.PublishedAt
+			src.PublishedAt = &t
+		}
+		if includeText {
+			src.Text = doc.Text
+		}
+		c := cand{src: src, excerpt: excerpt, score: p.score}
+		if wantRerank {
+			c.rerankText = doc.Title + "\n" + doc.Text
+		}
+		cands = append(cands, c)
+	}
+	// time-decay on /research sync — re-weight pooled cands before
+	// rerank. Sub-queries that hit recent docs bubble up first.
+	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
+	if decaySet && len(cands) > 0 {
+		now := time.Now()
+		for i := range cands {
+			cands[i].score *= decayMultiplier(cands[i].src.PublishedAt, now, decayHalfLife)
+		}
+		sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
+	}
+	if wantRerank && len(cands) > 1 {
+		rc := make([]rerank.Candidate, len(cands))
+		for i := range cands {
+			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
+		}
+		if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
+			reordered := make([]cand, 0, len(cands))
+			seen := make(map[int]bool, len(cands))
+			for _, id := range order {
+				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
+					reordered = append(reordered, cands[n])
+					seen[n] = true
+				}
+			}
+			for i, c := range cands {
+				if !seen[i] {
+					reordered = append(reordered, c)
+				}
+			}
+			cands = reordered
+		}
+	}
+	// MMR diversification on /research sync. /research aggregates
+	// across sub-queries — duplicate-ish docs accumulate fast. MMR after
+	// rerank, before truncation.
+	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
+	mmrFired := false
+	if mmrSet && len(cands) > 1 {
+		urls := make([]string, len(cands))
+		for i := range cands {
+			urls[i] = cands[i].src.URL
+		}
+		if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
+			reordered := make([]cand, len(order))
+			for i, idx := range order {
+				reordered[i] = cands[idx]
+			}
+			cands = reordered
+			mmrFired = true
+		}
+	}
+	if len(cands) > k {
+		cands = cands[:k]
+	}
+	sources := make([]answerSource, 0, len(cands))
+	var promptSources strings.Builder
+	for i, c := range cands {
+		// stamp ID = i+1 so the JSON response matches the [N]
+		// citation tokens we emit in the synth prompt below.
+		c.src.ID = i + 1
+		sources = append(sources, c.src)
+		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, c.excerpt)
+	}
+	// Sub-queries don't yield a
+	// single effectiveQuery, so "expansion fired" is approximated by intent:
+	// chat is up and expandMode requested it. Matches what warningsFor() uses
+	// to decide whether to flag a silent no-op.
+	denseReady := s.hnsw != nil && s.embedder != nil
+	expandFired := expandMode != "" && s.chat != nil
+	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, expandFired, wantRerank)
+	if mmrFired {
+		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
+	}
+	if decaySet {
+		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
+	}
+	if len(sources) == 0 {
+		writeJSON(w, http.StatusOK, researchResponse{
+			Query: q, Plan: subs, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
+			Answer:  "No matching sources for any sub-query.",
+			Sources: sources, Model: s.chat.Model(), Warnings: s.warningsFor(r),
+			TotalCandidates: len(best), Took: time.Since(start).String(),
+		})
+		return
+	}
+
+	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: researchSynthPrompt},
+		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Original question: " + q},
+	})
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, researchResponse{
+		Query: q, Plan: subs, Expand: normalizeExpandMode(expandMode), Retriever: retrieverLabel,
+		Answer: answer, Sources: sources,
+		Model: s.chat.Model(), Warnings: s.warningsFor(r),
+		TotalCandidates: len(best), Took: time.Since(start).String(),
+	})
+}
+
+// streamResearch is the SSE form of handleResearch. Event sequence:
+//
+//	plan    — sub-queries returned by the planner
+//	sources — deduped + ranked pool fed to the synthesizer
+//	chunk   — per chat delta during synth
+//	done    — final {"took": "..."}
+//	error   — terminal; either plan, retrieval, or synth failed
+func (s *pebbleHTTP) streamResearch(w http.ResponseWriter, r *http.Request, sc embed.StreamingChatClient, q string, k int, filt retrievalFilters, start time.Time) {
+	sse := newAnswerSSE(w, start)
+	if sse == nil {
+		writeProblem(w, http.StatusInternalServerError, "streaming requires http.Flusher")
+		return
+	}
+	// /research can sit on a single slow retrieve or chat call for
+	// 10–30 s. Without a keepalive, the browser's HTTP/2 stream times
+	// out as inactive and the request fails mid-pipeline. 7 s gives
+	// generous headroom under Safari's ~30 s idle limit.
+	stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
+	defer stopKA()
+	includeText := r.URL.Query().Get("include_text") == "true"
+
+	// surface silent no-ops upfront so SSE clients can render them
+	// while the plan call is still in flight. Fires before plan to give the
+	// fastest visual feedback when a misconfigured request arrives.
+	sse.warnings(s.warningsFor(r))
+	sse.phase("plan_start", map[string]any{"q": q, "model": sc.Model()})
+
+	// site= hint: LLM generates site-specific sub-queries with domain terminology
+	// rather than generic queries that miss small-site content in BM25 top-200.
+	planQ := q
+	if len(filt.sites) > 0 {
+		siteHints := make([]string, len(filt.sites))
+		for si, ss := range filt.sites {
+			siteHints[si] = ss.host
+			if ss.path != "" {
+				siteHints[si] += ss.path
+			}
+		}
+		planQ = q + "\n\nSite filter: " + strings.Join(siteHints, ", ") + ". Generate sub-queries using specific terminology likely found on this site."
+		if titles := s.getSiteTitles(r.Context(), filt.sites); len(titles) > 0 {
+			planQ += "\n\nExample pages on this site: " + strings.Join(titles, ", ") + "."
+		}
+	}
+	planRaw, err := s.doChat(r.Context(), sc, []embed.ChatMsg{
+		{Role: "system", Content: researchPlanPrompt},
+		{Role: "user", Content: planQ},
+	})
+	if err != nil {
+		sse.send(map[string]any{"type": "error", "phase": "plan", "error": err.Error()})
+		return
+	}
+	subs := parseSubQueries(planRaw, q)
+	{
+		maxSubs := 5
+		if len(filt.sites) > 0 && maxSubs > 3 {
+			maxSubs = 3 // site= budget: 3 sub-queries × fan-out-3 = one concurrent batch (~21s) + LLM fits in 60s
+		}
+		if len(subs) > maxSubs {
+			subs = subs[:maxSubs]
+		}
+	}
+	// per-sub-query expansion via retrieveWithExpansion.
+	// per-sub-query retriever dispatch (bm25 / dense / hybrid).
+	// read retriever + wantRerank early so the plan event can
+	// surface the full retriever label, not just expand.
+	// Default to hybrid + rerank on /research streaming — same rationale as /answer.
+	expandMode := r.URL.Query().Get("expand")
+	retrieverParam, wantRerank := s.applyLLMEndpointDefaults(
+		r.URL.Query().Get("retriever"),
+		r.URL.Query().Get("rerank"),
+	)
+	denseReady := s.hnsw != nil && s.embedder != nil
+	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, expandMode != "", wantRerank)
+
+	// surface the active expansion strategy on the plan event so
+	// UIs can render 'using paraphrase' or 'using HyDE' as soon as the plan
+	// arrives, before per-sub-query expansion fires.
+	planEvent := map[string]any{"type": "plan", "query": q, "plan": subs, "model": sc.Model()}
+	if mode := normalizeExpandMode(expandMode); mode != "" {
+		planEvent["expand"] = mode
+	}
+	if retrieverLabel != "" {
+		planEvent["retriever"] = retrieverLabel
+	}
+	sse.send(planEvent)
+	// Phase event mirrors /query's shape so the chat UI's timeline strip
+	// renders identical rows across all three synth endpoints. Co-exists
+	// with the legacy `type: plan` event above, which the CLI client at
+	// cmd/cosift/main.go keys on.
+	planPhase := map[string]any{"queries": subs, "retriever": retrieverLabel}
+	if mode := normalizeExpandMode(expandMode); mode != "" {
+		planPhase["expand"] = mode
+	}
+	sse.phase("plan", planPhase)
+
+	type ranked struct {
+		score float64
+		hit   index.Hit
+	}
+	type cand struct {
+		src        answerSource
+		excerpt    string
+		rerankText string
+		score      float64
+	}
+
+	// Multi-pass research loop. Each pass: retrieve for the current subs →
+	// fuse → materialize → rerank → mmr → synth → self-eval. Cumulative URL
+	// set across passes; new passes skip URLs already seen so retrieval
+	// widens rather than re-fetching the same docs. After synth, the model
+	// judges whether its draft is sufficient. If not, we use its
+	// refine_queries to drive the next pass. Cap at maxPasses to bound
+	// latency + LLM cost.
+	maxPasses := researchMaxPasses(r)
+	// site= constrains to a small domain: multi-pass BM25 at fetchK=300 costs
+	// 21s × passes beyond the 60s budget. Cap to 1 pass — a small site's corpus
+	// is fully represented in the first retrieval round anyway.
+	if len(filt.sites) > 0 && maxPasses > 1 {
+		maxPasses = 1
+	}
+	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
+	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
+	// Pre-build site boost map once per request (O(site_docs) famHost scan,
+	// cached for repeat queries). Nil when site= is absent or store unavailable.
+	// Built before perSub because its length equals the site doc count.
+	siteBoostIDs := s.getSiteBoostIDs(r.Context(), filt.sites)
+	perSub := k * 2
+	if perSub > 40 {
+		perSub = 40
+	}
+	if len(filt.sites) > 0 {
+		// Fetch deep enough to cover the whole site (×1.2 headroom); floor 200
+		// preserves the MaxScore-firing window for tiny site-key terms, ceiling
+		// 1000 bounds per-sub BM25 latency. Mitigated by Issue #2's MaxScore
+		// pre-check.
+		perSub = max(200, min(1000, int(math.Ceil(float64(len(siteBoostIDs))*1.2))))
+	}
+	keepCap := k
+	if wantRerank {
+		keepCap = s.rerankCandK
+		if keepCap < k {
+			keepCap = k
+		}
+	}
+
+	seenURLs := make(map[string]bool)
+	allCands := make([]cand, 0, k*maxPasses)
+	totalCandidates := 0 // counts URL contributions across passes for the sources event
+	var lastAnswer string
+	var finalRetrieverLabel string
+
+	for pass := 1; pass <= maxPasses; pass++ {
+		sse.phase("pass_start", map[string]any{"pass": pass, "queries": subs})
+
+		// Retrieval round for this pass. Sub-queries run concurrently with
+		// a bounded fan-out — they're independent calls to s.retrieve and
+		// the per-call cost dominates total pass latency. Serial execution
+		// of N sub-queries at ~12 s each pushed pass-1+pass-2 wall-clock
+		// past 60 s on this corpus, blowing the browser's hardcoded fetch
+		// timeout on long-streaming responses. With fan-out the pass
+		// shrinks to ~max(sub-query latency) plus the per-goroutine
+		// bookkeeping. sse.send is mutex-protected; bestMu protects best.
+		// seenURLs is read-only during this loop (last written at the end
+		// of the prior pass), so no synchronization is needed for it.
+		best := make(map[string]ranked, k*len(subs))
+		var bestMu sync.Mutex
+		var wg sync.WaitGroup
+		sem := make(chan struct{}, researchSubFanOut)
+		for _, sq := range subs {
+			sq := sq
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				sem <- struct{}{}
+				defer func() { <-sem }()
+				sse.phase("retrieving", map[string]any{"pass": pass, "query": sq})
+				hits, _, rerr := s.retrieveForSites(r.Context(), sq, perSub, retrieverParam, expandMode, siteBoostIDs, filt.sites)
+				if rerr != nil {
+					log.Printf("pebble-serve: /research sub-query %q failed: %v", sq, rerr)
+					sse.phase("expand", map[string]any{"pass": pass, "query": sq, "error": rerr.Error()})
+					return
+				}
+				bestMu.Lock()
+				for _, h := range hits {
+					if seenURLs[h.URL] {
+						continue
+					}
+					if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
+						best[h.URL] = ranked{score: h.Score, hit: h}
+					}
+				}
+				bestMu.Unlock()
+				sse.phase("expand", map[string]any{"pass": pass, "query": sq, "hits": len(hits)})
+			}()
+		}
+		wg.Wait()
+		totalCandidates += len(best)
+		pooled := make([]ranked, 0, len(best))
+		for _, v := range best {
+			pooled = append(pooled, v)
+		}
+		sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
+		sse.phase("fuse", map[string]any{"pass": pass, "unique_urls": len(pooled)})
+		// Truncate the sorted pool to keepCap before materialization. For site=
+		// queries the 50x boost (applied in Search before its descending sort)
+		// puts the site's matched docs at the front of pooled, so this cut keeps
+		// the boosted docs and bounds the LLM rerank candidate count to keepCap.
+		if len(pooled) > keepCap {
+			pooled = pooled[:keepCap]
+		}
+
+		// Materialize this pass's new candidates (filt-aware).
+		newCands := make([]cand, 0, len(pooled))
+		for _, p := range pooled {
+			doc, derr := s.store.GetDocByURL(r.Context(), p.hit.URL)
+			if derr != nil || doc == nil {
+				continue
+			}
+			if !filt.allow(doc.URL, doc.PublishedAt) {
+				continue
+			}
+			excerpt := textExcerpt(doc.Text, 1200)
+			src := answerSource{URL: doc.URL, Title: doc.Title, Excerpt: excerpt, Author: doc.Author}
+			if !doc.PublishedAt.IsZero() {
+				t := doc.PublishedAt
+				src.PublishedAt = &t
+			}
+			if includeText {
+				src.Text = doc.Text
+			}
+			c := cand{src: src, excerpt: excerpt, score: p.score}
+			if wantRerank {
+				c.rerankText = doc.Title + "\n" + doc.Text
+			}
+			newCands = append(newCands, c)
+		}
+		sse.phase("materialize", map[string]any{"pass": pass, "candidates": len(newCands)})
+
+		// Optional time-decay on this pass's new candidates.
+		passLabel := retrieverLabel
+		if decaySet && len(newCands) > 0 {
+			now := time.Now()
+			for i := range newCands {
+				newCands[i].score *= decayMultiplier(newCands[i].src.PublishedAt, now, decayHalfLife)
+			}
+			sort.SliceStable(newCands, func(i, j int) bool { return newCands[i].score > newCands[j].score })
+			passLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
+		}
+		if wantRerank && len(newCands) > 1 {
+			rc := make([]rerank.Candidate, len(newCands))
+			for i := range newCands {
+				rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: newCands[i].rerankText}
+			}
+			if order, rerr := s.doRerank(r.Context(), q, rc); rerr == nil && len(order) > 0 {
+				reordered := make([]cand, 0, len(newCands))
+				seenIdx := make(map[int]bool, len(newCands))
+				for _, id := range order {
+					if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(newCands) && !seenIdx[n] {
+						reordered = append(reordered, newCands[n])
+						seenIdx[n] = true
+					}
+				}
+				for i, c := range newCands {
+					if !seenIdx[i] {
+						reordered = append(reordered, c)
+					}
+				}
+				newCands = reordered
+			}
+			sse.phase("rerank", map[string]any{"pass": pass, "reranker": s.reranker.Name(), "candidates": len(newCands)})
+		}
+		if mmrSet && len(newCands) > 1 {
+			urls := make([]string, len(newCands))
+			for i := range newCands {
+				urls[i] = newCands[i].src.URL
+			}
+			if order := s.applyMMRPermutation(r.Context(), urls, q, mmrLambda); order != nil {
+				reordered := make([]cand, len(order))
+				for i, idx := range order {
+					reordered[i] = newCands[idx]
+				}
+				newCands = reordered
+				passLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
+				sse.phase("mmr", map[string]any{"pass": pass, "lambda": mmrLambda, "candidates": len(newCands)})
+			}
+		}
+		if len(newCands) > k {
+			newCands = newCands[:k]
+		}
+		// Promote into the cumulative pool.
+		for _, c := range newCands {
+			seenURLs[c.src.URL] = true
+			allCands = append(allCands, c)
+		}
+		finalRetrieverLabel = passLabel
+
+		// First-pass-empty path mirrors today's behavior: if pass 1 finds
+		// no usable sources at all, bail with empty=true.
+		if pass == 1 && len(allCands) == 0 {
+			sse.send(map[string]any{"type": "done", "took": time.Since(start).String(), "empty": true})
+			return
+		}
+		// Later-pass empty (no new sources added): synth would just rewrite
+		// the prior answer with no new signal. Stop and keep lastAnswer.
+		if pass > 1 && len(newCands) == 0 {
+			break
+		}
+
+		// Build the cumulative sources slice + prompt block, stamping IDs
+		// 1..N in the order URLs were promoted.
+		// For site= queries trim excerpts to 600 chars — reduces synthesis
+		// context from ~9.6k to ~4.8k chars, cutting synthesis latency ~40%.
+		synthExcerptLen := 1200
+		if len(filt.sites) > 0 {
+			synthExcerptLen = 600
+		}
+		cumulativeSources := make([]answerSource, 0, len(allCands))
+		var promptSources strings.Builder
+		for i, c := range allCands {
+			c.src.ID = i + 1
+			cumulativeSources = append(cumulativeSources, c.src)
+			ex := c.excerpt
+			if len(ex) > synthExcerptLen {
+				ex = ex[:synthExcerptLen]
+			}
+			fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, c.src.Title, c.src.URL, ex)
+		}
+		srcEvt := map[string]any{
+			"type": "sources", "sources": cumulativeSources,
+			"total_candidates": totalCandidates, "pass": pass,
+		}
+		if finalRetrieverLabel != "" {
+			srcEvt["retriever"] = finalRetrieverLabel
+		}
+		sse.send(srcEvt)
+
+		// Synth. Pass 1 uses the standard prompt; pass 2+ uses the refine
+		// prompt with the prior draft injected.
+		sse.phase("synth_start", map[string]any{
+			"pass": pass, "sources": len(cumulativeSources), "model": sc.Model(),
+		})
+		var synthMsgs []embed.ChatMsg
+		userMsg := "Sources:\n\n" + promptSources.String() + "Original question: " + q
+		if pass == 1 {
+			synthMsgs = []embed.ChatMsg{
+				{Role: "system", Content: researchSynthPrompt},
+				{Role: "user", Content: userMsg},
+			}
+		} else {
+			synthMsgs = []embed.ChatMsg{
+				{Role: "system", Content: researchRefineSynthPrompt},
+				{Role: "user", Content: userMsg + "\n\nYour prior draft answer:\n" + lastAnswer},
+			}
+		}
+		full, serr := s.doChatStream(r.Context(), sc, synthMsgs, sse.chunk)
+		if serr != nil {
+			sse.send(map[string]any{"type": "error", "phase": "synth", "error": serr.Error()})
+			return
+		}
+		lastAnswer = full
+
+		// On the final pass, skip self-eval — nothing to escalate to.
+		if pass == maxPasses {
+			break
+		}
+
+		// Self-evaluate. The model decides whether to escalate.
+		sse.phase("self_eval_start", map[string]any{"pass": pass})
+		evalUserMsg := fmt.Sprintf(
+			"Question: %s\n\nYour answer:\n%s\n\nSources used (id — title — url):\n%s",
+			q, lastAnswer, summarizeSourceList(cumulativeSources),
+		)
+		evalRaw, eerr := s.doChat(r.Context(), sc, []embed.ChatMsg{
+			{Role: "system", Content: selfEvalPrompt},
+			{Role: "user", Content: evalUserMsg},
+		})
+		if eerr != nil {
+			log.Printf("pebble-serve: /research self-eval pass %d failed: %v", pass, eerr)
+			sse.phase("self_eval", map[string]any{"pass": pass, "error": eerr.Error()})
+			break
+		}
+		ev, ok := parseSelfEval(evalRaw)
+		if !ok {
+			sse.phase("self_eval", map[string]any{"pass": pass, "parse_failed": true})
+			break
+		}
+		sse.phase("self_eval", map[string]any{
+			"pass": pass, "sufficient": ev.Sufficient,
+			"missing": ev.Missing, "refine_queries": ev.RefineQueries,
+		})
+		if ev.Sufficient {
+			break
+		}
+		// Blurb summarizing why we're escalating; UI renders inline.
+		blurbText := strings.TrimSpace(ev.Missing)
+		if blurbText == "" {
+			blurbText = "More depth needed"
+		}
+		blurbText += " — searching deeper."
+		sse.phase("blurb", map[string]any{"pass": pass, "text": blurbText})
+
+		// Drive the next pass with the refined queries the model proposed.
+		if len(ev.RefineQueries) == 0 {
+			break
+		}
+		subs = ev.RefineQueries
+		if len(subs) > 5 {
+			subs = subs[:5]
+		}
+	}
+
+	sse.done()
+}
+
+// summarizeSourceList formats sources into compact "[id] title — url" lines
+// for the self-eval user message. Avoids piping the full excerpts back into
+// the eval prompt — the eval is about coverage, not re-reading.
+func summarizeSourceList(srcs []answerSource) string {
+	var sb strings.Builder
+	for _, s := range srcs {
+		fmt.Fprintf(&sb, "[%d] %s — %s\n", s.ID, s.Title, s.URL)
+	}
+	return sb.String()
+}
diff --git a/cmd/cosift/serve_crawl.go b/cmd/cosift/serve_crawl.go
new file mode 100644
index 0000000..5039f13
--- /dev/null
+++ b/cmd/cosift/serve_crawl.go
@@ -0,0 +1,851 @@
+package main
+
+import (
+	"compress/gzip"
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// handleCrawlEnqueue accepts a single URL forwarded from a peer shard and
+// pushes it onto the local crawler's frontier. Auth via cfg.Cluster.
+// PeerAuthToken Bearer header.
+type crawlEnqueueReq struct {
+	URL string `json:"url"`
+}
+
+func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request) {
+	// Auth
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
+			return
+		}
+	}
+	if s.crawlSeed == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req crawlEnqueueReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"...\"}")
+		return
+	}
+	if err := s.crawlSeed(req.URL); err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"queued": req.URL})
+}
+
+// handleAllowDomain promotes a domain into the crawler's runtime dynamic
+// allowlist so subsequently-crawled URLs from it pass allowedDomain(). Used by
+// the HN/Reddit harvesters to grow the crawlable set organically once a
+// link-target domain recurs above their frequency threshold. Persisted so it
+// survives restart. Body: {"domain":"example.com"}.
+func (s *pebbleHTTP) handleAllowDomain(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
+			return
+		}
+	}
+	if s.crawlAllowDomain == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler")
+		return
+	}
+	var req struct {
+		Domain string `json:"domain"`
+	}
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 8<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.Domain == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"domain\": \"example.com\"}")
+		return
+	}
+	if err := s.crawlAllowDomain(req.Domain); err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"allowed": strings.ToLower(strings.TrimSpace(req.Domain))})
+}
+
+// handleFrontierClear wipes the entire frontier in a single Pebble
+// DeleteRange — every queued URL, primary + secondary index. Used when
+// the frontier is so polluted by spam-discovery crawls that
+// claim-time exclude rules can't drain it in reasonable time. Operator
+// re-seeds via seeds-file restart or /admin/sitemap-import calls.
+//
+// Sees the same PeerAuthToken as the other admin endpoints. Returns the
+// approximate count of deleted entries (DeleteRange is O(tombstones),
+// not O(N), so the count is reported as -1 to signal "swept range").
+func (s *pebbleHTTP) handleFrontierClear(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
+			return
+		}
+	}
+	if err := s.store.ClearFrontier(r.Context()); err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"cleared": true})
+}
+
+// handleFrontierPurgeHost deletes every queued frontier entry whose host
+// matches the supplied value. Used to drain blacklisted hosts that the
+// round-robin cursor would take days to chew through one-by-one (the
+// cursor visits each host once per cycle; a 5M-entry host needs 5M
+// cycles to drain, days of cycle time at typical claim rates).
+type frontierPurgeReq struct {
+	Host string `json:"host"`
+}
+
+func (s *pebbleHTTP) handleFrontierPurgeHost(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
+			return
+		}
+	}
+	var req frontierPurgeReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"host\": \"...\"}")
+		return
+	}
+	n, err := s.store.PurgeFrontierByHost(r.Context(), req.Host)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{"purged": n, "host": req.Host})
+}
+
+// handleSitemapImport fetches a sitemap.xml (or sitemap-index, one level
+// of recursion) and pushes every <loc> entry to the live frontier. Same
+// auth as crawl-enqueue. Synchronous — returns when all URLs are queued.
+// For very large sitemaps (>50K URLs) the call may take seconds.
+type sitemapImportReq struct {
+	URL string `json:"url"`
+}
+
+func (s *pebbleHTTP) handleSitemapImport(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedSitemap == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req sitemapImportReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"https://.../sitemap.xml\"}")
+		return
+	}
+	t0 := time.Now()
+	n, err := s.crawlSeedSitemap(r.Context(), req.URL)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	log.Printf("sitemap-import: queued %d URLs from %s in %s", n, req.URL, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"sitemap": req.URL,
+		"queued":  n,
+		"elapsed": time.Since(t0).String(),
+	})
+}
+
+// handleRecrawlSitemap fetches a sitemap.xml and calls Recrawl on every URL,
+// resetting done/errored entries back to queued so the crawler re-visits them.
+// Use this when a domain was previously blocked by include rules and its frontier
+// entries are stuck in errored state — sitemap-import's INSERT OR IGNORE won't
+// re-queue them, but Recrawl will.
+func (s *pebbleHTTP) handleRecrawlSitemap(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlRecrawl == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler")
+		return
+	}
+	var req sitemapImportReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
+		writeProblem(w, http.StatusBadRequest, `expected {"url": "https://.../sitemap.xml"}`)
+		return
+	}
+	// Fetch and parse the sitemap.
+	hc := &http.Client{Timeout: 20 * time.Second}
+	resp, err := hc.Get(req.URL)
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "fetch sitemap: "+err.Error())
+		return
+	}
+	defer resp.Body.Close()
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	// Extract <loc> entries — simple text scan avoids XML namespace headaches.
+	// Handles both pretty-printed (one tag per line) and compact sitemaps
+	// where <url><loc>…</loc></url> all appear on one line.
+	t0 := time.Now()
+	var reset, skipped int
+	src := string(raw)
+	for {
+		start := strings.Index(src, "<loc>")
+		if start < 0 {
+			break
+		}
+		src = src[start+len("<loc>"):]
+		end := strings.Index(src, "</loc>")
+		if end < 0 {
+			break
+		}
+		u := strings.TrimSpace(src[:end])
+		src = src[end+len("</loc>"):]
+		if err := s.crawlRecrawl(r.Context(), u); err != nil {
+			skipped++
+		} else {
+			reset++
+		}
+	}
+	log.Printf("recrawl-sitemap: reset %d, skipped %d from %s in %s", reset, skipped, req.URL, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"sitemap": req.URL,
+		"reset":   reset,
+		"skipped": skipped,
+		"elapsed": time.Since(t0).String(),
+	})
+}
+
+// handleCrawlNow runs the fetch-parse-index pipeline synchronously for one
+// or more URLs, bypassing the persistent frontier entirely. Use this when
+// the round-robin cursor in a large frontier (10M+) would take hours to
+// reach an explicitly-seeded URL — direct fetch + UpsertDocument + chunk +
+// embed all happen in this request. Returns per-URL outcomes.
+type crawlNowReq struct {
+	URLs []string `json:"urls"`
+	URL  string   `json:"url,omitempty"`
+}
+
+func (s *pebbleHTTP) handleCrawlNow(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlFetchNow == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req crawlNowReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20))
+	if err := json.Unmarshal(body, &req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "expected {\"urls\":[...]} or {\"url\":\"...\"}")
+		return
+	}
+	urls := req.URLs
+	if req.URL != "" {
+		urls = append(urls, req.URL)
+	}
+	if len(urls) == 0 {
+		writeProblem(w, http.StatusBadRequest, "no URLs in body")
+		return
+	}
+	results := make([]map[string]any, 0, len(urls))
+	t0 := time.Now()
+	for _, u := range urls {
+		err := s.crawlFetchNow(r.Context(), u)
+		out := map[string]any{"url": u}
+		if err != nil {
+			out["error"] = err.Error()
+		} else {
+			out["ok"] = true
+		}
+		results = append(results, out)
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"results": results,
+		"elapsed": time.Since(t0).String(),
+	})
+}
+
+// handleSitePack discovers a site's sitemaps + RSS feeds and bulk-enqueues
+// everything found. Targets the "I want to index THIS site" operator flow
+// without editing cosift.json + restarting + waiting for the cursor.
+// Discovery order:
+//  1. GET https://<host>/robots.txt — look for `Sitemap:` directives
+//  2. Fallback to canonical /sitemap.xml + /sitemap_index.xml
+//  3. RSS at common paths: /feed, /feed.xml, /rss, /rss.xml, /atom.xml, /feed/atom
+//
+// Each found resource is run through the existing SeedSitemap / SeedRSS
+// primitives. Returns per-resource counts.
+type sitePackReq struct {
+	Host string `json:"host"` // e.g. "example.com" or "blog.example.com" — no scheme
+}
+
+func (s *pebbleHTTP) handleSitePack(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedSitemap == nil || s.crawlSeedRSS == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req sitePackReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"example.com\"}")
+		return
+	}
+	host, ok := normalizeBareHost(req.Host)
+	if !ok {
+		writeProblem(w, http.StatusBadRequest, "host must be a bare hostname like example.com")
+		return
+	}
+	base := "https://" + host
+	hc := &http.Client{Timeout: 20 * time.Second}
+
+	type result struct {
+		Source  string `json:"source"` // "robots-sitemap" | "fallback-sitemap" | "rss"
+		URL     string `json:"url"`
+		Indexed int    `json:"indexed"`
+		Error   string `json:"error,omitempty"`
+	}
+	results := make([]result, 0, 8)
+	t0 := time.Now()
+
+	candidateSitemaps, fromRobots := discoverSitemaps(r.Context(), hc, base)
+	for _, su := range candidateSitemaps {
+		n, err := s.crawlSeedSitemap(r.Context(), su)
+		res := result{URL: su, Indexed: n}
+		if fromRobots {
+			res.Source = "robots-sitemap"
+		} else {
+			res.Source = "fallback-sitemap"
+		}
+		if err != nil {
+			res.Error = err.Error()
+		}
+		results = append(results, res)
+	}
+	// Step 3: common RSS paths.
+	for _, p := range []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/feed/atom"} {
+		fu := base + p
+		n, err := s.crawlSeedRSS(r.Context(), fu)
+		// Silently skip RSS paths that 404 / don't parse — most sites have
+		// only one of the listed paths.
+		if err != nil && n == 0 {
+			continue
+		}
+		results = append(results, result{Source: "rss", URL: fu, Indexed: n})
+	}
+
+	total := 0
+	for _, r := range results {
+		total += r.Indexed
+	}
+	log.Printf("site-pack: %s discovered %d resources, enqueued %d URLs in %s",
+		host, len(results), total, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"host":          host,
+		"resources":     len(results),
+		"total_indexed": total,
+		"elapsed":       time.Since(t0).String(),
+		"results":       results,
+	})
+}
+
+// normalizeBareHost strips scheme/trailing-slash from a host or URL and
+// returns the bare hostname. ok is false when the result still contains a
+// path segment (so callers can reject "example.com/foo" as a host).
+func normalizeBareHost(s string) (host string, ok bool) {
+	host = strings.TrimSpace(s)
+	host = strings.TrimPrefix(host, "https://")
+	host = strings.TrimPrefix(host, "http://")
+	host = strings.TrimSuffix(host, "/")
+	host = strings.ToLower(host)
+	if host == "" || strings.Contains(host, "/") {
+		return "", false
+	}
+	return host, true
+}
+
+// discoverSitemaps returns candidate sitemap URLs for a site, given its base
+// origin (e.g. "https://example.com"). It prefers Sitemap: directives in
+// robots.txt; when robots.txt yields none it falls back to a small ordered
+// list of canonical/CMS paths. fromRobots reports which source was used.
+func discoverSitemaps(ctx context.Context, hc *http.Client, base string) (sitemaps []string, fromRobots bool) {
+	if hc == nil {
+		hc = &http.Client{Timeout: 20 * time.Second}
+	}
+	if req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/robots.txt", nil); err == nil {
+		if rresp, err := hc.Do(req); err == nil {
+			if rresp.StatusCode < 400 {
+				rbody, _ := io.ReadAll(io.LimitReader(rresp.Body, 2<<20))
+				for _, line := range strings.Split(string(rbody), "\n") {
+					line = strings.TrimSpace(line)
+					if strings.HasPrefix(strings.ToLower(line), "sitemap:") {
+						if val := strings.TrimSpace(line[len("sitemap:"):]); val != "" {
+							sitemaps = append(sitemaps, val)
+						}
+					}
+				}
+			}
+			rresp.Body.Close()
+		}
+	}
+	if len(sitemaps) > 0 {
+		return sitemaps, true
+	}
+	// /sitemap.xml is the canonical spec but many CMSes (WordPress, Yoast,
+	// Ghost, Hugo themes) ship at non-canonical paths. Try a small ordered
+	// list before giving up: /sitemap.xml (most common), then WordPress's
+	// /wp-sitemap.xml + Yoast's per-content-type splits, then index variants.
+	for _, p := range []string{
+		"/sitemap.xml",
+		"/wp-sitemap.xml",    // WordPress 5.5+
+		"/sitemap_index.xml", // Yoast SEO
+		"/post-sitemap.xml",  // Yoast posts
+		"/page-sitemap.xml",  // Yoast pages
+		"/sitemap-index.xml", // some CMSes hyphenate
+		"/sitemap.xml.gz",    // gzipped variant (sitemap.go handles .gz)
+	} {
+		sitemaps = append(sitemaps, base+p)
+	}
+	return sitemaps, false
+}
+
+// parseLaneName maps a friendly lane name to a frontier lane byte. The empty
+// string and unknown values default to the high-priority submitted lane,
+// which is the point of /admin/site-submit: jump a site to the front.
+func parseLaneName(s string) byte {
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "refresh":
+		return store.LaneRefresh
+	case "discovered":
+		return store.LaneDiscovered
+	case "bulk":
+		return store.LaneBulk
+	default: // "", "priority", "submitted", or anything unrecognized
+		return store.LaneSubmitted
+	}
+}
+
+func laneName(lane byte) string {
+	switch lane {
+	case store.LaneSubmitted:
+		return "submitted"
+	case store.LaneRefresh:
+		return "refresh"
+	case store.LaneDiscovered:
+		return "discovered"
+	case store.LaneBulk:
+		return "bulk"
+	default:
+		return "submitted"
+	}
+}
+
+// handleSiteSubmit discovers every URL of a website (via robots.txt sitemaps,
+// then canonical fallbacks) and pushes them all onto the live crawl frontier
+// in a chosen priority lane — by default the high-priority "submitted" lane,
+// so the whole site jumps ahead of the generic discovery backlog. Same auth
+// as the other admin endpoints. Synchronous; large sitemaps take seconds.
+//
+// Body: {"host":"pilotprotocol.network", "lane":"priority"}
+//
+//	lane: "priority" (default) | "refresh" | "discovered" | "bulk"
+type siteSubmitReq struct {
+	Host string `json:"host"`
+	Lane string `json:"lane,omitempty"`
+}
+
+func (s *pebbleHTTP) handleSiteSubmit(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedSitemapLane == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req siteSubmitReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"example.com\", \"lane\":\"priority\"}")
+		return
+	}
+	host, ok := normalizeBareHost(req.Host)
+	if !ok {
+		writeProblem(w, http.StatusBadRequest, "host must be a bare hostname like example.com")
+		return
+	}
+	lane := parseLaneName(req.Lane)
+	base := "https://" + host
+	hc := &http.Client{Timeout: 20 * time.Second}
+	t0 := time.Now()
+
+	type result struct {
+		Source string `json:"source"` // "robots-sitemap" | "fallback-sitemap"
+		URL    string `json:"url"`
+		Queued int    `json:"queued"`
+		Error  string `json:"error,omitempty"`
+	}
+	candidateSitemaps, fromRobots := discoverSitemaps(r.Context(), hc, base)
+	source := "fallback-sitemap"
+	if fromRobots {
+		source = "robots-sitemap"
+	}
+	results := make([]result, 0, len(candidateSitemaps))
+	total := 0
+	for _, su := range candidateSitemaps {
+		n, err := s.crawlSeedSitemapLane(r.Context(), su, lane)
+		res := result{Source: source, URL: su, Queued: n}
+		if err != nil {
+			res.Error = err.Error()
+		}
+		total += n
+		results = append(results, res)
+	}
+	log.Printf("site-submit: %s → lane=%s discovered %d sitemap(s), queued %d URLs in %s",
+		host, laneName(lane), len(results), total, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"host":         host,
+		"lane":         laneName(lane),
+		"sitemaps":     len(results),
+		"total_queued": total,
+		"elapsed":      time.Since(t0).String(),
+		"results":      results,
+	})
+}
+
+// handleWETImportBulk fetches a CommonCrawl `wet.paths.gz` manifest,
+// takes the first N entries (or skip+take), and runs `/admin/wet-import`
+// against each one in parallel. Lets operators bulk-ingest a release with
+// one call instead of repeatedly POSTing per file. Synchronous — blocks
+// until all N finish, returns total docs indexed.
+//
+// Example body:
+//
+//	{"manifest_url":"https://data.commoncrawl.org/crawl-data/CC-MAIN-2024-51/wet.paths.gz",
+//	 "count":4, "skip":0, "concurrency":2, "lexical_only":true}
+type wetImportBulkReq struct {
+	ManifestURL string `json:"manifest_url"`
+	Count       int    `json:"count"`
+	Skip        int    `json:"skip,omitempty"`
+	Concurrency int    `json:"concurrency,omitempty"`
+	LexicalOnly bool   `json:"lexical_only,omitempty"`
+	DedupeFresh bool   `json:"dedupe_fresh,omitempty"`
+}
+
+func (s *pebbleHTTP) handleWETImportBulk(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedWET == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req wetImportBulkReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.ManifestURL == "" || req.Count <= 0 {
+		writeProblem(w, http.StatusBadRequest, "expected {\"manifest_url\":\"https://.../wet.paths.gz\",\"count\":N}")
+		return
+	}
+	if req.Concurrency <= 0 {
+		req.Concurrency = 2
+	}
+	if req.Concurrency > 8 {
+		req.Concurrency = 8 // beyond this the pebble write lock dominates anyway
+	}
+
+	// Fetch the manifest. wet.paths.gz is a gzipped newline-delimited list
+	// of relative paths under data.commoncrawl.org.
+	mreq, err := http.NewRequestWithContext(r.Context(), http.MethodGet, req.ManifestURL, http.NoBody)
+	if err != nil {
+		writeProblem(w, http.StatusBadRequest, "build manifest request: "+err.Error())
+		return
+	}
+	mreq.Header.Set("User-Agent", "cosift-bulk-import")
+	mresp, err := (&http.Client{Timeout: 60 * time.Second}).Do(mreq)
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "fetch manifest: "+err.Error())
+		return
+	}
+	defer mresp.Body.Close()
+	if mresp.StatusCode >= 400 {
+		writeProblem(w, http.StatusBadGateway, fmt.Sprintf("manifest http %d", mresp.StatusCode))
+		return
+	}
+	zr, err := gzip.NewReader(mresp.Body)
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "gunzip manifest: "+err.Error())
+		return
+	}
+	defer zr.Close()
+	mbody, err := io.ReadAll(io.LimitReader(zr, 32<<20))
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "read manifest: "+err.Error())
+		return
+	}
+	// Each line is a path like "crawl-data/CC-MAIN-.../wet/...warc.wet.gz".
+	// Prepend the data.commoncrawl.org base.
+	all := strings.Split(strings.TrimSpace(string(mbody)), "\n")
+	if req.Skip >= len(all) {
+		writeProblem(w, http.StatusBadRequest, fmt.Sprintf("skip=%d exceeds manifest size %d", req.Skip, len(all)))
+		return
+	}
+	end := req.Skip + req.Count
+	if end > len(all) {
+		end = len(all)
+	}
+	paths := all[req.Skip:end]
+
+	// Run imports in parallel, bounded by concurrency.
+	type result struct {
+		URL     string `json:"url"`
+		Indexed int    `json:"indexed"`
+		Elapsed string `json:"elapsed"`
+		Error   string `json:"error,omitempty"`
+	}
+	results := make([]result, len(paths))
+	sem := make(chan struct{}, req.Concurrency)
+	var wg sync.WaitGroup
+	t0 := time.Now()
+	for i, p := range paths {
+		i, p := i, strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			full := "https://data.commoncrawl.org/" + p
+			start := time.Now()
+			n, err := s.crawlSeedWET(r.Context(), full, req.DedupeFresh, req.LexicalOnly)
+			results[i].URL = full
+			results[i].Indexed = n
+			results[i].Elapsed = time.Since(start).Round(time.Second).String()
+			if err != nil {
+				results[i].Error = err.Error()
+			}
+		}()
+	}
+	wg.Wait()
+
+	total := 0
+	for _, r := range results {
+		total += r.Indexed
+	}
+	log.Printf("wet-import-bulk: indexed %d docs from %d WET files in %s", total, len(paths), time.Since(t0).Round(time.Second))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"manifest_url":  req.ManifestURL,
+		"files":         len(paths),
+		"total_indexed": total,
+		"elapsed":       time.Since(t0).String(),
+		"results":       results,
+	})
+}
+
+// handleWETImport streams a CommonCrawl WET file (gzipped, pre-extracted
+// plain text per URL) and runs each record through UpsertDocument + BM25
+// indexing + chunk + embed. Bypasses the fetch-and-parse pipeline entirely
+// because WET bodies are already extracted text — typically 50-100× faster
+// than open-web crawling.
+//
+// Example: POST {"url":"https://data.commoncrawl.org/crawl-data/CC-MAIN-2025-09/segments/.../wet/CC-MAIN-...wet.gz"}
+type wetImportReq struct {
+	URL         string `json:"url"`
+	DedupeFresh bool   `json:"dedupe_fresh,omitempty"` // skip URLs already-fresh in famDoc
+	LexicalOnly bool   `json:"lexical_only,omitempty"` // BM25-only ingest, defer dense embed to /admin/embed-backfill
+}
+
+func (s *pebbleHTTP) handleWETImport(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedWET == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req wetImportReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"url\":\"https://.../...warc.wet.gz\"}")
+		return
+	}
+	t0 := time.Now()
+	n, err := s.crawlSeedWET(r.Context(), req.URL, req.DedupeFresh, req.LexicalOnly)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	log.Printf("wet-import: indexed %d docs from %s in %s", n, req.URL, time.Since(t0).Round(time.Second))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"wet":     req.URL,
+		"indexed": n,
+		"elapsed": time.Since(t0).String(),
+	})
+}
+
+// handleFrontierDemoteHost re-keys every queued URL for a host into a
+// different lane. The escape hatch for the cloud.google.com problem:
+// 2.8M queued URLs on one host (65% of the queue) starve host-fair
+// claim slots from fresher lanes. Demote to lane 3 (bulk, 5% weight)
+// and lane 1/2 actually get the work.
+//
+// POST body: {"host": "cloud.google.com", "lane": 3}
+type frontierDemoteHostReq struct {
+	Host string `json:"host"`
+	Lane int    `json:"lane"`
+}
+
+func (s *pebbleHTTP) handleFrontierDemoteHost(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	ps, ok := any(s.store).(*store.PebbleStore)
+	if !ok {
+		writeProblem(w, http.StatusNotImplemented, "lanes are PebbleStore-only")
+		return
+	}
+	var req frontierDemoteHostReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.Host == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"host\":\"foo.com\",\"lane\":0..3}")
+		return
+	}
+	if req.Lane < 0 || req.Lane > 3 {
+		writeProblem(w, http.StatusBadRequest, "lane must be 0..3")
+		return
+	}
+	t0 := time.Now()
+	n, err := ps.DemoteHostToLane(r.Context(), req.Host, byte(req.Lane))
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	log.Printf("frontier-demote-host: moved %d URLs (%s -> lane %d) in %s", n, req.Host, req.Lane, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"host":    req.Host,
+		"lane":    req.Lane,
+		"moved":   n,
+		"elapsed": time.Since(t0).String(),
+	})
+}
+
+// handleFrontierPurgeStaleInFlight clears the stale 'i' secondary keys
+// left over from the pre-fix RecoverInFlight bug. Pre-fix, every restart
+// re-queued in-flight URLs via the LEGACY 'q' index only and skipped the
+// lane-aware 'i' delete, so each restart leaked the URL's lane-aware 'i'
+// key. GetLaneStats then reported impossibly-high in_flight counts
+// (>max_concurrent). Idempotent — re-running is a no-op once clean.
+func (s *pebbleHTTP) handleFrontierPurgeStaleInFlight(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	ps, ok := any(s.store).(*store.PebbleStore)
+	if !ok {
+		writeProblem(w, http.StatusNotImplemented, "PebbleStore-only")
+		return
+	}
+	t0 := time.Now()
+	n, err := ps.PurgeStaleInFlight(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	log.Printf("frontier-purge-stale-inflight: purged %d keys in %s", n, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{"purged": n, "elapsed": time.Since(t0).String()})
+}
+
+// handleRSSImport fetches an RSS 2.0 or Atom feed and pushes every <item>/
+// <entry> link to the live frontier. Same auth shape as sitemap-import.
+// Designed to be cron-friendly: idempotent against the frontier (re-seeding
+// the same feed only adds newly-listed items).
+type rssImportReq struct {
+	URL string `json:"url"`
+}
+
+func (s *pebbleHTTP) handleRSSImport(w http.ResponseWriter, r *http.Request) {
+	if want := s.cluster.PeerAuthToken; want != "" {
+		got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+		if got != want {
+			writeProblem(w, http.StatusUnauthorized, "missing or invalid admin token")
+			return
+		}
+	}
+	if s.crawlSeedRSS == nil {
+		writeProblem(w, http.StatusNotImplemented, "this shard has no in-serve crawler (-crawl-seeds-file not set)")
+		return
+	}
+	var req rssImportReq
+	body, _ := io.ReadAll(io.LimitReader(r.Body, 64<<10))
+	if err := json.Unmarshal(body, &req); err != nil || req.URL == "" {
+		writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"https://.../feed.xml\"}")
+		return
+	}
+	t0 := time.Now()
+	n, err := s.crawlSeedRSS(r.Context(), req.URL)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	log.Printf("rss-import: queued %d URLs from %s in %s", n, req.URL, time.Since(t0).Round(time.Millisecond))
+	writeJSON(w, http.StatusOK, map[string]any{
+		"feed":    req.URL,
+		"queued":  n,
+		"elapsed": time.Since(t0).String(),
+	})
+}
diff --git a/cmd/cosift/serve_helpers.go b/cmd/cosift/serve_helpers.go
new file mode 100644
index 0000000..38f38ac
--- /dev/null
+++ b/cmd/cosift/serve_helpers.go
@@ -0,0 +1,496 @@
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// retrievalFilters bundles the four post-retrieval predicates /search,
+// /answer, /find_similar, and /research all share. Centralized in
+// so /research's sync + stream paths stay in lockstep.
+type retrievalFilters struct {
+	include    []string
+	exclude    []string
+	sites      []siteScope
+	since      time.Time
+	until      time.Time
+	dateActive bool
+}
+
+func parseRetrievalFilters(r *http.Request) (retrievalFilters, error) {
+	f := retrievalFilters{
+		include: splitDomainsCSV(r.URL.Query().Get("include_domains")),
+		exclude: splitDomainsCSV(r.URL.Query().Get("exclude_domains")),
+		sites:   parseSiteScopes(r.URL.Query().Get("site")),
+	}
+	since, err := parseDateBound(r.URL.Query().Get("since"))
+	if err != nil {
+		return f, fmt.Errorf("since: %w", err)
+	}
+	until, err := parseDateBound(r.URL.Query().Get("until"))
+	if err != nil {
+		return f, fmt.Errorf("until: %w", err)
+	}
+	f.since = since
+	f.until = until
+	f.dateActive = !since.IsZero() || !until.IsZero()
+	return f, nil
+}
+
+// allow reports whether (url, publishedAt) clears the filter. publishedAt
+// can be the zero Time when the caller doesn't have it loaded; zero is
+// treated as "unknown" and dropped under any date filter.
+func (f retrievalFilters) allow(rawURL string, publishedAt time.Time) bool {
+	if len(f.include) > 0 || len(f.exclude) > 0 {
+		host := hostOf(rawURL)
+		if len(f.include) > 0 && !matchesAnyDomain(host, f.include) {
+			return false
+		}
+		if len(f.exclude) > 0 && matchesAnyDomain(host, f.exclude) {
+			return false
+		}
+	}
+	if len(f.sites) > 0 && !matchesAnySite(rawURL, f.sites) {
+		return false
+	}
+	if f.dateActive {
+		if publishedAt.IsZero() {
+			return false
+		}
+		if !f.since.IsZero() && publishedAt.Before(f.since) {
+			return false
+		}
+		if !f.until.IsZero() && publishedAt.After(f.until) {
+			return false
+		}
+	}
+	return true
+}
+
+func parseSubQueries(raw, fallback string) []string {
+	raw = strings.TrimSpace(raw)
+	for _, fence := range []string{"```json", "```"} {
+		raw = strings.TrimPrefix(raw, fence)
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	raw = strings.TrimSpace(raw)
+	startIdx := strings.Index(raw, "[")
+	endIdx := strings.LastIndex(raw, "]")
+	if startIdx >= 0 && endIdx > startIdx {
+		var arr []string
+		if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &arr); err == nil && len(arr) > 0 {
+			return arr
+		}
+	}
+	return []string{fallback}
+}
+
+func splitDomainsCSV(s string) []string {
+	if s == "" {
+		return nil
+	}
+	parts := strings.Split(s, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		t := strings.ToLower(strings.TrimSpace(p))
+		if t != "" {
+			out = append(out, t)
+		}
+	}
+	return out
+}
+
+func matchesAnyDomain(host string, patterns []string) bool {
+	host = strings.ToLower(host)
+	for _, p := range patterns {
+		if host == p || strings.HasSuffix(host, "."+p) {
+			return true
+		}
+	}
+	return false
+}
+
+func hostOf(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}
+
+// siteScope is one entry of the `site` search filter: a host (suffix-matched
+// on dot boundaries, same as include_domains) plus an optional URL path
+// prefix. A zero path matches the whole host; a non-empty path scopes results
+// to a section of the site (e.g. host "pilotprotocol.network" + path "/docs").
+type siteScope struct {
+	host string
+	path string // normalized, no trailing slash; "" = any path
+}
+
+// parseSiteScopes parses the `site` parameter — a CSV of host or host/path
+// (or full-URL) entries — into scopes. Examples of a single entry:
+//
+//	pilotprotocol.network              → whole host (and subdomains)
+//	pilotprotocol.network/docs         → only URLs under /docs
+//	https://pilotprotocol.network/docs → same (scheme tolerated)
+func parseSiteScopes(csv string) []siteScope {
+	if csv == "" {
+		return nil
+	}
+	var out []siteScope
+	for _, raw := range strings.Split(csv, ",") {
+		t := strings.TrimSpace(raw)
+		if t == "" {
+			continue
+		}
+		t = strings.TrimPrefix(t, "https://")
+		t = strings.TrimPrefix(t, "http://")
+		host, path := t, ""
+		if i := strings.IndexByte(t, '/'); i >= 0 {
+			host, path = t[:i], t[i:]
+		}
+		host = strings.ToLower(strings.TrimSpace(host))
+		path = strings.TrimRight(strings.TrimSpace(path), "/")
+		if host == "" {
+			continue
+		}
+		out = append(out, siteScope{host: host, path: path})
+	}
+	return out
+}
+
+// matchesAnySite reports whether rawURL falls within any of the scopes. Host
+// matching reuses include_domains semantics (exact or dot-boundary suffix);
+// path matching is a segment-boundary prefix so "/docs" matches "/docs" and
+// "/docs/x" but not "/docsearch".
+func matchesAnySite(rawURL string, scopes []siteScope) bool {
+	if len(scopes) == 0 {
+		return true
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	p := u.Path
+	for _, sc := range scopes {
+		if host != sc.host && !strings.HasSuffix(host, "."+sc.host) {
+			continue
+		}
+		if sc.path == "" || p == sc.path || strings.HasPrefix(p, sc.path+"/") {
+			return true
+		}
+	}
+	return false
+}
+
+// parseDateBound accepts the same forms as the SQLite-side server: empty
+// (zero time), RFC3339 ("2026-01-15T00:00:00Z"), or a bare date
+// ("2026-01-15", treated as UTC midnight).
+func parseDateBound(s string) (time.Time, error) {
+	if s == "" {
+		return time.Time{}, nil
+	}
+	if t, err := time.Parse(time.RFC3339, s); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse("2006-01-02", s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("expected YYYY-MM-DD or RFC3339, got %q", s)
+}
+
+// textExcerpt returns a body-prefix excerpt no longer than maxBytes, cut at a
+// word boundary if one is available within the trailing 16 bytes.
+// Static prefix (not query-aware) — matches the SQLite-side DocMeta.Excerpt
+// shape; a query-aware passage extractor is the path on the index
+// hot path and lives there, not in the HTTP handler.
+func textExcerpt(text string, maxBytes int) string {
+	if len(text) <= maxBytes {
+		return text
+	}
+	cut := text[:maxBytes]
+	for i := len(cut) - 1; i >= len(cut)-16 && i > 0; i-- {
+		if cut[i] == ' ' || cut[i] == '\n' || cut[i] == '\t' {
+			cut = cut[:i]
+			break
+		}
+	}
+	return strings.TrimSpace(cut) + "…"
+}
+
+// runPebbleInfo prints Pebble's built-in metrics for the store at -dir.
+// openPebbleOrFriendlyErr wraps store.OpenPebble with a more
+// helpful error message when the underlying failure is lock contention (a
+// pebble-serve / live crawl is holding the single-writer lock). Callers
+// using offline CLI subcommands (pebble-info, verify) hit this when run
+// during an active deployment.
+func openPebbleOrFriendlyErr(d string) (*store.PebbleStore, error) {
+	ps, err := store.OpenPebble(d)
+	if err != nil {
+		msg := err.Error()
+		if strings.Contains(msg, "lock") || strings.Contains(msg, "resource temporarily unavailable") {
+			return nil, fmt.Errorf("open pebble at %s: writer lock is held by another process (pebble-serve / crawl in flight); stop the running service first", d)
+		}
+		return nil, fmt.Errorf("open pebble at %s: %w", d, err)
+	}
+	return ps, nil
+}
+
+// runVerifyViaServer GETs pebble-serve's /verify endpoint and
+// renders the JSON body in the same shape runVerifyPebble's local path emits.
+// Lets `cosift verify -server URL` work against a running pebble-serve while
+// the writer lock is held by the crawl / serve process.
+func runVerifyViaServer(ctx context.Context, serverURL string, asJSON bool) error {
+	endpoint := strings.TrimRight(serverURL, "/") + "/verify"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("GET %s: %w", endpoint, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if asJSON {
+		fmt.Println(string(body))
+	} else {
+		var d struct {
+			OK                 bool  `json:"ok"`
+			IndexedDocsCounter int64 `json:"indexed_docs_counter"`
+			IndexedDocsScan    int64 `json:"indexed_docs_scan"`
+			IndexedDocsDrift   int64 `json:"indexed_docs_drift"`
+			SumDocLenCounter   int64 `json:"sum_doc_len_counter"`
+			SumDocLenScan      int64 `json:"sum_doc_len_scan"`
+			SumDocLenDrift     int64 `json:"sum_doc_len_drift"`
+		}
+		if err := json.Unmarshal(body, &d); err != nil {
+			return fmt.Errorf("decode %s: %w", endpoint, err)
+		}
+		fmt.Printf("pebble-serve: %s\n\n", serverURL)
+		fmt.Printf("  indexed_docs (counter): %d\n", d.IndexedDocsCounter)
+		fmt.Printf("  indexed_docs (scan):    %d\n", d.IndexedDocsScan)
+		fmt.Printf("  sum_doc_len  (counter): %d\n", d.SumDocLenCounter)
+		fmt.Printf("  sum_doc_len  (scan):    %d\n", d.SumDocLenScan)
+		if d.OK {
+			fmt.Println("\nOK: counters match the 'l' family scan.")
+		} else {
+			fmt.Printf("\nDRIFT: indexed_docs Δ=%+d, sum_doc_len Δ=%+d\n", d.IndexedDocsDrift, d.SumDocLenDrift)
+		}
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("counter drift detected (server returned %d)", resp.StatusCode)
+	}
+	return nil
+}
+
+// pebbleInfoJSON shapes the -json output of `cosift pebble-info`. Extracted
+// in so the shape can be unit-tested without spawning a subprocess.
+// Mirrors the /stats response — same jq filters compose against either source.
+func pebbleInfoJSON(path string, st store.Stats, sumLen, indexedDocs int64, meta index.HNSWMeta, hnswOK bool) map[string]any {
+	retrievers := []string{"bm25", "bm25-mlt"}
+	if hnswOK {
+		retrievers = append(retrievers, "dense", "hybrid")
+	}
+	out := map[string]any{
+		"path":         path,
+		"documents":    st.Documents,
+		"indexed_docs": indexedDocs,
+		"sum_doc_len":  sumLen,
+		"hnsw_loaded":  hnswOK,
+		"retrievers":   retrievers,
+	}
+	if indexedDocs > 0 {
+		out["avg_doc_len"] = float64(sumLen) / float64(indexedDocs)
+	}
+	if hnswOK {
+		out["vector_nodes"] = meta.NodeCount
+		out["vector_dim"] = meta.Dim
+	}
+	return out
+}
+
+// operator visibility into LSM levels, WAL state, on-disk
+// size, and compaction queue, surfaced via pebble.Metrics().String().
+func runPebbleInfo(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("pebble-info", flag.ExitOnError)
+	dir := fs.String("dir", "", "PebbleStore directory (defaults to <cfg.DataDir>/pebble)")
+	// Mirrors
+	// the shape /stats returns ( retrievers list,'s
+	// vector meta) so the same jq filters compose against either source.
+	asJSON := fs.Bool("json", false, "emit JSON instead of human-readable text (no pebble.Metrics dump)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	d := *dir
+	if d == "" {
+		d = filepath.Join(cfg.DataDir, "pebble")
+	}
+	ps, err := openPebbleOrFriendlyErr(d)
+	if err != nil {
+		return err
+	}
+	defer ps.Close()
+
+	st, err := ps.Stats(ctx)
+	if err != nil {
+		return err
+	}
+	sumLen, indexedDocs, _ := ps.CorpusStats(ctx)
+	meta, hnswOK, _ := index.LoadHNSWMeta(ctx, ps)
+
+	if *asJSON {
+		out := pebbleInfoJSON(d, st, sumLen, indexedDocs, meta, hnswOK)
+		blob, _ := json.MarshalIndent(out, "", "  ")
+		fmt.Println(string(blob))
+		return nil
+	}
+
+	fmt.Printf("PebbleStore: %s\n\n", d)
+	fmt.Printf("  documents:    %d\n", st.Documents)
+	// pebble-info already needs
+	// to read the store; reading them costs O(1) and gives operators 'is this
+	// store healthy' without round-tripping through pebble-serve /stats.
+	if indexedDocs > 0 {
+		fmt.Printf("  indexed_docs: %d\n", indexedDocs)
+		fmt.Printf("  sum_doc_len:  %d\n", sumLen)
+		fmt.Printf("  avg_doc_len:  %.2f\n", float64(sumLen)/float64(indexedDocs))
+	}
+	// when an HNSW meta blob is persisted, surface dim+nodes here
+	// too. Same cheap 20-byte read pebble-serve does at startup.
+	// Operators running pebble-info to inspect an offline store get the dense
+	// shape without opening pebble-serve.
+	// bm25/bm25-mlt
+	// always. dense/hybrid need the graph (meta present) AND, for non-URL
+	// modes, a runtime embedder — offline we can only check the persisted
+	// shape, so we report 'available with embedder' rather than asserting.
+	if hnswOK {
+		fmt.Printf("  vector_nodes: %d\n", meta.NodeCount)
+		fmt.Printf("  vector_dim:   %d\n", meta.Dim)
+	}
+	fmt.Println()
+	fmt.Println("--- retrievers ---")
+	fmt.Println("  bm25, bm25-mlt: always (BM25 index)")
+	if hnswOK {
+		fmt.Println("  dense:          available — /find_similar?url= works without embedder;")
+		fmt.Println("                  /search /answer /research need an embedder configured at server start")
+		fmt.Println("  hybrid:         available (same requirements as dense)")
+	} else {
+		fmt.Println("  dense, hybrid:  unavailable — no HNSW graph persisted in this store")
+		fmt.Println("                  (build embeddings during crawl with cfg.Embeddings.Model set)")
+	}
+	fmt.Println()
+	fmt.Println("--- pebble.Metrics ---")
+	fmt.Println(ps.Metrics().String())
+	return nil
+}
+
+// verify the running counters ('m'+indexed_docs,
+// 'm'+sum_doc_len) against an authoritative scan of the 'l' family.
+// Drift would mean a crash or bug left the counter inconsistent —
+// flagged here so it can be re-derived from the scan.
+func runVerifyPebble(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("verify", flag.ExitOnError)
+	dir := fs.String("dir", "", "PebbleStore directory (defaults to <cfg.DataDir>/pebble)")
+	// machine-readable output for CI integration.
+	asJSON := fs.Bool("json", false, "emit JSON report instead of human text (suitable for jq / CI)")
+	// -server URL routes the check through a running pebble-serve's
+	// /verify endpoint instead of opening the store directly. Useful when a
+	// crawl or pebble-serve is holding the writer lock.
+	serverURL := fs.String("server", "", "pebble-serve URL — when set, GETs /verify instead of opening the store")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *serverURL != "" {
+		return runVerifyViaServer(ctx, *serverURL, *asJSON)
+	}
+	d := *dir
+	if d == "" {
+		d = filepath.Join(cfg.DataDir, "pebble")
+	}
+	ps, err := openPebbleOrFriendlyErr(d)
+	if err != nil {
+		return err
+	}
+	defer ps.Close()
+
+	counterSum, counterCount, err := ps.CorpusStats(ctx)
+	if err != nil {
+		return fmt.Errorf("read counters: %w", err)
+	}
+	scanSum, scanCount, err := ps.SumDocLengths(ctx)
+	if err != nil {
+		return fmt.Errorf("scan 'l' family: %w", err)
+	}
+
+	driftCount := counterCount - scanCount
+	driftSum := counterSum - scanSum
+	ok := driftCount == 0 && driftSum == 0
+
+	if *asJSON {
+		out := map[string]any{
+			"path":                 d,
+			"ok":                   ok,
+			"indexed_docs_counter": counterCount,
+			"indexed_docs_scan":    scanCount,
+			"indexed_docs_drift":   driftCount,
+			"sum_doc_len_counter":  counterSum,
+			"sum_doc_len_scan":     scanSum,
+			"sum_doc_len_drift":    driftSum,
+		}
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(out)
+		if !ok {
+			return fmt.Errorf("counter drift detected")
+		}
+		return nil
+	}
+
+	fmt.Printf("PebbleStore: %s\n\n", d)
+	fmt.Printf("  indexed_docs (counter): %d\n", counterCount)
+	fmt.Printf("  indexed_docs (scan):    %d\n", scanCount)
+	fmt.Printf("  sum_doc_len  (counter): %d\n", counterSum)
+	fmt.Printf("  sum_doc_len  (scan):    %d\n", scanSum)
+	if ok {
+		fmt.Println("\nOK: counters match the 'l' family scan.")
+		return nil
+	}
+	fmt.Printf("\nDRIFT: indexed_docs Δ=%+d, sum_doc_len Δ=%+d\n", driftCount, driftSum)
+	return fmt.Errorf("counter drift detected")
+}
+
+func writeJSON(w http.ResponseWriter, status int, body any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(body)
+}
+
+func writeProblem(w http.ResponseWriter, status int, detail string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(map[string]any{
+		"error":  http.StatusText(status),
+		"status": status,
+		"detail": detail,
+	})
+}
diff --git a/cmd/cosift/serve_search.go b/cmd/cosift/serve_search.go
new file mode 100644
index 0000000..2042753
--- /dev/null
+++ b/cmd/cosift/serve_search.go
@@ -0,0 +1,2521 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/qexpand"
+	"github.com/pilot-protocol/cosift/internal/rerank"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// forwardURLToPeer POSTs a URL to peer's /admin/crawl-enqueue. Best-effort:
+// caller logs failures and the URL is dropped — no retry or persistent queue.
+var forwardHTTP = &http.Client{Timeout: 10 * time.Second}
+
+func (s *pebbleHTTP) forwardURLToPeer(ctx context.Context, rawURL, peerAddr string) error {
+	body, _ := json.Marshal(crawlEnqueueReq{URL: rawURL})
+	// peerAddr is host:port; assume http inside the cluster (mTLS / VPN
+	// would be a wrapper concern). Switch to https://... if peers expose TLS.
+	endpoint := "http://" + peerAddr + "/admin/crawl-enqueue"
+	if strings.HasPrefix(peerAddr, "http://") || strings.HasPrefix(peerAddr, "https://") {
+		endpoint = strings.TrimRight(peerAddr, "/") + "/admin/crawl-enqueue"
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t := s.cluster.PeerAuthToken; t != "" {
+		req.Header.Set("Authorization", "Bearer "+t)
+	}
+	resp, err := forwardHTTP.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 300 {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+		return fmt.Errorf("peer %s returned %d: %s", peerAddr, resp.StatusCode, b)
+	}
+	return nil
+}
+
+// scatterSearch fans /search out to every peer with the given params,
+// collects per-peer hit lists, RRF-merges them, and returns the merged
+// top-k. Used by /search, /answer, /research gateway modes.
+// includeText forces text inlining on the scatter so callers that need
+// passages (synth endpoints) get them in one round trip.
+func (s *pebbleHTTP) scatterSearch(ctx context.Context, q string, k int, perPeerK int, params url.Values, includeText bool) (hits []searchHit, warns []string, totalCandidates int) {
+	srcQ := url.Values{}
+	for kk, vv := range params {
+		srcQ[kk] = vv
+	}
+	srcQ.Set("q", q)
+	srcQ.Set("k", strconv.Itoa(perPeerK))
+	srcQ.Set("cluster_local", "1")
+	if includeText {
+		srcQ.Set("include_text", "true")
+	}
+	queryStr := srcQ.Encode()
+
+	type peerResult struct {
+		peerIdx int
+		hits    []searchHit
+		err     error
+	}
+	resCh := make(chan peerResult, len(s.cluster.Peers))
+	httpClient := &http.Client{Timeout: 15 * time.Second}
+	for i, peer := range s.cluster.Peers {
+		if peer == "" {
+			continue
+		}
+		i, peer := i, peer
+		go func() {
+			endpoint := "http://" + peer + "/search?" + queryStr
+			if strings.HasPrefix(peer, "http://") || strings.HasPrefix(peer, "https://") {
+				endpoint = strings.TrimRight(peer, "/") + "/search?" + queryStr
+			}
+			req, _ := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
+			if t := s.cluster.PeerAuthToken; t != "" {
+				req.Header.Set("Authorization", "Bearer "+t)
+			}
+			resp, err := httpClient.Do(req)
+			if err != nil {
+				resCh <- peerResult{peerIdx: i, err: err}
+				return
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode >= 300 {
+				resCh <- peerResult{peerIdx: i, err: fmt.Errorf("peer %s returned %d", peer, resp.StatusCode)}
+				return
+			}
+			var sr struct {
+				Hits []searchHit `json:"hits"`
+			}
+			body, _ := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
+			if err := json.Unmarshal(body, &sr); err != nil {
+				resCh <- peerResult{peerIdx: i, err: err}
+				return
+			}
+			resCh <- peerResult{peerIdx: i, hits: sr.Hits}
+		}()
+	}
+	lists := [][]index.Hit{}
+	hitFull := map[string]searchHit{}
+gather:
+	for i := 0; i < len(s.cluster.Peers); i++ {
+		if s.cluster.Peers[i] == "" {
+			continue
+		}
+		select {
+		case res := <-resCh:
+			if res.err != nil {
+				warns = append(warns, fmt.Sprintf("peer %d (%s) failed: %v", res.peerIdx, s.cluster.Peers[res.peerIdx], res.err))
+				continue
+			}
+			totalCandidates += len(res.hits)
+			peerList := make([]index.Hit, len(res.hits))
+			for j, h := range res.hits {
+				peerList[j] = index.Hit{URL: h.URL, Title: h.Title, Score: h.Score}
+				if _, ok := hitFull[h.URL]; !ok {
+					hitFull[h.URL] = h
+				}
+			}
+			lists = append(lists, peerList)
+		case <-ctx.Done():
+			warns = append(warns, "client context cancelled before all peers responded")
+			break gather
+		}
+	}
+	fused := rrfFuse(lists)
+	if len(fused) > k {
+		fused = fused[:k]
+	}
+	hits = make([]searchHit, 0, len(fused))
+	for _, f := range fused {
+		full, ok := hitFull[f.URL]
+		if !ok {
+			full = searchHit{URL: f.URL, Title: f.Title}
+		}
+		full.Score = f.Score
+		hits = append(hits, full)
+	}
+	return hits, warns, totalCandidates
+}
+
+// handleSearchGateway is the scatter-gather entry. It fans out the
+// search to every peer's /search (including its own shard via the peers[]
+// table), each peer over-fetches k*2 candidates locally, gateway RRF-merges
+// the per-peer lists, and returns the top-k. Slow / failing peers are
+// included in a warnings[] entry but don't block the response.
+func (s *pebbleHTTP) handleSearchGateway(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	k := 10
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
+			k = n
+		}
+	}
+	perPeerK := k * 2
+	if perPeerK < 20 {
+		perPeerK = 20
+	}
+	// delegated to shared scatterSearch helper.
+	hits, warns, total := s.scatterSearch(r.Context(), q, k, perPeerK, r.URL.Query(), r.URL.Query().Get("include_text") == "true")
+	resp := searchResponse{
+		Query:           q,
+		Retriever:       fmt.Sprintf("gateway:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
+		Hits:            hits,
+		TotalCandidates: total,
+		Warnings:        append(warns, s.warningsFor(r)...),
+		Took:            time.Since(start).String(),
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// numNonEmpty counts non-empty entries in a peer list (peers[i]="" means
+// "skip", typically MyShardID).
+func numNonEmpty(ps []string) int {
+	n := 0
+	for _, p := range ps {
+		if p != "" {
+			n++
+		}
+	}
+	return n
+}
+
+// handleFindSimilarGateway scatters /find_similar to every shard. The
+// owning shard (URL-mode) or every shard (text-mode) produces real
+// neighbors; others return empty. Gateway RRF-merges.
+func (s *pebbleHTTP) handleFindSimilarGateway(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	k := 10
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
+			k = n
+		}
+	}
+	perPeerK := k * 2
+	if perPeerK < 20 {
+		perPeerK = 20
+	}
+	srcQ := url.Values{}
+	for kk, vv := range r.URL.Query() {
+		srcQ[kk] = vv
+	}
+	srcQ.Set("k", strconv.Itoa(perPeerK))
+	srcQ.Set("cluster_local", "1")
+	queryStr := srcQ.Encode()
+
+	type peerResult struct {
+		peerIdx int
+		hits    []searchHit
+		err     error
+	}
+	resCh := make(chan peerResult, len(s.cluster.Peers))
+	httpClient := &http.Client{Timeout: 15 * time.Second}
+	for i, peer := range s.cluster.Peers {
+		if peer == "" {
+			continue
+		}
+		i, peer := i, peer
+		go func() {
+			endpoint := "http://" + peer + "/find_similar?" + queryStr
+			if strings.HasPrefix(peer, "http://") || strings.HasPrefix(peer, "https://") {
+				endpoint = strings.TrimRight(peer, "/") + "/find_similar?" + queryStr
+			}
+			req, _ := http.NewRequestWithContext(r.Context(), http.MethodGet, endpoint, http.NoBody)
+			if t := s.cluster.PeerAuthToken; t != "" {
+				req.Header.Set("Authorization", "Bearer "+t)
+			}
+			resp, err := httpClient.Do(req)
+			if err != nil {
+				resCh <- peerResult{peerIdx: i, err: err}
+				return
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode >= 300 {
+				resCh <- peerResult{peerIdx: i, err: fmt.Errorf("peer %s returned %d", peer, resp.StatusCode)}
+				return
+			}
+			var sr struct {
+				Hits []searchHit `json:"hits"`
+			}
+			body, _ := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
+			if err := json.Unmarshal(body, &sr); err != nil {
+				resCh <- peerResult{peerIdx: i, err: err}
+				return
+			}
+			resCh <- peerResult{peerIdx: i, hits: sr.Hits}
+		}()
+	}
+	lists := [][]index.Hit{}
+	hitFull := map[string]searchHit{}
+	warns := []string{}
+	total := 0
+gather:
+	for i := 0; i < len(s.cluster.Peers); i++ {
+		if s.cluster.Peers[i] == "" {
+			continue
+		}
+		select {
+		case res := <-resCh:
+			if res.err != nil {
+				warns = append(warns, fmt.Sprintf("peer %d (%s) failed: %v", res.peerIdx, s.cluster.Peers[res.peerIdx], res.err))
+				continue
+			}
+			total += len(res.hits)
+			peerList := make([]index.Hit, len(res.hits))
+			for j, h := range res.hits {
+				peerList[j] = index.Hit{URL: h.URL, Title: h.Title, Score: h.Score}
+				if _, ok := hitFull[h.URL]; !ok {
+					hitFull[h.URL] = h
+				}
+			}
+			lists = append(lists, peerList)
+		case <-r.Context().Done():
+			warns = append(warns, "client context cancelled")
+			break gather
+		}
+	}
+	fused := rrfFuse(lists)
+	if len(fused) > k {
+		fused = fused[:k]
+	}
+	out := make([]searchHit, 0, len(fused))
+	for _, f := range fused {
+		full, ok := hitFull[f.URL]
+		if !ok {
+			full = searchHit{URL: f.URL, Title: f.Title}
+		}
+		full.Score = f.Score
+		out = append(out, full)
+	}
+	writeJSON(w, http.StatusOK, searchResponse{
+		Query:           r.URL.Query().Get("url"),
+		Retriever:       fmt.Sprintf("gateway:find_similar:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
+		Hits:            out,
+		TotalCandidates: total,
+		Warnings:        append(warns, s.warningsFor(r)...),
+		Took:            time.Since(start).String(),
+	})
+}
+
+// handleResearchGateway runs the planner on the gateway, scatters
+// each sub-query to peers (collecting RRF-merged hits with text), then
+// runs a single research synth.
+func (s *pebbleHTTP) handleResearchGateway(w http.ResponseWriter, r *http.Request) {
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented, "/research requires cfg.Chat.Model on the gateway")
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	k := 8
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
+			k = n
+		}
+	}
+	// 1. Plan via chat.
+	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: researchPlanPrompt},
+		{Role: "user", Content: q},
+	})
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
+		return
+	}
+	subs := parseSubQueries(planRaw, q)
+	if len(subs) > 5 {
+		subs = subs[:5]
+	}
+	// 2. Scatter per sub-query. Dedup by URL, keep best fused score.
+	type ranked struct {
+		score float64
+		hit   searchHit
+	}
+	best := make(map[string]ranked, k*len(subs))
+	allWarns := []string{}
+	perSub := k * 2
+	for _, sq := range subs {
+		hits, warns, _ := s.scatterSearch(r.Context(), sq, perSub, perSub, r.URL.Query(), true)
+		allWarns = append(allWarns, warns...)
+		for _, h := range hits {
+			if prev, ok := best[h.URL]; !ok || h.Score > prev.score {
+				best[h.URL] = ranked{score: h.Score, hit: h}
+			}
+		}
+	}
+	pooled := make([]ranked, 0, len(best))
+	for _, v := range best {
+		pooled = append(pooled, v)
+	}
+	sort.Slice(pooled, func(i, j int) bool { return pooled[i].score > pooled[j].score })
+	if len(pooled) > k {
+		pooled = pooled[:k]
+	}
+	// 3. Build prompt.
+	sources := make([]answerSource, 0, len(pooled))
+	var promptSources strings.Builder
+	for i, p := range pooled {
+		src := answerSource{ID: i + 1, URL: p.hit.URL, Title: p.hit.Title, Excerpt: p.hit.Excerpt, Author: p.hit.Author, PublishedAt: p.hit.PublishedAt}
+		text := p.hit.Text
+		if text == "" {
+			text = p.hit.Excerpt
+		}
+		sources = append(sources, src)
+		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, p.hit.Title, p.hit.URL, text)
+	}
+	if len(sources) == 0 {
+		writeJSON(w, http.StatusOK, researchResponse{
+			Query: q, Plan: subs, Answer: "No matching sources across the cluster.",
+			Sources: sources, Model: s.chat.Model(),
+			Warnings: allWarns, Took: time.Since(start).String(),
+		})
+		return
+	}
+	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: researchSynthPrompt},
+		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Original question: " + q},
+	})
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, researchResponse{
+		Query: q, Plan: subs, Answer: answer, Sources: sources,
+		Model:     s.chat.Model(),
+		Retriever: fmt.Sprintf("gateway:rrf(%d-shard,%d-sub)", numNonEmpty(s.cluster.Peers), len(subs)),
+		Warnings:  allWarns, Took: time.Since(start).String(),
+	})
+}
+
+// handleAnswerGateway scatters /search across peers with include_text, runs
+// a SINGLE chat synth on the gateway.
+func (s *pebbleHTTP) handleAnswerGateway(w http.ResponseWriter, r *http.Request) {
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented, "/answer requires cfg.Chat.Model on the gateway")
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	k := 5
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
+			k = n
+		}
+	}
+	hits, warns, total := s.scatterSearch(r.Context(), q, k, k*2, r.URL.Query(), true)
+	if len(hits) == 0 {
+		writeJSON(w, http.StatusOK, answerResponse{
+			Query: q, Answer: "No matching sources across the cluster.",
+			Sources: []answerSource{}, Model: s.chat.Model(),
+			TotalCandidates: total, Warnings: warns, Took: time.Since(start).String(),
+		})
+		return
+	}
+	sources := make([]answerSource, 0, len(hits))
+	var promptSources strings.Builder
+	for i, h := range hits {
+		src := answerSource{ID: i + 1, URL: h.URL, Title: h.Title, Excerpt: h.Excerpt, Author: h.Author, PublishedAt: h.PublishedAt}
+		text := h.Text
+		if text == "" {
+			text = h.Excerpt
+		}
+		sources = append(sources, src)
+		fmt.Fprintf(&promptSources, "[%d] %s\n%s\n%s\n\n", i+1, h.Title, h.URL, text)
+	}
+	answer, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: answerSystemPrompt},
+		{Role: "user", Content: "Sources:\n\n" + promptSources.String() + "Question: " + q},
+	})
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, answerResponse{
+		Query: q, Answer: answer, Sources: sources, Model: s.chat.Model(),
+		Retriever:       fmt.Sprintf("gateway:rrf(%d-shard)", numNonEmpty(s.cluster.Peers)-len(warns)),
+		TotalCandidates: total, Warnings: warns, Took: time.Since(start).String(),
+	})
+}
+
+// searchHit is the minimal hit shape returned by pebble-serve's /search.
+// Intentionally narrower than the SQLite server's SearchHit — feature
+// parity (highlight, excerpt, calibration, paragraph filters) grows as
+// follow-up iters port each one through the Pebble side.
+type searchHit struct {
+	URL         string     `json:"url"`
+	Title       string     `json:"title"`
+	Score       float64    `json:"score"`
+	Excerpt     string     `json:"excerpt,omitempty"`
+	PublishedAt *time.Time `json:"published_at,omitempty"`
+	Author      string     `json:"author,omitempty"`
+	Text        string     `json:"text,omitempty"`
+}
+
+type searchResponse struct {
+	Query           string      `json:"query"`
+	EffectiveQuery  string      `json:"effective_query,omitempty"`
+	Expand          string      `json:"expand,omitempty"`
+	Retriever       string      `json:"retriever"`
+	Hits            []searchHit `json:"hits"`
+	TotalCandidates int         `json:"total_candidates,omitempty"`
+	Warnings        []string    `json:"warnings,omitempty"`
+	Took            string      `json:"took"`
+}
+
+// POST /search with JSON body — for callers whose query lists,
+// quoted phrases, or filter CSVs are awkward to URL-encode. Decodes into a
+// searchRequest, re-encodes as r.URL.RawQuery, then calls handleSearch so
+// every param the GET form supports works identically here. The hand-off
+// shape (URL.Values) is the cosift contract — easier to keep one parser
+// than to fork it across method handlers.
+type searchRequest struct {
+	Q              string `json:"q"`
+	K              int    `json:"k,omitempty"`
+	IncludeDomains string `json:"include_domains,omitempty"`
+	ExcludeDomains string `json:"exclude_domains,omitempty"`
+	Site           string `json:"site,omitempty"`
+	Since          string `json:"since,omitempty"`
+	Until          string `json:"until,omitempty"`
+	Sort           string `json:"sort,omitempty"`
+	Enrich         *bool  `json:"enrich,omitempty"`
+	IncludeText    bool   `json:"include_text,omitempty"`
+	Rerank         bool   `json:"rerank,omitempty"`
+	Expand         string `json:"expand,omitempty"`
+}
+
+func (s *pebbleHTTP) handleSearchPOST(w http.ResponseWriter, r *http.Request) {
+	var req searchRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+	v := url.Values{}
+	if req.Q != "" {
+		v.Set("q", req.Q)
+	}
+	if req.K > 0 {
+		v.Set("k", strconv.Itoa(req.K))
+	}
+	if req.IncludeDomains != "" {
+		v.Set("include_domains", req.IncludeDomains)
+	}
+	if req.ExcludeDomains != "" {
+		v.Set("exclude_domains", req.ExcludeDomains)
+	}
+	if req.Site != "" {
+		v.Set("site", req.Site)
+	}
+	if req.Since != "" {
+		v.Set("since", req.Since)
+	}
+	if req.Until != "" {
+		v.Set("until", req.Until)
+	}
+	if req.Sort != "" {
+		v.Set("sort", req.Sort)
+	}
+	if req.Enrich != nil && !*req.Enrich {
+		v.Set("enrich", "false")
+	}
+	if req.IncludeText {
+		v.Set("include_text", "true")
+	}
+	if req.Rerank {
+		v.Set("rerank", "true")
+	}
+	if req.Expand != "" {
+		v.Set("expand", req.Expand)
+	}
+	r.URL.RawQuery = v.Encode()
+	s.handleSearch(w, r)
+}
+
+type findSimilarRequest struct {
+	URL            string `json:"url,omitempty"`
+	Text           string `json:"text,omitempty"`  // content-based MLT, no source URL needed
+	Title          string `json:"title,omitempty"` // optional title-boost when using text mode
+	K              int    `json:"k,omitempty"`
+	Q              string `json:"q,omitempty"`
+	IncludeDomains string `json:"include_domains,omitempty"`
+	ExcludeDomains string `json:"exclude_domains,omitempty"`
+	Since          string `json:"since,omitempty"`
+	Until          string `json:"until,omitempty"`
+	IncludeText    bool   `json:"include_text,omitempty"`
+	Rerank         bool   `json:"rerank,omitempty"`
+}
+
+func (s *pebbleHTTP) handleFindSimilarPOST(w http.ResponseWriter, r *http.Request) {
+	var req findSimilarRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+	v := url.Values{}
+	if req.URL != "" {
+		v.Set("url", req.URL)
+	}
+	if req.Text != "" {
+		v.Set("text", req.Text)
+	}
+	if req.Title != "" {
+		v.Set("title", req.Title)
+	}
+	if req.K > 0 {
+		v.Set("k", strconv.Itoa(req.K))
+	}
+	if req.Q != "" {
+		v.Set("q", req.Q)
+	}
+	if req.IncludeDomains != "" {
+		v.Set("include_domains", req.IncludeDomains)
+	}
+	if req.ExcludeDomains != "" {
+		v.Set("exclude_domains", req.ExcludeDomains)
+	}
+	if req.Since != "" {
+		v.Set("since", req.Since)
+	}
+	if req.Until != "" {
+		v.Set("until", req.Until)
+	}
+	if req.IncludeText {
+		v.Set("include_text", "true")
+	}
+	if req.Rerank {
+		v.Set("rerank", "true")
+	}
+	r.URL.RawQuery = v.Encode()
+	s.handleFindSimilar(w, r)
+}
+
+func (s *pebbleHTTP) handleSearch(w http.ResponseWriter, r *http.Request) {
+	// When this process is in cluster
+	// gateway-mode AND the caller hasn't already set ?cluster_local=1
+	// (which is how the gateway tells peers "just do your local search,
+	// don't fan out again"), fan out to peers and RRF-merge their results.
+	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
+		s.handleSearchGateway(w, r)
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+	k := 10
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
+			k = n
+		}
+	}
+	// include_domains / exclude_domains — mirrors the
+	// SQLite-side server semantics (CSV, dot-boundary suffix match).
+	// since / until — ISO-date filters on doc.PublishedAt.
+	// When any filter is active we over-fetch so the post-filter has
+	// enough candidates to fill k; same brute-force shape SQLite used
+	// before it grew a proper index, fine at pebble-serve's scale.
+	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
+	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
+	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
+	if sinceErr != nil {
+		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
+		return
+	}
+	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
+	if untilErr != nil {
+		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
+		return
+	}
+	dateFilter := !since.IsZero() || !until.IsZero()
+	// site — scope results to one or more host[/path] sections, e.g.
+	// ?site=pilotprotocol.network/docs. Host-suffix + path-prefix match,
+	// ANDed with include/exclude. Applied post-retrieval like the domain
+	// filters, so it widens the over-fetch below.
+	sites := parseSiteScopes(r.URL.Query().Get("site"))
+	siteFilter := len(sites) > 0
+	// rerank widens both the fetch and the keep-cap before filtering,
+	// so the reranker sees a healthy candidate pool even with restrictive filters.
+	wantRerank := r.URL.Query().Get("rerank") == "true" && s.reranker != nil
+	keepCap := k
+	if wantRerank {
+		keepCap = s.rerankCandK
+		if keepCap < k {
+			keepCap = k
+		}
+	}
+	fetchK := keepCap
+	if len(include) > 0 || len(exclude) > 0 || dateFilter || siteFilter {
+		mult := 5
+		if dateFilter {
+			mult = 10
+		}
+		if siteFilter {
+			// site= uses a fixed pool regardless of k — fetching proportionally
+			// (k*100) hits the 2000-cap and spends 20s+ on BM25 for common terms.
+			// 300 candidates is enough to include a small site's relevant pages
+			// while keeping latency under ~3s even for high-frequency query terms.
+			fetchK = 300
+		} else {
+			fetchK = keepCap * mult
+			if fetchK > 500 {
+				fetchK = 500
+			}
+		}
+	} else if wantRerank {
+		fetchK = keepCap * 2
+	}
+	// expansion dispatch (bare / HyDE / paraphrase+RRF).
+	// Shared
+	// helper used by /answer + /research so all three get the same matrix.
+	// Reranker still scores against the original q regardless of strategy.
+	expandMode := r.URL.Query().Get("expand")
+	retrieverParam := r.URL.Query().Get("retriever")
+	denseReady := s.hnsw != nil && s.embedder != nil
+	hits, effectiveQuery, err := s.retrieve(r.Context(), q, fetchK, retrieverParam, expandMode)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	// enrich each surviving hit with Excerpt + PublishedAt + Author
+	// via a single GetDocByURL per hit. Cost: k extra Gets — block-cache hot,
+	// ~ms-scale at the k≤100 we accept. Opt out with ?enrich=false for callers
+	// that only need scoring. The date filter forces a doc fetch
+	// even when enrich=false, since PublishedAt lives on the gob.
+	// ?include_text=true inlines doc.Text on each hit so research
+	// pipelines avoid the N+1 round trip to /contents. Off by default —
+	// payload size grows linearly with k and average doc length.
+	enrich := r.URL.Query().Get("enrich") != "false"
+	if wantRerank {
+		enrich = true // rerank needs per-doc text — overrides enrich opt-out
+	}
+	// time-decay multiplier needs PublishedAt per hit, which lives
+	// behind the enrich flag. Force enrich when decay is requested so the
+	// signal is available downstream.
+	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
+	if decaySet {
+		enrich = true
+	}
+	includeText := r.URL.Query().Get("include_text") == "true"
+	out := make([]searchHit, 0, keepCap)
+	var rerankTexts []string
+	if wantRerank {
+		rerankTexts = make([]string, 0, keepCap)
+	}
+	for _, h := range hits {
+		if len(include) > 0 || len(exclude) > 0 {
+			host := hostOf(h.URL)
+			if len(include) > 0 && !matchesAnyDomain(host, include) {
+				continue
+			}
+			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
+				continue
+			}
+		}
+		if siteFilter && !matchesAnySite(h.URL, sites) {
+			continue
+		}
+		hit := searchHit{URL: h.URL, Title: h.Title, Score: h.Score}
+		if enrich || dateFilter || includeText {
+			doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
+			if derr != nil || doc == nil {
+				continue
+			}
+			if dateFilter {
+				if doc.PublishedAt.IsZero() {
+					continue // zero PublishedAt = unknown; drop under any date filter
+				}
+				if !since.IsZero() && doc.PublishedAt.Before(since) {
+					continue
+				}
+				if !until.IsZero() && doc.PublishedAt.After(until) {
+					continue
+				}
+			}
+			if enrich {
+				hit.Excerpt = textExcerpt(doc.Text, 320)
+				if !doc.PublishedAt.IsZero() {
+					t := doc.PublishedAt
+					hit.PublishedAt = &t
+				}
+				hit.Author = doc.Author
+			}
+			if includeText {
+				hit.Text = doc.Text
+			}
+			if wantRerank {
+				rerankTexts = append(rerankTexts, doc.Title+"\n"+doc.Text)
+			}
+		}
+		out = append(out, hit)
+		if len(out) >= keepCap {
+			break
+		}
+	}
+	// Runs BEFORE rerank so the rerank pool
+	// reflects both quality and recency — when fetchK is over-fetched, the
+	// reranker still sees the freshest-AND-most-relevant top-N. Hits without
+	// PublishedAt are left alone (no signal, no penalty). The pool re-sort
+	// re-aligns rerankTexts via a URL→text map so both knobs compose.
+	if decaySet {
+		var textByURL map[string]string
+		if wantRerank && len(rerankTexts) == len(out) {
+			textByURL = make(map[string]string, len(out))
+			for i, h := range out {
+				textByURL[h.URL] = rerankTexts[i]
+			}
+		}
+		applyTimeDecay(out, decayHalfLife, time.Now())
+		if textByURL != nil {
+			rerankTexts = rerankTexts[:0]
+			for _, h := range out {
+				rerankTexts = append(rerankTexts, textByURL[h.URL])
+			}
+		}
+	}
+	// rerank now that we have keepCap candidates with text.
+	if wantRerank && len(out) > 1 && len(rerankTexts) == len(out) {
+		cands := make([]rerank.Candidate, len(out))
+		for i := range out {
+			cands[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: rerankTexts[i]}
+		}
+		if order, rerr := s.doRerank(r.Context(), q, cands); rerr == nil && len(order) > 0 {
+			reordered := make([]searchHit, 0, len(out))
+			seen := make(map[int]bool, len(out))
+			for _, id := range order {
+				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(out) && !seen[n] {
+					reordered = append(reordered, out[n])
+					seen[n] = true
+				}
+			}
+			// Drop nothing: if the reranker omitted any IDs, append in original order.
+			for i, h := range out {
+				if !seen[i] {
+					reordered = append(reordered, h)
+				}
+			}
+			out = reordered
+		}
+	}
+	// MMR diversification. ?mmr=<lambda> (0..1) reorders the
+	// candidate pool to balance query relevance against diversity from
+	// previously-selected hits. Requires HNSW (per-hit vectors) and an
+	// embedder when the query vector wasn't already computed by the
+	// dense/hybrid path. Composes after rerank — rerank gives quality
+	// ordering, MMR diversifies it. warningsFor() handles the silent
+	// fall-through when MMR was requested but couldn't fire.
+	if mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr")); mmrSet && len(out) > 1 && s.hnsw != nil {
+		var qVec []float32
+		// Hybrid/dense path already embedded the query — try to reuse it.
+		// For BM25-only path we need to embed q if an embedder is configured.
+		if s.embedder != nil {
+			if vecs, embErr := s.embedder.Embed(r.Context(), []string{q}); embErr == nil && len(vecs) > 0 {
+				qVec = vecs[0]
+			}
+		}
+		if len(qVec) > 0 {
+			hitVecs := make([][]float32, len(out))
+			for i := range out {
+				if v, ok := s.hnsw.LookupVectorByURL(out[i].URL); ok {
+					hitVecs[i] = v
+				}
+			}
+			out = mmrSelect(qVec, out, hitVecs, mmrLambda)
+		}
+	}
+	if len(out) > k {
+		out = out[:k]
+	}
+	// Applies to
+	// the already-collected top-k pool; raise k to widen the pool before
+	// re-sorting if you need more candidates for date ordering.
+	switch r.URL.Query().Get("sort") {
+	case "date_desc":
+		sortHitsByDate(out, false)
+	case "date_asc":
+		sortHitsByDate(out, true)
+	}
+	// label centralized in buildRetrieverLabel so /answer
+	// and /research report the same vocabulary as /search.
+	retrieverLabel := s.buildRetrieverLabel(retrieverParam, expandMode, denseReady, effectiveQuery != q, wantRerank)
+	// mmr suffix when diversification actually fired (HNSW loaded
+	// + embedder available). Same conditional as the apply site above —
+	// the label tracks the real pipeline.
+	if mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr")); mmrSet && s.hnsw != nil && s.embedder != nil && mmrLambda < 1.0 {
+		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
+	}
+	// decay suffix when time-decay was applied.
+	if decaySet {
+		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
+	}
+	resp := searchResponse{
+		Query:     q,
+		Expand:    normalizeExpandMode(expandMode),
+		Retriever: retrieverLabel,
+		Hits:      out,
+		// total_candidates = BM25 candidates considered before
+		// filter (capped at fetchK). Operators tuning over-fetch can see
+		// whether their filter is dropping a lot — when out=k but
+		// total_candidates is close to fetchK, the filter is restrictive
+		// enough that you may want to raise k or relax filters.
+		TotalCandidates: len(hits),
+		Took:            time.Since(start).String(),
+	}
+	// surface the post-HyDE query when it actually changed, so callers
+	// can debug whether expand=true contributed any extra terms or returned q
+	// unchanged (chat down, empty passage, etc).
+	if effectiveQuery != q {
+		resp.EffectiveQuery = effectiveQuery
+	}
+	resp.Warnings = s.warningsFor(r)
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// sortHitsByDate orders hits by PublishedAt. Hits with no PublishedAt sink to
+// the end regardless of direction — they have no comparable value. asc=true
+// gives oldest-first; asc=false gives newest-first.
+func sortHitsByDate(hits []searchHit, asc bool) {
+	sort.SliceStable(hits, func(i, j int) bool {
+		ai := hits[i].PublishedAt != nil
+		aj := hits[j].PublishedAt != nil
+		if ai != aj {
+			return ai // hits with dates come before hits without
+		}
+		if !ai {
+			return false
+		}
+		if asc {
+			return hits[i].PublishedAt.Before(*hits[j].PublishedAt)
+		}
+		return hits[i].PublishedAt.After(*hits[j].PublishedAt)
+	})
+}
+
+// BM25-only "more like this". Mirrors EXA's /findSimilar shape but
+// stays dependency-free: no embeddings required. Algorithm is Lucene's MLT —
+// pick the source doc's top-N terms by tf·idf, build a query from them, run
+// the existing BM25 search, drop the source URL itself. With dense vectors
+// off the table on the Pebble path (HNSW indexing during crawl is the iter
+// follow-up), this is the cheapest credible /find_similar we can ship.
+func (s *pebbleHTTP) handleFindSimilar(w http.ResponseWriter, r *http.Request) {
+	// /find_similar runs locally with a
+	// known URL+vec, but in a cluster the URL's shard owns its vector and
+	// only one shard has it. We fan-out to all shards anyway: the owning
+	// shard does the real work; others either fall back to text-mode (if
+	// text supplied) or return empty.
+	if s.cluster.GatewayMode && s.cluster.IsClustered() && r.URL.Query().Get("cluster_local") != "1" {
+		s.handleFindSimilarGateway(w, r)
+		return
+	}
+	start := time.Now()
+	// accept either ?url= (existing behavior) or ?text= (content-
+	// based similarity for unindexed drafts). text path skips the source-URL
+	// exclusion since there's no source URL to exclude.
+	rawURL := r.URL.Query().Get("url")
+	rawText := r.URL.Query().Get("text")
+	if rawURL == "" && rawText == "" {
+		writeProblem(w, http.StatusBadRequest, "missing url or text parameter")
+		return
+	}
+	var srcTitle, srcText, srcURL string
+	if rawURL != "" {
+		decoded, err := url.QueryUnescape(rawURL)
+		if err != nil {
+			writeProblem(w, http.StatusBadRequest, "url parameter is not URL-encoded")
+			return
+		}
+		src, err := s.store.GetDocByURL(r.Context(), decoded)
+		if errors.Is(err, store.ErrNotFound) {
+			writeProblem(w, http.StatusNotFound, "url not in index")
+			return
+		}
+		if err != nil {
+			writeProblem(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		srcTitle, srcText, srcURL = src.Title, src.Text, src.URL
+	} else {
+		srcText = rawText
+		srcTitle = r.URL.Query().Get("title") // optional title boost when text mode
+	}
+	k := 10
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 100 {
+			k = n
+		}
+	}
+
+	// Tokenize title (×3) and body; mirror IndexDocument's title boost so
+	// title terms dominate the similarity query.
+	tf := make(map[string]int, 256)
+	for _, t := range index.Tokenize(srcTitle) {
+		tf[t] += 3
+	}
+	for _, t := range index.Tokenize(srcText) {
+		tf[t]++
+	}
+
+	_, n, _ := s.store.CorpusStats(r.Context())
+	if n <= 0 {
+		n = 1
+	}
+	logN := math.Log(float64(n))
+
+	type termScore struct {
+		term  string
+		score float64
+	}
+	scored := make([]termScore, 0, len(tf))
+	for term, freq := range tf {
+		info, ok, err := s.store.GetTermInfo(r.Context(), term)
+		if err != nil || !ok || info.DocFreq <= 0 {
+			continue
+		}
+		idf := logN - math.Log(float64(info.DocFreq))
+		if idf <= 0 {
+			continue // term appears in every doc; useless signal
+		}
+		scored = append(scored, termScore{term: term, score: float64(freq) * idf})
+	}
+	sort.Slice(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
+
+	topN := 10
+	if len(scored) < topN {
+		topN = len(scored)
+	}
+	if topN == 0 {
+		// previously referenced `decoded` (only in scope in the
+		// url-mode branch) — broken since's text-mode addition.
+		// Use srcURL when present, else echo the bare text/title as the
+		// source identifier so the empty response still tells the caller
+		// what was searched.
+		emptyQuery := srcURL
+		if emptyQuery == "" {
+			emptyQuery = srcTitle
+			if emptyQuery == "" {
+				emptyQuery = srcText
+			}
+		}
+		writeJSON(w, http.StatusOK, searchResponse{
+			Query: emptyQuery, Retriever: "bm25-mlt",
+			Hits: []searchHit{}, Took: time.Since(start).String(),
+		})
+		return
+	}
+	terms := make([]string, topN)
+	for i := 0; i < topN; i++ {
+		terms[i] = scored[i].term
+	}
+	queryStr := strings.Join(terms, " ")
+	// optional ?q= augments the auto-derived MLT query so callers
+	// can constrain "more like this URL" with an extra concept (e.g.
+	// /find_similar?url=...&q=pricing). Appended verbatim — supports the
+	// same quoted-phrase shape /search accepts.
+	if extra := strings.TrimSpace(r.URL.Query().Get("q")); extra != "" {
+		queryStr = queryStr + " " + extra
+	}
+
+	// scope MLT with the same retrieval filters /search and /answer
+	// accept. 'find pages similar to X but only on docs.example.com' is the
+	// archetype EXA findSimilar shape. Over-fetch enough that the source
+	// exclusion + filter still fills k.
+	include := splitDomainsCSV(r.URL.Query().Get("include_domains"))
+	exclude := splitDomainsCSV(r.URL.Query().Get("exclude_domains"))
+	since, sinceErr := parseDateBound(r.URL.Query().Get("since"))
+	if sinceErr != nil {
+		writeProblem(w, http.StatusBadRequest, "since: "+sinceErr.Error())
+		return
+	}
+	until, untilErr := parseDateBound(r.URL.Query().Get("until"))
+	if untilErr != nil {
+		writeProblem(w, http.StatusBadRequest, "until: "+untilErr.Error())
+		return
+	}
+	dateFilter := !since.IsZero() || !until.IsZero()
+	includeText := r.URL.Query().Get("include_text") == "true"
+	// ?rerank=true closes /find_similar's parity gap with the other
+	// retrieval endpoints. The MLT query is auto-derived from the source doc,
+	// so reranking the candidate neighbors against THAT query is exactly what
+	// EXA's findSimilar quality boost is doing under the hood.
+	wantRerank := r.URL.Query().Get("rerank") == "true" && s.reranker != nil
+	keepCap := k
+	if wantRerank {
+		keepCap = s.rerankCandK
+		if keepCap < k {
+			keepCap = k
+		}
+	}
+	fetchK := keepCap + 1 // +1 to absorb the source-URL exclusion
+	if len(include) > 0 || len(exclude) > 0 || dateFilter {
+		mult := 5
+		if dateFilter {
+			mult = 10
+		}
+		fetchK = keepCap * mult
+		if fetchK > 500 {
+			fetchK = 500
+		}
+	} else if wantRerank {
+		fetchK = keepCap * 2
+	}
+
+	// /find_similar?retriever=dense reuses the source's
+	// persisted vector (URL-mode) or embeds the user's text (text-mode) and
+	// runs an HNSW cosine search instead of BM25-MLT. ?retriever=hybrid
+	// runs BOTH BM25-MLT and dense, then RRF-fuses — the
+	// strongest "find similar" signal: lexical precision + semantic recall.
+	//
+	// Requires COSIFT_LOAD_HNSW=true at server start (for the graph);
+	// text-mode additionally needs a configured embedder. URL-mode dense
+	// works without an embedder — the source vector is already in the graph
+	// from indexing. Missing requirements fall through to BM25-MLT;
+	// warningsFor() flags it (carves out the URL-mode-no-embedder
+	// case so the warning isn't misleading).
+	retrieverParam := r.URL.Query().Get("retriever")
+	useDense := retrieverParam == "dense" && s.hnsw != nil
+	useHybrid := retrieverParam == "hybrid" && s.hnsw != nil
+	var (
+		hits       []index.Hit
+		denseFired bool
+		bm25Fired  bool
+	)
+	// Helper: fetch the query vector (URL-mode lookup, falling back to
+	// text-mode embed when allowed). Returns ok=false when neither path
+	// produced a usable vector.
+	getQueryVec := func() ([]float32, bool) {
+		if srcURL != "" {
+			if v, ok := s.hnsw.LookupVectorByURL(srcURL); ok {
+				return v, true
+			}
+		}
+		if s.embedder != nil && (srcText != "" || srcTitle != "") {
+			seed := strings.TrimSpace(srcTitle + " " + srcText)
+			if seed != "" {
+				vecs, embErr := s.embedder.Embed(r.Context(), []string{seed})
+				if embErr == nil && len(vecs) > 0 {
+					return vecs[0], true
+				}
+			}
+		}
+		return nil, false
+	}
+	if useDense {
+		if queryVec, ok := getQueryVec(); ok {
+			vhits := s.hnsw.Search(r.Context(), queryVec, fetchK)
+			hits = make([]index.Hit, len(vhits))
+			for i, vh := range vhits {
+				hits[i] = index.Hit{URL: vh.URL, Title: vh.Title, Score: vh.Score}
+			}
+			denseFired = true
+		}
+	} else if useHybrid {
+		// Run BM25-MLT and dense in series; RRF-fuse the two ranked lists.
+		// Each retriever votes fetchK candidates so the fused top-fetchK has
+		// room to balance both signals before filter/enrich/rerank.
+		bm25Hits, bm25Err := s.idx.Search(r.Context(), queryStr, fetchK)
+		if bm25Err != nil {
+			writeProblem(w, http.StatusInternalServerError, bm25Err.Error())
+			return
+		}
+		bm25Fired = true
+		if queryVec, ok := getQueryVec(); ok {
+			vhits := s.hnsw.Search(r.Context(), queryVec, fetchK)
+			denseHits := s.applyAuthorityToDense(vhits)
+			hits = rrfFuse([][]index.Hit{bm25Hits, denseHits})
+			if len(hits) > fetchK {
+				hits = hits[:fetchK]
+			}
+			denseFired = true
+		} else {
+			// Hybrid fell through to BM25-only (e.g. text-mode + no embedder).
+			// Keep the BM25 hits we already paid for.
+			hits = bm25Hits
+		}
+	}
+	if !denseFired && !bm25Fired {
+		var err error
+		hits, err = s.idx.Search(r.Context(), queryStr, fetchK)
+		if err != nil {
+			writeProblem(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+	}
+	type fsCand struct {
+		hit        searchHit
+		rerankText string
+	}
+	cands := make([]fsCand, 0, keepCap)
+	for _, h := range hits {
+		if srcURL != "" && h.URL == srcURL {
+			continue
+		}
+		if len(include) > 0 || len(exclude) > 0 {
+			host := hostOf(h.URL)
+			if len(include) > 0 && !matchesAnyDomain(host, include) {
+				continue
+			}
+			if len(exclude) > 0 && matchesAnyDomain(host, exclude) {
+				continue
+			}
+		}
+		hit := searchHit{URL: h.URL, Title: h.Title, Score: h.Score}
+		doc, derr := s.store.GetDocByURL(r.Context(), h.URL)
+		if derr != nil || doc == nil {
+			continue
+		}
+		if dateFilter {
+			if doc.PublishedAt.IsZero() {
+				continue
+			}
+			if !since.IsZero() && doc.PublishedAt.Before(since) {
+				continue
+			}
+			if !until.IsZero() && doc.PublishedAt.After(until) {
+				continue
+			}
+		}
+		hit.Excerpt = textExcerpt(doc.Text, 320)
+		if !doc.PublishedAt.IsZero() {
+			t := doc.PublishedAt
+			hit.PublishedAt = &t
+		}
+		hit.Author = doc.Author
+		if includeText {
+			hit.Text = doc.Text
+		}
+		c := fsCand{hit: hit}
+		if wantRerank {
+			c.rerankText = doc.Title + "\n" + doc.Text
+		}
+		cands = append(cands, c)
+		if len(cands) >= keepCap {
+			break
+		}
+	}
+	// Same pre-rerank placement as
+	// /search — re-sort cands by adjusted score so rerank sees a freshness-
+	// aware pool. Re-aligns rerankText too via URL→text map.
+	decayHalfLife, decaySet := parseDecayHalfLife(r.URL.Query().Get("decay"))
+	if decaySet && len(cands) > 0 {
+		now := time.Now()
+		for i := range cands {
+			cands[i].hit.Score *= decayMultiplier(cands[i].hit.PublishedAt, now, decayHalfLife)
+		}
+		sort.SliceStable(cands, func(i, j int) bool { return cands[i].hit.Score > cands[j].hit.Score })
+	}
+	if wantRerank && len(cands) > 1 {
+		rc := make([]rerank.Candidate, len(cands))
+		for i := range cands {
+			rc[i] = rerank.Candidate{ID: strconv.Itoa(i), Text: cands[i].rerankText}
+		}
+		if order, rerr := s.doRerank(r.Context(), queryStr, rc); rerr == nil && len(order) > 0 {
+			reordered := make([]fsCand, 0, len(cands))
+			seen := make(map[int]bool, len(cands))
+			for _, id := range order {
+				if n, perr := strconv.Atoi(id); perr == nil && n >= 0 && n < len(cands) && !seen[n] {
+					reordered = append(reordered, cands[n])
+					seen[n] = true
+				}
+			}
+			for i, c := range cands {
+				if !seen[i] {
+					reordered = append(reordered, c)
+				}
+			}
+			cands = reordered
+		}
+	}
+	// MMR diversification on /find_similar. Anchored at the
+	// source vector — URL-mode reuses the persisted vec, text-mode embeds
+	// the seed via getQueryVec (defined above for dense/hybrid dispatch).
+	// Without an explicit user query, the source IS the query — this is
+	// the right anchor for "find similar but diverse".
+	mmrLambda, mmrSet := parseMMRLambda(r.URL.Query().Get("mmr"))
+	mmrFired := false
+	if mmrSet && len(cands) > 1 && s.hnsw != nil {
+		if qVec, ok := getQueryVec(); ok {
+			hitVecs := make([][]float32, len(cands))
+			for i, c := range cands {
+				if v, vok := s.hnsw.LookupVectorByURL(c.hit.URL); vok {
+					hitVecs[i] = v
+				}
+			}
+			order := mmrOrder(qVec, hitVecs, mmrLambda)
+			if order != nil {
+				reordered := make([]fsCand, len(order))
+				for i, idx := range order {
+					reordered[i] = cands[idx]
+				}
+				cands = reordered
+				mmrFired = true
+			}
+		}
+	}
+	if len(cands) > k {
+		cands = cands[:k]
+	}
+	out := make([]searchHit, 0, len(cands))
+	for _, c := range cands {
+		out = append(out, c.hit)
+	}
+	// label tracks which retrievers actually fired.
+	//   bm25-mlt              — default, BM25 only
+	//   dense                 — ?retriever=dense fired (graph + vec found)
+	//   bm25-mlt+dense:rrf    — ?retriever=hybrid, both BM25 and dense fired
+	// If dense was requested but fell through (no graph, no vec), the label
+	// stays bm25-mlt and warningsFor() carries the reason.
+	retrieverLabel := "bm25-mlt"
+	switch {
+	case bm25Fired && denseFired:
+		retrieverLabel = "bm25-mlt+dense:rrf"
+	case denseFired:
+		retrieverLabel = "dense"
+	}
+	if wantRerank {
+		retrieverLabel += "+rerank:" + s.reranker.Name()
+	}
+	if mmrFired {
+		retrieverLabel += fmt.Sprintf("+mmr:%.2f", mmrLambda)
+	}
+	if decaySet {
+		retrieverLabel += fmt.Sprintf("+decay:%gd", decayHalfLife)
+	}
+	writeJSON(w, http.StatusOK, searchResponse{
+		Query:           queryStr,
+		Retriever:       retrieverLabel,
+		Hits:            out,
+		TotalCandidates: len(hits),
+		Warnings:        s.warningsFor(r),
+		Took:            time.Since(start).String(),
+	})
+}
+
+// doRerank wraps s.reranker.Rerank with attempt/failure counters.
+// Returning the original error unchanged so callers keep their existing
+// silent-fallback behavior; only the operator-side visibility changed.
+func (s *pebbleHTTP) doRerank(ctx context.Context, q string, cands []rerank.Candidate) ([]string, error) {
+	s.rerankAttempts.Add(1)
+	order, err := s.reranker.Rerank(ctx, q, cands)
+	if err != nil {
+		s.rerankFailures.Add(1)
+	}
+	return order, err
+}
+
+// applyBM25EnvOverrides reads COSIFT_BM25_K1 / _B and applies them
+// to idx. Shared by runPebbleServe and runQuery so both honor the same env.
+// Returns which knobs landed so callers can log selectively.
+type bm25EnvResult struct {
+	k1Set, bSet bool
+	k1Val, bVal float64
+	k1Bad, bBad string // non-empty when env was set but unparseable
+}
+
+func applyBM25EnvOverrides(idx *index.PebbleBM25) bm25EnvResult {
+	var out bm25EnvResult
+	if v := os.Getenv("COSIFT_BM25_K1"); v != "" {
+		if k1, err := strconv.ParseFloat(v, 64); err == nil && k1 > 0 {
+			idx.WithBM25Params(k1, 0)
+			out.k1Set, out.k1Val = true, k1
+		} else {
+			out.k1Bad = v
+		}
+	}
+	if v := os.Getenv("COSIFT_BM25_B"); v != "" {
+		if b, err := strconv.ParseFloat(v, 64); err == nil && b > 0 {
+			idx.WithBM25Params(0, b)
+			out.bSet, out.bVal = true, b
+		} else {
+			out.bBad = v
+		}
+	}
+	return out
+}
+
+// paraphraseQuery returns up to n paraphrases of q via the chat
+// client, parses the JSON array shape the SQLite-side paraphraser already
+// uses, and falls back to nil on any failure (no chat client, empty / malformed
+// reply, parse error). Caller decides what to do with an empty list — the
+// downstream RRF strategy treats it as 'no expansion, fall back to single
+// query'. No cache yet — keyed on (q, n) would need a different shape than
+// the HyDE cache; revisit when workload justifies.
+func (s *pebbleHTTP) paraphraseQuery(ctx context.Context, q string, n int) []string {
+	if s.chat == nil || n <= 0 {
+		return nil
+	}
+	s.paraMu.RLock()
+	if cached, ok := s.paraCache[q]; ok {
+		s.paraMu.RUnlock()
+		s.paraHits.Add(1)
+		return cached
+	}
+	s.paraMu.RUnlock()
+	s.paraMisses.Add(1)
+	const sys = `Generate paraphrases of a search query. Each paraphrase preserves the semantic intent but uses different vocabulary — different keywords that a target document might also use. Output ONLY a JSON array of strings.
+Example output for "go programming language": ["golang concurrent compiled language", "Google's systems programming language with goroutines"]`
+	resp, err := s.doChat(ctx, s.chat, []embed.ChatMsg{
+		{Role: "system", Content: sys},
+		{Role: "user", Content: fmt.Sprintf("Generate %d paraphrases of: %s", n, q)},
+	})
+	if err != nil {
+		return nil
+	}
+	raw := strings.TrimSpace(resp)
+	for _, fence := range []string{"```json", "```"} {
+		raw = strings.TrimPrefix(raw, fence)
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	raw = strings.TrimSpace(raw)
+	startIdx := strings.Index(raw, "[")
+	endIdx := strings.LastIndex(raw, "]")
+	if startIdx < 0 || endIdx <= startIdx {
+		return nil
+	}
+	var arr []string
+	if err := json.Unmarshal([]byte(raw[startIdx:endIdx+1]), &arr); err != nil {
+		return nil
+	}
+	out := make([]string, 0, len(arr))
+	for _, p := range arr {
+		t := strings.TrimSpace(p)
+		if t != "" {
+			out = append(out, t)
+		}
+	}
+	if len(out) > 0 {
+		s.paraMu.Lock()
+		if len(s.paraCache) >= s.paraCacheCap {
+			for k := range s.paraCache {
+				delete(s.paraCache, k)
+				break
+			}
+		}
+		s.paraCache[q] = out
+		s.paraMu.Unlock()
+	}
+	return out
+}
+
+// rrfFuse implements Reciprocal Rank Fusion across N ranked lists.
+// k=60 is the standard Cormack et al. constant; tweaking rarely changes top-k
+// ordering meaningfully. Each list contributes 1/(k + rank+1) to a URL's
+// fused score; URLs appearing in more lists at higher ranks rise. Returns
+// synthesized Hits ordered by fused score (URL/Title from first encounter,
+// Score = fused RRF score).
+func rrfFuse(lists [][]index.Hit) []index.Hit {
+	const fuseK = 60
+	type fused struct {
+		hit   index.Hit
+		score float64
+	}
+	scored := make(map[string]*fused, 64)
+	for _, list := range lists {
+		for rank, h := range list {
+			if existing, ok := scored[h.URL]; ok {
+				existing.score += 1.0 / float64(fuseK+rank+1)
+			} else {
+				scored[h.URL] = &fused{hit: h, score: 1.0 / float64(fuseK+rank+1)}
+			}
+		}
+	}
+	flat := make([]*fused, 0, len(scored))
+	for _, f := range scored {
+		flat = append(flat, f)
+	}
+	sort.Slice(flat, func(i, j int) bool { return flat[i].score > flat[j].score })
+	out := make([]index.Hit, len(flat))
+	for i, f := range flat {
+		out[i] = f.hit
+		out[i].Score = f.score
+	}
+	return out
+}
+
+// warningsFor surfaces silent no-ops that callers used to have to
+// derive from absent effective_query / retriever fields. Each warning is one
+// human-readable sentence; consumers programmatically inspect the slice.
+func (s *pebbleHTTP) warningsFor(r *http.Request) []string {
+	var w []string
+	if mode := r.URL.Query().Get("expand"); mode != "" {
+		if s.chat == nil {
+			w = append(w, "expand="+mode+" requested but no chat client configured (set cfg.Chat.Model)")
+		} else if normalizeExpandMode(mode) == "" {
+			// unknown expand value used to be silently ignored.
+			w = append(w, "expand="+mode+" is not a known strategy (try: hyde, paraphrase) — treated as no expansion")
+		}
+	}
+	if r.URL.Query().Get("rerank") == "true" && s.reranker == nil {
+		w = append(w, "rerank=true requested but no reranker configured (set cfg.Rerank.URL or cfg.Rerank.Enabled)")
+	}
+	// dense + hybrid retrievers need both a loaded HNSW graph
+	// and an embedder. Missing either falls through to BM25 with a warning.
+	// /find_similar?url=X URL-mode-dense reads the source vector
+	// directly from the graph via LookupVectorByURL — no embed RPC. Skip the
+	// embedder warning in that case (would otherwise be misleading: the
+	// request succeeded without an embedder).
+	// /find_similar falls back to bm25-mlt (not bm25), so the
+	// warning says so — caller's retriever label matches the warning text.
+	if rv := r.URL.Query().Get("retriever"); rv == "dense" || rv == "hybrid" {
+		isFindSimilar := strings.HasSuffix(r.URL.Path, "/find_similar")
+		isFindSimilarURLMode := isFindSimilar && r.URL.Query().Get("url") != ""
+		fallback := "BM25"
+		if isFindSimilar {
+			fallback = "BM25-MLT"
+		}
+		switch {
+		case s.hnsw == nil:
+			w = append(w, "retriever="+rv+" requested but HNSW graph not loaded (set COSIFT_LOAD_HNSW=true at server start) — fell back to "+fallback)
+		case s.embedder == nil && !isFindSimilarURLMode:
+			w = append(w, "retriever="+rv+" requested but no embedder configured (set cfg.Embeddings.Model) — fell back to "+fallback)
+		}
+	}
+	// MMR diversification needs HNSW + embedder. Bad values (non-
+	// float, out of [0,1]) fall through silently — flag them. Missing
+	// requirements also warn.
+	// /find_similar?url=X reuses the source vector from the graph
+	// for the MMR anchor — no embedder needed. Same carve-out as
+	// for the retriever warning.
+	if raw := r.URL.Query().Get("mmr"); raw != "" {
+		isFindSimilarURLMode := strings.HasSuffix(r.URL.Path, "/find_similar") && r.URL.Query().Get("url") != ""
+		if _, ok := parseMMRLambda(raw); !ok {
+			w = append(w, "mmr="+raw+" is not a float in [0,1] — diversification skipped")
+		} else if s.hnsw == nil {
+			w = append(w, "mmr requires HNSW graph (set COSIFT_LOAD_HNSW=true at server start) — diversification skipped")
+		} else if s.embedder == nil && !isFindSimilarURLMode {
+			w = append(w, "mmr requires an embedder to vectorize the query (set cfg.Embeddings.Model) — diversification skipped")
+		}
+	}
+	// invalid decay half-life flags loudly instead of silently
+	// being ignored. Empty value is fine (no decay requested).
+	if raw := r.URL.Query().Get("decay"); raw != "" {
+		if _, ok := parseDecayHalfLife(raw); !ok {
+			w = append(w, "decay="+raw+" is not a positive half-life in days (≤ 36500) — time-decay skipped")
+		}
+	}
+	// catch unknown ?sort= values (silently treated as relevance).
+	if sortVal := r.URL.Query().Get("sort"); sortVal != "" {
+		switch sortVal {
+		case "relevance", "date_desc", "date_asc":
+			// valid
+		default:
+			w = append(w, "sort="+sortVal+" is not a known mode (try: relevance, date_desc, date_asc) — treated as relevance")
+		}
+	}
+	// catch obviously-bad ?k= values that silently fell back to the
+	// per-endpoint default. Upper-bound clamping varies per endpoint so we
+	// flag only the universally-invalid cases (non-integer, zero, negative).
+	if kVal := r.URL.Query().Get("k"); kVal != "" {
+		if n, err := strconv.Atoi(kVal); err != nil || n <= 0 {
+			w = append(w, "k="+kVal+" is not a positive integer — using server default")
+		}
+	}
+	// Users sometimes
+	// pass 'https://example.com/foo' when 'example.com' was wanted; the
+	// dot-boundary matcher silently drops every result.
+	for _, key := range []string{"include_domains", "exclude_domains"} {
+		if v := r.URL.Query().Get(key); v != "" {
+			for _, d := range strings.Split(v, ",") {
+				d = strings.TrimSpace(d)
+				if strings.Contains(d, "://") || strings.Contains(d, "/") {
+					w = append(w, key+" entry "+d+" looks like a URL — pass a hostname (e.g., example.com), not a URL")
+					break // one warning per filter list is enough
+				}
+			}
+		}
+	}
+	if len(w) > 0 {
+		s.warningsEmitted.Add(1)
+	}
+	return w
+}
+
+// normalizeExpandMode returns the canonical strategy name for an `expand`
+// param value: "true" → "hyde" (alias), "hyde"/"paraphrase" pass through,
+// anything else → "" (so the response field doesn't echo a value that had
+// no effect).
+func normalizeExpandMode(raw string) string {
+	switch raw {
+	case "true", "hyde":
+		return "hyde"
+	case "paraphrase":
+		return "paraphrase"
+	default:
+		return ""
+	}
+}
+
+// parseDecayHalfLife parses ?decay= as a positive half-life in days. Returns
+// (halfLife, true) when valid. Empty / non-positive → (0, false) so caller
+// short-circuits. Bounded at 36500 (100 years) to avoid pathological values.
+func parseDecayHalfLife(raw string) (float64, bool) {
+	if raw == "" {
+		// Half-life from COSIFT_DEFAULT_DECAY_DAYS
+		// (default 180 days = 6 months). Docs older than 6 months get exponentially
+		// less weight in the score; recent docs surface naturally without keyword
+		// hacks like the JS regex. Set the env to 0 to disable globally.
+		// Explicit ?decay=N still wins.
+		def := 180.0
+		if v := os.Getenv("COSIFT_DEFAULT_DECAY_DAYS"); v != "" {
+			if f, err := strconv.ParseFloat(v, 64); err == nil && f >= 0 {
+				def = f
+			}
+		}
+		if def <= 0 {
+			return 0, false
+		}
+		return def, true
+	}
+	// explicit decay=0 disables (even if env default is on).
+	if raw == "0" {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(raw, 64)
+	if err != nil {
+		return 0, false
+	}
+	if math.IsNaN(v) || math.IsInf(v, 0) || v <= 0 || v > 36500 {
+		return 0, false
+	}
+	return v, true
+}
+
+// decayMultiplier computes the time-decay weight for a single PublishedAt.
+// Half-life H: doc from H days ago → 0.5x, 2H → 0.25x. Missing date or
+// non-positive halfLife → 1.0 (no decay).
+func decayMultiplier(publishedAt *time.Time, now time.Time, halfLifeDays float64) float64 {
+	if halfLifeDays <= 0 || publishedAt == nil || publishedAt.IsZero() {
+		return 1
+	}
+	ageDays := now.Sub(*publishedAt).Hours() / 24.0
+	if ageDays < 0 {
+		ageDays = 0 // future-dated → treated as fresh, never boosted
+	}
+	return math.Exp(-math.Ln2 * ageDays / halfLifeDays)
+}
+
+// applyTimeDecay multiplies each hit's Score by decayMultiplier and resorts.
+func applyTimeDecay(hits []searchHit, halfLifeDays float64, now time.Time) {
+	if halfLifeDays <= 0 || len(hits) == 0 {
+		return
+	}
+	for i := range hits {
+		hits[i].Score *= decayMultiplier(hits[i].PublishedAt, now, halfLifeDays)
+	}
+	sort.SliceStable(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
+}
+
+// parseMMRLambda parses ?mmr= as a float in [0, 1]. Returns (lambda, true)
+// when valid. Empty value or out-of-range returns (0, false) so the caller
+// can short-circuit without firing MMR.
+func parseMMRLambda(raw string) (float64, bool) {
+	if raw == "" {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(raw, 64)
+	if err != nil {
+		return 0, false
+	}
+	if v < 0 || v > 1 {
+		return 0, false
+	}
+	return v, true
+}
+
+// mmrOrder computes a permutation of [0..n-1] via Maximal Marginal Relevance
+// (Carbonell & Goldstein '98). At each step picks the candidate maximizing
+//
+//	score = λ · sim(q, c) - (1-λ) · max_{s∈selected} sim(c, s)
+//
+// Candidates whose vectors are missing (zero-length) score 0 on relevance
+// and 0 against any other missing-vec candidate — MMR can only diversify
+// what it can compare. lambda=1 → pure relevance (no diversification);
+// lambda=0 → pure diversity. 0.5–0.7 is a typical starting point.
+//
+// Returns the permutation as a []int; never drops anything. The caller
+// applies it to whatever URL-keyed slice it has.
+func mmrOrder(qVec []float32, hitVecs [][]float32, lambda float64) []int {
+	n := len(hitVecs)
+	if n == 0 {
+		return nil
+	}
+	selected := make([]bool, n)
+	order := make([]int, 0, n)
+	rel := make([]float64, n)
+	for i := range hitVecs {
+		if len(hitVecs[i]) > 0 {
+			rel[i] = float64(index.Dot(qVec, hitVecs[i]))
+		}
+	}
+	for len(order) < n {
+		bestI := -1
+		bestScore := -math.MaxFloat64
+		for i := 0; i < n; i++ {
+			if selected[i] {
+				continue
+			}
+			var maxSim float64
+			for _, j := range order {
+				if len(hitVecs[i]) == 0 || len(hitVecs[j]) == 0 {
+					continue
+				}
+				sim := float64(index.Dot(hitVecs[i], hitVecs[j]))
+				if sim > maxSim {
+					maxSim = sim
+				}
+			}
+			score := lambda*rel[i] - (1.0-lambda)*maxSim
+			if score > bestScore {
+				bestScore = score
+				bestI = i
+			}
+		}
+		if bestI < 0 {
+			break
+		}
+		selected[bestI] = true
+		order = append(order, bestI)
+	}
+	return order
+}
+
+// mmrSelect — thin /search wrapper around mmrOrder that operates on []searchHit
+// directly.
+func mmrSelect(qVec []float32, hits []searchHit, hitVecs [][]float32, lambda float64) []searchHit {
+	if len(hits) <= 1 || lambda >= 1.0 {
+		return hits
+	}
+	order := mmrOrder(qVec, hitVecs, lambda)
+	out := make([]searchHit, 0, len(hits))
+	for _, i := range order {
+		out = append(out, hits[i])
+	}
+	return out
+}
+
+// applyMMRPermutation embeds q, looks up per-URL vectors from HNSW, and
+// returns an MMR permutation. Returns nil when MMR can't fire (no graph,
+// no embedder, embed failed, λ≥1, single-element pool). Shared by /answer
+// and /research synth endpoints.
+func (s *pebbleHTTP) applyMMRPermutation(ctx context.Context, urls []string, q string, lambda float64) []int {
+	if s.hnsw == nil || s.embedder == nil || len(urls) <= 1 || lambda >= 1.0 {
+		return nil
+	}
+	vecs, err := s.embedder.Embed(ctx, []string{q})
+	if err != nil || len(vecs) == 0 {
+		return nil
+	}
+	hitVecs := make([][]float32, len(urls))
+	for i, u := range urls {
+		if v, ok := s.hnsw.LookupVectorByURL(u); ok {
+			hitVecs[i] = v
+		}
+	}
+	return mmrOrder(vecs[0], hitVecs, lambda)
+}
+
+// buildRetrieverLabel produces the human-readable retriever string
+// surfaced on /search, /answer, /research responses. Mirrors the
+// /search inline switch so all three endpoints report the same vocabulary.
+// expansionFired is the post-hoc "did expand=hyde/paraphrase actually run"
+// signal (effectiveQuery != q for /search & /answer; chat-present for
+// /research where sub-queries are individual). wantRerank appends the
+// "+rerank:<name>" suffix when rerank fired.
+func (s *pebbleHTTP) buildRetrieverLabel(retrieverParam, expandMode string, denseReady, expansionFired, wantRerank bool) string {
+	label := "bm25"
+	switch {
+	case retrieverParam == "dense" && denseReady:
+		label = "dense"
+	case retrieverParam == "hybrid" && denseReady:
+		label = "bm25+dense:rrf"
+	default:
+		switch expandMode {
+		case "paraphrase":
+			if expansionFired {
+				label = "bm25+paraphrase"
+			}
+		case "true", "hyde":
+			if expansionFired {
+				label = "bm25+hyde"
+			}
+		}
+	}
+	if wantRerank && s.reranker != nil {
+		label += "+rerank:" + s.reranker.Name()
+	}
+	return label
+}
+
+// applyLLMEndpointDefaults sets the retriever-choice and rerank defaults
+// the LLM-synthesized endpoints (/answer, /research) want. BM25 alone is
+// too slow at multi-million-doc corpora for synchronous LLM use, and the
+// reranker is the proven quality recovery layer — so /answer and /research
+// default to hybrid + rerank whenever the components are wired, while
+// /search keeps its plain-BM25 default for low-latency operator use.
+//
+// Operators retain full control: ?retriever=bm25|dense|hybrid and
+// ?rerank=true|false|1|0 override either default.
+func (s *pebbleHTTP) applyLLMEndpointDefaults(retrieverParam, rerankParam string) (string, bool) {
+	denseReady := s.hnsw != nil && s.embedder != nil
+	if retrieverParam == "" && denseReady {
+		retrieverParam = "hybrid"
+	}
+	wantRerank := s.reranker != nil
+	switch rerankParam {
+	case "false", "0":
+		wantRerank = false
+	case "true", "1":
+		wantRerank = s.reranker != nil
+	}
+	// When the vLLM probe says the backend is queue-deep, silently
+	// disable rerank. Operators can override by passing ?rerank=true
+	// explicitly — but with COSIFT_LLM_DEGRADE_QUEUE breached, the
+	// rerank call would just queue and time out anyway. Better to ship
+	// hybrid scores fast than block on a doomed LLM call.
+	if wantRerank && rerankParam == "" && s.llmProbe != nil && s.llmProbe.Loaded() {
+		wantRerank = false
+	}
+	return retrieverParam, wantRerank
+}
+
+// retrieve dispatches retriever choice (bm25 / dense / hybrid) and
+// then expansion (bare / HyDE / paraphrase+RRF) for BM25 paths. Shared by
+// /search, /answer, /research so all three endpoints get the same retriever
+// matrix. Dense / hybrid require both the loaded HNSW graph and an
+// embedder; missing either falls through to BM25 — warningsFor()
+// surfaces that to the client.
+// applyAuthorityToDense converts HNSW VectorHits into index.Hits with
+// the per-host authority multiplier applied to the score, AND re-sorts
+// the result by the new score so the rank order reflects authority.
+// RRF fusion downstream uses rank position, not raw score, so without
+// the re-sort the multiplier was a no-op (observed: 'What is the C10K
+// problem?' still cited 3 luanjiao.cfd subdomains after applying the
+// multiplier alone). PebbleBM25.Search already multiplies AND sorts
+// internally; this helper mirrors that contract for the dense path.
+func (s *pebbleHTTP) applyAuthorityToDense(vhits []index.VectorHit) []index.Hit {
+	out := make([]index.Hit, len(vhits))
+	for i, vh := range vhits {
+		score := vh.Score
+		if s.authority != nil {
+			score *= s.authority.Multiplier(hostFromURL(vh.URL))
+		}
+		out[i] = index.Hit{URL: vh.URL, Title: vh.Title, Score: score}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score })
+	return out
+}
+
+func (s *pebbleHTTP) retrieve(ctx context.Context, q string, fetchK int, retrieverParam, expandMode string) ([]index.Hit, string, error) {
+	return s.retrieveWith(ctx, s.idx, q, fetchK, retrieverParam, expandMode)
+}
+
+// retrieveWith is retrieve parameterized over the BM25 index, so callers that
+// need a transformed index (e.g. the site-boosted index from WithBoost) can
+// pass it in without shallow-copying the whole pebbleHTTP (which embeds
+// mutexes — copying it trips go vet's lock-copy check and is unsafe).
+func (s *pebbleHTTP) retrieveWith(ctx context.Context, idx *index.PebbleBM25, q string, fetchK int, retrieverParam, expandMode string) ([]index.Hit, string, error) {
+	denseReady := s.hnsw != nil && s.embedder != nil
+	switch {
+	case retrieverParam == "dense" && denseReady:
+		vecs, err := s.embedder.Embed(ctx, []string{q})
+		if err != nil {
+			return nil, "", fmt.Errorf("embedder: %w", err)
+		}
+		vhits := s.hnsw.Search(ctx, vecs[0], fetchK)
+		hits := s.applyAuthorityToDense(vhits)
+		return hits, q, nil
+	case retrieverParam == "hybrid" && denseReady:
+		bm25Hits, bm25Eff, bm25Err := s.retrieveWithExpansionIdx(ctx, idx, q, fetchK, expandMode)
+		if bm25Err != nil {
+			return nil, "", bm25Err
+		}
+		vecs, embErr := s.embedder.Embed(ctx, []string{q})
+		if embErr != nil {
+			return nil, "", fmt.Errorf("embedder: %w", embErr)
+		}
+		denseV := s.hnsw.Search(ctx, vecs[0], fetchK)
+		denseHits := s.applyAuthorityToDense(denseV)
+		hits := rrfFuse([][]index.Hit{bm25Hits, denseHits})
+		if len(hits) > fetchK {
+			hits = hits[:fetchK]
+		}
+		return hits, bm25Eff, nil
+	default:
+		return s.retrieveWithExpansionIdx(ctx, idx, q, fetchK, expandMode)
+	}
+}
+
+// getSiteBoostIDs returns a docID→50× multiplier map for all docs belonging
+// to the given site scopes. Populated lazily on first call; cached in
+// siteBoostCache thereafter. Using the famHost index so the scan is
+// O(site_docs) not O(corpus).
+func (s *pebbleHTTP) getSiteBoostIDs(ctx context.Context, scopes []siteScope) map[int64]float64 {
+	if len(scopes) == 0 || s.store == nil {
+		return nil
+	}
+	key := func() string {
+		parts := make([]string, len(scopes))
+		for i, sc := range scopes {
+			parts[i] = sc.host
+			if sc.path != "" {
+				parts[i] += sc.path
+			}
+		}
+		sort.Strings(parts)
+		return strings.Join(parts, "|")
+	}()
+	if cached, ok := s.siteBoostCache.Load(key); ok {
+		return cached.(map[int64]float64)
+	}
+	boost := make(map[int64]float64)
+	for _, sc := range scopes {
+		_ = s.store.IterHostDocIDs(ctx, sc.host, func(id int64) bool {
+			boost[id] = 50.0
+			return true
+		})
+	}
+	if len(boost) > 0 {
+		s.siteBoostCache.Store(key, boost)
+	}
+	return boost
+}
+
+// getSiteTitles returns up to maxSiteTitles real page titles from the given
+// site scopes, for injection into the research planner prompt so the LLM sees
+// the site's actual vocabulary. Bounded and cheap: a single early-stopping
+// famHost scan per scope plus <=10 GetDocMeta side-blob lookups. Cached in
+// siteTitleCache thereafter. Returns nil if nothing usable is found.
+func (s *pebbleHTTP) getSiteTitles(ctx context.Context, scopes []siteScope) []string {
+	if len(scopes) == 0 || s.store == nil {
+		return nil
+	}
+	key := func() string {
+		parts := make([]string, len(scopes))
+		for i, sc := range scopes {
+			parts[i] = sc.host
+			if sc.path != "" {
+				parts[i] += sc.path
+			}
+		}
+		sort.Strings(parts)
+		return strings.Join(parts, "|")
+	}()
+	if cached, ok := s.siteTitleCache.Load(key); ok {
+		return cached.([]string)
+	}
+	const maxSiteTitles = 10
+	titles := make([]string, 0, maxSiteTitles)
+	seen := make(map[string]struct{}, maxSiteTitles)
+	for _, sc := range scopes {
+		if len(titles) >= maxSiteTitles {
+			break
+		}
+		_ = s.store.IterHostDocIDs(ctx, sc.host, func(id int64) bool {
+			_, title, ok, err := s.store.GetDocMeta(ctx, id)
+			if err != nil {
+				return false // stop this scope on error
+			}
+			if !ok {
+				return true
+			}
+			t := strings.TrimSpace(title)
+			if t == "" {
+				return true
+			}
+			if _, dup := seen[t]; dup {
+				return true
+			}
+			seen[t] = struct{}{}
+			titles = append(titles, t)
+			return len(titles) < maxSiteTitles // stop once we have 10
+		})
+	}
+	if len(titles) > 0 {
+		s.siteTitleCache.Store(key, titles)
+	}
+	return titles
+}
+
+// hostPartitionReadEnabled reports whether site= queries should route to the
+// 'P'-family host partition via SearchInHost. Gated by
+// COSIFT_HOST_PARTITION_READ=1; off → the 50× boost path is used.
+func hostPartitionReadEnabled() bool {
+	return os.Getenv("COSIFT_HOST_PARTITION_READ") == "1"
+}
+
+// retrieveForSites resolves a site-scoped sub-query. When the host partition
+// is enabled and the scope is a single bare host, it scans only that host's
+// posting partition (O(site_docs)) via SearchInHost — the definitive fix for
+// the global-scan latency + recall problem. Otherwise (multi-site, path-scoped,
+// flag off, or partition not yet backfilled for the host) it falls back to the
+// 50× BM25 boost over the global index.
+func (s *pebbleHTTP) retrieveForSites(ctx context.Context, q string, fetchK int, retrieverParam, expandMode string, boostIDs map[int64]float64, scopes []siteScope) ([]index.Hit, string, error) {
+	if hostPartitionReadEnabled() && len(scopes) == 1 && scopes[0].path == "" && scopes[0].host != "" {
+		hits, err := s.idx.SearchInHost(ctx, q, scopes[0].host, fetchK)
+		if err == nil {
+			return hits, q, nil
+		}
+		if !errors.Is(err, index.ErrHostPartitionEmpty) {
+			return nil, "", err
+		}
+		// partition empty for this host → fall through to the boost path.
+	}
+	if len(boostIDs) == 0 {
+		return s.retrieve(ctx, q, fetchK, retrieverParam, expandMode)
+	}
+	return s.retrieveWith(ctx, s.idx.WithBoost(boostIDs), q, fetchK, retrieverParam, expandMode)
+}
+
+// retrieveWithExpansionIdx dispatches the BM25 call across the three
+// expansion strategies /search and /answer share — bare, HyDE, paraphrase+RRF.
+// Returns (hits, effectiveQuery, err). effectiveQuery == q when no expansion
+// fired (bare path, or expansion no-op'd because chat is down). It is
+// parameterized over the BM25 index so a transformed index (e.g. the
+// site-boosted one from WithBoost) can be passed in without shallow-copying the
+// whole pebbleHTTP struct (which embeds mutexes).
+func (s *pebbleHTTP) retrieveWithExpansionIdx(ctx context.Context, idx *index.PebbleBM25, q string, fetchK int, expandMode string) ([]index.Hit, string, error) {
+	switch expandMode {
+	case "paraphrase":
+		paras := s.paraphraseQuery(ctx, q, 3)
+		if len(paras) == 0 {
+			hits, err := idx.Search(ctx, q, fetchK)
+			return hits, q, err
+		}
+		queries := append([]string{q}, paras...)
+		lists := make([][]index.Hit, 0, len(queries))
+		for _, qq := range queries {
+			h, lerr := idx.Search(ctx, qq, fetchK)
+			if lerr != nil {
+				continue
+			}
+			lists = append(lists, h)
+		}
+		hits := rrfFuse(lists)
+		if len(hits) > fetchK {
+			hits = hits[:fetchK]
+		}
+		return hits, q + " | " + strings.Join(paras, " | "), nil
+	case "true", "hyde":
+		eq := s.expandQuery(ctx, q)
+		hits, err := idx.Search(ctx, eq, fetchK)
+		return hits, eq, err
+	case "entity":
+		// Rule-based query rewrite for question-form entity lookups
+		// ("who created X", "how tall is X", "what is the capital of
+		// X"). Strips the question form, appends canonical-attribute
+		// keywords to the original, single BM25 call. The earlier
+		// RRF-fusion variant regressed legitimate passes because
+		// equal-weight fusion across 4 queries pushed the original's
+		// top hit out of the reranker's top-30. Concat preserves the
+		// original signal while letting the new keywords add positive
+		// scoring contributions where biographical pages exist.
+		rewrites := qexpand.RewriteEntity(q)
+		if len(rewrites) == 0 {
+			hits, err := idx.Search(ctx, q, fetchK)
+			return hits, q, err
+		}
+		expanded := q + " " + strings.Join(rewrites, " ")
+		hits, err := idx.Search(ctx, expanded, fetchK)
+		return hits, expanded, err
+	default:
+		// Bare path runs entity-expansion when the query matches a
+		// question-form pattern. Earlier RRF-fuse experiment regressed
+		// 'who created the World Wide Web' (passing → bail) because
+		// equal-weight fusion across 4 queries pushed the
+		// previously-#1 Berners-Lee bio out of the reranker's top-30.
+		// Concat-into-single-query avoids the fusion-weight problem:
+		// the rewrites only add positive scoring contributions for
+		// pages that contain the canonical-attribute words ("creator,"
+		// "inventor"), without diluting the original query's ranking
+		// signal. Operators can disable with COSIFT_DISABLE_ENTITY_EXPAND=1.
+		if os.Getenv("COSIFT_DISABLE_ENTITY_EXPAND") == "" {
+			if rewrites := qexpand.RewriteEntity(q); len(rewrites) > 0 {
+				expanded := q + " " + strings.Join(rewrites, " ")
+				hits, err := idx.Search(ctx, expanded, fetchK)
+				return hits, expanded, err
+			}
+		}
+		hits, err := idx.Search(ctx, q, fetchK)
+		return hits, q, err
+	}
+}
+
+// expandQuery returns q + " " + a HyDE-generated passage when a chat client is
+// configured. On any error (no chat client, chat call fails, empty passage),
+// returns q unchanged so callers can compose safely without explicit guards.
+// added a bounded in-memory cache to skip the chat call on
+// repeat queries — important for /research?expand=true where the same
+// sub-query rephrasing can fire across many similar research requests.
+func (s *pebbleHTTP) expandQuery(ctx context.Context, q string) string {
+	if s.chat == nil {
+		return q
+	}
+	s.hydeMu.RLock()
+	cached, hit := s.hydeCache[q]
+	s.hydeMu.RUnlock()
+	if hit {
+		s.hydeHits.Add(1)
+		return q + " " + cached
+	}
+	s.hydeMisses.Add(1)
+	passage, err := s.doChat(ctx, s.chat, []embed.ChatMsg{
+		{Role: "system", Content: hydeSystemPrompt},
+		{Role: "user", Content: q},
+	})
+	if err != nil {
+		return q
+	}
+	passage = strings.TrimSpace(passage)
+	if passage == "" {
+		return q
+	}
+	s.hydeMu.Lock()
+	if len(s.hydeCache) >= s.hydeCacheCap {
+		// Bounded: drop one arbitrary entry (Go map range is randomized).
+		for k := range s.hydeCache {
+			delete(s.hydeCache, k)
+			break
+		}
+	}
+	s.hydeCache[q] = passage
+	s.hydeMu.Unlock()
+	return q + " " + passage
+}
+
+// queryPlanPrompt is the LLM-orchestrator prompt. The model
+// analyzes the user's natural-language query, classifies intent, and
+// outputs a structured retrieval plan: expanded queries (3 variations
+// covering different angles), recency window, retriever choice, and
+// optionally a domain allowlist. The plan then drives a broader RRF-
+// fused retrieval. JSON schema is strict; sanity-checked on receive.
+const queryPlanPrompt = `You analyze a user's natural-language search query and output a JSON retrieval plan.
+
+Output ONLY a JSON object (no markdown, no prose, no commentary) with this exact shape:
+{
+  "intent": "current_news" | "factual_lookup" | "comparison" | "research" | "findability",
+  "queries": ["expanded query 1", "expanded query 2", "expanded query 3"],
+  "since_days": null | <positive integer, days lookback from today>,
+  "retriever": "bm25" | "dense" | "hybrid",
+  "decay_days": <positive integer half-life> | null
+}
+
+Guidance:
+- "queries": 3 paraphrases/expansions of the original. Include entity names, synonyms, related terms. e.g. "latest news from Romania" → ["Romania political news 2026", "Romanian government coalition PSD AUR", "Bucharest current events"]
+- "since_days": only set when the query implies recency (latest/current/recent/today/news). null otherwise.
+- "decay_days": half-life in days. For current-events queries use 60. For factual/research queries use null or 365.
+- "retriever": hybrid is the safe default. Use "bm25" only for exact-term lookups (model names, error codes). Use "dense" only for purely conceptual queries with no keyword overlap.
+- DO NOT invent include_domains. The corpus chooses sources, not you.
+
+Examples:
+Input: "latest news from Romania"
+{"intent":"current_news","queries":["Romania political news 2026","Romanian PSD AUR coalition government","Bucharest economy and politics current"],"since_days":90,"retriever":"hybrid","decay_days":60}
+
+Input: "what is BM25 ranking"
+{"intent":"factual_lookup","queries":["BM25 Okapi ranking function formula","BM25 information retrieval probabilistic","term frequency inverse document frequency BM25"],"since_days":null,"retriever":"hybrid","decay_days":null}
+
+Input: "compare HNSW vs IVF"
+{"intent":"comparison","queries":["HNSW vs IVF approximate nearest neighbor","Hierarchical Navigable Small World versus Inverted File","ANN index tradeoffs HNSW IVF performance"],"since_days":null,"retriever":"hybrid","decay_days":730}`
+
+// queryPlan captures the LLM's structured retrieval plan. Sanitized on
+// parse — out-of-range values clipped to safe defaults.
+type queryPlan struct {
+	Intent    string   `json:"intent"`
+	Queries   []string `json:"queries"`
+	SinceDays *int     `json:"since_days"`
+	Retriever string   `json:"retriever"`
+	DecayDays *int     `json:"decay_days"`
+}
+
+// parseQueryPlan extracts the planner's JSON output and applies guard-rails.
+// Returns a plan with sane defaults if the LLM produces garbage — never
+// errors at the caller, so /query always succeeds.
+func parseQueryPlan(raw, original string) queryPlan {
+	defaultPlan := queryPlan{
+		Intent:    "research",
+		Queries:   []string{original},
+		Retriever: "hybrid",
+	}
+	// Strip code fences if the LLM wrapped output despite the prompt.
+	raw = strings.TrimSpace(raw)
+	if strings.HasPrefix(raw, "```") {
+		if i := strings.Index(raw, "\n"); i > 0 {
+			raw = raw[i+1:]
+		}
+		raw = strings.TrimSuffix(raw, "```")
+	}
+	var p queryPlan
+	if err := json.Unmarshal([]byte(raw), &p); err != nil {
+		return defaultPlan
+	}
+	if len(p.Queries) == 0 {
+		p.Queries = []string{original}
+	}
+	if len(p.Queries) > 5 {
+		p.Queries = p.Queries[:5]
+	}
+	switch p.Retriever {
+	case "bm25", "dense", "hybrid":
+		// ok
+	default:
+		p.Retriever = "hybrid"
+	}
+	if p.SinceDays != nil && (*p.SinceDays <= 0 || *p.SinceDays > 3650) {
+		p.SinceDays = nil
+	}
+	if p.DecayDays != nil && (*p.DecayDays <= 0 || *p.DecayDays > 3650) {
+		p.DecayDays = nil
+	}
+	if p.Intent == "" {
+		p.Intent = "research"
+	}
+	return p
+}
+
+// handleQuery is the LLM-orchestrated search/research endpoint.
+// Flow: (1) LLM planner analyzes intent + emits expanded queries, (2) each
+// expansion runs through /search internals (hybrid + decay), (3) RRF-fuse
+// the per-expansion result lists, (4) optional rerank, (5) synth with
+// citations. Returns the plan in the response so callers can see what was
+// done.
+//
+// Effectively: /research planner-style decomposition, but optimized for
+// QUERY EXPANSION not sub-question DECOMPOSITION. The two are different —
+// research splits multi-faceted questions ("compare X and Y" → "X facts",
+// "Y facts", "X vs Y tradeoffs"); query expands a single intent into
+// paraphrases that catch corpus phrasing variation.
+func (s *pebbleHTTP) handleQuery(w http.ResponseWriter, r *http.Request) {
+	if s.chat == nil {
+		writeProblem(w, http.StatusNotImplemented, "/query requires cfg.Chat.Model")
+		return
+	}
+	start := time.Now()
+	q := r.URL.Query().Get("q")
+	if q == "" {
+		writeProblem(w, http.StatusBadRequest, "missing q parameter")
+		return
+	}
+
+	// SSE opt-in. Stream phase events through the planner → expand → fuse
+	// → synth pipeline so operators see the LLM's sub-queries, per-expansion
+	// hit counts, fusion result, and synth tokens as they're produced.
+	wantStream := wantsSSE(r)
+	var sse *answerSSE
+	var streamChat embed.StreamingChatClient
+	if wantStream {
+		if sc, ok := s.chat.(embed.StreamingChatClient); ok {
+			if a := newAnswerSSE(w, start); a != nil {
+				sse, streamChat = a, sc
+				stopKA := sse.startKeepalive(r.Context(), 7*time.Second)
+				defer stopKA()
+				sse.phase("plan_start", map[string]any{"q": q, "model": s.chat.Model()})
+			}
+		}
+	}
+
+	// Step 1: LLM planner.
+	planRaw, err := s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+		{Role: "system", Content: queryPlanPrompt},
+		{Role: "user", Content: q},
+	})
+	if err != nil {
+		if sse != nil {
+			sse.errorEvt("plan: " + err.Error())
+			return
+		}
+		writeProblem(w, http.StatusBadGateway, "plan: "+err.Error())
+		return
+	}
+	plan := parseQueryPlan(planRaw, q)
+	if sse != nil {
+		details := map[string]any{
+			"intent":    plan.Intent,
+			"retriever": plan.Retriever,
+			"queries":   plan.Queries,
+		}
+		if plan.SinceDays != nil {
+			details["since_days"] = *plan.SinceDays
+		}
+		if plan.DecayDays != nil {
+			details["decay_days"] = *plan.DecayDays
+		}
+		sse.phase("plan", details)
+	}
+
+	// Step 2: run each expanded query through retrieval, dedupe by URL.
+	type hitInfo struct {
+		hit    index.Hit
+		bestRR float64 // best reciprocal rank across expansions
+	}
+	byURL := make(map[string]*hitInfo, 64)
+	fetchK := 30
+	for _, sub := range plan.Queries {
+		hits, _, err := s.retrieve(r.Context(), sub, fetchK, plan.Retriever, "")
+		if err != nil {
+			if sse != nil {
+				sse.phase("expand", map[string]any{"query": sub, "error": err.Error()})
+			}
+			continue
+		}
+		for rank, h := range hits {
+			rr := 1.0 / float64(60+rank) // RRF with k=60
+			if existing, ok := byURL[h.URL]; ok {
+				if rr > existing.bestRR {
+					existing.bestRR = rr
+				}
+			} else {
+				byURL[h.URL] = &hitInfo{hit: h, bestRR: rr}
+			}
+		}
+		if sse != nil {
+			sse.phase("expand", map[string]any{"query": sub, "hits": len(hits)})
+		}
+	}
+
+	// Step 3: sort by fused RRF score with optional per-host boosts.
+	fused := make([]index.Hit, 0, len(byURL))
+	for _, v := range byURL {
+		h := v.hit
+		h.Score = v.bestRR
+		if len(s.hostBoosts) > 0 {
+			h.Score *= index.HostBoostFor(h.URL, s.hostBoosts)
+		}
+		fused = append(fused, h)
+	}
+	sort.Slice(fused, func(i, j int) bool { return fused[i].Score > fused[j].Score })
+	if sse != nil {
+		sse.phase("fuse", map[string]any{"unique_urls": len(fused)})
+	}
+
+	// Step 4: optional since filter + decay (decay handled by /answer-style
+	// path when we materialize). Apply since filter here on PublishedAt.
+	keep := 10
+	if v := r.URL.Query().Get("k"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20 {
+			keep = n
+		}
+	}
+	var since time.Time
+	if plan.SinceDays != nil {
+		since = time.Now().AddDate(0, 0, -(*plan.SinceDays))
+	}
+
+	// Step 5: enrich + filter by since.
+	type cand struct {
+		src     answerSource
+		excerpt string
+		text    string
+	}
+	cands := make([]cand, 0, keep)
+	for _, h := range fused {
+		if len(cands) >= keep {
+			break
+		}
+		doc, err := s.store.GetDocByURL(r.Context(), h.URL)
+		if err != nil || doc == nil {
+			continue
+		}
+		if !since.IsZero() && (doc.PublishedAt.IsZero() || doc.PublishedAt.Before(since)) {
+			continue
+		}
+		c := cand{
+			src: answerSource{
+				ID:    len(cands) + 1,
+				URL:   h.URL,
+				Title: h.Title,
+			},
+			excerpt: textExcerpt(doc.Text, 320),
+			text:    doc.Text,
+		}
+		if !doc.PublishedAt.IsZero() {
+			t := doc.PublishedAt
+			c.src.PublishedAt = &t
+		}
+		cands = append(cands, c)
+	}
+	if sse != nil {
+		sse.phase("materialize", map[string]any{"candidates": len(cands)})
+	}
+
+	// Step 6: synth via answerSystemPrompt.
+	var promptSrcs strings.Builder
+	sources := make([]answerSource, 0, len(cands))
+	for i, c := range cands {
+		c.src.ID = i + 1
+		c.src.Excerpt = c.excerpt
+		fmt.Fprintf(&promptSrcs, "[%d] %s\n%s\n\n", c.src.ID, c.src.URL, truncateForPromptLite(c.text, 1200))
+		sources = append(sources, c.src)
+	}
+
+	if sse != nil {
+		sse.sources(q, sources, s.chat.Model(), "query:planner+hybrid+rrf", len(fused))
+		if len(cands) == 0 {
+			sse.chunk("No sources matched the planner's expanded queries within the chosen time window.")
+			sse.done()
+			return
+		}
+		sse.phase("synth_start", map[string]any{"sources": len(sources), "model": streamChat.Model()})
+		_, cerr := s.doChatStream(r.Context(), streamChat, []embed.ChatMsg{
+			{Role: "system", Content: answerSystemPrompt},
+			{Role: "user", Content: "Sources:\n\n" + promptSrcs.String() + "Question: " + q},
+		}, sse.chunk)
+		if cerr != nil {
+			sse.errorEvt("synth: " + cerr.Error())
+			return
+		}
+		sse.done()
+		return
+	}
+
+	answerText := ""
+	if len(cands) > 0 {
+		answerText, err = s.doChat(r.Context(), s.chat, []embed.ChatMsg{
+			{Role: "system", Content: answerSystemPrompt},
+			{Role: "user", Content: "Sources:\n\n" + promptSrcs.String() + "Question: " + q},
+		})
+		if err != nil {
+			writeProblem(w, http.StatusBadGateway, "synth: "+err.Error())
+			return
+		}
+	} else {
+		answerText = "No sources matched the planner's expanded queries within the chosen time window."
+	}
+
+	writeJSON(w, http.StatusOK, map[string]any{
+		"query":   q,
+		"plan":    plan,
+		"answer":  answerText,
+		"sources": sources,
+		"model":   s.chat.Model(),
+		"took":    time.Since(start).String(),
+	})
+}
+
+func (s *pebbleHTTP) handleContents(w http.ResponseWriter, r *http.Request) {
+	rawURL := r.URL.Query().Get("url")
+	if rawURL == "" {
+		writeProblem(w, http.StatusBadRequest, "missing url parameter")
+		return
+	}
+	decoded, err := url.QueryUnescape(rawURL)
+	if err != nil {
+		writeProblem(w, http.StatusBadRequest, "url parameter is not URL-encoded")
+		return
+	}
+	doc, err := s.store.GetDocByURL(r.Context(), decoded)
+	if errors.Is(err, store.ErrNotFound) {
+		writeProblem(w, http.StatusNotFound, "url not in index")
+		return
+	}
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"url":          doc.URL,
+		"title":        doc.Title,
+		"text":         doc.Text,
+		"author":       doc.Author,
+		"published_at": doc.PublishedAt,
+		"fetched_at":   doc.FetchedAt,
+	})
+}
+
+// POST /contents — batch URL → document. Up to 100 URLs in
+// one round-trip. URLs not in the index emit a {url, found:false, error}
+// stub in the same slot so the caller can correlate positionally. Wire
+// shape (results+took, per-item found+cached+lang) mirrors the SQLite-side
+// /contents POST so `cosift contents <url> <url>` works against either
+// backend without code changes.
+type contentsBatchReq struct {
+	URLs []string `json:"urls"`
+}
+
+type contentsBatchItem struct {
+	URL         string    `json:"url"`
+	Found       bool      `json:"found"`
+	Title       string    `json:"title,omitempty"`
+	Text        string    `json:"text,omitempty"`
+	Lang        string    `json:"lang,omitempty"`
+	Cached      bool      `json:"cached"`
+	Author      string    `json:"author,omitempty"`
+	PublishedAt time.Time `json:"published_at,omitempty"`
+	FetchedAt   time.Time `json:"fetched_at,omitempty"`
+	Error       string    `json:"error,omitempty"`
+}
+
+func (s *pebbleHTTP) handleContentsBatch(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	var req contentsBatchReq
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		writeProblem(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
+		return
+	}
+	if len(req.URLs) == 0 {
+		writeProblem(w, http.StatusBadRequest, "urls: must be a non-empty array")
+		return
+	}
+	if len(req.URLs) > 100 {
+		writeProblem(w, http.StatusBadRequest, "urls: at most 100 per request")
+		return
+	}
+	results := make([]contentsBatchItem, 0, len(req.URLs))
+	for _, u := range req.URLs {
+		item := contentsBatchItem{URL: u}
+		doc, err := s.store.GetDocByURL(r.Context(), u)
+		switch {
+		case errors.Is(err, store.ErrNotFound):
+			item.Error = "not in index"
+		case err != nil:
+			item.Error = err.Error()
+		case doc == nil:
+			item.Error = "not in index"
+		default:
+			item.Found = true
+			item.Cached = true
+			item.Title = doc.Title
+			item.Text = doc.Text
+			item.Lang = doc.Lang
+			item.Author = doc.Author
+			item.PublishedAt = doc.PublishedAt
+			item.FetchedAt = doc.FetchedAt
+		}
+		results = append(results, item)
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"results": results,
+		"took":    time.Since(start).String(),
+	})
+}
diff --git a/cmd/cosift/serve_setup.go b/cmd/cosift/serve_setup.go
new file mode 100644
index 0000000..25369a1
--- /dev/null
+++ b/cmd/cosift/serve_setup.go
@@ -0,0 +1,1418 @@
+package main
+
+import (
+	"context"
+	_ "embed"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/crawler"
+
+	"github.com/pilot-protocol/cosift/internal/answercache"
+	"github.com/pilot-protocol/cosift/internal/authority"
+	"github.com/pilot-protocol/cosift/internal/chatgate"
+	"github.com/pilot-protocol/cosift/internal/config"
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/index"
+	"github.com/pilot-protocol/cosift/internal/rerank"
+	"github.com/pilot-protocol/cosift/internal/sla"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func runPebbleServe(ctx context.Context, cfg *config.Config, args []string) error {
+	fs := flag.NewFlagSet("pebble-serve", flag.ExitOnError)
+	dir := fs.String("dir", "", "PebbleStore directory (required; the SQLite cfg.DataDir is ignored)")
+	addr := fs.String("addr", cfg.Server.Addr, "listen address (defaults to server.addr from cosift.json)")
+	// -crawl-seeds-file lets pebble-serve run an in-process crawler
+	// against the same PebbleStore. Eliminates the writer-lock dance between
+	// cosift-serve and cosift-crawl — search + crawl + index growth all in one
+	// process, one binary, one lock. The crawler runs as a goroutine for the
+	// lifetime of the server.
+	crawlSeeds := fs.String("crawl-seeds-file", "", "if set, run a continuous in-process crawler against these seeds while serving")
+	crawlCheckpoint := fs.Duration("crawl-checkpoint", 60*time.Second, "HNSW persist cadence for the in-serve crawler")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *dir == "" {
+		return errors.New("pebble-serve: -dir is required")
+	}
+	if *addr == "" {
+		*addr = "127.0.0.1:7777"
+	}
+
+	ps, err := store.OpenPebble(*dir)
+	if err != nil {
+		return fmt.Errorf("open pebble: %w", err)
+	}
+	defer ps.Close()
+
+	// log corpus size at startup so operators see whether the
+	// store opened with the expected doc count (silent open would force them
+	// to curl /stats just to confirm a restart loaded the right data dir).
+	if _, indexedDocs, err := ps.CorpusStats(ctx); err == nil && indexedDocs > 0 {
+		log.Printf("pebble-serve: opened store with %d indexed docs", indexedDocs)
+	}
+
+	// Meta is 20 bytes (dim+nodeCount); first-entry probe is the fallback
+	// when meta is absent but vector entries exist (partial persist).
+	// We never load the full graph here — gigabytes of RAM at 10M-vector scale.
+	hasVectors := false
+	var vectorDim, vectorNodes int
+	if meta, ok, err := index.LoadHNSWMeta(ctx, ps); err == nil && ok {
+		hasVectors = true
+		vectorDim = meta.Dim
+		vectorNodes = meta.NodeCount
+	} else {
+		_ = ps.IterateVectorNodes(ctx, func(_ uint64, _ []byte) bool {
+			hasVectors = true
+			return false
+		})
+	}
+	// Loading is
+	// gigabytes of RAM at production scale (10M vectors × 1536 dim ≈ 60GB),
+	// so it's opt-in. When the env var isn't set we just keep the cheap meta
+	// snapshot from and operators see has_vectors=true.
+	var hnswGraph *index.HNSW
+	if hasVectors && os.Getenv("COSIFT_LOAD_HNSW") == "true" {
+		g, ok, err := index.LoadHNSW(ctx, ps)
+		switch {
+		case err != nil:
+			log.Printf("pebble-serve: COSIFT_LOAD_HNSW=true but LoadHNSW failed: %v", err)
+		case !ok:
+			log.Printf("pebble-serve: COSIFT_LOAD_HNSW=true but no HNSW meta on store")
+		default:
+			hnswGraph = g
+			log.Printf("pebble-serve: HNSW graph loaded into memory: %d nodes, dim=%d", g.Len(), vectorDim)
+			// Default 50 from NewHNSW
+			// underfits big graphs grown via AddPassage; raising to ~200
+			// restored Recall@10 from 0.47 to ~0.85 on the 800K-vector
+			// production corpus without a graph rebuild.
+			if v := os.Getenv("COSIFT_HNSW_EF_SEARCH"); v != "" {
+				if n, err := strconv.Atoi(v); err == nil && n > 0 {
+					g.SetEfSearch(n)
+					log.Printf("pebble-serve: HNSW efSearch override = %d", n)
+				}
+			}
+			// if a PQ codebook + codes exist in this store, wire
+			// them so /search uses asymmetric PQ distance (much faster on
+			// large graphs). When absent, search falls back to raw vectors.
+			//
+			// bench-pq exposed that PQ on this dim=768 corpus drops
+			// Recall@10 from ~0.89 to ~0.60 — the 32× compression has a
+			// recall cost that may exceed the speed benefit. Operators can
+			// keep the codes on disk but disable the runtime path by setting
+			// COSIFT_DISABLE_PQ=true.
+			if os.Getenv("COSIFT_DISABLE_PQ") == "true" {
+				log.Printf("pebble-serve: PQ disabled via COSIFT_DISABLE_PQ — using raw vectors for search")
+			} else if cb, cbOK, _ := index.LoadPQCodebook(ctx, ps); cbOK {
+				codes := make([][]uint16, g.Len())
+				loaded := 0
+				if err := ps.IteratePQCodes(ctx, func(nodeID uint64, blob []byte) bool {
+					if int(nodeID) >= len(codes) {
+						return true
+					}
+					// codebook-aware decode handles both byte-packed
+					// (new, K≤256) and uint16 LE (legacy) blob shapes.
+					code, err := cb.DecodeCodeBlob(blob)
+					if err != nil || len(code) != cb.M {
+						return true
+					}
+					codes[int(nodeID)] = code
+					loaded++
+					return true
+				}); err != nil {
+					log.Printf("pebble-serve: PQ codes iterate failed: %v — falling back to raw vectors", err)
+				} else {
+					g.UsePQ(cb, codes)
+					log.Printf("pebble-serve: PQ search enabled (codebook: dim=%d M=%d K=%d, codes=%d/%d)",
+						cb.Dim, cb.M, cb.K, loaded, g.Len())
+				}
+			}
+		}
+	}
+	if hasVectors {
+		if vectorNodes > 0 {
+			suffix := "(graph not loaded into memory; set COSIFT_LOAD_HNSW=true to load)"
+			if hnswGraph != nil {
+				suffix = "(graph loaded)"
+			}
+			log.Printf("pebble-serve: HNSW index present: %d nodes, dim=%d %s", vectorNodes, vectorDim, suffix)
+		} else {
+			log.Printf("pebble-serve: HNSW vector entries present (no meta blob — partial persist?)")
+		}
+	}
+
+	// configurable HyDE + paraphrase cache caps (defaults 256).
+	hydeCap := 256
+	if v := os.Getenv("COSIFT_HYDE_CACHE_SIZE"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			hydeCap = n
+			log.Printf("pebble-serve: HyDE cache size override = %d", n)
+		} else {
+			// surface unparseable env vars as warnings at startup
+			// instead of silently keeping defaults. Catches typos like
+			// COSIFT_HYDE_CACHE_SIZE="512m" or negative values.
+			log.Printf("pebble-serve: WARN COSIFT_HYDE_CACHE_SIZE=%q is not a positive integer — using default %d", v, hydeCap)
+		}
+	}
+	paraCap := 256
+	if v := os.Getenv("COSIFT_PARA_CACHE_SIZE"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			paraCap = n
+			log.Printf("pebble-serve: paraphrase cache size override = %d", n)
+		} else {
+			log.Printf("pebble-serve: WARN COSIFT_PARA_CACHE_SIZE=%q is not a positive integer — using default %d", v, paraCap)
+		}
+	}
+	idx := index.NewPebbleBM25(ps)
+	// Wikipedia / kernel.org / arxiv float
+	// above spam directories in the long tail. Optional Tranco /
+	// Majestic CSVs at COSIFT_TRANCO_CSV / COSIFT_MAJESTIC_CSV improve
+	// the score; with no CSVs the embedded whitelist + TLD heuristics
+	// still produce useful signals.
+	scorer := authority.New()
+	if v := os.Getenv("COSIFT_AUTHORITY_ALPHA"); v != "" {
+		if a, err := strconv.ParseFloat(v, 64); err == nil && a >= 0 {
+			scorer = scorer.WithAlpha(a)
+		}
+	}
+	if path := os.Getenv("COSIFT_TRANCO_CSV"); path != "" {
+		if f, err := os.Open(path); err == nil {
+			n, _ := scorer.LoadTranco(f)
+			_ = f.Close()
+			log.Printf("pebble-serve: authority Tranco loaded (%d entries from %s)", n, path)
+		} else {
+			log.Printf("pebble-serve: WARN COSIFT_TRANCO_CSV=%q: %v", path, err)
+		}
+	}
+	if path := os.Getenv("COSIFT_MAJESTIC_CSV"); path != "" {
+		if f, err := os.Open(path); err == nil {
+			n, _ := scorer.LoadMajestic(f)
+			_ = f.Close()
+			log.Printf("pebble-serve: authority Majestic loaded (%d entries from %s)", n, path)
+		} else {
+			log.Printf("pebble-serve: WARN COSIFT_MAJESTIC_CSV=%q: %v", path, err)
+		}
+	}
+	idx = idx.WithAuthority(scorer)
+	log.Printf("pebble-serve: authority scoring enabled (alpha=%.2f, trusted=%d, tranco=%d, majestic=%d)",
+		scorer.Alpha(), scorer.Stats().TrustedHosts, scorer.Stats().TrancoEntries, scorer.Stats().MajesticEntries)
+	// COSIFT_BM25_K1 / COSIFT_BM25_B override per instance.
+	// Shared with runQuery via applyBM25EnvOverrides so any PebbleBM25 built
+	// from CLI or server honors the same env.
+	o := applyBM25EnvOverrides(idx)
+	if o.k1Set {
+		log.Printf("pebble-serve: BM25 k1 override = %.2f", o.k1Val)
+	}
+	if o.bSet {
+		log.Printf("pebble-serve: BM25 b override = %.2f", o.bVal)
+	}
+	if o.k1Bad != "" {
+		log.Printf("pebble-serve: WARN COSIFT_BM25_K1=%q is not a positive float — using default %.2f", o.k1Bad, idx.K1())
+	}
+	if o.bBad != "" {
+		log.Printf("pebble-serve: WARN COSIFT_BM25_B=%q is not a positive float — using default %.2f", o.bBad, idx.B())
+	}
+	srv := &pebbleHTTP{
+		store:        ps,
+		idx:          idx,
+		hydeCache:    make(map[string]string, hydeCap),
+		hydeCacheCap: hydeCap,
+		paraCache:    make(map[string][]string, paraCap),
+		paraCacheCap: paraCap,
+		hasVectors:   hasVectors,
+		vectorDim:    vectorDim,
+		vectorNodes:  vectorNodes,
+		hnsw:         hnswGraph,
+		started:      time.Now(),
+		authority:    scorer,
+		// capture doc count at startup so /stats can compute
+		// per-minute crawl rate without long-term counters.
+		startupDocs: vectorNodes, // placeholder; overwritten below
+	}
+	// Read actual doc count at startup for crawl-rate baseline. Uses
+	// CorpusStats (the atomic-mirror indexed_docs counter) — NOT
+	// Stats.Documents which scans the raw 'd' family. The two diverge
+	// in two ways:
+	//
+	//   - Stats.Documents may include partially-upserted rows that the
+	//     indexed_docs counter hasn't seen yet, leaving startupDocs >
+	//     indexedDocs and freezing docs_added_since_start at 0 forever.
+	//   - On a cold corpus, Stats does a 12s scan that returns the
+	//     count AT END-OF-SCAN, by which time the crawler has already
+	//     added 50+ more docs, pushing startupDocs above the post-boot
+	//     indexed_docs atomic.
+	//
+	// CorpusStats reads the atomic directly (or bootstraps it from the
+	// indexed_docs meta key on first call), giving an exact match with
+	// the indexedDocs value the rate calc uses below.
+	if _, count, err := ps.CorpusStats(ctx); err == nil && count > 0 {
+		srv.startupDocs = int(count)
+	} else {
+		srv.startupDocs = 0
+	}
+	// stamp cluster config onto srv early so /search's
+	// gateway-mode check fires even on nodes that don't run a crawler.
+	if err := cfg.Cluster.Validate(); err != nil {
+		log.Printf("pebble-serve: cluster config invalid (%v) — running single-node", err)
+	} else {
+		srv.cluster = cfg.Cluster
+		if cfg.Cluster.IsClustered() {
+			role := "leaf"
+			if cfg.Cluster.GatewayMode {
+				role = "gateway+leaf"
+			}
+			log.Printf("pebble-serve: cluster mode (role=%s, shard=%d/%d, peers=%d)",
+				role, cfg.Cluster.MyShardID, cfg.Cluster.NumShards, len(cfg.Cluster.Peers))
+		}
+	}
+	// Uses the same OpenAI-compatible chat
+	// client the SQLite-side server uses; works against OpenAI, Together,
+	// Azure, llama.cpp, vLLM, Ollama, anything speaking /v1/chat/completions.
+	// anonymous auth when cfg.Chat.URL is set (self-hosted, no key
+	// needed). Required only for hosted defaults (empty URL = api.openai.com).
+	// Shared between answer + rerank pools so an
+	// /answer burst can't also exhaust the rerank pool's goroutines.
+	srv.chatGate = chatgate.New(
+		envIntDefault("COSIFT_LLM_GATE_RERANK", 8),
+		envIntDefault("COSIFT_LLM_GATE_ANSWER", 4),
+	)
+	// /answer response cache. Default TTL 60s + 1024-entry cap is a
+	// reasonable starting point for a workload with a few hundred
+	// recurring queries; raise via COSIFT_ANSWER_CACHE_TTL_SEC for
+	// stable corpora, disable with =0 if every query is unique.
+	srv.answerCache = answercache.New(
+		time.Duration(envIntDefault("COSIFT_ANSWER_CACHE_TTL_SEC", 60))*time.Second,
+		envIntDefault("COSIFT_ANSWER_CACHE_CAP", 1024),
+	)
+	srv.researchCache = answercache.New(
+		time.Duration(envIntDefault("COSIFT_RESEARCH_CACHE_TTL_SEC", 600))*time.Second,
+		envIntDefault("COSIFT_RESEARCH_CACHE_CAP", 256),
+	)
+	// Query log (observability). COSIFT_QUERY_LOG=/path enables JSONL append
+	// of every query-endpoint request. Disabled if unset.
+	if qp := os.Getenv("COSIFT_QUERY_LOG"); qp != "" {
+		if f, err := os.OpenFile(qp, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644); err == nil {
+			srv.qlogFile = f
+			log.Printf("pebble-serve: query log enabled → %s", qp)
+		} else {
+			log.Printf("pebble-serve: query log open failed (%s): %v", qp, err)
+		}
+	}
+	// Feedback rate limiter — always on (stricter than global). Per-client via
+	// XFF. Override COSIFT_FEEDBACK_RPM / _BURST.
+	srv.fbRL = &rateLimiter{
+		rpm:       float64(envIntDefault("COSIFT_FEEDBACK_RPM", 20)),
+		burst:     float64(envIntDefault("COSIFT_FEEDBACK_BURST", 5)),
+		whitelist: map[string]bool{},
+	}
+	// Feedback log. COSIFT_FEEDBACK_LOG=/path, or defaults beside the query log.
+	if fp := feedbackLogPath(); fp != "" {
+		if f, err := os.OpenFile(fp, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644); err == nil {
+			srv.fbFile = f
+			log.Printf("pebble-serve: feedback log enabled → %s", fp)
+		} else {
+			log.Printf("pebble-serve: feedback log open failed (%s): %v", fp, err)
+		}
+	}
+	// Tries the vLLM /metrics endpoint at the chat origin (works for
+	// self-hosted vLLM, no-op against OpenAI). Threshold default 8 — that
+	// matches the answer-pool cap so when the chat backend is already
+	// saturated we stop trying to add more LLM work.
+	if cfg.Chat.URL != "" {
+		probeURL := chatgate.VLLMMetricsURLFrom(cfg.Chat.URL)
+		threshold := envIntDefault("COSIFT_LLM_DEGRADE_QUEUE", 8)
+		interval := envDurationMsDefault("COSIFT_LLM_PROBE_MS", 2*time.Second)
+		srv.llmProbe = chatgate.NewLoadProbe(probeURL, threshold, interval)
+		go srv.llmProbe.Run(ctx)
+		log.Printf("pebble-serve: vLLM load probe started (url=%s, threshold=%d, interval=%s)", probeURL, threshold, interval)
+	}
+	// Defaults match the
+	// June-3 baseline measurements; ops can override per env. Violation
+	// log defaults under cfg.DataDir so it co-locates with the rest of
+	// the operational state.
+	slaLogPath := os.Getenv("COSIFT_SLA_LOG_PATH")
+	if slaLogPath == "" && cfg.DataDir != "" {
+		slaLogPath = filepath.Join(cfg.DataDir, "sla-violations.jsonl")
+	}
+	slaTargets := []sla.Target{
+		{Endpoint: "/search", P95: envDurationMsDefault("COSIFT_SLA_SEARCH_P95_MS", 1500*time.Millisecond), P99: envDurationMsDefault("COSIFT_SLA_SEARCH_P99_MS", 4*time.Second), MaxErrorRate: 0.02},
+		{Endpoint: "/answer", P95: envDurationMsDefault("COSIFT_SLA_ANSWER_P95_MS", 8*time.Second), P99: envDurationMsDefault("COSIFT_SLA_ANSWER_P99_MS", 20*time.Second), MaxErrorRate: 0.05},
+		{Endpoint: "/research", P95: envDurationMsDefault("COSIFT_SLA_RESEARCH_P95_MS", 30*time.Second), P99: envDurationMsDefault("COSIFT_SLA_RESEARCH_P99_MS", 60*time.Second), MaxErrorRate: 0.05},
+		{Endpoint: "/healthz", P95: 50 * time.Millisecond, P99: 200 * time.Millisecond, MaxErrorRate: 0.01},
+		{Endpoint: "/stats", P95: 200 * time.Millisecond, P99: time.Second, MaxErrorRate: 0.01},
+	}
+	slaWindow := envDurationMsDefault("COSIFT_SLA_WINDOW_MS", 5*time.Minute)
+	slaEvalInterval := envDurationMsDefault("COSIFT_SLA_EVAL_MS", 30*time.Second)
+	if m, err := sla.New(slaTargets, slaWindow, slaLogPath); err == nil {
+		srv.sla = m
+		go srv.sla.Run(ctx.Done(), slaEvalInterval)
+		log.Printf("pebble-serve: SLA monitor started (window=%s, eval=%s, log=%s)", slaWindow, slaEvalInterval, slaLogPath)
+	} else {
+		log.Printf("pebble-serve: SLA monitor disabled: %v", err)
+	}
+	// Holds the unwrapped OpenAIChatClient so the rerank
+	// wrapper below can sit directly on top of the inner client (single
+	// gate acquisition per call) instead of stacking on srv.chatSafe.
+	var rawChatForRerank embed.ChatClient
+	if cfg.Chat.Model != "" {
+		apiKey := resolveAPIKey("chat")
+		rawChat := embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Chat.Model)
+		rawChatForRerank = rawChat
+		srv.chatSafe = chatgate.NewSafeChat(rawChat, chatgate.Options{
+			Gate:          srv.chatGate,
+			Kind:          chatgate.KindAnswer,
+			StageDeadline: envDurationMsDefault("COSIFT_LLM_DEADLINE_ANSWER_MS", 30*time.Second),
+			MaxRetries:    envIntDefault("COSIFT_LLM_RETRIES", 1),
+		})
+		srv.chat = srv.chatSafe
+		auth := "anonymous"
+		if apiKey != "" {
+			auth = "bearer-token"
+		}
+		st := srv.chatGate.Stats()
+		log.Printf("pebble-serve: /answer enabled (chat model=%s, auth=%s, gate=answer:%d/rerank:%d)",
+			cfg.Chat.Model, auth, st.AnswerCap, st.RerankCap)
+	}
+	// Same anonymous-when-URL
+	// semantics as chat above.
+	if cfg.Embeddings.Model != "" {
+		apiKey := resolveAPIKey("embed")
+		// when cfg.Embeddings.URLs is set, build one client per
+		// URL and wrap them in a RoundRobinEmbedder so cosift fans embed
+		// requests across multiple backends. Falls back to single-URL
+		// behavior when URLs is empty.
+		urls := cfg.Embeddings.URLs
+		if len(urls) == 0 {
+			urls = []string{cfg.Embeddings.URL}
+		}
+		clients := make([]embed.Embedder, 0, len(urls))
+		for _, u := range urls {
+			clients = append(clients, embed.NewOpenAIClient(apiKey, u, cfg.Embeddings.Model, cfg.Embeddings.Dim))
+		}
+		var base embed.Embedder = embed.NewRoundRobinEmbedder(clients)
+		// Both search and
+		// crawler embed paths share the cache layer — same text returns
+		// instantly on re-fetch / re-query, no ollama call.
+		cacheNote := ""
+		if cfg.Embeddings.CacheDir != "" {
+			base = embed.NewCachedEmbedder(base, cfg.Embeddings.CacheDir)
+			cacheNote = fmt.Sprintf(", cache=%s", cfg.Embeddings.CacheDir)
+		}
+		srv.embedder = base
+		auth := "anonymous"
+		if apiKey != "" {
+			auth = "bearer-token"
+		}
+		log.Printf("pebble-serve: embedder configured (model=%s, dim=%d, auth=%s, backends=%d%s)",
+			cfg.Embeddings.Model, cfg.Embeddings.Dim, auth, len(clients), cacheNote)
+	}
+	// wire rerank.Reranker when cfg.Rerank is configured. Two paths
+	// (matching the SQLite-side): cfg.Rerank.URL → HTTPReranker (Cohere/Voyage/
+	// Jina/TEI wire shape); otherwise cfg.Chat.Model present → LLMReranker
+	// (RankGPT-style listwise via chat). /search?rerank=true gates the wrapper.
+	if cfg.Rerank.URL != "" {
+		apiKey := cfg.Rerank.APIKey
+		if apiKey == "" {
+			apiKey = os.Getenv("COHERE_API_KEY")
+			if apiKey == "" {
+				apiKey = os.Getenv("VOYAGE_API_KEY")
+			}
+		}
+		srv.reranker = rerank.NewHTTPReranker(cfg.Rerank.URL, apiKey, cfg.Rerank.Model)
+		log.Printf("pebble-serve: rerank enabled (http: %s, model=%s)", cfg.Rerank.URL, cfg.Rerank.Model)
+	} else if cfg.Rerank.Enabled && srv.chat != nil {
+		// when cfg.Rerank.Model is set and differs from cfg.Chat.Model,
+		// build a separate chat client for rerank pointing at the same /v1 URL
+		// but with the alternate model. Lets operators dedicate a small/fast
+		// model (e.g. qwen3.5:0.8b) to rerank so it doesn't queue behind chat
+		// generations on the same model. Falls back to the main chat client
+		// when Rerank.Model is empty or equal to Chat.Model.
+		var rerankInner embed.ChatClient = rawChatForRerank
+		if cfg.Rerank.Model != "" && cfg.Rerank.Model != cfg.Chat.Model {
+			apiKey := resolveAPIKey("chat")
+			rerankInner = embed.NewOpenAIChat(apiKey, cfg.Chat.URL, cfg.Rerank.Model)
+			log.Printf("pebble-serve: rerank using dedicated chat (model=%s, url=%s)", cfg.Rerank.Model, cfg.Chat.URL)
+		}
+		srv.rerankSafe = chatgate.NewSafeChat(rerankInner, chatgate.Options{
+			Gate:          srv.chatGate,
+			Kind:          chatgate.KindRerank,
+			StageDeadline: envDurationMsDefault("COSIFT_LLM_DEADLINE_RERANK_MS", 5*time.Second),
+			MaxRetries:    envIntDefault("COSIFT_LLM_RETRIES", 1),
+		})
+		srv.reranker = rerank.NewLLMReranker(srv.rerankSafe)
+		log.Printf("pebble-serve: rerank enabled (llm: %s, gated)", srv.rerankSafe.Model())
+	}
+	srv.rerankCandK = cfg.Rerank.CandidateK
+	if srv.rerankCandK <= 0 {
+		srv.rerankCandK = 20
+	}
+	srv.hostBoosts = cfg.Defaults.HostBoosts
+
+	mux := http.NewServeMux()
+	// Built once from env;
+	// nil when disabled (COSIFT_RATELIMIT_RPM unset or 0). Wraps every route
+	// below — including /healthz so monitoring hits are budgeted too;
+	// operators wanting unlimited probes should set
+	// COSIFT_RATELIMIT_WHITELIST to include their monitoring source.
+	srv.rl = newRateLimiterFromEnv()
+	if srv.rl != nil {
+		log.Printf("pebble-serve: rate limit active (rpm=%.0f burst=%.0f whitelist=%v)", srv.rl.rpm, srv.rl.burst, srv.rl.whitelistList())
+	}
+	wrap := func(h http.HandlerFunc) http.HandlerFunc { return srv.count(srv.rateLimit(h)) }
+	// qwrap adds query logging (innermost, so it sees the real status+bytes) for
+	// the user-facing query endpoints — the observability substrate we lacked.
+	qwrap := func(h http.HandlerFunc) http.HandlerFunc { return srv.count(srv.rateLimit(srv.qlog(h))) }
+	// awrap = wrap + admin auth. All /admin/* routes go through this so the
+	// peer-token gate is enforced at the mux level (belt-and-suspenders with any
+	// per-handler check), closing gaps where a handler forgets to inline it.
+	awrap := func(h http.HandlerFunc) http.HandlerFunc { return srv.count(srv.rateLimit(srv.requireAdmin(h))) }
+	// landing page at / and OpenAPI 3.1 spec at /openapi.json.
+	// Both embedded into the binary at build time — operators get a single
+	// self-contained executable, no separate static-asset deployment.
+	mux.HandleFunc("GET /", wrap(srv.handleLanding))
+	// Streams /answer or /research SSE
+	// into a multi-turn conversation view with citation rendering.
+	mux.HandleFunc("GET /chat", wrap(srv.handleChat))
+	mux.HandleFunc("GET /openapi.json", wrap(srv.handleOpenAPI))
+	// Swagger UI bundled into the binary. /docs renders the
+	// interactive spec; /docs/swagger-ui.css and /docs/swagger-ui-bundle.js
+	// serve the dist assets locally so the page works air-gapped (no CDN).
+	mux.HandleFunc("GET /docs", wrap(srv.handleSwaggerUI))
+	mux.HandleFunc("GET /docs/{file...}", wrap(srv.handleSwaggerAsset))
+	mux.HandleFunc("GET /healthz", wrap(srv.handleHealthz))
+	mux.HandleFunc("GET /find", qwrap(srv.handleFind))
+	mux.HandleFunc("GET /stats", wrap(srv.handleStats))
+	mux.HandleFunc("GET /domains", wrap(srv.handleDomains))
+	// frontier queue visibility — counts by status + top-N
+	// hosts in queue. Distinct from /domains which is over INDEXED docs.
+	mux.HandleFunc("GET /queue", wrap(srv.handleQueue))
+	// Single-node deployments still expose
+	// this; it's just unused. Authenticated by cfg.Cluster.PeerAuthToken
+	// (Bearer); when token is empty, requests from any source are accepted.
+	mux.HandleFunc("POST /admin/crawl-enqueue", awrap(srv.handleCrawlEnqueue))
+	mux.HandleFunc("POST /admin/allow-domain", awrap(srv.handleAllowDomain))
+	mux.HandleFunc("POST /admin/frontier-purge-host", awrap(srv.handleFrontierPurgeHost))
+	mux.HandleFunc("POST /admin/frontier-clear", awrap(srv.handleFrontierClear))
+	mux.HandleFunc("POST /admin/frontier-demote-host", awrap(srv.handleFrontierDemoteHost))
+	mux.HandleFunc("POST /admin/frontier-purge-stale-inflight", awrap(srv.handleFrontierPurgeStaleInFlight))
+	mux.HandleFunc("POST /admin/rss-import", awrap(srv.handleRSSImport))
+	mux.HandleFunc("POST /admin/crawl-now", awrap(srv.handleCrawlNow))
+	mux.HandleFunc("POST /admin/wet-import", awrap(srv.handleWETImport))
+	mux.HandleFunc("POST /admin/wet-import-bulk", awrap(srv.handleWETImportBulk))
+	mux.HandleFunc("POST /admin/site-pack", awrap(srv.handleSitePack))
+	mux.HandleFunc("POST /admin/site-submit", awrap(srv.handleSiteSubmit))
+	mux.HandleFunc("POST /admin/embed-backfill", awrap(srv.handleEmbedBackfill))
+	mux.HandleFunc("POST /admin/host-backfill", awrap(srv.handleHostBackfill))
+	mux.HandleFunc("GET /admin/eval-quick", awrap(srv.handleEvalQuick))
+	mux.HandleFunc("POST /admin/hnsw-compact", awrap(srv.handleHNSWCompact))
+	mux.HandleFunc("GET /query", qwrap(srv.handleQuery))
+	mux.HandleFunc("POST /query", qwrap(srv.handleQuery))
+	// import a sitemap.xml (or sitemap-index) and push every
+	// listed URL into the live frontier.
+	mux.HandleFunc("POST /admin/sitemap-import", awrap(srv.handleSitemapImport))
+	mux.HandleFunc("POST /admin/recrawl-sitemap", awrap(srv.handleRecrawlSitemap))
+	// PQ training admin endpoint. Same auth as crawl-enqueue
+	// (Bearer cfg.Cluster.PeerAuthToken). Runs synchronously — for the
+	// 224K-vec corpus we have today it takes ~minutes; operator-only.
+	mux.HandleFunc("POST /admin/pq-train", awrap(srv.handlePQTrain))
+	// backfill-only — re-encode every node that doesn't have a
+	// code yet against the existing codebook. No retrain. Fast.
+	mux.HandleFunc("POST /admin/pq-encode", awrap(srv.handlePQEncode))
+	// Creates a hard-linked, consistent
+	// snapshot dir that's safe to tar without racing background compactions.
+	mux.HandleFunc("POST /admin/checkpoint", awrap(srv.handleCheckpoint))
+	mux.HandleFunc("GET /search", qwrap(srv.handleSearch))
+	mux.HandleFunc("POST /search", qwrap(srv.handleSearchPOST))
+	mux.HandleFunc("GET /contents", wrap(srv.handleContents))
+	mux.HandleFunc("POST /contents", wrap(srv.handleContentsBatch))
+	mux.HandleFunc("GET /verify", wrap(srv.handleVerify))
+	mux.HandleFunc("GET /metrics", wrap(srv.handleMetrics))
+	mux.HandleFunc("GET /admin/query-log", awrap(srv.handleQueryLog))
+	mux.HandleFunc("POST /feedback", wrap(srv.handleFeedback))
+	mux.HandleFunc("GET /admin/feedback", awrap(srv.handleFeedbackList))
+	mux.HandleFunc("GET /sla", wrap(srv.handleSLA))
+	mux.HandleFunc("GET /admin/domains-audit", awrap(srv.handleDomainsAudit))
+	mux.HandleFunc("GET /find_similar", qwrap(srv.handleFindSimilar))
+	mux.HandleFunc("POST /find_similar", qwrap(srv.handleFindSimilarPOST))
+	mux.HandleFunc("GET /answer", qwrap(srv.handleAnswer))
+	mux.HandleFunc("POST /answer", qwrap(srv.handleAnswerPOST))
+	mux.HandleFunc("GET /research", qwrap(srv.handleResearch))
+	mux.HandleFunc("POST /research", qwrap(srv.handleResearchPOST))
+
+	httpSrv := &http.Server{
+		Addr:              *addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 10 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		WriteTimeout:      60 * time.Second,
+		IdleTimeout:       120 * time.Second,
+		MaxHeaderBytes:    1 << 20, // 1 MB
+	}
+
+	log.Printf("pebble-serve: listening on %s (PebbleStore at %s)", *addr, *dir)
+	// Production state observed on GH200: after a restart series that
+	// healed HNSW zombies, the meta counters showed 0 while the 'l'
+	// family had 10.87M entries — degraded BM25 length normalization.
+	// Off the hot path so HNSW load + first request aren't blocked.
+	go func() {
+		// Brief settle so an active IndexDocument can populate counters
+		// the cheap way before we kick the O(N) scan.
+		time.Sleep(10 * time.Second)
+		sumLen, count, recomputed, err := ps.BootstrapCorpusStats(ctx)
+		if err != nil {
+			log.Printf("pebble-serve: corpus stats bootstrap failed: %v", err)
+			return
+		}
+		if recomputed {
+			log.Printf("pebble-serve: corpus stats bootstrapped (indexed_docs=%d, sum_doc_len=%d)", count, sumLen)
+		}
+	}()
+	// Without this the
+	// subdomain-density penalty in authority.Score never fires — we
+	// saw 6.68M .cfd/.sbs hosts on GH200 sitting at score 0.20 instead
+	// of dropping to "block." Single-pass iteration is ~18s at 9M hosts;
+	// runs off the hot path so HNSW load isn't blocked. Re-runs every
+	// 6h to track ongoing crawl growth without operator intervention.
+	go func() {
+		// Initial settle so the rest of startup completes first.
+		time.Sleep(15 * time.Second)
+		refresh := func() {
+			counts := map[string]int{}
+			err := ps.IterateDomains(ctx, func(host string, _ int) bool {
+				// Use the Scorer's eTLD+1 logic so the bucket keys we
+				// produce here match what Score() looks up — without this
+				// the publicsuffix-aware Scorer keys on "bbc.co.uk" while
+				// the bootstrap keys on "co.uk" and the lookup misses.
+				e := authority.ETLD1(host)
+				if e == "" {
+					return true
+				}
+				counts[e]++
+				return true
+			})
+			if err != nil {
+				log.Printf("pebble-serve: subdomain-count refresh: %v", err)
+				return
+			}
+			srv.authority.SetSubdomainCounts(counts)
+			log.Printf("pebble-serve: authority subdomain counts refreshed (%d eTLD+1 buckets)", len(counts))
+		}
+		refresh()
+		// Re-scan periodically to absorb ongoing crawl growth.
+		t := time.NewTicker(6 * time.Hour)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				refresh()
+			}
+		}
+	}()
+	// Only
+	// listens on loopback so it can never be exposed publicly through the
+	// reverse proxy. Set COSIFT_PPROF_ADDR=127.0.0.1:6060 to enable.
+	if pprofAddr := os.Getenv("COSIFT_PPROF_ADDR"); pprofAddr != "" {
+		// optional mutex / block sampling so the
+		// dense-search-latency investigation has data. Off by default
+		// because each sample is a couple-µs overhead per contended
+		// lock event. Set rate ≥1 to enable; 1000 means "sample 1 in
+		// 1000 lock waits" which is plenty for diagnosis.
+		if v := os.Getenv("COSIFT_MUTEX_PROFILE_FRACTION"); v != "" {
+			if n, err := strconv.Atoi(v); err == nil && n > 0 {
+				runtime.SetMutexProfileFraction(n)
+				log.Printf("pebble-serve: mutex profile fraction = %d", n)
+			}
+		}
+		if v := os.Getenv("COSIFT_BLOCK_PROFILE_RATE"); v != "" {
+			if n, err := strconv.Atoi(v); err == nil && n > 0 {
+				runtime.SetBlockProfileRate(n)
+				log.Printf("pebble-serve: block profile rate = %d", n)
+			}
+		}
+		pprofMux := http.NewServeMux()
+		pprofMux.HandleFunc("/debug/pprof/", pprof.Index)
+		pprofMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		pprofMux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+		go func() {
+			log.Printf("pebble-serve: pprof listening on %s", pprofAddr)
+			// Bare ListenAndServe uses zero-value Server with no timeouts —
+			// a slow-reader on the pprof port can pin goroutines indefinitely.
+			// WriteTimeout is generous because CPU profile and trace endpoints
+			// stream for 30s+ by design.
+			srv := &http.Server{
+				Addr:              pprofAddr,
+				Handler:           pprofMux,
+				ReadHeaderTimeout: 10 * time.Second,
+				ReadTimeout:       30 * time.Second,
+				WriteTimeout:      5 * time.Minute,
+				IdleTimeout:       120 * time.Second,
+				MaxHeaderBytes:    1 << 20,
+			}
+			if err := srv.ListenAndServe(); err != nil {
+				log.Printf("pebble-serve: pprof exited: %v", err)
+			}
+		}()
+	}
+	go func() {
+		<-ctx.Done()
+		log.Printf("pebble-serve: shutting down")
+		shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		_ = httpSrv.Shutdown(shutCtx)
+	}()
+
+	// Reuses ps + srv.hnsw + srv.embedder. The
+	// crawler.Crawler uses concurrency from cfg.Crawler.MaxConcurrent. Vectors
+	// land into the SAME HNSW pointer that /search reads from, so freshly
+	// crawled content is searchable as soon as AddPassage returns.
+	// Checkpoint goroutine persists the graph every crawlCheckpoint.
+	// collect a WaitGroup of crawler goroutines so the final
+	// HNSW persist runs BEFORE ps.Close() — fixes a 'pebble: closed' panic.
+	var crawlWG sync.WaitGroup
+	if *crawlSeeds != "" {
+		if err := srv.startInProcessCrawl(ctx, ps, *crawlSeeds, *crawlCheckpoint, cfg, &crawlWG); err != nil {
+			log.Printf("pebble-serve: in-process crawler not started: %v", err)
+		}
+	}
+
+	servErr := httpSrv.ListenAndServe()
+	// wait for crawler + checkpoint goroutines to finish their
+	// final persist before the deferred ps.Close() runs. Otherwise Persist
+	// can race ps.Close() and panic.
+	crawlWG.Wait()
+	if servErr != nil && servErr != http.ErrServerClosed {
+		return servErr
+	}
+	return nil
+}
+
+// startInProcessCrawl wires the crawler-inside-serve flow. Bumps
+// srv.hnsw to a fresh empty graph if none was loaded (so dense retrieval is
+// live as soon as the first passage lands), then runs the crawler in a
+// goroutine for the server's lifetime.
+func (s *pebbleHTTP) startInProcessCrawl(ctx context.Context, ps *store.PebbleStore, seedsFile string, ckpEvery time.Duration, cfg *config.Config, wg *sync.WaitGroup) error {
+	if s.embedder == nil {
+		return errors.New("crawl requires embedder configuration (cfg.Embeddings.Model)")
+	}
+	buf, err := os.ReadFile(seedsFile)
+	if err != nil {
+		return fmt.Errorf("read seeds file: %w", err)
+	}
+	var seeds []string
+	for _, line := range strings.Split(string(buf), "\n") {
+		t := strings.TrimSpace(line)
+		if t == "" || strings.HasPrefix(t, "#") {
+			continue
+		}
+		seeds = append(seeds, t)
+	}
+	if len(seeds) == 0 {
+		return errors.New("seeds file empty after parsing")
+	}
+
+	// Ensure HNSW is non-nil so the bridge can call AddPassage. /search reads
+	// the same pointer, so growth is immediately searchable.
+	if s.hnsw == nil {
+		s.hnsw = index.NewHNSW(s.embedder.Dim())
+		log.Printf("in-serve crawler: created fresh HNSW (dim=%d)", s.embedder.Dim())
+	}
+
+	c := crawler.NewWithBackend(cfg.Crawler, ps, index.NewPebbleBM25(ps))
+	// crawler embedder is wrapped in a semaphore-throttled
+	// wrapper so the bulk crawl can never queue enough requests at
+	// ollama to starve interactive /search?retriever=dense calls.
+	// Default cap = 8 (well under OLLAMA_NUM_PARALLEL=32); operator
+	// override via COSIFT_CRAWL_EMBED_CONCURRENCY.
+	crawlEmbedCap := 8
+	if v := os.Getenv("COSIFT_CRAWL_EMBED_CONCURRENCY"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			crawlEmbedCap = n
+		}
+	}
+	// Coalesces many
+	// per-doc embed calls into one larger inner.Embed per ~20 ms window
+	// (up to maxBatch=128 texts). Search bypasses the batcher (uses
+	// s.embedder directly) so interactive latency stays sub-100 ms.
+	// Operator overrides:
+	//   COSIFT_EMBED_BATCH=128       — texts per inner call
+	//   COSIFT_EMBED_BATCH_WAIT_MS=20 — drain timer in ms
+	batchMax := 128
+	if v := os.Getenv("COSIFT_EMBED_BATCH"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			batchMax = n
+		}
+	}
+	batchWait := 20 * time.Millisecond
+	if v := os.Getenv("COSIFT_EMBED_BATCH_WAIT_MS"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			batchWait = time.Duration(n) * time.Millisecond
+		}
+	}
+	batchedEmbedder := embed.NewBatchingEmbedder(s.embedder, batchMax, batchWait)
+	crawlEmbedder := embed.NewThrottledEmbedder(batchedEmbedder, crawlEmbedCap)
+	log.Printf("pebble-serve: crawler embed = throttle(%d) → batch(max=%d, wait=%s) → round-robin → backends; search bypasses batcher",
+		crawlEmbedCap, batchMax, batchWait)
+	c = c.WithEmbedder(crawlEmbedder)
+	c = c.WithPassageWriter(&hnswPassageWriter{ps: ps, hnsw: s.hnsw})
+	// Single-node config
+	// (NumShards <= 1) makes route fn a no-op (every URL ownsLocally).
+	// s.cluster was stamped at server-init time so search-only nodes also
+	// see it; we just need the router here.
+	if cfg.Cluster.IsClustered() {
+		c = c.WithRouter(
+			func(url string) (bool, string) {
+				if cfg.Cluster.OwnsURL(url) {
+					return true, ""
+				}
+				return false, cfg.Cluster.PeerForURL(url)
+			},
+			s.forwardURLToPeer,
+		)
+		log.Printf("in-serve crawler: cluster mode (shard=%d/%d, peers=%d)",
+			cfg.Cluster.MyShardID, cfg.Cluster.NumShards, len(cfg.Cluster.Peers))
+	}
+	// Expose Seed so /admin/crawl-enqueue can hand off forwarded URLs.
+	s.crawlSeed = c.Seed
+	// expose SeedSitemap so /admin/sitemap-import can push
+	// sitemap-discovered URLs into the live frontier.
+	s.crawlSeedSitemap = c.SeedSitemap
+	// expose SeedSitemapLane so /admin/site-submit can push a whole site's
+	// URLs into a chosen priority lane (default: submitted/priority).
+	s.crawlSeedSitemapLane = c.SeedSitemapLane
+	// expose Recrawl so /admin/recrawl-sitemap can reset done/errored URLs.
+	s.crawlRecrawl = c.Recrawl
+	s.crawlSeedRSS = c.SeedRSS
+	s.crawlFetchNow = c.FetchAndIndexNow
+	s.crawlSeedWET = c.SeedWET
+	s.crawlAllowDomain = c.AddAllowedDomain
+	for _, u := range seeds {
+		// Only seed locally-owned URLs in cluster mode; the rest get forwarded.
+		if cfg.Cluster.IsClustered() && !cfg.Cluster.OwnsURL(u) {
+			if err := s.forwardURLToPeer(ctx, u, cfg.Cluster.PeerForURL(u)); err != nil {
+				log.Printf("in-serve crawler: forward initial seed %s: %v", u, err)
+			}
+			continue
+		}
+		_ = c.Seed(u)
+	}
+	log.Printf("in-serve crawler: %d seeds queued (concurrency=%d, depth=%d, checkpoint=%s)",
+		len(seeds), cfg.Crawler.MaxConcurrent, cfg.Crawler.MaxDepth, ckpEvery)
+	s.crawlActive = true
+
+	// Checkpoint goroutine: incremental persist via PersistFrom — each
+	// tick only writes nodes [lastN, n). Shutdown does a full persist
+	// from 0 so any backlinks added to older nodes get refreshed.
+	// Seeding lastN from the loaded graph means the first checkpoint
+	// after restart only writes nodes added during this run; the prior
+	// N are already on disk.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		t := time.NewTicker(ckpEvery)
+		defer t.Stop()
+		lastN := s.hnsw.Len()
+		if lastN > 0 {
+			log.Printf("in-serve crawler: checkpoint baseline = %d nodes (loaded from disk)", lastN)
+		}
+		for {
+			select {
+			case <-ctx.Done():
+				n := s.hnsw.Len()
+				if n > 0 {
+					t0 := time.Now()
+					log.Printf("in-serve crawler: final HNSW persist at shutdown (%d nodes, full)", n)
+					if err := s.hnsw.Persist(context.Background(), ps); err != nil {
+						log.Printf("in-serve crawler: final HNSW persist failed: %v", err)
+					} else {
+						log.Printf("in-serve crawler: final HNSW persist complete in %s", time.Since(t0))
+					}
+				}
+				return
+			case <-t.C:
+				n := s.hnsw.Len()
+				if n == 0 || n == lastN {
+					continue
+				}
+				// graph can shrink (e.g., /admin/hnsw-compact rewrites
+				// indices and writes a smaller meta). When that happens, lastN
+				// from before the compaction is stale and > n; PersistFrom(lastN)
+				// would be a no-op forever, stranding any new AddPassages until
+				// shutdown. The compact handler does its own full Persist so disk
+				// is already in sync; we just need to resync lastN here.
+				if n < lastN {
+					lastN = n
+					continue
+				}
+				t0 := time.Now()
+				if err := s.hnsw.PersistFrom(context.Background(), ps, lastN); err != nil {
+					log.Printf("in-serve crawler: HNSW persist (incremental from %d) failed: %v", lastN, err)
+					continue
+				}
+				// alongside HNSW node writes, persist any newly-
+				// encoded PQ codes for nodes [lastN, n). Skipped silently
+				// when no codebook is loaded.
+				pqWritten := 0
+				if s.hnsw.HasPQ() {
+					if w, err := s.hnsw.PersistPQCodesFrom(context.Background(), ps, lastN); err != nil {
+						log.Printf("in-serve crawler: PQ codes persist failed: %v", err)
+					} else {
+						pqWritten = w
+					}
+				}
+				if pqWritten > 0 {
+					log.Printf("in-serve crawler: HNSW checkpoint at %d nodes (+%d incremental, +%d PQ codes, took %s)",
+						n, n-lastN, pqWritten, time.Since(t0))
+				} else {
+					log.Printf("in-serve crawler: HNSW checkpoint at %d nodes (+%d incremental, took %s)",
+						n, n-lastN, time.Since(t0))
+				}
+				lastN = n
+			}
+		}
+	}()
+
+	// Crawler goroutine — restarts automatically when new items appear in the
+	// frontier after a drain. This ensures that site-submit / recrawl-sitemap
+	// URLs submitted while the crawler is idle are picked up without a service
+	// restart. Poll every 5s after drain; resume as soon as queued > 0.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			log.Printf("in-serve crawler: starting")
+			if err := c.Run(ctx); err != nil {
+				log.Printf("in-serve crawler: exited with error: %v", err)
+				return
+			}
+			log.Printf("in-serve crawler: frontier drained, waiting for new work…")
+			// Poll until the parent context is cancelled or new items appear.
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-time.After(5 * time.Second):
+				}
+				fs, err := s.store.GetFrontierStats(ctx)
+				if err != nil {
+					continue
+				}
+				if fs.Queued > 0 {
+					log.Printf("in-serve crawler: %d new frontier items, restarting", fs.Queued)
+					break
+				}
+			}
+			if ctx.Err() != nil {
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+// pebbleHTTP holds the read-side handles. Stays small on purpose — the
+// SQLite-side Server struct accumulated a lot of config knobs over many iters;
+// the Pebble surface starts minimal and grows feature-by-feature.
+type pebbleHTTP struct {
+	store       *store.PebbleStore
+	idx         *index.PebbleBM25
+	chat        embed.ChatClient // nil when cfg.Chat.Model is unset; /answer returns 501
+	reranker    rerank.Reranker  // nil when no rerank is configured; ?rerank=true is a no-op then
+	rerankCandK int              // candidates pulled for rerank; default 20
+
+	// Bounded LLM concurrency + circuit breaker. chatGate is shared
+	// between the answer and rerank pools so a burst on one side can't
+	// also pin the goroutines of the other (the auto-sitemap-style leak
+	// that took the box down). chatSafe wraps the answer-class client
+	// (used for /answer, planner, HyDE, paraphrase) with KindAnswer;
+	// rerankSafe wraps the per-passage scoring client with KindRerank.
+	// Nil = uninitialized (tests / mocks).
+	chatGate   *chatgate.Gate
+	chatSafe   *chatgate.SafeChat
+	rerankSafe *chatgate.SafeChat
+
+	// TTL'd response cache + singleflight for /answer. A popular query
+	// during a load spike costs one hybrid + rerank + LLM call; the
+	// next 100 callers hit the cache in microseconds OR join the
+	// in-flight call (sub-second). Disabled when ttl=0.
+	answerCache *answercache.Cache
+
+	// /research SSE stream cache. Captures and replays the full SSE byte
+	// stream so repeat research queries skip retrieval + LLM entirely.
+	researchCache *answercache.Cache
+
+	// Query log: appends one JSON line per query-endpoint request so we have
+	// real observability into what users ask, what returns empty, and latency.
+	// nil file = disabled (COSIFT_QUERY_LOG unset). See querylog.go.
+	qlogFile *os.File
+	qlogMu   sync.Mutex
+
+	// Feedback log: appends one JSON line per /feedback rating, correlated to a
+	// query by qid. The real usefulness signal (penalize/reward). See feedback.go.
+	fbFile *os.File
+	fbMu   sync.Mutex
+	// fbRL rate-limits /feedback per real client (XFF) — always on, stricter than
+	// the global limiter, since feedback is public/unauthed and abusable.
+	fbRL *rateLimiter
+
+	// Per-site docID boost map cache. key = canonical site string (e.g.
+	// "pilotprotocol.network"). Populated lazily on first site= query;
+	// subsequent calls are sub-microsecond map lookups.
+	siteBoostCache sync.Map // map[string]map[int64]float64
+
+	// Per-site page-title cache for planner context. key = canonical site
+	// string (same scheme as siteBoostCache). value = []string (<=10 titles).
+	// Populated lazily on first site= research query.
+	siteTitleCache sync.Map // map[string][]string
+
+	// Polls vLLM /metrics. When num_requests_waiting exceeds
+	// COSIFT_LLM_DEGRADE_QUEUE, optional LLM stages (rerank, HyDE,
+	// paraphrase) are silently skipped — graceful degradation rather
+	// than queue-then-timeout.
+	llmProbe *chatgate.LoadProbe
+
+	// Per-host authority Scorer; multiplies BM25 / hybrid scores by
+	// (1 + alpha * Score(host)). Wikipedia / kernel.org / arxiv float
+	// over the spam directories in the long tail of the crawl.
+	authority *authority.Scorer
+
+	// Per-endpoint latency + error-rate tracking against
+	// operator-configured targets. Violations are appended to disk
+	// (COSIFT_SLA_LOG_PATH) and surfaced on /sla.
+	sla *sla.Monitor
+
+	// hostBoosts is the operator-configured host-suffix →
+	// multiplier map, applied to fused retrieval scores by /query.
+	// Empty / nil = no boosts (fast path).
+	hostBoosts map[string]float64
+
+	// Nil = disabled.
+	rl *rateLimiter
+
+	// Empty cluster cfg = single-node, no-ops below.
+	cluster config.Cluster
+	// crawlSeed is set after startInProcessCrawl runs so /admin/crawl-enqueue
+	// can hand off forwarded URLs into the in-process frontier. Nil when no
+	// in-serve crawler is wired.
+	crawlSeed func(url string) error
+	// crawlSeedSitemap wraps Crawler.SeedSitemap so the /admin/
+	// sitemap-import endpoint can push sitemap URLs into the live frontier.
+	crawlSeedSitemap func(ctx context.Context, url string) (int, error)
+	// crawlSeedSitemapLane is like crawlSeedSitemap but lets the caller pick
+	// the frontier lane — used by /admin/site-submit to land a site's URLs
+	// in the high-priority submitted lane.
+	crawlSeedSitemapLane func(ctx context.Context, url string, lane byte) (int, error)
+	crawlSeedRSS         func(ctx context.Context, url string) (int, error)
+	crawlFetchNow        func(ctx context.Context, url string) error
+	crawlSeedWET         func(ctx context.Context, url string, dedupeFresh, lexicalOnly bool) (int, error)
+	// crawlAllowDomain promotes a domain into the crawler's runtime dynamic
+	// allowlist (used by /admin/allow-domain for organic HN/Reddit growth).
+	crawlAllowDomain func(domain string) error
+	crawlRecrawl     func(ctx context.Context, url string) error
+
+	// doc count at startup so /stats can report crawl rate
+	// without persistent counter tables. docs_added = current - startup,
+	// rate = docs_added / uptime.
+	startupDocs int
+	// crawlActive = true when in-serve crawler is running.
+	crawlActive bool
+
+	// /research?expand=true issues a
+	// chat call PER sub-query, so a sticky workload (repeated queries, slow
+	// sub-query rephrasings from the planner) hits the chat provider many
+	// times for the same passage. Cap at 256 entries; on overflow drop one
+	// arbitrary entry (Go map iteration order is randomized, good enough).
+	hydeMu       sync.RWMutex
+	hydeCache    map[string]string
+	hydeCacheCap int // env-configurable via COSIFT_HYDE_CACHE_SIZE
+
+	// PQStatus's lock contended
+	// hard with the crawler's AddPassage writers — observed /stats
+	// latency varied 0.2 s to 5.5 s. 5 s TTL keeps the cache effective
+	// for the landing page (polls every ~30 s) without serving very
+	// stale numbers.
+	//
+	// After TTL, return stale body
+	// immediately and refresh async; only the first-ever call pays the
+	// cold-cache cost. statsRefreshing is a single-flight guard so we
+	// don't fan out N background refreshes when a burst arrives stale.
+	statsBodyMu     sync.Mutex
+	statsBodyBlob   []byte
+	statsBodyAt     time.Time
+	statsRefreshing atomic.Bool
+
+	// /research?expand=paraphrase fans
+	// out 3 paraphrases × N sub-queries — same hot path as HyDE but each
+	// miss is 3x larger by output volume. Keyed on q only (fixed n=3 today).
+	paraMu       sync.RWMutex
+	paraCache    map[string][]string
+	paraCacheCap int // env-configurable via COSIFT_PARA_CACHE_SIZE
+	paraHits     atomic.Int64
+	paraMisses   atomic.Int64
+
+	// atomic counters surfaced on /metrics so operators can size the
+	// HyDE cache against a real workload — if misses dominate, raise the cap
+	// or move to an L2 store; if hits dominate at a tight working set, the
+	// cap is sufficient and you can mostly forget about it.
+	hydeHits   atomic.Int64
+	hydeMisses atomic.Int64
+
+	// snapshot at startup of whether persisted HNSW vectors exist
+	// in the 'v' family. Cheap peek (first-hit short-circuit). Surfaced on
+	// /stats so operators know whether the store is ready for the future
+	// ?retriever=dense path without needing to grep pebble-info.
+	hasVectors bool
+	// when an HNSW meta blob is persisted (the normal case),
+	// surface dim + node count too. Cheap 20-byte read at startup.
+	vectorDim   int
+	vectorNodes int
+	// Loaded
+	// only when COSIFT_LOAD_HNSW=true at startup (gigabytes RAM at scale).
+	// Nil = graph not loaded; /search?retriever=dense returns a warning.
+	hnsw *index.HNSW
+	// Required by
+	// /search?retriever=dense alongside s.hnsw. Built at startup when
+	// cfg.Embeddings.Model is set. Nil → ?retriever=dense warns + falls back.
+	embedder embed.Embedder
+
+	// Rerank failures fall back
+	// to BM25 order silently — that's the right reliability move, but without
+	// a counter operators can't tell whether their LLM/HTTP reranker is
+	// healthy or quietly broken.
+	rerankAttempts atomic.Int64
+	rerankFailures atomic.Int64
+
+	// chat call attempt + failure counters across HyDE expansion,
+	// /answer synth (sync + SSE), and /research plan + synth (sync + SSE).
+	// A spike in failures (provider 429s, network blips) is the clearest
+	// early signal that synth endpoints are degraded.
+	chatAttempts atomic.Int64
+	chatFailures atomic.Int64
+
+	// count responses that carried at least one warning.
+	// Operators alerting on this catch 'a deploy started sending malformed
+	// requests' without having to parse response bodies.
+	warningsEmitted atomic.Int64
+
+	// /metrics divides this by
+	// chatAttempts to give mean chat latency, separated from the
+	// per-endpoint duration. Diagnoses 'where did the seconds go' on a
+	// slow /research stream: chat-side or retrieval-side?
+	chatDurationNanos atomic.Int64
+
+	// per-endpoint request counters + duration sums via a
+	// counting middleware wrapping every mux entry. sync.Map keeps the hot
+	// path lock-free; /metrics reads via Range. Path is the label so a
+	// misrouted call (404) doesn't get counted under an existing endpoint.
+	// rate(sum)/rate(count) over the duration sum gives mean latency in PromQL.
+	requestCounts sync.Map // map[string]*endpointMetrics
+
+	started time.Time
+}
+
+type endpointMetrics struct {
+	count    atomic.Int64
+	sumNanos atomic.Int64
+}
+
+// count is the request-counting middleware. Bumps a per-path
+// {count, sumNanos} struct, lazily created on first request to a path. Hot
+// path is sync.Map.Load + two atomic Adds — no contention even under high
+// RPS. Duration is sampled after the handler returns so streaming endpoints
+// account for their full open connection time.
+// envIntDefault reads name as an int; falls back to def when unset or
+// unparseable. Used by gate / deadline knobs so operators can tune via
+// env without a config-file roundtrip.
+func envIntDefault(name string, def int) int {
+	if v := os.Getenv(name); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			return n
+		}
+	}
+	return def
+}
+
+// envDurationMsDefault reads name as a millisecond integer; returns def
+// on missing/invalid. Chose ms over Go's "2s" parse to keep operator
+// muscle memory consistent with the other COSIFT_*_MS knobs.
+func envDurationMsDefault(name string, def time.Duration) time.Duration {
+	if v := os.Getenv(name); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			return time.Duration(n) * time.Millisecond
+		}
+	}
+	return def
+}
+
+// rateLimiter is a per-IP token-bucket limiter. Active when
+// COSIFT_RATELIMIT_RPM > 0; nil = disabled. Whitelisted IPs bypass entirely.
+type rateLimiter struct {
+	rpm       float64
+	burst     float64
+	whitelist map[string]bool
+	buckets   sync.Map // string IP → *rateLimitBucket
+}
+
+type rateLimitBucket struct {
+	mu     sync.Mutex
+	tokens float64
+	last   time.Time
+}
+
+// newRateLimiterFromEnv reads COSIFT_RATELIMIT_RPM / _BURST / _WHITELIST.
+// Returns nil when RPM is unset or non-positive — limiting is off by default
+// so the no-config self-host story stays simple.
+func newRateLimiterFromEnv() *rateLimiter {
+	rpmStr := os.Getenv("COSIFT_RATELIMIT_RPM")
+	if rpmStr == "" {
+		return nil
+	}
+	rpm, err := strconv.ParseFloat(rpmStr, 64)
+	if err != nil || rpm <= 0 {
+		return nil
+	}
+	burst := 10.0
+	if v := os.Getenv("COSIFT_RATELIMIT_BURST"); v != "" {
+		if b, err := strconv.ParseFloat(v, 64); err == nil && b > 0 {
+			burst = b
+		}
+	}
+	wl := map[string]bool{}
+	for _, ip := range strings.Split(os.Getenv("COSIFT_RATELIMIT_WHITELIST"), ",") {
+		ip = strings.TrimSpace(ip)
+		if ip != "" {
+			wl[ip] = true
+		}
+	}
+	return &rateLimiter{rpm: rpm, burst: burst, whitelist: wl}
+}
+
+// whitelistList returns the whitelisted IPs as a slice (for logging only).
+func (rl *rateLimiter) whitelistList() []string {
+	if rl == nil {
+		return nil
+	}
+	out := make([]string, 0, len(rl.whitelist))
+	for ip := range rl.whitelist {
+		out = append(out, ip)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// allow returns whether the request from ip may proceed. Side-effects: drains
+// one token from the IP's bucket on success.
+func (rl *rateLimiter) allow(ip string) bool {
+	if rl == nil {
+		return true
+	}
+	if rl.whitelist[ip] {
+		return true
+	}
+	bv, _ := rl.buckets.LoadOrStore(ip, &rateLimitBucket{tokens: rl.burst, last: time.Now()})
+	b := bv.(*rateLimitBucket)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	now := time.Now()
+	elapsed := now.Sub(b.last).Seconds()
+	b.tokens += elapsed * rl.rpm / 60.0
+	if b.tokens > rl.burst {
+		b.tokens = rl.burst
+	}
+	b.last = now
+	if b.tokens < 1 {
+		return false
+	}
+	b.tokens--
+	return true
+}
+
+// stripPort drops ":port" from a "host:port" or "[v6]:port" RemoteAddr. Falls
+// back to the input on parse failure (so we still get SOME per-client key).
+func stripPort(remoteAddr string) string {
+	host, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		return remoteAddr
+	}
+	return host
+}
+
+// rateLimit is the HTTP middleware that gates each request through the per-IP
+// limiter. Returns 429 with a JSON problem doc + Retry-After hint when the
+// bucket is empty. No-op when s.rl is nil.
+//
+// requireAdmin gates a handler on the peer auth token (cfg.Cluster.PeerAuthToken,
+// sent as "Authorization: Bearer <token>"). When the token is empty — the
+// single-node default — the check is skipped and any caller is accepted. Applied
+// at the mux level via awrap to every /admin/* route so no admin handler can ship
+// without auth, even if it forgets the per-handler check.
+func (s *pebbleHTTP) requireAdmin(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if want := s.cluster.PeerAuthToken; want != "" {
+			got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+			if got != want {
+				writeProblem(w, http.StatusUnauthorized, "missing or invalid peer token")
+				return
+			}
+		}
+		h(w, r)
+	}
+}
+
+// X-Forwarded-For is honored ONLY when the request came from a configured
+// trusted proxy; otherwise clients could spoof their IP by setting the header.
+// For self-host with cosift directly on the public network, leave
+// cfg.Server.TrustedProxies empty (default) and the RemoteAddr is used.
+func (s *pebbleHTTP) rateLimit(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ip := stripPort(r.RemoteAddr)
+		if !s.rl.allow(ip) {
+			w.Header().Set("Retry-After", "60")
+			writeProblem(w, http.StatusTooManyRequests, "rate limit exceeded for ip="+ip)
+			return
+		}
+		h(w, r)
+	}
+}
+
+func (s *pebbleHTTP) count(h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		key := r.URL.Path
+		v, ok := s.requestCounts.Load(key)
+		if !ok {
+			v, _ = s.requestCounts.LoadOrStore(key, &endpointMetrics{})
+		}
+		m := v.(*endpointMetrics)
+		m.count.Add(1)
+		start := time.Now()
+		// Wrap the writer so the SLA monitor knows whether the call
+		// succeeded; default status to 200 because handlers that just
+		// w.Write() never call WriteHeader explicitly.
+		sw := &statusCapturingWriter{ResponseWriter: w, status: 200}
+		h(sw, r)
+		dur := time.Since(start)
+		m.sumNanos.Add(dur.Nanoseconds())
+		if s.sla != nil {
+			s.sla.Observe(key, dur, sw.status < 500)
+		}
+	}
+}
+
+// statusCapturingWriter lets the count middleware record the response
+// status for the SLA observer. The middleware treats 5xx as failures
+// for SLA purposes; 4xx are client errors and don't count against us.
+type statusCapturingWriter struct {
+	http.ResponseWriter
+	status      int
+	wroteHeader bool
+	bytes       int64 // response body bytes — used by the query logger as an empty-result proxy
+}
+
+func (s *statusCapturingWriter) WriteHeader(code int) {
+	if !s.wroteHeader {
+		s.status = code
+		s.wroteHeader = true
+	}
+	s.ResponseWriter.WriteHeader(code)
+}
+
+func (s *statusCapturingWriter) Write(p []byte) (int, error) {
+	if !s.wroteHeader {
+		s.wroteHeader = true
+	}
+	n, err := s.ResponseWriter.Write(p)
+	s.bytes += int64(n)
+	return n, err
+}
+
+// Flush passes through to support SSE / streaming handlers.
+func (s *statusCapturingWriter) Flush() {
+	if f, ok := s.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+//go:embed assets/landing.html
+var landingHTML []byte
+
+//go:embed assets/chat.html
+var chatHTML []byte
+
+//go:embed assets/openapi.json
+var openapiJSON []byte
+
+//go:embed assets/swagger/index.html
+var swaggerIndexHTML []byte
+
+//go:embed assets/swagger/swagger-ui.css
+var swaggerUICSS []byte
+
+//go:embed assets/swagger/swagger-ui-bundle.js
+var swaggerUIJS []byte
+
+// handleLanding serves the self-host dashboard. Only matches exact
+// "/" — Go 1.22's ServeMux routes sub-paths to the longest match, so this
+// won't accidentally swallow /search etc.
+func (s *pebbleHTTP) handleLanding(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=60")
+	w.Write(landingHTML)
+}
+
+// handleChat serves the multi-turn chat UI. JS calls /answer
+// or /research with stream=true and renders SSE events incrementally.
+func (s *pebbleHTTP) handleChat(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=60")
+	_, _ = w.Write(chatHTML)
+}
+
+func (s *pebbleHTTP) handleOpenAPI(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	w.Write(openapiJSON)
+}
+
+func (s *pebbleHTTP) handleSwaggerUI(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "public, max-age=300")
+	w.Write(swaggerIndexHTML)
+}
+
+func (s *pebbleHTTP) handleSwaggerAsset(w http.ResponseWriter, r *http.Request) {
+	file := r.PathValue("file")
+	switch file {
+	case "swagger-ui.css":
+		w.Header().Set("Content-Type", "text/css")
+		w.Header().Set("Cache-Control", "public, max-age=86400")
+		w.Write(swaggerUICSS)
+	case "swagger-ui-bundle.js":
+		w.Header().Set("Content-Type", "application/javascript")
+		w.Header().Set("Cache-Control", "public, max-age=86400")
+		w.Write(swaggerUIJS)
+	default:
+		http.NotFound(w, r)
+	}
+}
diff --git a/cmd/cosift/serve_stats.go b/cmd/cosift/serve_stats.go
new file mode 100644
index 0000000..b3b3e4b
--- /dev/null
+++ b/cmd/cosift/serve_stats.go
@@ -0,0 +1,615 @@
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/embed"
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+func (s *pebbleHTTP) handleHealthz(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, http.StatusOK, map[string]any{"status": "ok"})
+}
+
+// handleDomains returns indexed hosts by doc count.
+//
+// Query params:
+//   - top:    legacy alias for limit (capped at 500). Preserved for the
+//     landing page contract.
+//   - q:      substring filter (case-insensitive) on the host name.
+//   - offset: pagination offset (default 0).
+//   - limit:  page size (default 50, capped at 500).
+//
+// Response:
+//
+//	{"total": N, "offset": M, "limit": L, "domains": [...]}
+func (s *pebbleHTTP) handleDomains(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query().Get("q")
+	offset := 0
+	if v := r.URL.Query().Get("offset"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n >= 0 {
+			offset = n
+		}
+	}
+	limit := 50
+	if v := r.URL.Query().Get("limit"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
+			limit = n
+		}
+	}
+	// Legacy 'top' alias from When set, behaves like limit + no
+	// search/offset; lands as the first page.
+	if v := r.URL.Query().Get("top"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
+			limit = n
+			offset = 0
+		}
+	}
+	domains, total, err := s.store.ListDomains(r.Context(), q, offset, limit)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"total":   total,
+		"offset":  offset,
+		"limit":   limit,
+		"q":       q,
+		"domains": domains,
+	})
+}
+
+// handleQueue surfaces frontier queue depth + top-N hosts currently queued
+// for crawl. Fills the gap left by /domains, which only shows
+// already-indexed hosts.
+func (s *pebbleHTTP) handleQueue(w http.ResponseWriter, r *http.Request) {
+	topN := 25
+	if v := r.URL.Query().Get("top"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
+			topN = n
+		}
+	}
+	fs, err := s.store.GetFrontierStats(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	hosts, err := s.store.TopQueuedHosts(r.Context(), topN)
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	body := map[string]any{
+		"queued":    fs.Queued,
+		"in_flight": fs.InFlight,
+		"done":      fs.Done,
+		"errored":   fs.Errored,
+		"top_hosts": hosts,
+	}
+	// Lane breakdown: PebbleStore-only (SQLite Store has no lanes). When the
+	// store is a PebbleStore, surface the per-lane queued/in_flight counts so
+	// operators can see whether the weighted RR is actually draining RSS
+	// (lane 1) ahead of bulk (lane 3).
+	if ps, ok := any(s.store).(*store.PebbleStore); ok {
+		if ls, lerr := ps.GetLaneStats(r.Context()); lerr == nil {
+			laneNames := [4]string{"submitted", "refresh", "discovered", "bulk"}
+			lanesOut := make([]map[string]any, 0, 4)
+			for i, n := range laneNames {
+				lanesOut = append(lanesOut, map[string]any{
+					"lane":      i,
+					"name":      n,
+					"queued":    ls.Lanes[i].Queued,
+					"in_flight": ls.Lanes[i].InFlight,
+				})
+			}
+			body["lanes"] = lanesOut
+			body["legacy_queued"] = ls.LegacyQueued
+			body["legacy_in_flight"] = ls.LegacyInFlight
+		}
+	}
+	writeJSON(w, http.StatusOK, body)
+}
+
+// handleDomainsAudit streams the entire 'h' family as JSONL, one
+// {host, count, authority, etld1, tld} record per line. Single pass
+// over the index — at 9M hosts on the GH200 this is ~30 seconds of
+// streaming, vs the 18000 full scans that brute-paginating /domains
+// would force. Read-only; safe to run live.
+//
+// Query params:
+//
+//	min_count=N — emit only hosts with count >= N (default 1)
+//	limit=N     — cap the number of emitted rows (default unlimited)
+//	below=F     — emit only hosts with authority score < F (filter
+//	              for the blocklist proposal stream; default 1.0)
+//	above=F     — emit only hosts with score >= F (filter for the
+//	              authority-list inspection; default 0.0)
+func (s *pebbleHTTP) handleDomainsAudit(w http.ResponseWriter, r *http.Request) {
+	minCount := 1
+	if v := r.URL.Query().Get("min_count"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			minCount = n
+		}
+	}
+	limit := -1
+	if v := r.URL.Query().Get("limit"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			limit = n
+		}
+	}
+	below := 1.0
+	if v := r.URL.Query().Get("below"); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			below = f
+		}
+	}
+	above := 0.0
+	if v := r.URL.Query().Get("above"); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			above = f
+		}
+	}
+	w.Header().Set("Content-Type", "application/x-ndjson")
+	w.Header().Set("Cache-Control", "no-store")
+	enc := json.NewEncoder(w)
+	emitted := 0
+	flusher, _ := w.(http.Flusher)
+	flushEvery := 256
+	err := s.store.IterateDomains(r.Context(), func(host string, count int) bool {
+		if count < minCount {
+			return true
+		}
+		var score float64 = 0.5
+		if s.authority != nil {
+			score = s.authority.Score(host)
+		}
+		if score >= below || score < above {
+			return true
+		}
+		rec := map[string]any{
+			"host":      host,
+			"count":     count,
+			"authority": score,
+			"recommend": classify(score),
+		}
+		if err := enc.Encode(rec); err != nil {
+			return false
+		}
+		emitted++
+		if limit > 0 && emitted >= limit {
+			return false
+		}
+		if flusher != nil && emitted%flushEvery == 0 {
+			flusher.Flush()
+		}
+		return true
+	})
+	if err != nil {
+		// Best-effort: log and stop; we may have already flushed
+		// partial output, so don't try to writeProblem.
+		log.Printf("handleDomainsAudit: %v", err)
+	}
+	if flusher != nil {
+		flusher.Flush()
+	}
+}
+
+// handleSLA returns the current SLA evaluation snapshot. Read-only.
+// Surfaces per-endpoint p50/p95/p99 + error rate, configured targets,
+// and the most recent violations. Operators consume this in dashboards
+// + on-call paging.
+func (s *pebbleHTTP) handleSLA(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	if s.sla == nil {
+		_, _ = w.Write([]byte(`{"enabled":false}`))
+		return
+	}
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	_ = enc.Encode(s.sla.Snapshot())
+}
+
+func (s *pebbleHTTP) handleStats(w http.ResponseWriter, r *http.Request) {
+	// stale-while-revalidate cache.
+	//   - Fresh hit (< TTL): return cached, X-Cache: HIT.
+	//   - Stale (have cache, past TTL but under maxStale): return cached,
+	//     X-Cache: STALE, kick a background refresh (single-flight).
+	//   - Very stale (past maxStale): fall through to synchronous compute
+	//     — protects against the case where a refresh goroutine
+	//     hangs under HNSW lock contention and the cache shows ancient
+	//     values (we saw a startup-time vec=654 served for ~15 min).
+	//   - Cold (no cache): compute synchronously, X-Cache: MISS.
+	const statsBodyTTL = 5 * time.Second
+	const statsBodyMaxStale = 60 * time.Second
+	s.statsBodyMu.Lock()
+	body := s.statsBodyBlob
+	age := time.Since(s.statsBodyAt)
+	cached := body != nil && !s.statsBodyAt.IsZero()
+	s.statsBodyMu.Unlock()
+
+	if cached && age < statsBodyTTL {
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("X-Cache", "HIT")
+		_, _ = w.Write(body)
+		return
+	}
+	if cached && age < statsBodyMaxStale {
+		if s.statsRefreshing.CompareAndSwap(false, true) {
+			go func() {
+				defer s.statsRefreshing.Store(false)
+				ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+				defer cancel()
+				if newBody, err := s.buildStatsBody(ctx); err == nil {
+					s.statsBodyMu.Lock()
+					s.statsBodyBlob = newBody
+					s.statsBodyAt = time.Now()
+					s.statsBodyMu.Unlock()
+				}
+			}()
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("X-Cache", "STALE")
+		_, _ = w.Write(body)
+		return
+	}
+
+	// Cold path: compute synchronously and populate the cache.
+	newBody, err := s.buildStatsBody(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	s.statsBodyMu.Lock()
+	s.statsBodyBlob = newBody
+	s.statsBodyAt = time.Now()
+	s.statsBodyMu.Unlock()
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("X-Cache", "MISS")
+	_, _ = w.Write(newBody)
+}
+
+// buildStatsBody collects every signal /stats surfaces and marshals it.
+// Heavy paths: PebbleStore.Stats (d-family scan, partially counter-cached
+// since), HNSW.PQStatus (O(N) under h.mu read lock that
+// contends with crawler writers). Called by handleStats both
+// synchronously (cold) and from a background goroutine (SWR refresh).
+func (s *pebbleHTTP) buildStatsBody(ctx context.Context) ([]byte, error) {
+	// /stats was observed taking 31s under load
+	// when the SWR cache exceeded maxStale: store.Stats() iterates the
+	// entire 'd' family (10.87M entries) every cold compute. The
+	// CorpusStats counter is an O(1) atomic read mirror of the persisted
+	// indexed_docs meta — equivalent doc count for any operator-facing
+	// purpose, microseconds instead of seconds. Terms isn't populated in
+	// the Pebble path (always 0) so dropping the scan loses nothing.
+	sumLen, indexedDocs, err := s.store.CorpusStats(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var avg float64
+	if indexedDocs > 0 {
+		avg = float64(sumLen) / float64(indexedDocs)
+	}
+	// surface config knobs that affect retrieval/synth quality so
+	// operators can verify env overrides took effect from one endpoint.
+	out := map[string]any{
+		"documents":    indexedDocs,
+		"terms":        int64(0),
+		"indexed_docs": indexedDocs,
+		"sum_doc_len":  sumLen,
+		"avg_doc_len":  avg,
+		"uptime":       time.Since(s.started).String(),
+		"backend":      "pebble",
+		"bm25_k1":      s.idx.K1(),
+		"bm25_b":       s.idx.B(),
+	}
+	if s.reranker != nil {
+		out["reranker"] = s.reranker.Name()
+		out["rerank_candidate_k"] = s.rerankCandK
+	}
+	if s.chat != nil {
+		out["chat_model"] = s.chat.Model()
+		// surface the cache caps so operators can verify
+		// COSIFT_HYDE_CACHE_SIZE / _PARA_CACHE_SIZE overrides took effect.
+		out["hyde_cache_size"] = s.hydeCacheCap
+		out["paraphrase_cache_size"] = s.paraCacheCap
+	}
+	// Bounded LLM concurrency + circuit-breaker state. Surfaced so
+	// operators can see whether the gate is shedding load (rejected > 0)
+	// or holding requests (in_flight = cap) and whether the breaker has
+	// tripped (state != "closed").
+	if s.chatGate != nil {
+		out["llm_gate"] = s.chatGate.Stats()
+	}
+	if s.chatSafe != nil {
+		out["llm_answer_circuit"] = s.chatSafe.CircuitState()
+	}
+	if s.rerankSafe != nil {
+		out["llm_rerank_circuit"] = s.rerankSafe.CircuitState()
+	}
+	if s.answerCache != nil {
+		out["answer_cache"] = s.answerCache.Stats()
+	}
+	if s.llmProbe != nil {
+		out["llm_load"] = s.llmProbe.Stats()
+	}
+	if s.sla != nil {
+		out["sla"] = s.sla.Snapshot()
+	}
+	if s.authority != nil {
+		out["authority"] = s.authority.Stats()
+	}
+	// embed cache hit/miss counters so operators can see how
+	// often re-fetches are skipping ollama. The cache wraps the round-
+	// robin embedder when cfg.Embeddings.CacheDir is set; type-assert
+	// down to extract the counters.
+	if ce, ok := s.embedder.(*embed.CachedEmbedder); ok {
+		hits := ce.Hits()
+		misses := ce.Misses()
+		total := hits + misses
+		hitRate := 0.0
+		if total > 0 {
+			hitRate = 100 * float64(hits) / float64(total)
+		}
+		out["embed_cache"] = map[string]any{
+			"hits":         hits,
+			"misses":       misses,
+			"hit_rate_pct": hitRate,
+		}
+	}
+	// signal whether the store has HNSW vectors persisted, and
+	// (when meta is available) surface dim + node count.
+	// when the graph is loaded in memory, report s.hnsw.Len()
+	// instead of the startup-cached vectorNodes count — otherwise /stats
+	// shows the corpus frozen in time while the in-serve crawler keeps
+	// growing the graph.
+	// "% of the indexed web" novelty stat. The web is too big to
+	// give a meaningful number, but Common Crawl's monthly snapshot is
+	// ~3.5 B unique pages — a recognizable reference point. Operators can
+	// override with COSIFT_WEB_DENOMINATOR if they want a different scale
+	// (e.g. their own intranet, a TLD slice, etc.).
+	webDenom := int64(3_500_000_000)
+	if v := os.Getenv("COSIFT_WEB_DENOMINATOR"); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+			webDenom = n
+		}
+	}
+	out["web_pct"] = 100.0 * float64(indexedDocs) / float64(webDenom)
+	out["web_denominator"] = webDenom
+	out["has_vectors"] = s.hasVectors
+	switch {
+	case s.hnsw != nil:
+		out["vector_nodes"] = s.hnsw.Len()
+		out["vector_dim"] = s.vectorDim
+	case s.vectorNodes > 0:
+		out["vector_nodes"] = s.vectorNodes
+		out["vector_dim"] = s.vectorDim
+	}
+	// whether the graph is loaded into memory for dense retrieval.
+	out["hnsw_loaded"] = s.hnsw != nil
+	// PQ status — operator-facing visibility into compression
+	// state. Only present when the graph is loaded; nil otherwise.
+	if s.hnsw != nil {
+		pq := s.hnsw.PQStatus()
+		// coverage is over VALID nodes (vec != nil), not raw total —
+		// zombie slots from pre partial persists inflate the total
+		// without being searchable. NodesTotal still surfaced for context.
+		denom := pq.NodesValid
+		if denom == 0 {
+			denom = pq.NodesTotal // avoid div-by-zero on a fresh load
+		}
+		coverage := 0.0
+		if denom > 0 {
+			coverage = 100 * float64(pq.NodesWithCode) / float64(denom)
+		}
+		zombies := pq.NodesTotal - pq.NodesValid
+		pqInfo := map[string]any{
+			"enabled":         pq.Enabled,
+			"nodes_with_code": pq.NodesWithCode,
+			"nodes_valid":     pq.NodesValid,
+			"nodes_total":     pq.NodesTotal,
+			"zombie_nodes":    zombies,
+			"coverage_pct":    coverage,
+		}
+		if pq.Enabled {
+			pqInfo["dim"] = pq.Dim
+			pqInfo["m"] = pq.M
+			pqInfo["k"] = pq.K
+		}
+		out["pq"] = pqInfo
+	}
+	// Clients can read
+	// this once instead of probing ?retriever=dense + parsing the warning.
+	// bm25 always — index is always available. dense/hybrid require a
+	// loaded graph; the embedder lets dense/hybrid handle text-mode and
+	// new queries. /find_similar URL-mode dense works without embedder, but
+	// that's an endpoint-specific carve-out — keep this list general.
+	retrievers := []string{"bm25", "bm25-mlt"}
+	if s.hnsw != nil && s.embedder != nil {
+		retrievers = append(retrievers, "dense", "hybrid")
+	} else if s.hnsw != nil {
+		// Graph but no embedder — only URL-mode /find_similar can use dense.
+		retrievers = append(retrievers, "dense:find_similar_url_only")
+	}
+	out["retrievers"] = retrievers
+	// crawl-rate surface for the in-serve crawler.
+	// Computes docs added since process start and per-minute rate.
+	if s.crawlActive {
+		uptimeMin := time.Since(s.started).Minutes()
+		added := int(indexedDocs) - s.startupDocs
+		if added < 0 {
+			added = 0
+		}
+		rate := 0.0
+		if uptimeMin > 0 {
+			rate = float64(added) / uptimeMin
+		}
+		out["crawl_active"] = true
+		out["docs_added_since_start"] = added
+		out["docs_per_minute"] = rate
+	}
+	return json.Marshal(out)
+}
+
+// Prometheus-format scrape endpoint. Hand-written plain text
+// (no client_golang dep) — exposition format is simple enough that
+// pulling in a dep just to print four gauges isn't justified.
+// Quantities chosen to be O(1) reads (CorpusStats counters + uptime);
+// frontier scans deliberately excluded so scrape latency stays flat
+// regardless of corpus size.
+func (s *pebbleHTTP) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	sumLen, count, err := s.store.CorpusStats(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	var avg float64
+	if count > 0 {
+		avg = float64(sumLen) / float64(count)
+	}
+	uptime := time.Since(s.started).Seconds()
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprintf(w, "# HELP cosift_indexed_docs Number of documents passed through IndexDocument.\n")
+	fmt.Fprintf(w, "# TYPE cosift_indexed_docs gauge\n")
+	fmt.Fprintf(w, "cosift_indexed_docs %d\n", count)
+	fmt.Fprintf(w, "# HELP cosift_sum_doc_len_total Sum of indexed document lengths in tokens.\n")
+	fmt.Fprintf(w, "# TYPE cosift_sum_doc_len_total counter\n")
+	fmt.Fprintf(w, "cosift_sum_doc_len_total %d\n", sumLen)
+	fmt.Fprintf(w, "# HELP cosift_avg_doc_len Average document length in tokens (derived).\n")
+	fmt.Fprintf(w, "# TYPE cosift_avg_doc_len gauge\n")
+	fmt.Fprintf(w, "cosift_avg_doc_len %.2f\n", avg)
+	fmt.Fprintf(w, "# HELP cosift_uptime_seconds Seconds since pebble-serve started.\n")
+	fmt.Fprintf(w, "# TYPE cosift_uptime_seconds counter\n")
+	fmt.Fprintf(w, "cosift_uptime_seconds %.0f\n", uptime)
+	// crawl_active gauges whether
+	// the crawler goroutine is running; docs_added is monotonic
+	// since process start; docs_per_minute is the rolling rate.
+	crawlActiveGauge := 0
+	if s.crawlActive {
+		crawlActiveGauge = 1
+	}
+	fmt.Fprintf(w, "# HELP cosift_crawl_active 1 if in-serve crawler is running, 0 otherwise.\n")
+	fmt.Fprintf(w, "# TYPE cosift_crawl_active gauge\n")
+	fmt.Fprintf(w, "cosift_crawl_active %d\n", crawlActiveGauge)
+	if s.crawlActive {
+		added := count - int64(s.startupDocs)
+		if added < 0 {
+			added = 0
+		}
+		fmt.Fprintf(w, "# HELP cosift_crawl_docs_added_total Documents added by the in-serve crawler since process start.\n")
+		fmt.Fprintf(w, "# TYPE cosift_crawl_docs_added_total counter\n")
+		fmt.Fprintf(w, "cosift_crawl_docs_added_total %d\n", added)
+		rate := 0.0
+		if uptime > 0 {
+			rate = float64(added) / (uptime / 60)
+		}
+		fmt.Fprintf(w, "# HELP cosift_crawl_docs_per_minute Recent crawl rate (docs added per minute, averaged since process start).\n")
+		fmt.Fprintf(w, "# TYPE cosift_crawl_docs_per_minute gauge\n")
+		fmt.Fprintf(w, "cosift_crawl_docs_per_minute %.2f\n", rate)
+	}
+	// HyDE cache effectiveness. Hits/misses both monotonic so
+	// Prometheus rate() over these gives cache pressure under load.
+	fmt.Fprintf(w, "# HELP cosift_hyde_cache_hits_total HyDE cache hits (expandQuery served from memory).\n")
+	fmt.Fprintf(w, "# TYPE cosift_hyde_cache_hits_total counter\n")
+	fmt.Fprintf(w, "cosift_hyde_cache_hits_total %d\n", s.hydeHits.Load())
+	fmt.Fprintf(w, "# HELP cosift_hyde_cache_misses_total HyDE cache misses (expandQuery called the LLM).\n")
+	fmt.Fprintf(w, "# TYPE cosift_hyde_cache_misses_total counter\n")
+	fmt.Fprintf(w, "cosift_hyde_cache_misses_total %d\n", s.hydeMisses.Load())
+	fmt.Fprintf(w, "# HELP cosift_paraphrase_cache_hits_total Paraphrase cache hits (paraphraseQuery served from memory).\n")
+	fmt.Fprintf(w, "# TYPE cosift_paraphrase_cache_hits_total counter\n")
+	fmt.Fprintf(w, "cosift_paraphrase_cache_hits_total %d\n", s.paraHits.Load())
+	fmt.Fprintf(w, "# HELP cosift_paraphrase_cache_misses_total Paraphrase cache misses (paraphraseQuery called the LLM).\n")
+	fmt.Fprintf(w, "# TYPE cosift_paraphrase_cache_misses_total counter\n")
+	fmt.Fprintf(w, "cosift_paraphrase_cache_misses_total %d\n", s.paraMisses.Load())
+	fmt.Fprintf(w, "# HELP cosift_rerank_attempts_total Rerank calls invoked (any endpoint with ?rerank=true).\n")
+	fmt.Fprintf(w, "# TYPE cosift_rerank_attempts_total counter\n")
+	fmt.Fprintf(w, "cosift_rerank_attempts_total %d\n", s.rerankAttempts.Load())
+	fmt.Fprintf(w, "# HELP cosift_rerank_failures_total Rerank calls that returned an error (silently fell back to BM25 order).\n")
+	fmt.Fprintf(w, "# TYPE cosift_rerank_failures_total counter\n")
+	fmt.Fprintf(w, "cosift_rerank_failures_total %d\n", s.rerankFailures.Load())
+	fmt.Fprintf(w, "# HELP cosift_chat_attempts_total Chat-client calls invoked (HyDE, /answer synth, /research plan + synth).\n")
+	fmt.Fprintf(w, "# TYPE cosift_chat_attempts_total counter\n")
+	fmt.Fprintf(w, "cosift_chat_attempts_total %d\n", s.chatAttempts.Load())
+	fmt.Fprintf(w, "# HELP cosift_chat_failures_total Chat-client calls that returned an error.\n")
+	fmt.Fprintf(w, "# TYPE cosift_chat_failures_total counter\n")
+	fmt.Fprintf(w, "cosift_chat_failures_total %d\n", s.chatFailures.Load())
+	fmt.Fprintf(w, "# HELP cosift_chat_duration_seconds_sum Cumulative wall-clock spent inside chat-client calls.\n")
+	fmt.Fprintf(w, "# TYPE cosift_chat_duration_seconds_sum counter\n")
+	fmt.Fprintf(w, "cosift_chat_duration_seconds_sum %.6f\n", float64(s.chatDurationNanos.Load())/1e9)
+	fmt.Fprintf(w, "# HELP cosift_warnings_emitted_total Responses that carried at least one warning (misconfigured request).\n")
+	fmt.Fprintf(w, "# TYPE cosift_warnings_emitted_total counter\n")
+	fmt.Fprintf(w, "cosift_warnings_emitted_total %d\n", s.warningsEmitted.Load())
+	// HNSW vector index shape. Live count when graph is
+	// loaded; startup-cached otherwise.
+	vectorNodesLive := s.vectorNodes
+	if s.hnsw != nil {
+		vectorNodesLive = s.hnsw.Len()
+	}
+	if vectorNodesLive > 0 {
+		fmt.Fprintf(w, "# HELP cosift_vector_nodes Number of HNSW vector nodes in memory (or persisted when not loaded).\n")
+		fmt.Fprintf(w, "# TYPE cosift_vector_nodes gauge\n")
+		fmt.Fprintf(w, "cosift_vector_nodes %d\n", vectorNodesLive)
+		fmt.Fprintf(w, "# HELP cosift_vector_dim Embedding dimension of the persisted HNSW index.\n")
+		fmt.Fprintf(w, "# TYPE cosift_vector_dim gauge\n")
+		fmt.Fprintf(w, "cosift_vector_dim %d\n", s.vectorDim)
+	}
+	// PromQL
+	// rate(cosift_request_duration_seconds_sum) / rate(cosift_requests_total)
+	// gives mean latency in any window. Labels = path; misrouted calls (404)
+	// don't share a label with any handled path.
+	fmt.Fprintf(w, "# HELP cosift_requests_total HTTP requests served, by endpoint.\n")
+	fmt.Fprintf(w, "# TYPE cosift_requests_total counter\n")
+	fmt.Fprintf(w, "# HELP cosift_request_duration_seconds_sum Cumulative request duration, by endpoint.\n")
+	fmt.Fprintf(w, "# TYPE cosift_request_duration_seconds_sum counter\n")
+	s.requestCounts.Range(func(k, v any) bool {
+		path := k.(string)
+		m := v.(*endpointMetrics)
+		fmt.Fprintf(w, "cosift_requests_total{endpoint=%q} %d\n", path, m.count.Load())
+		fmt.Fprintf(w, "cosift_request_duration_seconds_sum{endpoint=%q} %.6f\n", path, float64(m.sumNanos.Load())/1e9)
+		return true
+	})
+}
+
+// HTTP form of `cosift verify`. Same comparison (running
+// counters vs authoritative 'l' family scan) on the already-open store, so
+// monitoring can poll without contending for the single-writer lock or
+// shelling into the container. 503 on drift makes this composable with
+// Kubernetes liveness / load-balancer health checks.
+func (s *pebbleHTTP) handleVerify(w http.ResponseWriter, r *http.Request) {
+	counterSum, counterCount, err := s.store.CorpusStats(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	scanSum, scanCount, err := s.store.SumDocLengths(r.Context())
+	if err != nil {
+		writeProblem(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	driftCount := counterCount - scanCount
+	driftSum := counterSum - scanSum
+	ok := driftCount == 0 && driftSum == 0
+	body := map[string]any{
+		"ok":                   ok,
+		"indexed_docs_counter": counterCount,
+		"indexed_docs_scan":    scanCount,
+		"indexed_docs_drift":   driftCount,
+		"sum_doc_len_counter":  counterSum,
+		"sum_doc_len_scan":     scanSum,
+		"sum_doc_len_drift":    driftSum,
+	}
+	status := http.StatusOK
+	if !ok {
+		status = http.StatusServiceUnavailable
+	}
+	writeJSON(w, status, body)
+}
diff --git a/cmd/cosift/zz_round3_test.go b/cmd/cosift/zz_round3_test.go
index 5b2231d..c5e44f9 100644
--- a/cmd/cosift/zz_round3_test.go
+++ b/cmd/cosift/zz_round3_test.go
@@ -53,7 +53,7 @@ type openaiMock struct {
 //   - embeddings return one vector of `embedDim` floats per input (a hash of
 //     the text bytes scaled to [0,1])
 //
-// Override per-test via SetChatResponse / SetEmbedDim.
+// Override per-test via SetChatResponse.
 func openaiTestServer(t *testing.T) *openaiMock {
 	t.Helper()
 	m := &openaiMock{embedDim: 8}
@@ -134,13 +134,6 @@ func (m *openaiMock) SetChatResponses(seq ...string) {
 	m.chatRespQueue = append([]string{}, seq...)
 }
 
-// SetEmbedDim swaps the embedding vector dimensionality.
-func (m *openaiMock) SetEmbedDim(d int) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.embedDim = d
-}
-
 // ChatCalls returns the number of /v1/chat/completions requests served.
 func (m *openaiMock) ChatCalls() int { m.mu.Lock(); defer m.mu.Unlock(); return m.chatCalls }
 
diff --git a/cmd/cosift/zz_round4_test.go b/cmd/cosift/zz_round4_test.go
index 92b38ea..06ece1a 100644
--- a/cmd/cosift/zz_round4_test.go
+++ b/cmd/cosift/zz_round4_test.go
@@ -899,10 +899,23 @@ func TestRunAnswerEvalNoKey(t *testing.T) {
 func TestRunAnswerEvalBadCorpus(t *testing.T) {
 	_ = os.Unsetenv("OPENAI_API_KEY")
 	_ = os.Unsetenv("OPENAI")
+	// Self-contained: write a valid query set so the queries load (which runs
+	// first) succeeds, then point -corpus at a missing file so the corpus load
+	// is the branch that fails. -chat-url satisfies the API-key gate without a
+	// real key; no -dry-run so execution proceeds past the queries load to the
+	// in-process corpus load.
+	tmp := t.TempDir()
+	queriesPath := filepath.Join(tmp, "queries.json")
+	mustWriteJSON(t, queriesPath, map[string]any{
+		"name": "bad-corpus",
+		"queries": []any{
+			map[string]any{"text": "first query", "relevant": []string{"https://example.com/a"}},
+		},
+	})
 	err := runAnswerEval(context.Background(), []string{
 		"-corpus", "/no/such/corpus.json",
-		"-queries", "testdata/eval/queries.json",
-		"-dry-run",
+		"-queries", queriesPath,
+		"-chat-url", "http://127.0.0.1:1",
 	})
 	if err == nil || !strings.Contains(err.Error(), "load corpus") {
 		t.Errorf("got %v", err)
diff --git a/deploy/README.md b/deploy/README.md
new file mode 100644
index 0000000..36b1916
--- /dev/null
+++ b/deploy/README.md
@@ -0,0 +1,35 @@
+# cosift deploy — content pipeline (GH200 production)
+
+Operational artifacts for the running cosift backend (`cosift.pilotprotocol.network`,
+binary `/home/ubuntu/cosift`, service `cosift-serve`). Captured from the server so the
+content pipeline is reproducible from git, not only on the box.
+
+## Layout
+- `harvesters/` — scripts in `/usr/local/bin/` that feed the crawl frontier via the
+  cosift admin API (`/admin/crawl-enqueue`, `/admin/rss-import`, `/admin/wet-import-bulk`,
+  `/admin/site-submit`). All free/OSS, stdlib-only.
+- `systemd/` — `.service` (oneshot) + `.timer` units that schedule the harvesters.
+- `lists/` — source lists the harvesters read (`cosift-feeds.txt` RSS feeds,
+  `cosift-subreddits.txt`, `cosift-sites.txt`).
+
+## Sources & cadence
+| Harvester | Source | Schedule |
+|---|---|---|
+| rss-refresh | `cosift-feeds.txt` (RSS/Atom incl. macrumors, tomshardware, arXiv cs.*) | 30 min |
+| arxiv-harvest | arXiv export API (recent papers per cs.* category) | daily 05:30 |
+| hf-harvest | HF Hub API top-likes models/datasets/spaces + blog | daily 06:30 |
+| hf-full-harvest | full HF Hub enumeration (cursor-paginated, resumable) | every 3h |
+| hn-harvest | HN Firebase API + external-link domain promotion | 4×/day |
+| reddit-harvest | per-subreddit Atom RSS + link promotion (rate-limited by reddit) | 4×/day |
+| wet-refresh | CommonCrawl WET (rotating offset, lexical-only) | daily 03:00 |
+| pypi/hf-task/github-topic | popular PyPI / HF-by-task / GitHub-by-topic | weekly |
+| sitemap-refresh | (disabled — net-negative on single box) | — |
+
+## Notes
+- Curated ingester: `disable_link_following=true` + `include_domains` allowlist in
+  `cosift.json` (on the server). `/admin/allow-domain` grows it at runtime from HN/Reddit
+  link-domain frequency (persisted to `cosift-dynamic-domains.txt`).
+- Env (systemd drop-ins on server): `COSIFT_HOST_PARTITION[_READ]=1`, `COSIFT_QUERY_LOG`,
+  `COSIFT_DYNAMIC_DOMAINS_FILE`, WET/feedback tunables.
+- To re-deploy: copy `harvesters/*` → `/usr/local/bin/`, `systemd/*` → `/etc/systemd/system/`,
+  `lists/*` → `/home/ubuntu/`, then `systemctl daemon-reload && systemctl enable --now <timer>`.
diff --git a/deploy/harvesters/cosift-arxiv-harvest.sh b/deploy/harvesters/cosift-arxiv-harvest.sh
new file mode 100755
index 0000000..b794891
--- /dev/null
+++ b/deploy/harvesters/cosift-arxiv-harvest.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+# cosift-arxiv-harvest — enqueue the latest arXiv papers per category into the
+# crawl frontier. THE always-new anchor for the curated corpus.
+#
+# Why this exists: cosift runs link-following OFF (curated ingester), so it only
+# indexes URLs explicitly fed to it. arXiv's RSS endpoints are dead (0 items),
+# but the export API (Atom) reliably returns recent submissions. arXiv publishes
+# ~500-1000 papers/day across cs.* — all in the quality allowlist — so feeding
+# each new paper's abs URL guarantees a continuous stream of new documents.
+#
+# Frontier dedup means re-running is safe: only genuinely-new papers (the day's
+# submissions) actually enter the queue; the rest are no-ops.
+set -u
+
+ADDR="http://127.0.0.1:7777"
+API="https://export.arxiv.org/api/query"
+TOKEN=$(python3 -c "import json;c=json.load(open('/home/ubuntu/cosift.json'));print(c.get('cluster',{}).get('peer_auth_token',''))" 2>/dev/null || echo "")
+CATS="${COSIFT_ARXIV_CATS:-cs.AI cs.LG cs.CL cs.CV cs.DC cs.SE cs.CR cs.IR cs.NE cs.DS cs.DB cs.RO stat.ML}"
+MAXR="${COSIFT_ARXIV_MAX:-100}"
+
+total=0
+for cat in $CATS; do
+  # arXiv asks for >=3s between API hits; -L follows the http->https redirect.
+  xml=$(curl -sL --max-time 90 "${API}?search_query=cat:${cat}&sortBy=submittedDate&sortOrder=descending&max_results=${MAXR}")
+  urls=$(printf '%s' "$xml" | python3 -c "
+import sys,re
+ids=re.findall(r'<id>(https?://arxiv\.org/abs/[^<]+)</id>', sys.stdin.read())
+for u in ids: print(u)
+")
+  n=0
+  while IFS= read -r u; do
+    [ -z "$u" ] && continue
+    curl -s -o /dev/null --max-time 30 -X POST "$ADDR/admin/crawl-enqueue" \
+      -H "Content-Type: application/json" \
+      ${TOKEN:+-H "Authorization: Bearer $TOKEN"} \
+      -d "{\"url\":\"$u\"}"
+    n=$((n+1))
+  done <<< "$urls"
+  echo "cosift-arxiv-harvest: $cat -> enqueued $n"
+  total=$((total + n))
+  sleep 3
+done
+echo "cosift-arxiv-harvest: total enqueued=$total at $(date -u +%FT%TZ)"
diff --git a/deploy/harvesters/cosift-compact.sh b/deploy/harvesters/cosift-compact.sh
new file mode 100755
index 0000000..b87a569
--- /dev/null
+++ b/deploy/harvesters/cosift-compact.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# cosift-compact.sh — invoke hnsw-compact when zombies exceed a threshold.
+# Run from cosift-compact.timer (weekly recommended).
+set -e
+TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+THRESHOLD_PCT=30
+# /admin/hnsw-compact is gated by cluster.peer_auth_token — read it from the
+# same config the server uses so this keeps working once the token is set.
+TOKEN=$(python3 -c "import json; print(json.load(open('/home/ubuntu/cosift.json')).get('cluster',{}).get('peer_auth_token',''))" 2>/dev/null || echo "")
+STATS=$(curl -s --max-time 30 http://127.0.0.1:7777/stats)
+if [ -z "$STATS" ]; then
+  echo "${TS} stats fetch failed" | sudo tee -a /var/log/cosift-compact.log >/dev/null
+  exit 0
+fi
+PCT=$(echo "$STATS" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+t=d[\"pq\"][\"nodes_total\"]
+z=d[\"pq\"][\"zombie_nodes\"]
+if t==0: print(0); exit()
+print(int(100*z/t))
+")
+echo "${TS} zombies=${PCT}% threshold=${THRESHOLD_PCT}%" | sudo tee -a /var/log/cosift-compact.log >/dev/null
+if [ "$PCT" -lt "$THRESHOLD_PCT" ]; then
+  echo "${TS} below threshold, skipping compact" | sudo tee -a /var/log/cosift-compact.log >/dev/null
+  exit 0
+fi
+RESULT=$(curl -s --max-time 1800 -X POST ${TOKEN:+-H "Authorization: Bearer $TOKEN"} http://127.0.0.1:7777/admin/hnsw-compact)
+echo "${TS} compact result: ${RESULT}" | sudo tee -a /var/log/cosift-compact.log >/dev/null
diff --git a/deploy/harvesters/cosift-eval.sh b/deploy/harvesters/cosift-eval.sh
new file mode 100755
index 0000000..05503f0
--- /dev/null
+++ b/deploy/harvesters/cosift-eval.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Iter 500: daily eval smoke run. Hits /admin/eval-quick, appends one line
+# per run to /var/log/cosift-eval.log with timestamp + JSON summary. Lets
+# operators grep historical answer rates and spot quality regressions
+# tied to corpus growth or code changes.
+set -e
+TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+RESULT=$(curl -s --max-time 300 http://127.0.0.1:7777/admin/eval-quick)
+# Compact one-line JSON summary (drop the verbose per-query results for the log).
+SUMMARY=$(echo "$RESULT" | python3 -c '
+import json,sys
+try:
+    d=json.load(sys.stdin)
+    out={k:d[k] for k in ["queries","answered","no_info","empty","errors","answer_rate_pct","avg_latency_ms","total_elapsed","chat_model"] if k in d}
+    print(json.dumps(out))
+except Exception as e:
+    print("{\"error\":\"%s\"}" % e)
+')
+echo "${TS} ${SUMMARY}" | sudo tee -a /var/log/cosift-eval.log >/dev/null
+echo "${TS} ${SUMMARY}"
diff --git a/deploy/harvesters/cosift-github-topic-harvest.py b/deploy/harvesters/cosift-github-topic-harvest.py
new file mode 100755
index 0000000..9449a02
--- /dev/null
+++ b/deploy/harvesters/cosift-github-topic-harvest.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""cosift-github-topic-harvest — index top-starred GitHub repos per topic.
+
+READMEs are the canonical install/usage docs for tools/libraries. Harvesting by
+topic warms the corpus with high-quality, actively-maintained repos. github.com
+is allowlisted. The GitHub search API is unauthenticated here (≈10 req/min), so
+we space topic queries. Stdlib only.
+"""
+import json, os, urllib.request, urllib.parse, time
+
+ADDR = "http://127.0.0.1:7777"
+PER = int(os.environ.get("COSIFT_GH_TOPIC_LIMIT", "30"))
+SPACING = float(os.environ.get("COSIFT_GH_SPACING", "7"))  # unauth search ~10/min
+TOPICS = os.environ.get("COSIFT_GH_TOPICS",
+    "machine-learning deep-learning computer-vision nlp llm ocr object-detection "
+    "speech-recognition reinforcement-learning robotics jetson embedded "
+    "kubernetes devops observability rust golang database "
+    "web-framework security cryptography").split()
+
+def token():
+    try: return json.load(open("/home/ubuntu/cosift.json")).get("cluster",{}).get("peer_auth_token","")
+    except Exception: return ""
+TOKEN = token()
+
+def get_json(url):
+    req = urllib.request.Request(url, headers={"User-Agent":"cosift-gh-topic/1.0",
+        "Accept":"application/vnd.github+json"})
+    with urllib.request.urlopen(req, timeout=30) as r: return json.loads(r.read())
+
+def post(path, payload):
+    req = urllib.request.Request(ADDR+path, data=json.dumps(payload).encode(),
+        method="POST", headers={"Content-Type":"application/json"})
+    if TOKEN: req.add_header("Authorization","Bearer "+TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r: return r.status
+    except Exception as e: return str(e)
+
+def main():
+    total = 0
+    for topic in TOPICS:
+        url = ("https://api.github.com/search/repositories?q=topic:"
+               + urllib.parse.quote(topic) + f"&sort=stars&order=desc&per_page={PER}")
+        try:
+            res = get_json(url)
+        except Exception as e:
+            print(f"gh-topic-harvest: {topic} failed: {e}"); time.sleep(SPACING); continue
+        items = res.get("items", [])
+        n = 0
+        for it in items:
+            u = it.get("html_url")
+            if u: post("/admin/crawl-enqueue", {"url": u}); n += 1
+        print(f"gh-topic-harvest: {topic} -> enqueued {n}")
+        total += n
+        time.sleep(SPACING)
+    print(f"gh-topic-harvest: total {total} at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-hf-full-harvest.py b/deploy/harvesters/cosift-hf-full-harvest.py
new file mode 100755
index 0000000..0634228
--- /dev/null
+++ b/deploy/harvesters/cosift-hf-full-harvest.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""cosift-hf-full-harvest — enumerate the ENTIRE HuggingFace Hub and enqueue
+every model, dataset, and Space for indexing. "As many as possible" coverage.
+
+Walks the Hub via cursor pagination (Link: rel="next" header), limit=1000/page.
+RESUMABLE: the per-kind cursor is checkpointed to STATE after every page, so a
+run picks up exactly where the last left off — across crashes, restarts, and the
+per-run budget. When a kind is fully enumerated it resets (next cycle re-walks to
+pick up newly-published repos).
+
+Reality check: enumeration is fast, but the crawler then has to fetch+embed each
+page at its per-host rate — full-Hub coverage is a multi-week grind on one box.
+The frontier is the buffer; we throttle enqueue per run so we don't build an
+absurd backlog faster than it can be reasoned about. Stdlib only.
+"""
+import json, os, sys, time, urllib.request
+
+ADDR = "http://127.0.0.1:7777"
+HF = "https://huggingface.co"
+STATE = "/home/ubuntu/cosift-hf-cursor.json"
+BUDGET = int(os.environ.get("COSIFT_HF_FULL_BUDGET", "50000"))  # max enqueues per run
+PAGE = int(os.environ.get("COSIFT_HF_FULL_PAGE", "1000"))
+KINDS = [("models", "/"), ("datasets", "/datasets/"), ("spaces", "/spaces/")]
+
+def token():
+    try: return json.load(open("/home/ubuntu/cosift.json")).get("cluster",{}).get("peer_auth_token","")
+    except Exception: return ""
+TOKEN = token()
+
+def http(url):
+    """GET returning (json_body, next_url_or_None) using the Link rel=next header."""
+    req = urllib.request.Request(url, headers={"User-Agent":"cosift-hf-full/1.0"})
+    with urllib.request.urlopen(req, timeout=60) as r:
+        body = json.loads(r.read())
+        nxt = None
+        link = r.headers.get("Link", "")
+        for part in link.split(","):
+            if 'rel="next"' in part:
+                s = part.find("<"); e = part.find(">")
+                if s != -1 and e != -1:
+                    nxt = part[s+1:e]
+        return body, nxt
+
+def post(path, payload):
+    req = urllib.request.Request(ADDR+path, data=json.dumps(payload).encode(),
+        method="POST", headers={"Content-Type":"application/json"})
+    if TOKEN: req.add_header("Authorization","Bearer "+TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r: return r.status
+    except Exception: return 0
+
+def load_state():
+    if os.path.exists(STATE):
+        try: return json.load(open(STATE))
+        except Exception: pass
+    return {}
+
+def main():
+    st = load_state()
+    enq = 0
+    for kind, prefix in KINDS:
+        if enq >= BUDGET:
+            break
+        ck = "cursor_" + kind
+        # Resume from saved cursor; empty/"DONE" means start fresh this cycle.
+        cur = st.get(ck) or ""
+        url = f"{HF}/api/{kind}?limit={PAGE}" + (f"&cursor={cur}" if cur else "")
+        page_n = 0
+        while enq < BUDGET:
+            try:
+                items, nxt = http(url)
+            except Exception as e:
+                print(f"hf-full: {kind} page failed: {e}", file=sys.stderr); break
+            for it in items:
+                i = it.get("id")
+                if i:
+                    post("/admin/crawl-enqueue", {"url": f"{HF}{prefix}{i}"})
+                    enq += 1
+            page_n += 1
+            if not nxt:
+                st[ck] = ""  # fully enumerated; restart next cycle for new repos
+                print(f"hf-full: {kind} FULLY enumerated this cycle")
+                break
+            # checkpoint the cursor from the next URL so we resume mid-kind.
+            cidx = nxt.find("cursor=")
+            st[ck] = nxt[cidx+7:] if cidx != -1 else ""
+            url = nxt
+            if enq >= BUDGET:
+                print(f"hf-full: {kind} budget hit at page {page_n}, cursor saved")
+                break
+        json.dump(st, open(STATE, "w"))
+    print(f"hf-full: enqueued {enq} this run at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-hf-harvest.py b/deploy/harvesters/cosift-hf-harvest.py
new file mode 100755
index 0000000..43bb95a
--- /dev/null
+++ b/deploy/harvesters/cosift-hf-harvest.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""cosift-hf-harvest — index HuggingFace via the public Hub API (free, OSS).
+
+Enqueues the most-liked models, datasets, and Spaces (popularity = quality, vs
+sort=lastModified which is auto-uploaded checkpoint spam), plus recent blog
+posts. huggingface.co is already in the allowlist, so no domain promotion is
+needed — just enqueue. Stdlib only.
+
+URL shapes: model <id> -> /<id>; dataset -> /datasets/<id>; space -> /spaces/<id>.
+"""
+import json, os, sys, time, urllib.request
+
+ADDR = "http://127.0.0.1:7777"
+HF = "https://huggingface.co"
+LIMIT = int(os.environ.get("COSIFT_HF_LIMIT", "100"))
+SORT = os.environ.get("COSIFT_HF_SORT", "likes")
+
+def token():
+    try:
+        return json.load(open("/home/ubuntu/cosift.json")).get("cluster", {}).get("peer_auth_token", "")
+    except Exception:
+        return ""
+TOKEN = token()
+
+def get_json(url, timeout=40):
+    req = urllib.request.Request(url, headers={"User-Agent": "cosift-hf/1.0"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read())
+
+def post(path, payload):
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(ADDR + path, data=data, method="POST",
+                                 headers={"Content-Type": "application/json"})
+    if TOKEN:
+        req.add_header("Authorization", "Bearer " + TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status
+    except Exception as e:
+        return str(e)
+
+def harvest(kind, path_prefix):
+    """kind: api endpoint (models/datasets/spaces); path_prefix: URL prefix."""
+    try:
+        items = get_json(f"{HF}/api/{kind}?sort={SORT}&direction=-1&limit={LIMIT}")
+    except Exception as e:
+        print(f"hf-harvest: {kind} failed: {e}", file=sys.stderr); return 0
+    n = 0
+    for it in items:
+        i = it.get("id")
+        if not i:
+            continue
+        post("/admin/crawl-enqueue", {"url": f"{HF}{path_prefix}{i}"}); n += 1
+    print(f"hf-harvest: {kind} -> enqueued {n} (sort={SORT})")
+    return n
+
+def main():
+    total = 0
+    total += harvest("models", "/")
+    total += harvest("datasets", "/datasets/")
+    total += harvest("spaces", "/spaces/")
+    # Blog index (posts are also covered by the blog RSS feed).
+    post("/admin/crawl-enqueue", {"url": f"{HF}/blog"})
+    total += 1
+    print(f"hf-harvest: total enqueued={total} at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-hf-task-harvest.py b/deploy/harvesters/cosift-hf-task-harvest.py
new file mode 100755
index 0000000..66b0403
--- /dev/null
+++ b/deploy/harvesters/cosift-hf-task-harvest.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""cosift-hf-task-harvest — index top HF models per ML task/pipeline tag.
+
+Complements cosift-hf-harvest.py (which does global top-by-likes and misses
+niche-but-relevant models). Harvesting top-N PER TASK surfaces the best model
+for each capability (object-detection, OCR/image-to-text, ASR, etc.) — e.g.
+license-plate / DETR detectors that never crack the global likes leaderboard.
+huggingface.co is allowlisted. Stdlib only.
+"""
+import json, os, urllib.request, urllib.parse, time
+
+ADDR = "http://127.0.0.1:7777"
+HF = "https://huggingface.co"
+PER = int(os.environ.get("COSIFT_HF_TASK_LIMIT", "50"))
+TASKS = os.environ.get("COSIFT_HF_TASKS",
+    "object-detection image-segmentation image-classification image-to-text "
+    "automatic-speech-recognition text-to-speech text-generation text-classification "
+    "token-classification question-answering summarization translation "
+    "sentence-similarity feature-extraction zero-shot-classification "
+    "text-to-image image-to-image depth-estimation").split()
+
+def token():
+    try: return json.load(open("/home/ubuntu/cosift.json")).get("cluster",{}).get("peer_auth_token","")
+    except Exception: return ""
+TOKEN = token()
+
+def get_json(url):
+    req = urllib.request.Request(url, headers={"User-Agent":"cosift-hf-task/1.0"})
+    with urllib.request.urlopen(req, timeout=40) as r: return json.loads(r.read())
+
+def post(path, payload):
+    req = urllib.request.Request(ADDR+path, data=json.dumps(payload).encode(),
+        method="POST", headers={"Content-Type":"application/json"})
+    if TOKEN: req.add_header("Authorization","Bearer "+TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r: return r.status
+    except Exception as e: return str(e)
+
+def main():
+    total = 0
+    for task in TASKS:
+        url = f"{HF}/api/models?filter={urllib.parse.quote(task)}&sort=likes&direction=-1&limit={PER}"
+        try: items = get_json(url)
+        except Exception as e:
+            print(f"hf-task-harvest: {task} failed: {e}"); continue
+        n = 0
+        for it in items:
+            i = it.get("id")
+            if i: post("/admin/crawl-enqueue", {"url": f"{HF}/{i}"}); n += 1
+        print(f"hf-task-harvest: {task} -> enqueued {n}")
+        total += n
+        time.sleep(1)
+    print(f"hf-task-harvest: total {total} at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-hn-harvest.py b/deploy/harvesters/cosift-hn-harvest.py
new file mode 100755
index 0000000..fcec725
--- /dev/null
+++ b/deploy/harvesters/cosift-hn-harvest.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""cosift-hn-harvest — index Hacker News via the official Firebase API.
+
+Enqueues each story's HN discussion page (news.ycombinator.com, allowlisted)
+AND its external link. External links live on arbitrary domains, so we grow the
+crawl allowlist ORGANICALLY: a link-target domain that recurs on HN >= THRESHOLD
+times is promoted via /admin/allow-domain, then its URLs are enqueued. One-off
+junk domains never cross the threshold and stay filtered.
+
+Stdlib only (urllib + json) — no pip deps. Frequency state persists across runs.
+"""
+import json, os, sys, time, urllib.request, urllib.parse
+from collections import Counter
+
+ADDR = "http://127.0.0.1:7777"
+HN = "https://hacker-news.firebaseio.com/v0"
+STATE = "/home/ubuntu/cosift-hn-domains.json"   # {domain: count, "_promoted":[...]}
+THRESHOLD = int(os.environ.get("COSIFT_HN_PROMOTE_THRESHOLD", "3"))
+PER_LIST = int(os.environ.get("COSIFT_HN_PER_LIST", "60"))
+LISTS = ["topstories", "beststories", "newstories", "askstories", "showstories"]
+
+def token():
+    try:
+        return json.load(open("/home/ubuntu/cosift.json")).get("cluster", {}).get("peer_auth_token", "")
+    except Exception:
+        return ""
+TOKEN = token()
+
+def get(url, timeout=30):
+    req = urllib.request.Request(url, headers={"User-Agent": "cosift-hn/1.0"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read()
+
+def post(path, payload):
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(ADDR + path, data=data, method="POST",
+                                 headers={"Content-Type": "application/json"})
+    if TOKEN:
+        req.add_header("Authorization", "Bearer " + TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status
+    except Exception as e:
+        return str(e)
+
+def host_of(u):
+    try:
+        h = urllib.parse.urlparse(u).hostname or ""
+        return h.lower().lstrip(".")
+    except Exception:
+        return ""
+
+def etld1(h):
+    # crude eTLD+1 (good enough for promotion granularity)
+    parts = h.split(".")
+    return ".".join(parts[-2:]) if len(parts) >= 2 else h
+
+def main():
+    state = {}
+    if os.path.exists(STATE):
+        try: state = json.load(open(STATE))
+        except Exception: state = {}
+    counts = Counter({k: v for k, v in state.items() if not k.startswith("_")})
+    promoted = set(state.get("_promoted", []))
+
+    # 1. Collect story ids across lists.
+    ids = []
+    seen = set()
+    for lst in LISTS:
+        try:
+            arr = json.loads(get(f"{HN}/{lst}.json"))
+        except Exception as e:
+            print(f"hn-harvest: {lst} failed: {e}", file=sys.stderr); continue
+        for i in (arr or [])[:PER_LIST]:
+            if i not in seen:
+                seen.add(i); ids.append(i)
+    print(f"hn-harvest: {len(ids)} unique stories")
+
+    hn_pages, ext_urls = [], []
+    for i in ids:
+        try:
+            it = json.loads(get(f"{HN}/item/{i}.json"))
+        except Exception:
+            continue
+        if not it:
+            continue
+        hn_pages.append(f"https://news.ycombinator.com/item?id={i}")
+        u = it.get("url")
+        if u and u.startswith("http"):
+            ext_urls.append(u)
+
+    # 2. Count external-link domains; promote those crossing THRESHOLD.
+    newly_promoted = []
+    for u in ext_urls:
+        d = etld1(host_of(u))
+        if d:
+            counts[d] += 1
+    for d, n in counts.items():
+        if n >= THRESHOLD and d not in promoted:
+            r = post("/admin/allow-domain", {"domain": d})
+            if r in (200, "200") or str(r) == "200":
+                promoted.add(d); newly_promoted.append(d)
+    print(f"hn-harvest: promoted {len(newly_promoted)} new domains: {newly_promoted[:20]}")
+
+    # 3. Enqueue HN pages (always) + external urls whose domain is promoted.
+    nq = 0
+    for u in hn_pages:
+        post("/admin/crawl-enqueue", {"url": u}); nq += 1
+    ne = 0
+    for u in ext_urls:
+        if etld1(host_of(u)) in promoted:
+            post("/admin/crawl-enqueue", {"url": u}); ne += 1
+    print(f"hn-harvest: enqueued {nq} HN pages + {ne} external links")
+
+    # 4. Persist counts + promotions.
+    out = dict(counts); out["_promoted"] = sorted(promoted)
+    json.dump(out, open(STATE, "w"))
+    print(f"hn-harvest: done at {time.strftime('%FT%TZ', time.gmtime())} "
+          f"({len(promoted)} domains promoted total)")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-pypi-harvest.py b/deploy/harvesters/cosift-pypi-harvest.py
new file mode 100755
index 0000000..ccf11e1
--- /dev/null
+++ b/deploy/harvesters/cosift-pypi-harvest.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""cosift-pypi-harvest — index the most-used PyPI packages' project pages.
+
+Pre-indexing for baseline coverage: the top-N packages by download count (from
+hugovk's well-known list) cover the common Python ecosystem so /search and
+/research can answer "what does package X do / how to install it". The long tail
+of niche packages is handled live by /find. pypi.org is allowlisted; we enqueue
+project pages and the crawler indexes the description + README. Stdlib only.
+"""
+import json, os, urllib.request, time
+
+ADDR = "http://127.0.0.1:7777"
+TOP = int(os.environ.get("COSIFT_PYPI_TOP", "1000"))
+LIST = "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
+
+def token():
+    try: return json.load(open("/home/ubuntu/cosift.json")).get("cluster",{}).get("peer_auth_token","")
+    except Exception: return ""
+TOKEN = token()
+
+def post(path, payload):
+    req = urllib.request.Request(ADDR+path, data=json.dumps(payload).encode(),
+        method="POST", headers={"Content-Type":"application/json"})
+    if TOKEN: req.add_header("Authorization","Bearer "+TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r: return r.status
+    except Exception as e: return str(e)
+
+def main():
+    req = urllib.request.Request(LIST, headers={"User-Agent":"cosift-pypi/1.0"})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        rows = json.loads(r.read()).get("rows", [])
+    n = 0
+    for row in rows[:TOP]:
+        name = row.get("project")
+        if not name: continue
+        post("/admin/crawl-enqueue", {"url": f"https://pypi.org/project/{name}/"})
+        n += 1
+    print(f"pypi-harvest: enqueued {n} package pages (top {TOP}) at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-reddit-harvest.py b/deploy/harvesters/cosift-reddit-harvest.py
new file mode 100755
index 0000000..fe6f022
--- /dev/null
+++ b/deploy/harvesters/cosift-reddit-harvest.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""cosift-reddit-harvest — index Reddit via per-subreddit RSS/Atom (free, OSS).
+
+Reddit's .json API serves HTML to datacenter IPs, but the .rss (Atom) endpoint
+works. Per subreddit we enqueue the comment page (reddit.com, allowlisted) and
+the post's external link, with the SAME organic domain-promotion as HN: an
+external domain recurring >= THRESHOLD times is promoted via /admin/allow-domain.
+
+Stdlib only. Evaluation notes (logged): per-feed item count + HTTP status, so we
+can see reddit's reliability/rate-limiting vs HN over time.
+"""
+import json, os, re, sys, time, urllib.request, urllib.parse
+from collections import Counter
+
+ADDR = "http://127.0.0.1:7777"
+SUBS_FILE = "/home/ubuntu/cosift-subreddits.txt"
+STATE = "/home/ubuntu/cosift-reddit-domains.json"
+THRESHOLD = int(os.environ.get("COSIFT_REDDIT_PROMOTE_THRESHOLD", "3"))
+SPACING = float(os.environ.get("COSIFT_REDDIT_SPACING", "2"))
+
+def token():
+    try:
+        return json.load(open("/home/ubuntu/cosift.json")).get("cluster", {}).get("peer_auth_token", "")
+    except Exception:
+        return ""
+TOKEN = token()
+
+def get(url, timeout=30):
+    req = urllib.request.Request(url, headers={"User-Agent": "cosift-reddit/1.0"})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return r.read().decode("utf-8", "replace"), r.status
+    except Exception as e:
+        return "", str(e)
+
+def post(path, payload):
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(ADDR + path, data=data, method="POST",
+                                 headers={"Content-Type": "application/json"})
+    if TOKEN:
+        req.add_header("Authorization", "Bearer " + TOKEN)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status
+    except Exception as e:
+        return str(e)
+
+def host_of(u):
+    try:
+        return (urllib.parse.urlparse(u).hostname or "").lower().lstrip(".")
+    except Exception:
+        return ""
+
+def etld1(h):
+    p = h.split(".")
+    return ".".join(p[-2:]) if len(p) >= 2 else h
+
+def main():
+    if not os.path.exists(SUBS_FILE):
+        print(f"reddit-harvest: {SUBS_FILE} missing", file=sys.stderr); return
+    subs = [l.strip() for l in open(SUBS_FILE) if l.strip() and not l.startswith("#")]
+
+    state = {}
+    if os.path.exists(STATE):
+        try: state = json.load(open(STATE))
+        except Exception: state = {}
+    counts = Counter({k: v for k, v in state.items() if not k.startswith("_")})
+    promoted = set(state.get("_promoted", []))
+
+    comment_pages, ext_urls = [], []
+    for sub in subs:
+        url = f"https://www.reddit.com/r/{sub}/.rss"
+        body, status = get(url)
+        # Atom entries: comment-page link + external [link] href in content.
+        cpages = re.findall(r'<link href="(https://www\.reddit\.com/r/[^"]+)"', body)
+        exts = re.findall(r'href="([^"]+)">\[link\]</a>', body)
+        comment_pages.extend(cpages)
+        ext_urls.extend(e for e in exts if e.startswith("http"))
+        print(f"reddit-harvest: r/{sub} -> {len(cpages)} threads, {len(exts)} ext (http={status})")
+        time.sleep(SPACING)  # reddit politeness
+
+    # Count + promote external domains.
+    newly = []
+    for u in ext_urls:
+        d = etld1(host_of(u))
+        if d:
+            counts[d] += 1
+    for d, n in counts.items():
+        if n >= THRESHOLD and d not in promoted:
+            if str(post("/admin/allow-domain", {"domain": d})) == "200":
+                promoted.add(d); newly.append(d)
+    print(f"reddit-harvest: promoted {len(newly)} new domains: {newly[:20]}")
+
+    # Enqueue comment pages (allowlisted) + external urls from promoted domains.
+    nq = ne = 0
+    for u in set(comment_pages):
+        post("/admin/crawl-enqueue", {"url": u}); nq += 1
+    for u in set(ext_urls):
+        if etld1(host_of(u)) in promoted:
+            post("/admin/crawl-enqueue", {"url": u}); ne += 1
+    print(f"reddit-harvest: enqueued {nq} threads + {ne} external links")
+
+    out = dict(counts); out["_promoted"] = sorted(promoted)
+    json.dump(out, open(STATE, "w"))
+    print(f"reddit-harvest: done at {time.strftime('%FT%TZ', time.gmtime())}")
+
+if __name__ == "__main__":
+    main()
diff --git a/deploy/harvesters/cosift-sitemap-refresh.sh b/deploy/harvesters/cosift-sitemap-refresh.sh
new file mode 100755
index 0000000..277fdfe
--- /dev/null
+++ b/deploy/harvesters/cosift-sitemap-refresh.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# cosift-sitemap-refresh — re-discover + queue sitemaps for curated quality
+# hosts (cosift-sites.txt) so newly-published pages enter the crawl frontier.
+# Companion to cosift-rss-refresh.sh: RSS catches the latest-N article links
+# fast (every 30min); this sweeps each host's full sitemap a few times a day to
+# catch pages RSS missed. The crawler's refetch window (COSIFT_REFETCH_AFTER_*)
+# + frontier dedup mean re-submitting mostly queues genuinely-new URLs.
+#
+# Runs SEQUENTIALLY with spacing — site-submit pulls full sitemaps (can be
+# large), so we keep it gentle to avoid a crawl/index burst.
+set -u
+
+SITES=/home/ubuntu/cosift-sites.txt
+ADDR="http://127.0.0.1:7777"
+LANE="${COSIFT_SITEMAP_LANE:-refresh}"
+SPACING="${COSIFT_SITEMAP_SPACING:-3}"   # seconds between hosts
+TOKEN=$(python3 -c "import json; c=json.load(open('/home/ubuntu/cosift.json')); print(c.get('cluster',{}).get('peer_auth_token',''))" 2>/dev/null || echo "")
+
+if [ ! -f "$SITES" ]; then
+  echo "cosift-sitemap-refresh: $SITES not found; nothing to do"
+  exit 0
+fi
+
+total=0
+while IFS= read -r HOST; do
+  case "$HOST" in ''|\#*) continue;; esac
+  HOST=$(echo "$HOST" | tr -d '[:space:]')
+  [ -z "$HOST" ] && continue
+  RESP=$(curl -s --max-time 300 -X POST "$ADDR/admin/site-submit" \
+    -H "Content-Type: application/json" \
+    ${TOKEN:+-H "Authorization: Bearer $TOKEN"} \
+    -d "{\"host\":\"$HOST\",\"lane\":\"$LANE\"}")
+  Q=$(echo "$RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('total_queued','?'))" 2>/dev/null || echo "?")
+  echo "cosift-sitemap-refresh: $HOST -> queued $Q (lane=$LANE)"
+  [ "$Q" != "?" ] && [ -n "$Q" ] && total=$((total + Q))
+  sleep "$SPACING"
+done < "$SITES"
+
+echo "cosift-sitemap-refresh: done, total queued=$total at $(date -u +%FT%TZ)"
diff --git a/deploy/harvesters/cosift-wet-refresh.sh b/deploy/harvesters/cosift-wet-refresh.sh
new file mode 100755
index 0000000..960498a
--- /dev/null
+++ b/deploy/harvesters/cosift-wet-refresh.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+# cosift-wet-refresh — rotating CommonCrawl WET ingest.
+#
+# Walks a CC release's ~100k-file manifest one slice per run so each run
+# ingests NEW files instead of re-reading 0-9 (the old script hardcoded
+# skip=0,count=10 and dedup'd to ~zero net docs every week). State (release +
+# next offset) persists in STATE_FILE. When a newer CC release appears, the
+# offset resets to 0 so we migrate to the freshest crawl automatically.
+#
+# Tunables (env):
+#   COSIFT_WET_COUNT         files per run        (default 100 ≈ 2.1M docs)
+#   COSIFT_WET_CONCURRENCY   parallel imports     (default 4, handler caps 8)
+#   COSIFT_WET_LEXICAL_ONLY  skip embeddings      (default true — fast BM25 ingest)
+set -u
+
+ADDR="http://127.0.0.1:7777"
+STATE_FILE="/home/ubuntu/cosift-wet-state.txt"
+COUNT="${COSIFT_WET_COUNT:-50}"
+CONCURRENCY="${COSIFT_WET_CONCURRENCY:-2}"
+LEXICAL_ONLY="${COSIFT_WET_LEXICAL_ONLY:-true}"
+MANIFEST_FILES=100000   # CC releases ship ~100k WET files
+
+TOKEN=$(python3 -c "import json; c=json.load(open('/home/ubuntu/cosift.json')); print(c.get('cluster',{}).get('peer_auth_token',''))" 2>/dev/null || echo "")
+
+# Latest CC release id (e.g. CC-MAIN-2026-21).
+LATEST=$(curl -s --max-time 30 https://data.commoncrawl.org/crawl-data/index.html \
+  | grep -oE 'CC-MAIN-[0-9]{4}-[0-9]{2}' | sort -u | tail -1)
+if [ -z "$LATEST" ]; then
+  echo "cosift-wet-refresh: could not determine latest release; aborting"
+  exit 1
+fi
+
+# Persisted (release, offset). Reset to 0 when the release rolls over.
+STATE_RELEASE=""
+OFFSET=0
+if [ -f "$STATE_FILE" ]; then
+  read -r STATE_RELEASE OFFSET < "$STATE_FILE" || true
+  [ -z "${OFFSET:-}" ] && OFFSET=0
+fi
+if [ "$STATE_RELEASE" != "$LATEST" ]; then
+  echo "cosift-wet-refresh: release rolled '${STATE_RELEASE:-none}' -> $LATEST, resetting offset to 0"
+  OFFSET=0
+fi
+if [ "$OFFSET" -ge "$MANIFEST_FILES" ]; then
+  echo "cosift-wet-refresh: walked entire $LATEST manifest, wrapping offset to 0"
+  OFFSET=0
+fi
+
+JSON=$(printf '{"manifest_url":"https://data.commoncrawl.org/crawl-data/%s/wet.paths.gz","count":%d,"skip":%d,"concurrency":%d,"lexical_only":%s}' \
+  "$LATEST" "$COUNT" "$OFFSET" "$CONCURRENCY" "$LEXICAL_ONLY")
+echo "cosift-wet-refresh: release=$LATEST skip=$OFFSET count=$COUNT concurrency=$CONCURRENCY"
+
+# Advance the offset BEFORE the import. wet-import-bulk is idempotent (URL
+# dedup), so the worst case from advancing early is skipping a slice on a hard
+# failure — acceptable in a 100k-file manifest. The failure we actually hit was
+# the opposite: a successful server-side import whose response the loaded box
+# dropped, leaving the offset stuck and re-reading the same slice forever.
+# Advancing first makes the walk monotonic regardless of response delivery.
+NEXT=$((OFFSET + COUNT))
+echo "$LATEST $NEXT" > "$STATE_FILE"
+
+RESP=$(curl -s --retry 2 --retry-delay 30 --max-time 7200 -X POST "$ADDR/admin/wet-import-bulk" \
+  -H 'Content-Type: application/json' \
+  ${TOKEN:+-H "Authorization: Bearer $TOKEN"} \
+  -d "$JSON")
+
+TOTAL=$(echo "$RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('total_indexed',''))" 2>/dev/null || echo "")
+if [ -z "$TOTAL" ]; then
+  # Import may have succeeded server-side even if we couldn't read the response
+  # (heavy write bursts can drop the client connection). Offset already advanced,
+  # so the walk continues; check the cosift-serve journal for the real result.
+  echo "cosift-wet-refresh: response unreadable (import may have succeeded server-side); offset already advanced $OFFSET -> $NEXT. Check 'journalctl -u cosift-serve' for 'wet-import-bulk: indexed'. Response head:"
+  echo "$RESP" | head -c 500
+  exit 0
+fi
+
+echo "cosift-wet-refresh: indexed=$TOTAL, advanced offset $OFFSET -> $NEXT, done at $(date -u +%FT%TZ)"
diff --git a/deploy/lists/cosift-feeds.txt b/deploy/lists/cosift-feeds.txt
new file mode 100644
index 0000000..c28f235
--- /dev/null
+++ b/deploy/lists/cosift-feeds.txt
@@ -0,0 +1,254 @@
+https://www.reddit.com/r/programming/.rss
+https://www.reddit.com/r/golang/.rss
+https://www.reddit.com/r/rust/.rss
+https://www.reddit.com/r/MachineLearning/.rss
+https://www.reddit.com/r/LocalLLaMA/.rss
+https://www.reddit.com/r/Python/.rss
+https://www.reddit.com/r/javascript/.rss
+https://www.reddit.com/r/node/.rss
+https://www.reddit.com/r/reactjs/.rss
+https://www.reddit.com/r/dotnet/.rss
+https://www.reddit.com/r/cpp/.rss
+https://www.reddit.com/r/java/.rss
+https://www.reddit.com/r/django/.rss
+https://www.reddit.com/r/flask/.rss
+https://www.reddit.com/r/learnprogramming/.rss
+https://www.reddit.com/r/compsci/.rss
+https://www.reddit.com/r/datascience/.rss
+https://www.reddit.com/r/ArtificialIntelligence/.rss
+https://www.reddit.com/r/OpenAI/.rss
+https://www.reddit.com/r/Anthropic/.rss
+https://www.reddit.com/r/ClaudeAI/.rss
+https://www.reddit.com/r/singularity/.rss
+https://www.reddit.com/r/StableDiffusion/.rss
+https://www.reddit.com/r/selfhosted/.rss
+https://www.reddit.com/r/homelab/.rss
+https://www.reddit.com/r/sysadmin/.rss
+https://www.reddit.com/r/linux/.rss
+https://www.reddit.com/r/linuxquestions/.rss
+https://www.reddit.com/r/security/.rss
+https://www.reddit.com/r/netsec/.rss
+https://www.reddit.com/r/cybersecurity/.rss
+https://www.reddit.com/r/science/.rss
+https://www.reddit.com/r/askscience/.rss
+https://www.reddit.com/r/space/.rss
+https://www.reddit.com/r/physics/.rss
+https://www.reddit.com/r/biology/.rss
+https://www.reddit.com/r/chemistry/.rss
+https://www.reddit.com/r/math/.rss
+https://www.reddit.com/r/worldnews/.rss
+https://www.reddit.com/r/news/.rss
+https://www.reddit.com/r/technology/.rss
+https://www.reddit.com/r/futurology/.rss
+https://www.reddit.com/r/todayilearned/.rss
+https://www.reddit.com/r/explainlikeimfive/.rss
+https://www.reddit.com/r/AskHistorians/.rss
+https://www.reddit.com/r/history/.rss
+https://www.reddit.com/r/philosophy/.rss
+https://www.reddit.com/r/books/.rss
+https://www.reddit.com/r/movies/.rss
+https://www.reddit.com/r/television/.rss
+https://www.reddit.com/r/gaming/.rss
+https://www.reddit.com/r/Games/.rss
+https://www.reddit.com/r/boardgames/.rss
+https://www.reddit.com/r/sports/.rss
+https://www.reddit.com/r/Fitness/.rss
+https://www.reddit.com/r/food/.rss
+https://www.reddit.com/r/Cooking/.rss
+https://www.reddit.com/r/photography/.rss
+https://www.reddit.com/r/business/.rss
+https://www.reddit.com/r/economics/.rss
+https://www.reddit.com/r/investing/.rss
+https://www.reddit.com/r/personalfinance/.rss
+https://www.reddit.com/r/Entrepreneur/.rss
+https://www.reddit.com/r/startups/.rss
+https://www.reddit.com/r/stocks/.rss
+https://www.reddit.com/r/Health/.rss
+https://www.reddit.com/r/medicine/.rss
+https://www.reddit.com/r/nutrition/.rss
+https://www.reddit.com/r/mentalhealth/.rss
+https://www.reddit.com/r/travel/.rss
+https://www.reddit.com/r/geography/.rss
+https://www.reddit.com/r/geopolitics/.rss
+https://www.reddit.com/r/environment/.rss
+https://www.reddit.com/r/climate/.rss
+https://www.reddit.com/r/sociology/.rss
+https://www.reddit.com/r/psychology/.rss
+https://www.reddit.com/r/Education/.rss
+https://www.reddit.com/r/dataisbeautiful/.rss
+https://www.reddit.com/r/mildlyinteresting/.rss
+https://www.reddit.com/r/art/.rss
+https://www.reddit.com/r/Music/.rss
+https://www.reddit.com/r/writing/.rss
+https://www.reddit.com/r/literature/.rss
+https://www.reddit.com/r/Cars/.rss
+https://www.reddit.com/r/formula1/.rss
+https://www.reddit.com/r/aviation/.rss
+https://www.reddit.com/r/Bitcoin/.rss
+https://www.reddit.com/r/CryptoCurrency/.rss
+https://www.reddit.com/r/ethereum/.rss
+https://www.reddit.com/r/apple/.rss
+https://www.reddit.com/r/android/.rss
+https://www.reddit.com/r/3Dprinting/.rss
+https://www.reddit.com/r/electronics/.rss
+https://www.reddit.com/r/arduino/.rss
+https://www.reddit.com/r/raspberry_pi/.rss
+https://www.reddit.com/r/buildapc/.rss
+https://www.reddit.com/r/AskReddit/.rss
+https://www.reddit.com/r/IAmA/.rss
+https://www.reddit.com/r/nosleep/.rss
+https://www.reddit.com/r/UpliftingNews/.rss
+https://www.reddit.com/r/Documentaries/.rss
+https://hnrss.org/newest
+https://hnrss.org/frontpage
+https://hnrss.org/best
+https://hnrss.org/ask
+https://hnrss.org/show
+https://hnrss.org/jobs
+https://lobste.rs/rss
+https://lwn.net/headlines/newrss
+https://www.phoronix.com/rss.php
+https://arxiv.org/rss/cs.LG
+https://arxiv.org/rss/cs.CL
+https://arxiv.org/rss/cs.AI
+https://arxiv.org/rss/cs.DC
+https://arxiv.org/rss/cs.CR
+https://arxiv.org/rss/cs.IR
+https://arxiv.org/rss/cs.SE
+https://arxiv.org/rss/cs.PL
+https://arxiv.org/rss/cs.DS
+https://arxiv.org/rss/cs.DB
+https://arxiv.org/rss/cs.OS
+https://arxiv.org/rss/cs.NI
+https://arxiv.org/rss/cs.CV
+https://arxiv.org/rss/cs.RO
+https://arxiv.org/rss/stat.ML
+https://arxiv.org/rss/math.PR
+https://arxiv.org/rss/math.ST
+https://arxiv.org/rss/physics.comp-ph
+https://www.docker.com/blog/feed/
+https://aws.amazon.com/blogs/aws/feed/
+https://stackoverflow.blog/feed/
+https://hackaday.com/feed/
+https://www.theverge.com/rss/index.xml
+https://techcrunch.com/feed/
+https://feeds.arstechnica.com/arstechnica/index
+https://www.wired.com/feed/rss
+https://feeds.feedburner.com/oreilly/radar
+https://blog.golang.org/feed.atom
+https://blog.rust-lang.org/feed.xml
+https://kubernetes.io/feed.xml
+https://blog.cloudflare.com/rss/
+https://github.blog/feed/
+https://news.ycombinator.com/rss
+https://martinfowler.com/feed.atom
+https://overreacted.io/rss.xml
+https://kentcdodds.com/blog/rss.xml
+https://www.joelonsoftware.com/feed/
+https://blog.pragmaticengineer.com/rss/
+https://www.allthingsdistributed.com/atom.xml
+https://eng.uber.com/feed/
+https://netflixtechblog.com/feed
+https://engineering.linkedin.com/blog.rss.html
+https://medium.com/feed/airbnb-engineering
+https://slack.engineering/feed/
+https://dropbox.tech/feed
+https://eng.lyft.com/feed
+https://blog.discord.com/feed
+https://shopify.engineering/blog.atom
+https://medium.com/feed/pinterest-engineering
+https://engineering.fb.com/feed/
+https://opensource.googleblog.com/feeds/posts/default
+https://huggingface.co/blog/feed.xml
+https://openai.com/blog/rss.xml
+https://blog.langchain.dev/rss/
+https://blog.cohere.com/rss/
+https://www.databricks.com/blog/feed
+https://qdrant.tech/feed/
+https://vercel.com/atom.xml
+https://supabase.com/feed.xml
+https://feeds.bbci.co.uk/news/rss.xml
+https://feeds.bbci.co.uk/news/world/rss.xml
+https://feeds.bbci.co.uk/news/business/rss.xml
+https://feeds.bbci.co.uk/news/technology/rss.xml
+https://feeds.bbci.co.uk/news/science_and_environment/rss.xml
+https://feeds.bbci.co.uk/news/health/rss.xml
+https://www.reuters.com/rssFeed/topNews
+https://www.theguardian.com/world/rss
+https://www.theguardian.com/technology/rss
+https://www.theguardian.com/science/rss
+https://www.nytimes.com/services/xml/rss/nyt/HomePage.xml
+https://www.nytimes.com/services/xml/rss/nyt/World.xml
+https://www.nytimes.com/services/xml/rss/nyt/Technology.xml
+https://www.nytimes.com/services/xml/rss/nyt/Science.xml
+https://www.npr.org/rss/rss.php?id=1001
+https://www.npr.org/rss/rss.php?id=1004
+https://www.npr.org/rss/rss.php?id=1019
+https://www.economist.com/sections/science-technology/rss.xml
+https://www.aljazeera.com/xml/rss/all.xml
+https://www.spiegel.de/international/index.rss
+https://www.cdc.gov/podcasts/feed.xml
+https://tools.cdc.gov/podcasts/feed.aspx
+https://www.fda.gov/about-fda/contact-fda/stay-informed/rss-feeds/press-releases/rss.xml
+https://www.nasa.gov/news-release/feed/
+https://blogs.nasa.gov/feed/
+https://feeds.nature.com/nature/rss/current
+https://www.nature.com/subjects/computer-science.rss
+https://feeds.feedburner.com/PubmedTopReads
+https://venturebeat.com/feed/
+https://thenextweb.com/feed
+https://www.engadget.com/rss.xml
+https://gizmodo.com/rss
+https://lifehacker.com/rss
+https://mashable.com/feeds/rss/tech
+https://hbr.org/feed
+https://www.fastcompany.com/latest/rss
+https://feeds.feedburner.com/businessinsider
+https://www.bloomberg.com/feed/podcast/index.rss
+https://www.ft.com/rss/home
+https://blog.acolyer.org/feed/
+https://research.google/blog/rss/
+https://blogs.nvidia.com/feed/
+https://about.fb.com/feed/
+https://feeds.gawker.com/jalopnik/full
+https://torrentfreak.com/feed/
+https://www.smashingmagazine.com/feed/
+https://css-tricks.com/feed/
+https://www.freecodecamp.org/news/rss/
+https://medium.com/feed/better-programming
+https://medium.com/feed/javascript-scene
+https://daringfireball.net/feeds/main
+https://www.macrumors.com/macrumors.xml
+https://www.androidauthority.com/feed/
+https://9to5mac.com/feed/
+https://www.theinformation.com/feed
+https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml
+https://www.theguardian.com/international/rss
+https://feeds.npr.org/1001/rss.xml
+https://moxie.foxnews.com/google-publisher/latest.xml
+https://feeds.feedburner.com/TheHackersNews
+https://krebsonsecurity.com/feed/
+https://www.bleepingcomputer.com/feed/
+https://devblogs.microsoft.com/feed/
+https://blog.jetbrains.com/feed/
+https://www.nature.com/nature.rss
+https://www.science.org/rss/news_current.xml
+https://phys.org/rss-feed/
+https://www.quantamagazine.org/feed/
+https://www.newscientist.com/feed/home/
+https://rss.arxiv.org/rss/cs.AI
+https://rss.arxiv.org/rss/cs.LG
+https://rss.arxiv.org/rss/cs.CL
+https://rss.arxiv.org/rss/cs.DC
+https://rss.arxiv.org/rss/cs.SE
+https://bair.berkeley.edu/blog/feed.xml
+https://blog.research.google/feeds/posts/default
+https://deepmind.google/blog/rss.xml
+https://lwn.net/headlines/rss
+https://www.infoq.com/feed/
+https://rss.arxiv.org/rss/cs.CV
+https://rss.arxiv.org/rss/cs.NE
+https://rss.arxiv.org/rss/stat.ML
+https://rss.arxiv.org/rss/cs.IR
+https://rss.arxiv.org/rss/cs.CR
+https://www.tomshardware.com/feeds/all
diff --git a/deploy/lists/cosift-sites.txt b/deploy/lists/cosift-sites.txt
new file mode 100644
index 0000000..7ac7023
--- /dev/null
+++ b/deploy/lists/cosift-sites.txt
@@ -0,0 +1,28 @@
+# cosift-sites.txt — curated quality hosts whose sitemaps are re-submitted
+# periodically by cosift-sitemap-refresh.timer so newly-published pages enter
+# the crawl frontier. Bare hostnames, one per line. '#' comments allowed.
+# Pick hosts with frequently-updated sitemaps (news/tech/docs) that don't
+# hard-block bots.
+#
+# News / journalism
+arstechnica.com
+theverge.com
+techcrunch.com
+wired.com
+theguardian.com
+bbc.com
+apnews.com
+# Tech / engineering blogs
+stackoverflow.blog
+github.blog
+blog.cloudflare.com
+netflixtechblog.com
+# Science / research
+nature.com
+quantamagazine.org
+phys.org
+# AI / ML
+huggingface.co
+# Docs (update regularly, high authority)
+developer.mozilla.org
+kubernetes.io
diff --git a/deploy/lists/cosift-subreddits.txt b/deploy/lists/cosift-subreddits.txt
new file mode 100644
index 0000000..b2e918e
--- /dev/null
+++ b/deploy/lists/cosift-subreddits.txt
@@ -0,0 +1,25 @@
+# Quality subreddits for cosift-reddit-harvest. One per line, '#' comments.
+programming
+MachineLearning
+LocalLLaMA
+compsci
+datascience
+science
+askscience
+technology
+hardware
+selfhosted
+devops
+rust
+golang
+python
+cpp
+java
+javascript
+ExperiencedDevs
+ArtificialIntelligence
+AskHistorians
+math
+statistics
+netsec
+cybersecurity
diff --git a/deploy/scripts/README.md b/deploy/scripts/README.md
new file mode 100644
index 0000000..bd16231
--- /dev/null
+++ b/deploy/scripts/README.md
@@ -0,0 +1,131 @@
+# cosift deploy — PULL-based self-update (signed releases)
+
+How a new cosift build reaches the GH200 **without CI ever holding the
+production SSH key**. CI builds, signs, and publishes a GitHub Release; the
+box pulls, verifies the signature against a public key baked locally, snapshots,
+atomically swaps the binary, restarts, and health-gates with auto-rollback.
+
+```
+  tag push (vX.Y.Z)                         GH200 box (every ~5 min)
+        │                                          │
+  ┌─────▼─────────────────────┐            ┌───────▼────────────────────────┐
+  │ .github/workflows/         │            │ cosift-self-update.timer        │
+  │   deploy.yml               │            │   → cosift-self-update.service  │
+  │ verify → build → sign      │  Release   │   → cosift-self-update.sh       │
+  │ (minisign, secret key)     │ ─────────► │ poll latest release             │
+  │ upload binary+.sha256+.sig │  assets    │ verify sha256 + minisign(pub)   │
+  └────────────────────────────┘            │ snapshot → swap → restart       │
+                                            │ health-gate /healthz, rollback  │
+                                            └─────────────────────────────────┘
+```
+
+No SSH key, no box credentials, ever live in GitHub Actions. The trust anchor
+is the minisign keypair: the **private** key lives only as a GitHub secret; the
+**public** key is baked on the box and is the only thing that authorizes an
+install.
+
+## One-time setup
+
+### 1. Generate the minisign keypair (do this locally, once)
+
+```bash
+minisign -G -p minisign.pub -s minisign.key
+# -G generates a keypair. Choose a passphrase OR use -W for an unencrypted
+# secret key. CI signs non-interactively, so the secret key stored in the
+# GitHub secret must be UNENCRYPTED — generate with -W:
+minisign -G -W -p minisign.pub -s minisign.key
+```
+
+This produces two files:
+- `minisign.pub`  — public key (safe to commit / publish / bake on the box)
+- `minisign.key`  — secret key (NEVER commit; goes into the GitHub secret)
+
+### 2. Store the secret key as a GitHub Actions secret
+
+Repo → Settings → Secrets and variables → Actions → New repository secret:
+
+- **Name:** `MINISIGN_SECRET_KEY`
+- **Value:** the full contents of `minisign.key`
+
+`deploy.yml` writes it to a temp file, signs with `minisign -S -W`, and deletes
+it. Because the workflow signs non-interactively, the secret key must be the
+**unencrypted** form (`minisign -G -W`).
+
+> Anyone who can edit `deploy.yml` and trigger it can sign a release. Protect
+> the workflow with branch/tag protection and required reviews accordingly.
+
+### 3. Bake the PUBLIC key on the box
+
+```bash
+sudo mkdir -p /etc/cosift
+sudo cp minisign.pub /etc/cosift/minisign.pub
+sudo chmod 0644 /etc/cosift/minisign.pub
+```
+
+The self-updater verifies every download against `/etc/cosift/minisign.pub`
+and refuses to install anything that doesn't verify.
+
+### 4. Install the self-update script + timer on the box
+
+```bash
+# script (the serve unit expects it under /home/ubuntu/scripts/)
+sudo install -o ubuntu -g ubuntu -m 0755 \
+  deploy/scripts/cosift-self-update.sh /home/ubuntu/scripts/cosift-self-update.sh
+
+# the snapshot script it reuses must also be present + executable
+sudo install -o ubuntu -g ubuntu -m 0755 \
+  scripts/snapshot.sh /home/ubuntu/scripts/snapshot.sh
+
+# systemd units
+sudo cp deploy/systemd/cosift-self-update.service /etc/systemd/system/
+sudo cp deploy/systemd/cosift-self-update.timer   /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now cosift-self-update.timer
+
+# verify
+systemctl status cosift-self-update.timer
+systemctl list-timers cosift-self-update.timer
+```
+
+Dependencies on the box: `curl`, `jq`, `sha256sum` (coreutils), `minisign`,
+`systemctl`. Install minisign with `sudo apt-get install -y minisign`.
+
+## Cutting a release
+
+```bash
+git tag v0.1.0
+git push origin v0.1.0
+```
+
+The `deploy.yml` workflow runs `verify` (vet + race tests + smoke), `build`
+(static `linux/arm64`, sha256, minisign signature), and `release` (uploads
+`cosift-linux-arm64`, `.sha256`, `.minisig` as Release assets). Within ~5 min
+the box's timer fires, verifies, and rolls the update out. `workflow_dispatch`
+builds + signs the artifact for inspection but does **not** cut a Release
+(only tag pushes do).
+
+## Operating the self-updater
+
+```bash
+# trigger a poll right now (don't wait for the timer)
+sudo systemctl start cosift-self-update.service
+
+# watch what it did
+journalctl -u cosift-self-update.service -n 200 --no-pager
+
+# pause / resume automatic polling
+sudo systemctl disable --now cosift-self-update.timer
+sudo systemctl enable  --now cosift-self-update.timer
+```
+
+Safety properties baked into `cosift-self-update.sh`:
+- `set -euo pipefail` + `flock` (no concurrent runs racing the swap).
+- Installs only if sha256 **and** minisign signature verify against the baked
+  public key; also cross-checks the downloaded binary's `version` against the
+  release tag.
+- Pre-restart GCS snapshot via `scripts/snapshot.sh`.
+- Atomic swap: stage `cosift.new` → keep `cosift.prev` → `mv` over `cosift`.
+- Health gate: polls `/healthz` every 10s for up to 10 min (the listener binds
+  only after the ~4-5 min HNSW load, so a 200 is true readiness).
+- Auto-rollback: on health-gate failure, restores `cosift.prev`, restarts, and
+  exits non-zero.
diff --git a/deploy/scripts/cosift-self-update.sh b/deploy/scripts/cosift-self-update.sh
new file mode 100755
index 0000000..6a3df33
--- /dev/null
+++ b/deploy/scripts/cosift-self-update.sh
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+# cosift-self-update.sh — PULL-based self-updater for the cosift production box.
+#
+# Polls the latest GitHub Release for a newer cosift binary, verifies it
+# (sha256 + minisign signature against a public key baked on the box),
+# snapshots the index to GCS, atomically swaps the binary, restarts the
+# service, and HEALTH-GATES the new process with automatic rollback.
+#
+# CI never holds the SSH key: the box pulls, CI only pushes signed releases.
+#
+# Run by cosift-self-update.timer every ~5 min. Logs to journald.
+#
+# Exit codes:
+#   0  no update needed, or update applied and healthy
+#   1  update attempted but failed (rolled back where possible)
+#   2  precondition / environment error (no binary, no pubkey, missing deps)
+#
+# Configurable via environment (defaults target the GH200 prod layout):
+#   COSIFT_REPO          pilot-protocol/cosift
+#   COSIFT_BIN           /home/ubuntu/cosift
+#   COSIFT_SERVICE       cosift-serve
+#   COSIFT_PUBKEY        /etc/cosift/minisign.pub
+#   COSIFT_HEALTH_URL    http://127.0.0.1:7777/healthz
+#   COSIFT_ASSET         cosift-linux-arm64
+#   COSIFT_SNAPSHOT      /home/ubuntu/scripts/snapshot.sh   (best-effort)
+#   COSIFT_HEALTH_TIMEOUT_S  600   (10 min; HNSW load is ~4-5 min)
+#   COSIFT_HEALTH_INTERVAL_S 10
+#   GITHUB_TOKEN         (optional; raises the GitHub API rate limit)
+set -euo pipefail
+
+REPO="${COSIFT_REPO:-pilot-protocol/cosift}"
+BIN="${COSIFT_BIN:-/home/ubuntu/cosift}"
+SERVICE="${COSIFT_SERVICE:-cosift-serve}"
+PUBKEY="${COSIFT_PUBKEY:-/etc/cosift/minisign.pub}"
+HEALTH_URL="${COSIFT_HEALTH_URL:-http://127.0.0.1:7777/healthz}"
+ASSET="${COSIFT_ASSET:-cosift-linux-arm64}"
+SNAPSHOT="${COSIFT_SNAPSHOT:-/home/ubuntu/scripts/snapshot.sh}"
+HEALTH_TIMEOUT_S="${COSIFT_HEALTH_TIMEOUT_S:-600}"
+HEALTH_INTERVAL_S="${COSIFT_HEALTH_INTERVAL_S:-10}"
+API="https://api.github.com/repos/${REPO}/releases/latest"
+
+log() { echo "self-update: $*"; }
+err() { echo "self-update: ERROR: $*" >&2; }
+
+# --- Concurrency lock: never let two runs race the binary swap. ----------
+# flock on a dedicated FD; -n means fail fast if another run holds it.
+LOCKFILE="/run/cosift-self-update.lock"
+exec 9>"$LOCKFILE" 2>/dev/null || exec 9>"/tmp/cosift-self-update.lock"
+if ! flock -n 9; then
+  log "another self-update run holds the lock; skipping this tick"
+  exit 0
+fi
+
+# --- Preconditions -------------------------------------------------------
+for dep in curl jq sha256sum minisign systemctl; do
+  if ! command -v "$dep" >/dev/null 2>&1; then
+    err "missing required dependency: $dep"
+    exit 2
+  fi
+done
+if [[ ! -x "$BIN" ]]; then
+  err "current binary not found or not executable: $BIN"
+  exit 2
+fi
+if [[ ! -f "$PUBKEY" ]]; then
+  err "minisign public key not found: $PUBKEY (bake it on the box first)"
+  exit 2
+fi
+
+# --- Determine running vs. latest version --------------------------------
+running="$("$BIN" version 2>/dev/null | tr -d '[:space:]' || true)"
+if [[ -z "$running" ]]; then
+  err "could not read running version via '$BIN version'; aborting (fail-safe)"
+  exit 2
+fi
+log "running version: $running"
+
+auth_args=()
+if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+  auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
+fi
+
+release_json="$(curl -fsSL "${auth_args[@]}" \
+  -H "Accept: application/vnd.github+json" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "$API" 2>/dev/null || true)"
+if [[ -z "$release_json" ]]; then
+  err "failed to query latest release from $API"
+  exit 1
+fi
+
+latest_tag="$(printf '%s' "$release_json" | jq -r '.tag_name // empty')"
+if [[ -z "$latest_tag" ]]; then
+  err "latest release has no tag_name (response truncated below)"
+  printf '%s\n' "$release_json" | head -c 400 >&2
+  exit 1
+fi
+log "latest release tag: $latest_tag"
+
+# Compare. We strip a leading 'v' so a binary stamped '1.2.3' matches tag
+# 'v1.2.3'. If they already match, nothing to do.
+norm() { printf '%s' "${1#v}"; }
+if [[ "$(norm "$running")" == "$(norm "$latest_tag")" ]]; then
+  log "already at latest ($running); nothing to do"
+  exit 0
+fi
+log "update available: $running -> $latest_tag"
+
+# --- Resolve asset download URLs ----------------------------------------
+asset_url() {
+  printf '%s' "$release_json" \
+    | jq -r --arg n "$1" '.assets[] | select(.name == $n) | .browser_download_url' \
+    | head -n1
+}
+bin_url="$(asset_url "$ASSET")"
+sha_url="$(asset_url "${ASSET}.sha256")"
+sig_url="$(asset_url "${ASSET}.minisig")"
+if [[ -z "$bin_url" || -z "$sha_url" || -z "$sig_url" ]]; then
+  err "release $latest_tag missing one of: $ASSET / .sha256 / .minisig"
+  exit 1
+fi
+
+# --- Download into a private temp dir -----------------------------------
+tmp="$(mktemp -d)"
+cleanup() { rm -rf "$tmp"; }
+trap cleanup EXIT
+
+log "downloading $ASSET, .sha256, .minisig"
+curl -fsSL "${auth_args[@]}" -o "$tmp/$ASSET"          "$bin_url"
+curl -fsSL "${auth_args[@]}" -o "$tmp/${ASSET}.sha256" "$sha_url"
+curl -fsSL "${auth_args[@]}" -o "$tmp/${ASSET}.minisig" "$sig_url"
+
+# --- VERIFY sha256 -------------------------------------------------------
+# The .sha256 file is `sha256sum` output: "<hex>  <name>". Recompute and
+# compare against the hex column rather than trusting the filename column.
+expected_sha="$(awk '{print $1}' "$tmp/${ASSET}.sha256")"
+actual_sha="$(sha256sum "$tmp/$ASSET" | awk '{print $1}')"
+if [[ -z "$expected_sha" || "$expected_sha" != "$actual_sha" ]]; then
+  err "sha256 mismatch (expected=$expected_sha actual=$actual_sha) — refusing"
+  exit 1
+fi
+log "sha256 OK ($actual_sha)"
+
+# --- VERIFY minisign signature against the box-baked public key ----------
+# This is the trust anchor: only releases signed by the matching secret key
+# (held as a GitHub secret in CI) are accepted. A good sha256 alone is not
+# enough — an attacker who can serve assets could forge a matching checksum.
+if ! minisign -V -p "$PUBKEY" -m "$tmp/$ASSET" -x "$tmp/${ASSET}.minisig"; then
+  err "minisign signature verification FAILED — refusing to install"
+  exit 1
+fi
+log "minisign signature OK (verified against $PUBKEY)"
+
+# Sanity: the downloaded binary should report the version we expect.
+chmod +x "$tmp/$ASSET"
+dl_version="$("$tmp/$ASSET" version 2>/dev/null | tr -d '[:space:]' || true)"
+if [[ -n "$dl_version" && "$(norm "$dl_version")" != "$(norm "$latest_tag")" ]]; then
+  err "downloaded binary reports '$dl_version' but release tag is '$latest_tag' — refusing"
+  exit 1
+fi
+
+# --- Pre-restart GCS snapshot (best-effort but logged) -------------------
+if [[ -x "$SNAPSHOT" ]]; then
+  log "taking pre-restart snapshot via $SNAPSHOT"
+  if ! "$SNAPSHOT"; then
+    err "snapshot failed; continuing with update (index is also recoverable from prior snapshot)"
+  fi
+else
+  log "snapshot script not executable at $SNAPSHOT; skipping pre-restart snapshot"
+fi
+
+# --- Atomic swap ---------------------------------------------------------
+# Stage alongside the live binary so the final mv is a same-filesystem
+# rename (atomic). Keep the previous binary for rollback.
+new="${BIN}.new"
+prev="${BIN}.prev"
+cp -f "$tmp/$ASSET" "$new"
+chmod +x "$new"
+log "staged new binary at $new"
+
+cp -f "$BIN" "$prev"        # snapshot current binary for rollback
+mv -f "$new" "$BIN"         # atomic rename over the live path
+log "swapped in $latest_tag (previous kept at $prev)"
+
+# --- Restart -------------------------------------------------------------
+log "restarting $SERVICE"
+systemctl restart "$SERVICE"
+
+# --- HEALTH GATE ---------------------------------------------------------
+# The listener only binds AFTER the ~4-5 min synchronous HNSW load, so a
+# 200 from /healthz is a true readiness signal. Poll up to the timeout.
+log "health-gating $HEALTH_URL (timeout ${HEALTH_TIMEOUT_S}s, every ${HEALTH_INTERVAL_S}s)"
+deadline=$(( $(date +%s) + HEALTH_TIMEOUT_S ))
+healthy=0
+while (( $(date +%s) < deadline )); do
+  if curl -fsS --max-time 5 "$HEALTH_URL" >/dev/null 2>&1; then
+    healthy=1
+    break
+  fi
+  sleep "$HEALTH_INTERVAL_S"
+done
+
+if (( healthy == 1 )); then
+  log "healthy on $latest_tag — update complete"
+  exit 0
+fi
+
+# --- AUTO-ROLLBACK -------------------------------------------------------
+err "new version $latest_tag did not become healthy within ${HEALTH_TIMEOUT_S}s — rolling back to $running"
+if [[ -f "$prev" ]]; then
+  mv -f "$prev" "$BIN"
+  systemctl restart "$SERVICE"
+  # Give the rolled-back process a chance to come back so the box isn't
+  # left dark. Best-effort; we still exit non-zero to flag the failure.
+  rb_deadline=$(( $(date +%s) + HEALTH_TIMEOUT_S ))
+  while (( $(date +%s) < rb_deadline )); do
+    if curl -fsS --max-time 5 "$HEALTH_URL" >/dev/null 2>&1; then
+      err "rolled back to $running and healthy again"
+      exit 1
+    fi
+    sleep "$HEALTH_INTERVAL_S"
+  done
+  err "rollback restarted $SERVICE but it is not yet healthy — manual attention needed"
+else
+  err "no previous binary at $prev to roll back to — manual attention needed"
+fi
+exit 1
diff --git a/deploy/systemd/cosift-arxiv-harvest.service b/deploy/systemd/cosift-arxiv-harvest.service
new file mode 100644
index 0000000..b674488
--- /dev/null
+++ b/deploy/systemd/cosift-arxiv-harvest.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Cosift arXiv harvest (enqueue latest papers per category)
+After=cosift-serve.service
+Wants=cosift-serve.service
+
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-arxiv-harvest.sh
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-arxiv-harvest.timer b/deploy/systemd/cosift-arxiv-harvest.timer
new file mode 100644
index 0000000..447b1b5
--- /dev/null
+++ b/deploy/systemd/cosift-arxiv-harvest.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift arXiv harvest — daily (arXiv announces ~once/day on weekdays)
+
+[Timer]
+OnCalendar=*-*-* 05:30:00
+Persistent=true
+AccuracySec=5min
+
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-github-topic-harvest.service b/deploy/systemd/cosift-github-topic-harvest.service
new file mode 100644
index 0000000..d5e128e
--- /dev/null
+++ b/deploy/systemd/cosift-github-topic-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift github-topic pre-index harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-github-topic-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-github-topic-harvest.timer b/deploy/systemd/cosift-github-topic-harvest.timer
new file mode 100644
index 0000000..c57c9c1
--- /dev/null
+++ b/deploy/systemd/cosift-github-topic-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift github-topic pre-index schedule
+[Timer]
+OnCalendar=Wed *-*-* 07:00:00
+Persistent=true
+AccuracySec=10min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-hf-full-harvest.service b/deploy/systemd/cosift-hf-full-harvest.service
new file mode 100644
index 0000000..9527dc9
--- /dev/null
+++ b/deploy/systemd/cosift-hf-full-harvest.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Cosift HF FULL Hub enumeration harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+Environment=COSIFT_HF_FULL_BUDGET=50000
+ExecStart=/usr/local/bin/cosift-hf-full-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-hf-full-harvest.timer b/deploy/systemd/cosift-hf-full-harvest.timer
new file mode 100644
index 0000000..b7a67ef
--- /dev/null
+++ b/deploy/systemd/cosift-hf-full-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift HF full enumeration schedule
+[Timer]
+OnCalendar=*-*-* 00/3:00:00
+Persistent=true
+AccuracySec=10min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-hf-harvest.service b/deploy/systemd/cosift-hf-harvest.service
new file mode 100644
index 0000000..e99cdd8
--- /dev/null
+++ b/deploy/systemd/cosift-hf-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift HuggingFace harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-hf-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-hf-harvest.timer b/deploy/systemd/cosift-hf-harvest.timer
new file mode 100644
index 0000000..30bd08b
--- /dev/null
+++ b/deploy/systemd/cosift-hf-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift HuggingFace harvest schedule
+[Timer]
+OnCalendar=*-*-* 06:30:00
+Persistent=true
+AccuracySec=5min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-hf-task-harvest.service b/deploy/systemd/cosift-hf-task-harvest.service
new file mode 100644
index 0000000..0df319e
--- /dev/null
+++ b/deploy/systemd/cosift-hf-task-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift hf-task pre-index harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-hf-task-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-hf-task-harvest.timer b/deploy/systemd/cosift-hf-task-harvest.timer
new file mode 100644
index 0000000..3bd5d37
--- /dev/null
+++ b/deploy/systemd/cosift-hf-task-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift hf-task pre-index schedule
+[Timer]
+OnCalendar=Tue *-*-* 07:00:00
+Persistent=true
+AccuracySec=10min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-hn-harvest.service b/deploy/systemd/cosift-hn-harvest.service
new file mode 100644
index 0000000..b0a2461
--- /dev/null
+++ b/deploy/systemd/cosift-hn-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift Hacker News harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-hn-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-hn-harvest.timer b/deploy/systemd/cosift-hn-harvest.timer
new file mode 100644
index 0000000..b797fa7
--- /dev/null
+++ b/deploy/systemd/cosift-hn-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift Hacker News harvest schedule
+[Timer]
+OnCalendar=*-*-* 02,08,14,20:00:00
+Persistent=true
+AccuracySec=5min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-pypi-harvest.service b/deploy/systemd/cosift-pypi-harvest.service
new file mode 100644
index 0000000..be50e1e
--- /dev/null
+++ b/deploy/systemd/cosift-pypi-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift pypi pre-index harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-pypi-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-pypi-harvest.timer b/deploy/systemd/cosift-pypi-harvest.timer
new file mode 100644
index 0000000..7fa251b
--- /dev/null
+++ b/deploy/systemd/cosift-pypi-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift pypi pre-index schedule
+[Timer]
+OnCalendar=Mon *-*-* 07:00:00
+Persistent=true
+AccuracySec=10min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-reddit-harvest.service b/deploy/systemd/cosift-reddit-harvest.service
new file mode 100644
index 0000000..1b0dc95
--- /dev/null
+++ b/deploy/systemd/cosift-reddit-harvest.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift Reddit harvest
+After=cosift-serve.service
+Wants=cosift-serve.service
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-reddit-harvest.py
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-reddit-harvest.timer b/deploy/systemd/cosift-reddit-harvest.timer
new file mode 100644
index 0000000..98af143
--- /dev/null
+++ b/deploy/systemd/cosift-reddit-harvest.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Cosift Reddit harvest schedule
+[Timer]
+OnCalendar=*-*-* 04,10,16,22:00:00
+Persistent=true
+AccuracySec=5min
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-rss-refresh.service b/deploy/systemd/cosift-rss-refresh.service
new file mode 100644
index 0000000..75ae97c
--- /dev/null
+++ b/deploy/systemd/cosift-rss-refresh.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Cosift RSS refresh (re-import every tracked feed)
+After=cosift-serve.service
+Wants=cosift-serve.service
+
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/home/ubuntu/cosift-rss-refresh.sh
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-rss-refresh.timer b/deploy/systemd/cosift-rss-refresh.timer
new file mode 100644
index 0000000..2c057e3
--- /dev/null
+++ b/deploy/systemd/cosift-rss-refresh.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Cosift RSS refresh — every 30 minutes
+
+[Timer]
+OnBootSec=10min
+OnUnitActiveSec=30min
+AccuracySec=1min
+
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-self-update.service b/deploy/systemd/cosift-self-update.service
new file mode 100644
index 0000000..747a482
--- /dev/null
+++ b/deploy/systemd/cosift-self-update.service
@@ -0,0 +1,33 @@
+[Unit]
+Description=cosift PULL-based self-updater (verify + snapshot + swap + health-gate + rollback)
+# Don't poll until the box is on the network and the serve unit is up; the
+# updater health-gates the same service it restarts.
+After=network-online.target cosift-serve.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=ubuntu
+WorkingDirectory=/home/ubuntu
+# Public verification key baked on the box. The matching private key lives
+# only as the MINISIGN_SECRET_KEY GitHub Actions secret (CI signs releases).
+Environment=COSIFT_REPO=pilot-protocol/cosift
+Environment=COSIFT_BIN=/home/ubuntu/cosift
+Environment=COSIFT_SERVICE=cosift-serve
+Environment=COSIFT_PUBKEY=/etc/cosift/minisign.pub
+Environment=COSIFT_HEALTH_URL=http://127.0.0.1:7777/healthz
+Environment=COSIFT_SNAPSHOT=/home/ubuntu/scripts/snapshot.sh
+# Optional: drop a GITHUB_TOKEN here (or in a drop-in) to raise the API rate
+# limit. Public releases work unauthenticated, so it's not required.
+# Environment=GITHUB_TOKEN=
+ExecStart=/home/ubuntu/scripts/cosift-self-update.sh
+# The health gate plus a possible rollback can each take the full ~10 min
+# HNSW load window; give the unit generous headroom before systemd kills it.
+TimeoutStartSec=1500
+# A failed self-update (non-zero exit) is informational, not fatal for the
+# box — the running service is unaffected (or already rolled back).
+SuccessExitStatus=0
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/systemd/cosift-self-update.timer b/deploy/systemd/cosift-self-update.timer
new file mode 100644
index 0000000..f2e5fb2
--- /dev/null
+++ b/deploy/systemd/cosift-self-update.timer
@@ -0,0 +1,16 @@
+[Unit]
+Description=Poll for a newer signed cosift release every ~5 min
+
+[Timer]
+# First poll 2 min after boot (let cosift-serve finish its HNSW load), then
+# every 5 min. RandomizedDelaySec smears load off the GitHub API and avoids
+# a thundering-herd if multiple boxes ever run this.
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=60
+AccuracySec=30s
+Persistent=true
+Unit=cosift-self-update.service
+
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-sitemap-refresh.service b/deploy/systemd/cosift-sitemap-refresh.service
new file mode 100644
index 0000000..8340a9d
--- /dev/null
+++ b/deploy/systemd/cosift-sitemap-refresh.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Cosift sitemap refresh (re-submit curated quality hosts)
+After=cosift-serve.service
+Wants=cosift-serve.service
+
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-sitemap-refresh.sh
+StandardOutput=journal
+StandardError=journal
diff --git a/deploy/systemd/cosift-sitemap-refresh.timer b/deploy/systemd/cosift-sitemap-refresh.timer
new file mode 100644
index 0000000..4a61a1e
--- /dev/null
+++ b/deploy/systemd/cosift-sitemap-refresh.timer
@@ -0,0 +1,11 @@
+[Unit]
+Description=Cosift sitemap refresh — every 6 hours
+
+[Timer]
+OnBootSec=20min
+OnUnitActiveSec=6h
+AccuracySec=5min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/deploy/systemd/cosift-wet-refresh.service b/deploy/systemd/cosift-wet-refresh.service
new file mode 100644
index 0000000..617e991
--- /dev/null
+++ b/deploy/systemd/cosift-wet-refresh.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Cosift WET ingest from latest CommonCrawl
+After=cosift-serve.service
+Requires=cosift-serve.service
+
+[Service]
+Type=oneshot
+User=ubuntu
+ExecStart=/usr/local/bin/cosift-wet-refresh.sh
diff --git a/deploy/systemd/cosift-wet-refresh.timer b/deploy/systemd/cosift-wet-refresh.timer
new file mode 100644
index 0000000..5342d70
--- /dev/null
+++ b/deploy/systemd/cosift-wet-refresh.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run cosift-wet-refresh weekly
+
+[Timer]
+OnCalendar=*-*-* 03:00:00
+Persistent=true
+Unit=cosift-wet-refresh.service
+
+[Install]
+WantedBy=timers.target
diff --git a/docs/DIAGRAMS-contrib-network.md b/docs/DIAGRAMS-contrib-network.md
new file mode 100644
index 0000000..882f929
--- /dev/null
+++ b/docs/DIAGRAMS-contrib-network.md
@@ -0,0 +1,734 @@
+# Cosift Open Contribution Network — Flow Diagrams
+
+Companion to `PLAN-contrib-network.md`. Each diagram answers one question and
+references the workstreams (WS0-WS15) implementing it. Mermaid renders natively
+on GitHub.
+
+Diagram index:
+
+1. [System overview — layers and trust boundaries](#1-system-overview)
+2. [End-to-end story — developer publishes → AI agent retrieves](#2-end-to-end-story)
+3. [Component map of the broker](#3-component-map-of-the-broker)
+4. [Publisher: claim + verify a docs prefix](#4-publisher-claim-and-verify)
+5. [Publisher: heartbeat + lease lifecycle](#5-publisher-heartbeats-and-lease-state)
+6. [Publisher: lease state machine](#6-lease-state-machine)
+7. [Worker: install → register → ready](#7-worker-install-and-register)
+8. [Worker: fetch contribution lifecycle](#8-fetch-contribution-lifecycle)
+9. [Worker: embed contribution lifecycle](#9-embed-contribution-lifecycle)
+10. [Frontier lanes and work distribution](#10-frontier-lanes-and-work-distribution)
+11. [Quarantine → verify → promote pipeline](#11-quarantine-verify-promote-pipeline)
+12. [Retrieval path with gating](#12-retrieval-path-with-gating)
+13. [Transport binding — Pilot tunnel identity check (I9)](#13-transport-binding-i9)
+14. [Principal reputation state machine](#14-principal-reputation)
+15. [Multi-broker federation (forward-compat)](#15-multi-broker-federation-future)
+
+---
+
+## 1. System overview
+
+Three planes — humans/agents at the top, Pilot overlay in the middle, the broker (cosift on GH200) at the bottom. Every contribution byte crosses the middle plane (**I9**).
+
+```mermaid
+flowchart TB
+    subgraph H[Humans and agents]
+        DEV[Developer<br/>publishes docs]
+        AI[Claude Code / AI agent<br/>retrieves answers]
+    end
+
+    subgraph A[Apps on contributor / consumer machines]
+        PUB[cosift-node app<br/>publisher role]
+        WRK[cosift-node app<br/>worker role + bundled embedder]
+        MCP[cosift MCP shim]
+    end
+
+    subgraph P[Pilot overlay - tunnels authenticated by ed25519 pubkey]
+        T1((overlay tunnel))
+        T2((overlay tunnel))
+        T3((overlay tunnel))
+    end
+
+    subgraph B[Broker - cosift on GH200]
+        LIS[Pilot listener<br/>virt port 7700]
+        RT[Router + middleware<br/>envelope verify, peer-pubkey binding]
+        BL[Business logic<br/>collections - leases - principals - verifier]
+        ST[(Pebble store<br/>P, C, A, S, W, docs, frontier, HNSW)]
+        VLM[vLLM<br/>chat]
+        EMB[Ollama replicas x8<br/>nomic-embed-text-v1.5]
+    end
+
+    DEV --> PUB
+    AI --> MCP
+
+    PUB --> T1
+    WRK --> T2
+    MCP --> T3
+
+    T1 --> LIS
+    T2 --> LIS
+    T3 --> LIS
+
+    LIS --> RT --> BL --> ST
+    BL --> VLM
+    BL --> EMB
+
+    style P fill:#eef,stroke:#88a,stroke-width:2px
+    style B fill:#efe,stroke:#8a8,stroke-width:2px
+```
+
+Trust boundaries: every arrow crossing into the broker is overlay-authenticated; the envelope signing key equals the tunnel peer pubkey.
+
+---
+
+## 2. End-to-end story
+
+The full loop from a dev shipping docs to an AI agent retrieving an answer. Time flows top-to-bottom; only happy-path events shown.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    actor DEV as Developer
+    participant PUB as cosift-node (publisher)
+    participant ORG as docs.foo.dev origin
+    participant BRK as Cosift broker
+    participant WRK as cosift-node (worker, elsewhere)
+    participant MCP as cosift MCP shim
+    actor AI as Claude Code user
+
+    DEV->>PUB: install via pilotctl appstore
+    PUB->>BRK: register principal (overlay, signed)
+    PUB->>BRK: claim_prefix("docs.foo.dev")
+    BRK-->>PUB: challenge token T
+    PUB->>DEV: place T at /.well-known/cosift-verify.txt
+    DEV->>ORG: deploy T
+    BRK->>ORG: GET /.well-known (verification worker)
+    ORG-->>BRK: T matches
+    BRK->>BRK: collection verified, active
+
+    PUB->>BRK: submit URLs (lane 0)
+    BRK->>BRK: push to frontier lane 0
+
+    WRK->>BRK: claim work (kinds: fetch, embed)
+    BRK-->>WRK: leased URLs and chunks
+    WRK->>ORG: fetch pages
+    WRK->>WRK: embed chunks (bundled sidecar)
+    WRK->>BRK: submit signed results
+    BRK->>BRK: quarantine, verify, promote
+
+    Note over PUB,BRK: heartbeat every 6h keeps lease active
+
+    AI->>MCP: cosift_search("how to use foo v2.3")
+    MCP->>BRK: /search (overlay)
+    BRK->>BRK: BM25 + HNSW + rerank + judge<br/>+ gate (collection state, banned principals)
+    BRK-->>MCP: hits with provenance
+    MCP-->>AI: results in Claude Code
+```
+
+The lease is what makes "active only while installed" true: the developer's app stops heartbeating when uninstalled, lease lapses, docs leave the index.
+
+---
+
+## 3. Component map of the broker
+
+Zoom into one cosift instance. Routes split by auth surface; background loops sit on the side.
+
+```mermaid
+flowchart LR
+    subgraph IN[Inbound]
+        OVL[Overlay listener<br/>port 7700]
+        LOOP[Loopback HTTP :7777<br/>admin only, not exposed]
+    end
+
+    subgraph MW[Middleware]
+        ENV[Envelope verify +<br/>peer-pubkey binding]
+        ADM[Bearer admin]
+    end
+
+    subgraph RT[Routes]
+        PR[/pub/* publisher API/]
+        WR[/work/* worker API/]
+        SR[/search /query /answer /research/]
+        PUB[/queue /healthz /stats /metrics/]
+        AR[/admin/* operator/]
+    end
+
+    subgraph BL[Core logic]
+        COL[collections + leases]
+        PRN[principal registry + reputation]
+        WL[work-lease + frontier lanes]
+        VR[verification scheduler]
+        RET[retrieval + filter + judge]
+    end
+
+    subgraph ST[Pebble families]
+        FAM["'P' principals<br/>'C' collections<br/>'A' attribution<br/>'S' staging<br/>'W' work leases<br/>'d/u/i/h' docs<br/>'f' frontier (lane-prefixed)<br/>'v/q' HNSW + PQ"]
+    end
+
+    subgraph BG[Background loops]
+        SWP[lease sweeper]
+        REV[re-verify domain]
+        VWK[verification workers]
+        REC[re-crawl scheduler]
+        SNP[snapshot timer]
+    end
+
+    OVL --> ENV
+    LOOP --> ADM
+    ENV --> PR
+    ENV --> WR
+    ENV --> SR
+    ENV --> PUB
+    ADM --> AR
+
+    PR --> COL
+    WR --> WL
+    SR --> RET
+    AR --> WL
+    AR --> PRN
+
+    COL --> ST
+    PRN --> ST
+    WL --> ST
+    VR --> ST
+    RET --> ST
+
+    SWP --> COL
+    REV --> COL
+    VWK --> VR
+    REC --> WL
+    SNP --> ST
+```
+
+`/admin/*` routes only exist on loopback HTTP behind the operator's admin token — the overlay listener refuses them outright (route allowlist).
+
+---
+
+## 4. Publisher claim and verify
+
+Domain ownership without a central account system. The well-known token is the proof; the broker re-verifies every 30 days.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant DEV as Developer
+    participant APP as cosift-node app
+    participant BRK as Broker
+    participant ORG as docs.foo.dev
+
+    DEV->>APP: node.claim_prefix(prefix="docs.foo.dev")
+    APP->>APP: build envelope (kind=pub_claim,<br/>principal=K_pub, seq=N)
+    APP->>BRK: POST /pub/claim (overlay, envelope)
+    BRK->>BRK: verify envelope, allocate collection_id,<br/>store record state=pending,<br/>generate token T
+    BRK-->>APP: { collection_id, token T, expires }
+    APP-->>DEV: "Place T at https://docs.foo.dev/.well-known/cosift-verify.txt"
+
+    DEV->>ORG: deploy verification file
+    Note over BRK: verification worker tick (every 60s)
+    BRK->>ORG: GET /.well-known/cosift-verify.txt
+    ORG-->>BRK: contents = T (match)
+    BRK->>BRK: state = verified -> active<br/>verified_at = now
+
+    APP->>BRK: GET /pub/verify-status (envelope)
+    BRK-->>APP: { state: "active" }
+    APP-->>DEV: "Verified. You may submit."
+
+    Note over BRK: every 30 days re-verify. failure leads to state=unverified<br/>(serving continues, writes blocked)
+```
+
+Conflicting claims: longest-prefix nesting; first verified wins. A second principal claiming the same exact prefix → 409 immediately.
+
+---
+
+## 5. Publisher heartbeats and lease state
+
+Nonce-chained heartbeats (**D1**) — the response carries the next nonce; replay-proof without inbound delivery. App uninstall = identity destroyed = heartbeats stop = lease lapses.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant APP as cosift-node (publisher)
+    participant BRK as Broker
+
+    Note over APP,BRK: every 6h while installed
+
+    APP->>APP: load nonce_N from local state
+    APP->>APP: sign envelope (kind=heartbeat,<br/>seq=S, body=nonce_N)
+    APP->>BRK: POST /pub/heartbeat (overlay, envelope)
+    BRK->>BRK: verify envelope, assert nonce_N matches<br/>expected pending_nonce, assert seq advances
+    BRK->>BRK: update last_renewal, generate nonce_{N+1}, store as pending
+    BRK-->>APP: { next_nonce: nonce_{N+1}, lease_state: active }
+    APP->>APP: persist nonce_{N+1}
+
+    Note over APP,BRK: no heartbeat for 72h leads to grace.<br/>plus 7d to suspended. plus 60d to purged
+```
+
+Sweeper goroutine ticks once a minute; transitions are flag flips on the `'C'` record + an in-memory suspended-set update consumed by retrieval gating (WS6).
+
+---
+
+## 6. Lease state machine
+
+Two state machines live on a collection: the **lease** (presence-driven, this diagram) and the **verification** state (active vs unverified, which gates writes but never serving).
+
+```mermaid
+stateDiagram-v2
+    [*] --> pending : POST /pub/claim
+    pending --> verified : well-known token matches
+    pending --> [*] : token expires (24h)
+    verified --> active : first heartbeat received
+    active --> grace : 72h no renewal
+    grace --> active : renewal arrives
+    grace --> suspended : 7d total no renewal<br/>(remove from retrieval)
+    suspended --> active : renewal arrives<br/>(instant reactivation, no re-crawl)
+    suspended --> purged : 60d total no renewal<br/>(walk attribution, delete docs)
+    purged --> [*]
+
+    note right of active
+      serving normally
+      writes accepted
+    end note
+    note right of grace
+      score multiplier 0.5
+      writes accepted
+    end note
+    note right of suspended
+      docs hidden in retrieval
+      writes blocked
+      still on disk
+    end note
+```
+
+Generous timings: a laptop on vacation doesn't nuke a publisher's docs.
+
+---
+
+## 7. Worker install and register
+
+Self-contained — no ollama, no python. The bundle ships the embed runtime and model weights.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    actor OP as Operator
+    participant CLI as pilotctl
+    participant SUP as appstore supervisor
+    participant APP as cosift-node binary
+    participant SIDE as llama.cpp sidecar
+    participant BRK as Broker
+
+    OP->>CLI: pilotctl appstore install io.pilot.cosift-node
+    CLI->>SUP: fetch bundle (~307 MB)
+    SUP->>SUP: verify bundle sha256 vs catalogue<br/>verify each data entry sha256 vs manifest
+    SUP->>APP: spawn cosift-node --identity $APP/identity.json
+    APP->>APP: load or create identity (ed25519)
+    APP->>SIDE: spawn embed-runtime<br/>--model models/nomic-...gguf<br/>--uds $APP/embed.sock
+    SIDE-->>APP: ready
+    APP->>SIDE: embed(fixture_text)
+    SIDE-->>APP: vector v
+    APP->>APP: assert cosine(v, fixture_vector) at least 0.9999
+    APP->>BRK: POST /work/register (overlay, envelope<br/>kind=register, reports model_blob_sha256)
+    BRK->>BRK: store principal record 'P' family<br/>tier=new, sampling=1.0
+    BRK-->>APP: { ok, principal_id }
+    APP-->>SUP: ready signal
+    SUP-->>CLI: state: ready
+    CLI-->>OP: installed and ready
+```
+
+If the fixture check fails, the app stays alive for `node.status` but `node.worker_start` returns `runtime_not_ready` — never enters the loop with a tampered model.
+
+---
+
+## 8. Fetch contribution lifecycle
+
+Claim → fetch → submit → quarantine → sampled verify → promote (or reject). Politeness stays server-controlled.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant WRK as Worker app
+    participant BRK as Broker
+    participant ORG as Target site
+    participant VR as Verifier (background)
+
+    WRK->>BRK: POST /work/claim<br/>{kinds: [fetch], lanes: [1,2,3], n: 10}
+    BRK->>BRK: drain lanes by weighted RR<br/>respect host fairness + politeness deadline
+    BRK->>BRK: write 'W' leases (TTL 10min)
+    BRK-->>WRK: [{claim_id, url, robots_ok, deadline, max_body}, ...]
+
+    loop per claim
+        WRK->>ORG: GET url (own egress)
+        ORG-->>WRK: HTML/PDF
+        WRK->>WRK: extract title/text/lang/published_at
+        WRK->>BRK: POST /work/submit<br/>(envelope, claim_id,<br/>raw_sha256, extracted)
+        BRK->>BRK: schema check, body size check,<br/>content-type allowlist
+        BRK->>BRK: stage in 'S' family
+        BRK-->>WRK: { staged: true }
+    end
+
+    Note over BRK,VR: sampling rate r per principal tier<br/>r=1.0 fresh, 0.05 trusted
+
+    VR->>VR: pick fraction r of staged items
+    VR->>ORG: re-fetch url (same path)
+    VR->>VR: normalize via parse.go pipeline<br/>(drop tags, boilerplate, whitespace)
+    VR->>VR: 64-bit simhash compare<br/>(match at most 6, partial at most 14, else mismatch)
+    VR->>VR: authority check (Score(host) floor)<br/>judge gate (LLM, fail-open)<br/>ContentSHA dedup
+
+    alt all gates pass
+        VR->>VR: UpsertDocument + IndexDocument<br/>write attribution row 'A'<br/>bump principal verified_ok
+    else any gate fails or mismatch pattern
+        VR->>VR: discard from 'S'<br/>bump principal verified_fail<br/>(pattern leads to tier drop or ban)
+    end
+```
+
+Same churn-aware grading prevents single-doc slashing — a contributor goes down only on a *pattern* of mismatches.
+
+---
+
+## 9. Embed contribution lifecycle
+
+Verification is sharper here than for content — same model + same blob = deterministic match, cosine ≥ 0.9999.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant WRK as Worker app
+    participant SIDE as Bundled embed sidecar
+    participant BRK as Broker
+    participant EMB as Broker's local embedder
+    participant VR as Verifier
+
+    WRK->>BRK: POST /work/claim<br/>{kinds: [embed],<br/>model_blob_sha256: 970aa74c...,<br/>n: 100}
+    BRK->>BRK: assert blob sha matches index pin<br/>(if not -> 409, never hand work)
+    BRK->>BRK: scan docs lacking vectors<br/>chunk via internal/index/chunk.go<br/>truncate via truncateForEmbedLite
+    BRK-->>WRK: [{claim_id, doc_id, offset, length, text}, ...]
+
+    loop per chunk
+        WRK->>SIDE: embed(text)
+        SIDE-->>WRK: vector (768 float32)
+    end
+
+    WRK->>BRK: POST /work/submit<br/>(envelope, claim_id, vectors)
+    BRK->>BRK: dim check, finite check,<br/>unit-normalize server-side
+    BRK->>BRK: stage in 'S' family
+
+    VR->>VR: pick fraction r of vectors
+    VR->>EMB: re-embed(same text)
+    EMB-->>VR: vector v_local
+    VR->>VR: cosine(v_submitted, v_local) at least 0.9999?
+    alt all sampled pass
+        VR->>VR: HNSW.AddPassageBatch<br/>write attribution row 'A'<br/>bump verified_ok
+    else any fails
+        VR->>VR: reject whole batch<br/>bump verified_fail<br/>(tier drop on pattern)
+    end
+```
+
+Server defines the text (**I6**) — contributors never get to choose the input, which collapses the verification problem into "did you compute the right vector for this text."
+
+---
+
+## 10. Frontier lanes and work distribution
+
+The frontier is a single keyspace with a lane byte; weighted drain prevents bulk from starving submissions.
+
+```mermaid
+flowchart LR
+    subgraph SRC[Sources]
+        S0[publisher submit<br/>POST /pub/submit]
+        S1[RSS / sitemap refresh<br/>internal/crawler/rss.go]
+        S2[crawler discovery]
+        S3[WET / bulk import]
+    end
+
+    subgraph FR[Frontier 'f' family - lane-prefixed]
+        L0["lane 0 submitted<br/>weight 50"]
+        L1["lane 1 refresh<br/>weight 30"]
+        L2["lane 2 discovered<br/>weight 15"]
+        L3["lane 3 bulk<br/>weight 5"]
+    end
+
+    subgraph CLM[ClaimWork - weighted RR]
+        WRR["pop by lane weights<br/>host fairness within lane<br/>per-lane cursor"]
+    end
+
+    subgraph CONS[Consumers]
+        SC[server-side crawler<br/>internal/crawler]
+        WC[external workers<br/>via /work/claim]
+    end
+
+    subgraph LEA[Work leases 'W' family]
+        WL["claim_id ->{principal, expiry, attempts}<br/>TTL 10min fetch / 5min embed"]
+    end
+
+    S0 --> L0
+    S1 --> L1
+    S2 --> L2
+    S3 --> L3
+
+    L0 --> WRR
+    L1 --> WRR
+    L2 --> WRR
+    L3 --> WRR
+
+    WRR --> SC
+    WRR --> WC
+    WC -.lease.-> WL
+    WL -.expiry -> attempts++.-> WRR
+
+    style L0 fill:#dfd
+    style L3 fill:#fdd
+
+    L0 -. "lane 0 fetch -> trusted-tier only<br/>(or server-side)" .- SC
+```
+
+If lane 0 is empty its share redistributes to other lanes; deterministic weighted RR keeps statistical fairness across many claims.
+
+---
+
+## 11. Quarantine, verify, promote pipeline
+
+The atomic gate: nothing reaches the main index without passing every check for its kind.
+
+```mermaid
+flowchart TB
+    SUB["POST /work/submit<br/>envelope verified"]
+    SCH["schema + size + content-type<br/>fail fast"]
+    STG[("'S' family<br/>quarantine, TTL 14d")]
+    SCD["verification scheduler<br/>picks at rate r per principal tier"]
+
+    subgraph FETCH[Fetch path]
+        REF["re-fetch via crawler stack<br/>+ remote_fetcher CF transport"]
+        NORM["normalize via parse.go<br/>drop tags, boilerplate, collapseWS"]
+        SIM["64-bit simhash compare<br/>match, partial, mismatch"]
+    end
+
+    subgraph EMBED[Embed path]
+        RCMP["recompute via local embedder<br/>same blob sha"]
+        COS{"cosine at least 0.9999?"}
+    end
+
+    subgraph GATES[Promotion gates - both paths]
+        AUTH["authority.Score(host)<br/>lane-0 exempt, else floor 0.2"]
+        JDG["judge LLM relevance<br/>fail-open"]
+        DEDUP["ContentSHA exact-dup check"]
+    end
+
+    PROM["UpsertDocument + IndexDocument<br/>write 'A' attribution row<br/>bump verified_ok"]
+    REJ["discard from 'S'<br/>bump verified_fail<br/>pattern leads to tier drop or ban then purge"]
+
+    SUB --> SCH
+    SCH -- ok --> STG
+    SCH -- bad --> REJ
+    STG --> SCD
+
+    SCD -- fetch kind --> REF --> NORM --> SIM
+    SCD -- embed kind --> RCMP --> COS
+
+    SIM -- match or partial --> AUTH
+    SIM -- mismatch --> REJ
+    COS -- yes --> AUTH
+    COS -- no --> REJ
+
+    AUTH --> JDG --> DEDUP
+    DEDUP -- new --> PROM
+    DEDUP -- exact-dup --> REJ
+
+    style REJ fill:#fee
+    style PROM fill:#efe
+```
+
+`partial` for content is neutral (page churn — ads, timestamps); only a *pattern* of mismatches drops a tier.
+
+---
+
+## 12. Retrieval path with gating
+
+One choke point covers `/search`, `/query`, `/answer`, `/research`: the post-`retrieve()` filter loop. Suspended collections + banned principals filtered there cheaply via the docMeta v2 packed blob.
+
+```mermaid
+flowchart TB
+    Q["/query  /search  /answer  /research"]
+    PARSE["parse params: q, k, filters,<br/>decay, mode, max_passes"]
+
+    subgraph RTV[retrieve and rerank]
+        BM[BM25 lexical]
+        DN[HNSW dense ANN]
+        FUSE[hybrid fuse RRF]
+        RR["rerank Cohere or Voyage<br/>or LLMReranker"]
+    end
+
+    subgraph GATE[post-retrieve gate - the single choke point]
+        DM["lookup docMeta v2<br/>collection_id + principal"]
+        SUS{"collection<br/>suspended?"}
+        BAN{"principal<br/>banned?"}
+        GR{"collection<br/>in grace?"}
+    end
+
+    JDG["judge LLM relevance<br/>drop low-score hits"]
+    MMR[MMR diversify]
+    SYN["synthesize answer<br/>multi-pass research"]
+    OUT["SSE stream:<br/>phase events + answer + sources"]
+
+    Q --> PARSE --> BM --> FUSE
+    PARSE --> DN --> FUSE
+    FUSE --> RR --> DM
+    DM --> SUS
+    SUS -- yes --> DROP1["drop hit"]
+    SUS -- no --> BAN
+    BAN -- yes --> DROP2["drop hit"]
+    BAN -- no --> GR
+    GR -- yes --> DECAY["score x 0.5<br/>reuses decay multiplier"]
+    GR -- no --> KEEP["keep at full score"]
+    KEEP --> JDG
+    DECAY --> JDG
+    JDG --> MMR --> SYN --> OUT
+
+    style GATE fill:#ffe,stroke:#aa6
+```
+
+Same machinery for every retrieval endpoint — invariant **I8** enforced by sharing one function, not four.
+
+---
+
+## 13. Transport binding (I9)
+
+The contribution-path proof: the envelope's signing key must equal the Pilot tunnel's authenticated peer pubkey. Forge either side, broker refuses.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant APP as Worker / publisher app<br/>(principal pubkey K)
+    participant D1 as Pilot daemon<br/>(contributor side)
+    participant D2 as Pilot daemon<br/>(broker side, on GH200)
+    participant LIS as Broker pilot_listener
+    participant MW as Envelope middleware
+    participant H as Handler
+
+    APP->>D1: net.dial(broker_overlay_addr, 7700)
+    D1->>D2: tunnel handshake (Pilot mutual auth)
+    Note over D1,D2: cryptographically proves<br/>peer pubkey is K
+    D2->>LIS: accept stream, remote_pubkey = K
+    LIS->>LIS: stamp ctx[pilotPeerPubKey] = K
+
+    APP->>APP: build envelope (PrincipalPub = K, sig by K)
+    APP->>LIS: POST /work/claim (envelope in header)
+    LIS->>MW: call handler with ctx
+    MW->>MW: verify envelope sig (ed25519)
+    MW->>MW: assert envelope.PrincipalPub == ctx.pilotPeerPubKey
+    alt match
+        MW->>MW: assert seq advances, persist new seq
+        MW->>H: invoke with principal in ctx
+        H-->>APP: 200 OK
+    else mismatch
+        MW->>MW: bump Sybil counter on K<br/>(forger used wrong key, real K caught)
+        MW-->>APP: 401 envelope_peer_mismatch
+    end
+
+    Note over LIS,MW: bypass only via Contrib.AllowHTTPDev=true<br/>and 127.0.0.1 (dev only)
+```
+
+Public HTTPS via Caddy 404s these routes — overlay is the only network-reachable path.
+
+---
+
+## 14. Principal reputation
+
+Sampling rate is the lever — fresh keys verified at 100%, earned trust lowers it; mismatch patterns drop you back or ban you.
+
+```mermaid
+stateDiagram-v2
+    [*] --> new : POST /work/register
+    new --> basic : 200+ verified_ok<br/>AND age 7d+<br/>AND 0 mismatches / 30d
+    basic --> trusted : 2000+ verified_ok<br/>AND age 30d+<br/>AND operator approves
+    basic --> new : mismatch pattern<br/>(3+ in 24h)
+    trusted --> basic : mismatch pattern
+    new --> banned : mismatch pattern<br/>OR operator action
+    basic --> banned : mismatch pattern persists
+    trusted --> banned : mismatch pattern persists
+    banned --> [*] : purge all attributed docs<br/>(WS1.3 PurgePrincipal)
+
+    note right of new
+      sampling 1.0
+      lanes 2, 3
+      200 URL / 5k chunks per day
+    end note
+    note right of basic
+      sampling 0.25
+      lanes 1, 2, 3
+      2k URL / 50k chunks per day
+    end note
+    note right of trusted
+      sampling 0.05 (floor)
+      lanes 0, 1, 2, 3
+      10k URL / 250k chunks per day
+    end note
+```
+
+Expected un-caught bad submissions per burned identity ≈ (1−r)/r → 0 / 3 / 19. Purge (**I4**) reverses survivors.
+
+---
+
+## 15. Multi-broker federation (future)
+
+What v2 looks like if **D10** says go. Phase 1-5 keeps the runway clear (WS11a) but ships single-broker.
+
+```mermaid
+flowchart TB
+    subgraph DIR[Pilot directory - list-agents]
+        D["keyword: cosift-broker<br/>each broker advertises<br/>region, capacity, age"]
+    end
+
+    subgraph B[Broker pool on Pilot overlay]
+        BA[Broker A<br/>GH200 USA]
+        BB[Broker B<br/>Jetson EU]
+        BC[Broker C<br/>operator X JP]
+    end
+
+    subgraph APPS[Worker / publisher apps]
+        W1[node W1]
+        W2[node W2]
+        P1[node P1]
+    end
+
+    APPS -.discover.-> DIR
+    DIR -.candidates.-> APPS
+    W1 -- "pick lowest-latency<br/>broker_overlay_addrs plural" --> BA
+    W2 --> BB
+    P1 --> BC
+
+    subgraph GOSS[Inter-broker gossip - signed]
+        G1[principal records<br/>self-signed by principal<br/>portable across brokers]
+        G2[collection claims<br/>well-known token<br/>re-verifiable per broker]
+        G3[reputation exports<br/>signed counter deltas<br/>opt-in import]
+        G4["attribution<br/>doc carries principal_sig + broker_sig"]
+    end
+
+    BA <-->|signed sync| BB
+    BB <-->|signed sync| BC
+    BA <-->|signed sync| BC
+
+    BA -.uses.-> GOSS
+    BB -.uses.-> GOSS
+    BC -.uses.-> GOSS
+
+    subgraph FRO[Frontier sharding - reuses existing code]
+        SH["URL hash FNV-32 -> shard<br/>internal/config/cluster.go<br/>ForwardFn for cross-broker forward"]
+    end
+
+    BA -.shard.-> SH
+    BB -.shard.-> SH
+    BC -.shard.-> SH
+
+    style GOSS fill:#eef,stroke:#88a,stroke-dasharray:5 5
+    style DIR fill:#fef,stroke:#a8a
+```
+
+The portability invariants from WS11a are what make this additive instead of a rewrite: principals, collections, attribution, and frontier sharding all already work cross-broker; v2 just adds discovery + gossip + import logic.
+
+---
+
+## Reading order if you only have ten minutes
+
+1. **Diagram 1** — see the three planes and where Pilot sits.
+2. **Diagram 2** — see the developer-to-AI-agent loop end to end.
+3. **Diagram 13** — see the I9 transport binding (the security spine).
+4. **Diagram 11** — see the verification pipeline (the trust spine).
+5. **Diagram 6** — see the lease lifecycle (the adoption flywheel).
diff --git a/docs/ENV.md b/docs/ENV.md
new file mode 100644
index 0000000..17d4cf8
--- /dev/null
+++ b/docs/ENV.md
@@ -0,0 +1,246 @@
+# cosift — Environment Variable Reference
+
+This is the complete, code-verified reference for every runtime environment
+variable the cosift engine reads. Defaults and effects below were read directly
+from source (file:line cited in each row), not inferred.
+
+**How to read this doc:**
+
+- **Two configuration surfaces.** Env vars (this doc) are the *operator-tuning*
+  surface — set them in the systemd unit / shell / container env. They sit
+  *on top of* a second, separate surface: the JSON config file
+  (`cosift.json`, typically `/home/ubuntu/cosift.json` in prod) parsed by
+  `internal/config/config.go`. Things like the listen address, data dir,
+  embeddings/chat URLs + models, rerank config, cluster topology, proxies,
+  and the admin/peer-auth tokens live in `cosift.json`. This doc does **not**
+  fully document `cosift.json`; it only notes where an env var overrides or
+  interacts with a config field. The two should not be confused.
+- **Helper semantics** (matters for what counts as a valid override):
+  - `envIntDefault` (server, `serve_setup.go:1140`) — int; **must be `> 0`**, else falls back to the default. Empty/zero/negative/garbage → default.
+  - `envDurationMsDefault` (server, `serve_setup.go:1152`) — integer **milliseconds**; must be `> 0`, else default. (Note: ms, not Go `"2s"` syntax.)
+  - `envInt` (Pebble, `store/pebble.go:250`) — int; must be `> 0`, else default.
+  - `envIntCrawler` (crawler, `crawler.go:635`) — int; must be `>= 0` (zero is a *valid* value here), else default.
+  - `envFloatCrawler` (crawler, `crawler.go:964`) — float; must be `>= 0`, else default.
+  - `firstEnv` (`main.go:381`) / `resolveAPIKey` (`main.go:355`) — return the first non-empty value among a list of names.
+  - Inline `os.Getenv` checks vary; each row states the exact behavior.
+- **Booleans are string-literal compares**, not generic parsing. Where a var is
+  documented as `bool`, the code compares against an exact string (e.g.
+  `== "1"`, `== "true"`, `== "false"`). Any other value behaves as "unset".
+  The "default" column states the effective behavior when unset.
+
+---
+
+## SECRETS — read this first
+
+cosift has **no env var that holds the server's admin password.** Be precise
+about what each "token/key" var actually does:
+
+| Var | What it really is | Secret? |
+|---|---|---|
+| `COSIFT_ADMIN_TOKEN` | **CLIENT-side** Bearer for the `cosift admin …` subcommands (`cmd_admin.go`) — the value the *CLI* sends to a remote box's `/admin/*`. It is **NOT** the server's auth gate. | Yes (client credential) |
+| `COSIFT_EMBED_API_KEY` / `COSIFT_CHAT_API_KEY` | Provider API keys for the embed / chat slots (`resolveAPIKey`). Fall back to `OPENAI_API_KEY` → `OPENAI`. | Yes |
+| `OPENAI_API_KEY` / `OPENAI` | OpenAI(-compatible) key; `OPENAI` is the accepted short form. | Yes |
+| `COHERE_API_KEY` / `VOYAGE_API_KEY` | Rerank provider keys (HTTP reranker, when `cfg.Rerank.URL` is set and `cfg.Rerank.APIKey` is empty). | Yes |
+
+**Server admin auth is NOT an env var.** The production `pebble-serve` admin
+gate is `cluster.peer_auth_token` in `cosift.json` (and the legacy `serve`
+path uses `server.admin_token`). See the **secret-ownership matrix in
+[docs/HANDOVER.md §1](HANDOVER.md)** for the authoritative ownership classes,
+rotation policy, and the current-state caveats (e.g. `peer_auth_token` empty in
+prod = admin gate skipped).
+
+**Production uses LOCAL models.** In prod, the embeddings and chat URLs in
+`cosift.json` point at `127.0.0.1` (self-hosted vLLM / Ollama / TEI). Local
+OpenAI-compatible endpoints accept anonymous requests, so
+`OPENAI_API_KEY` / `OPENAI` / `COHERE_API_KEY` / `VOYAGE_API_KEY` /
+`COSIFT_EMBED_API_KEY` / `COSIFT_CHAT_API_KEY` are **unset and unused in
+production.** They exist only for operators who self-host against a keyed
+OpenAI-compatible provider.
+
+**Raw-query persistence is DISABLED in prod for confidentiality.**
+`COSIFT_QUERY_LOG` and `COSIFT_FEEDBACK_LOG` enable on-box append of raw
+queries / feedback to JSONL. Per **[docs/HANDOVER.md §2](HANDOVER.md)** these
+are removed/unset at handover so no raw query text is persisted on the box. Do
+not re-enable them without the confidentiality decision in that section.
+
+---
+
+## Server & Network
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `PORT` | int (string) | unset → no override | 12-factor port. When set, server binds `0.0.0.0:$PORT` (overrides `cosift.json` `server.addr`). For Cloud Run / Fly / Heroku / Render. | `internal/config/config.go:499` |
+| `COSIFT_LISTEN` | string | unset → use `cosift.json` `server.addr` | Full listen address escape hatch (e.g. `127.0.0.1:8080`). Overrides `PORT`. | `internal/config/config.go:502` |
+| `COSIFT_DATA_DIR` | string | unset → `cosift.json` `data_dir` (or built-in default) | Overrides the data directory (Pebble store + operational state). | `internal/config/config.go:505` |
+
+> `cosift.json` is auto-augmented by a best-effort load of `./.env` at startup
+> (`config.go:475`, `LoadDotEnv`), which sets any `KEY=VALUE` lines into the
+> process env *only if not already set*.
+
+---
+
+## Auth & Secrets
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_ADMIN_TOKEN` | string (SECRET, client-side) | unset → no Bearer sent | Token the `cosift admin …` CLI subcommands send as the admin Bearer to a remote box. **Not** a server gate. | `cmd/cosift/cmd_admin.go:96,135,332,415,537`; `cmd_maintenance.go:618` |
+| `COSIFT_EMBED_API_KEY` | string (SECRET) | unset → fall back to `OPENAI_API_KEY` → `OPENAI` → anonymous | API key for the **embed** slot. | `cmd/cosift/main.go:359` (`resolveAPIKey("embed")`) |
+| `COSIFT_CHAT_API_KEY` | string (SECRET) | unset → fall back to `OPENAI_API_KEY` → `OPENAI` → anonymous | API key for the **chat** slot. | `cmd/cosift/main.go:361` (`resolveAPIKey("chat")`) |
+| `OPENAI_API_KEY` | string (SECRET) | unset → try `OPENAI` → anonymous | OpenAI(-compatible) key; common fallback for both embed and chat slots, and read directly by `index` / `eval` / `serve` subcommands. | `main.go:368`; `cmd_index.go:155,612`; `cmd_eval.go:272,335,356,375,1316`; `cmd_serve.go:136`; `cmd_maintenance.go:562,609,610` |
+| `OPENAI` | string (SECRET) | unset → anonymous | Accepted short form of `OPENAI_API_KEY`. Checked only after `OPENAI_API_KEY` is empty. | `main.go:368`; same call sites as `OPENAI_API_KEY` |
+| `COHERE_API_KEY` | string (SECRET) | unset → try `VOYAGE_API_KEY` → anonymous | API key for the HTTP reranker when `cfg.Rerank.URL` is set and `cfg.Rerank.APIKey` is empty (checked first). | `serve_setup.go:431`; `cmd_serve.go:179` (via `firstEnv`) |
+| `VOYAGE_API_KEY` | string (SECRET) | unset → anonymous | HTTP reranker key, checked only after `COHERE_API_KEY` is empty. | `serve_setup.go:433`; `cmd_serve.go:179` (via `firstEnv`) |
+
+---
+
+## Logging / Observability / Retention
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_QUERY_LOG` | string (path) | unset → query logging disabled (no-op) | Path to JSONL file; when set, every query-endpoint request is appended. Also gates the feedback log default location and the `/admin/query-log` tail endpoint. **Disabled in prod — see HANDOVER §2.** | `serve_setup.go:308`; `querylog.go:96`; `feedback.go:37,147` |
+| `COSIFT_FEEDBACK_LOG` | string (path) | unset → falls back to `feedback-log.jsonl` beside `COSIFT_QUERY_LOG`; if that's also unset → disabled | Path for the feedback JSONL log. **Disabled in prod — see HANDOVER §2.** | `feedback.go:34`; `serve_setup.go` (via `feedbackLogPath()`) |
+| `COSIFT_SLA_LOG_PATH` | string (path) | unset → `<DataDir>/sla-violations.jsonl` (if DataDir set) | Where SLA-violation records are written. | `serve_setup.go:348` |
+| `COSIFT_SLA_SEARCH_P95_MS` | duration (ms) | `1500` (1.5s) | P95 SLA target for `/search`. | `serve_setup.go:353` |
+| `COSIFT_SLA_SEARCH_P99_MS` | duration (ms) | `4000` (4s) | P99 SLA target for `/search`. | `serve_setup.go:353` |
+| `COSIFT_SLA_ANSWER_P95_MS` | duration (ms) | `8000` (8s) | P95 SLA target for `/answer`. | `serve_setup.go:354` |
+| `COSIFT_SLA_ANSWER_P99_MS` | duration (ms) | `20000` (20s) | P99 SLA target for `/answer`. | `serve_setup.go:354` |
+| `COSIFT_SLA_RESEARCH_P95_MS` | duration (ms) | `30000` (30s) | P95 SLA target for `/research`. | `serve_setup.go:355` |
+| `COSIFT_SLA_RESEARCH_P99_MS` | duration (ms) | `60000` (60s) | P99 SLA target for `/research`. | `serve_setup.go:355` |
+| `COSIFT_SLA_WINDOW_MS` | duration (ms) | `300000` (5min) | Rolling window over which SLA percentiles are computed. | `serve_setup.go:359` |
+| `COSIFT_SLA_EVAL_MS` | duration (ms) | `30000` (30s) | How often the SLA monitor re-evaluates targets. | `serve_setup.go:360` |
+| `COSIFT_WEB_DENOMINATOR` | int (int64) | `3500000000` (3.5B, Common Crawl est.) | Denominator for the `/stats` "% of indexed web" novelty figure; must be `> 0`. | `serve_stats.go:380` |
+| `COSIFT_DEBUG_UPSERT` | bool (`"1"`) | unset → off | When `"1"`, prints `upsert-new: id=… url=…` to stderr for each newly-inserted document. Debug only. | `store/pebble.go:298,1413` |
+
+> Note: `/healthz` (P95 50ms / P99 200ms) and `/stats` (P95 200ms / P99 1s) SLA
+> targets are **hardcoded** (`serve_setup.go:356-357`) and not env-tunable.
+
+---
+
+## Rate Limiting
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_RATELIMIT_RPM` | float | unset / `<= 0` → **global limiter disabled** | Per-IP token-bucket refill rate (requests/min). Enabling this turns on the global rate limiter. | `serve_setup.go:1180` |
+| `COSIFT_RATELIMIT_BURST` | float | `10` (only when RPM set) | Token-bucket burst capacity for the global limiter; must be `> 0`. | `serve_setup.go:1189` |
+| `COSIFT_RATELIMIT_WHITELIST` | csv | empty → no whitelist | Comma-separated IPs that bypass the global limiter entirely. | `serve_setup.go:1195` |
+| `COSIFT_FEEDBACK_RPM` | int | `20` | Per-client RPM for the **always-on** `/feedback` limiter (stricter than global); must be `> 0`. | `serve_setup.go:319` |
+| `COSIFT_FEEDBACK_BURST` | int | `5` | Burst capacity for the `/feedback` limiter; must be `> 0`. | `serve_setup.go:320` |
+
+---
+
+## Retrieval & Ranking (BM25, decay, authority, expansion, caches)
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_BM25_K1` | float | `1.2` (PebbleBM25 default) | BM25 term-frequency saturation `k1`; applied only if parseable and `> 0`. | `serve_search.go:1350` |
+| `COSIFT_BM25_B` | float | `0.75` (PebbleBM25 default) | BM25 length-normalization `b`; applied only if parseable and `> 0`. | `serve_search.go:1358` |
+| `COSIFT_BM25_MIN_IDF` | float | `0.5` | IDF floor below which a query term is dropped as a stopword. `0` disables pruning. Must be `>= 0`. | `internal/index/pebble_bm25.go:33` |
+| `COSIFT_BM25_DISABLE_MAXSCORE` | bool (non-empty disables) | unset → MaxScore optimization **enabled** | Any non-empty value disables the WAND/MaxScore early-termination optimization for benchmark-grade lossless ranking. | `internal/index/pebble_bm25.go:204` |
+| `COSIFT_DEFAULT_DECAY_DAYS` | float | `180` (6-month half-life) | Default recency half-life (days) applied when a request has no explicit `?decay=`. `0` disables decay globally. Must be `>= 0`; explicit `?decay=N` still wins. | `serve_search.go:1592` |
+| `COSIFT_AUTHORITY_ALPHA` | float | scorer's built-in default (`authority.New()`) | Authority-score blend weight; applied only if parseable and `>= 0`. | `serve_setup.go:188` |
+| `COSIFT_TRANCO_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Tranco rankings CSV to enrich authority scoring. | `serve_setup.go:193` |
+| `COSIFT_MAJESTIC_CSV` | string (path) | unset → embedded whitelist + TLD heuristics only | Path to a Majestic Million CSV to enrich authority scoring. | `serve_setup.go:202` |
+| `COSIFT_DISABLE_ENTITY_EXPAND` | bool (non-empty disables) | unset → entity expansion **enabled** | When unset, query is expanded with canonical-attribute rewrites (`qexpand.RewriteEntity`). Any non-empty value disables it. | `serve_search.go:2064` |
+| `COSIFT_HYDE_CACHE_SIZE` | int | `256` | Capacity of the HyDE hypothetical-document cache; must be `> 0` (warns on bad value). | `serve_setup.go:161` |
+| `COSIFT_PARA_CACHE_SIZE` | int | `256` | Capacity of the paraphrase cache; must be `> 0` (warns on bad value). | `serve_setup.go:173` |
+| `COSIFT_ANSWER_CACHE_TTL_SEC` | int (seconds) | `60` | `/answer` response-cache TTL; must be `> 0` (use `0`/unset semantics via the answer-cache impl to effectively disable). | `serve_setup.go:299` |
+| `COSIFT_ANSWER_CACHE_CAP` | int | `1024` | `/answer` response-cache entry cap; must be `> 0`. | `serve_setup.go:300` |
+| `COSIFT_RESEARCH_CACHE_TTL_SEC` | int (seconds) | `600` (10min) | `/research` response-cache TTL; must be `> 0`. | `serve_setup.go:303` |
+| `COSIFT_RESEARCH_CACHE_CAP` | int | `256` | `/research` response-cache entry cap; must be `> 0`. | `serve_setup.go:304` |
+| `COSIFT_HOST_PARTITION` | bool (`"1"`) | unset → off | **Write path:** when `"1"`, host-partitioned postings are written on index (~2x posting write-amplification). | `store/pebble.go:3570` |
+| `COSIFT_HOST_PARTITION_READ` | bool (`"1"`) | unset → off | **Read path:** when `"1"`, `site=` queries route to the `'P'`-family host partition (`SearchInHost`); off → the 50× host-boost path is used. | `serve_search.go:1976` |
+
+---
+
+## HNSW & Embeddings
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_LOAD_HNSW` | bool (`"true"`) | unset → graph **not** loaded into RAM (cheap meta only) | When `"true"`, loads the full HNSW vector graph into memory at startup (gigabytes at scale). Opt-in. | `serve_setup.go:90` |
+| `COSIFT_HNSW_EF_SEARCH` | int | `50` (NewHNSW default) | HNSW `efSearch` override (raising to ~200 restored Recall@10); applied only when the graph is loaded and value `> 0`. | `serve_setup.go:104` |
+| `COSIFT_DISABLE_PQ` | bool (`"true"`) | unset → PQ used if codebook present | When `"true"`, disables the runtime Product-Quantization search path even if PQ codes exist on disk (uses raw vectors). | `serve_setup.go:119` |
+| `COSIFT_HNSW_CHECKPOINT_SEC` | int (seconds) | `60` | Interval at which the in-crawl HNSW graph is checkpointed (`cosift crawl`). Override applied only if value `>= 5`. | `cmd/cosift/cmd_crawl.go:168` |
+| `COSIFT_CRAWL_EMBED_CONCURRENCY` | int | `8` | Max concurrent embed calls from the in-serve crawler (throttle to protect interactive `/search`); must be `> 0`. | `serve_setup.go:742` |
+| `COSIFT_EMBED_BATCH` | int | `128` | Max texts coalesced per inner embed call (batching embedder); must be `> 0`. | `serve_setup.go:755` |
+| `COSIFT_EMBED_BATCH_WAIT_MS` | duration (ms) | `20` | Batching-embedder drain timer; must be `> 0`. | `serve_setup.go:761` |
+
+---
+
+## LLM (gates, deadlines, retries, degrade)
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_LLM_GATE_RERANK` | int | `8` | Concurrent-call cap for the rerank pool in the shared chat gate; must be `> 0`. | `serve_setup.go:291` |
+| `COSIFT_LLM_GATE_ANSWER` | int | `4` | Concurrent-call cap for the answer pool in the shared chat gate; must be `> 0`. | `serve_setup.go:292` |
+| `COSIFT_LLM_DEADLINE_ANSWER_MS` | duration (ms) | `30000` (30s) | Per-stage deadline for the `/answer` chat call; must be `> 0`. | `serve_setup.go:379` |
+| `COSIFT_LLM_DEADLINE_RERANK_MS` | duration (ms) | `5000` (5s) | Per-stage deadline for the LLM rerank call; must be `> 0`. | `serve_setup.go:454` |
+| `COSIFT_LLM_RETRIES` | int | `1` | Max retries for gated chat calls (both answer and rerank); must be `> 0`. | `serve_setup.go:380,455` |
+| `COSIFT_LLM_DEGRADE_QUEUE` | int | `8` | vLLM queue-depth threshold above which the server sheds LLM work (degrade mode). Only active when `cfg.Chat.URL` is set. Must be `> 0`. | `serve_setup.go:338` |
+| `COSIFT_LLM_PROBE_MS` | duration (ms) | `2000` (2s) | Interval for the vLLM `/metrics` load probe. Only active when `cfg.Chat.URL` is set; must be `> 0`. | `serve_setup.go:339` |
+| `COSIFT_CHAT_MAX_TOKENS` | int | `2048` | Max-tokens cap on chat generation (prevents runaway/thinking loops); must be `> 0`. | `internal/embed/chat.go:95` |
+| `COSIFT_THINKING` | bool (`"1"`) | unset → thinking **disabled** for Qwen3 models | When `"1"`, re-enables the model's "thinking" mode (only affects Qwen3* chat templates via vLLM). | `internal/embed/chat.go:112` |
+
+---
+
+## Crawler & Content
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_FETCH_HEADER_TIMEOUT_MS` | duration (ms) | `6000` (6s) | HTTP response-header timeout per fetch; must be `> 0`. | `internal/crawler/crawler.go:254` |
+| `COSIFT_FETCH_TIMEOUT_MS` | duration (ms) | `12000` (12s) | Overall per-fetch HTTP timeout; must be `> 0`. | `internal/crawler/crawler.go:310` |
+| `COSIFT_MAX_CONNS_PER_HOST` | int | `128` | `MaxConnsPerHost` / `MaxIdleConnsPerHost` for the crawler transport; must be `> 0`. | `internal/crawler/crawler.go:266` |
+| `COSIFT_AUTO_SITEMAP_CONCURRENCY` | int | `16` | Cap on concurrent background auto-sitemap discoveries; must be `> 0`. | `internal/crawler/crawler.go:332` |
+| `COSIFT_DYNAMIC_DOMAINS_FILE` | string (path) | unset → none | Path to a file of dynamic (JS-rendered) domains loaded at crawler init. | `internal/crawler/crawler.go:342` |
+| `COSIFT_DIRECT_HOSTS` | csv | unset → built-in `defaultDirectHosts` list | Comma-separated hosts that bypass the remote fetcher and fetch directly. Setting it **replaces** the default list. | `internal/crawler/remote_fetcher.go:70` |
+| `COSIFT_CRAWL_PDF` | bool (`"false"` disables) | unset → PDF parsing **enabled** (sandboxed) | Set to `"false"` to disable sandboxed PDF parsing. Any other value leaves it on. | `internal/crawler/crawler.go:1155` |
+| `COSIFT_REFETCH_AFTER_HOURS` | int (hours) | `0` → disabled (every revisit issues a conditional GET) | Skip re-fetching a healthy URL fetched within this window. Also defines the "fresh" window for prefer-new (defaults to 24h there) and the WET fresh window. Must be `> 0`. | `crawler.go:1107,1492`; `wet.go:94` |
+| `COSIFT_PREFER_NEW_URLS` | bool (`"1"`) | unset → off | When `"1"`, the frontier prefers never-seen URLs over recently-fetched ones (one `GetDocByURL` per candidate). | `internal/crawler/crawler.go:1488` |
+| `COSIFT_ZOMBIE_RECLAIM` | bool (`"1"`) | unset → off | When `"1"`, marks a re-crawled URL's prior chunk generation invalid in the HNSW graph before adding fresh vectors. | `crawler.go:666,1320`; `wet.go:309` |
+| `COSIFT_EMBED_DECOUPLE_WORKERS` | int | `0` → decoupled embed pipeline **off** | Number of dedicated embed-worker goroutines that drain the crawl embed queue (decouples crawl from embed/HNSW-write latency). `0` keeps the synchronous path. Must be `>= 0`. | `internal/crawler/crawler.go:586` |
+| `COSIFT_EMBED_DECOUPLE_BUFFER` | int | `4096` | Buffer size of the decoupled embed queue (only when workers `> 0`). Must be `>= 0`. | `internal/crawler/crawler.go:588` |
+| `COSIFT_HOSTSWEEP_DISABLED` | bool (`"1"`) | unset → sweeper **enabled** | When `"1"`, disables the self-cleaning host sweeper entirely. | `internal/crawler/crawler.go:868` |
+| `COSIFT_HOSTSWEEP_INTERVAL_SEC` | int (seconds) | `600` (10min) | Host-sweeper run interval; floored at 30s. Must be `>= 0`. | `internal/crawler/crawler.go:871` |
+| `COSIFT_HOSTSWEEP_MIN_ATTEMPTS` | int | `100` | Min fetch attempts before a host is eligible for purge/demote. Must be `>= 0`. | `internal/crawler/crawler.go:875` |
+| `COSIFT_HOSTSWEEP_DEAD_RATE` | float | `0.20` | Success-rate threshold below which a host is purged. Must be `>= 0`. | `internal/crawler/crawler.go:876` |
+| `COSIFT_HOSTSWEEP_WEAK_RATE` | float | `0.50` | Success-rate threshold below which a host is demoted (between dead and weak). Must be `>= 0`. | `internal/crawler/crawler.go:877` |
+| `COSIFT_WET_BATCH_SIZE` | int | `32` | Records per batch when ingesting Common Crawl WET archives; must be `> 0`. | `internal/crawler/wet.go:101` |
+| `COSIFT_WET_WORKERS` | int | `8` | Worker goroutines for WET ingest; must be `> 0`. | `internal/crawler/wet.go:107` |
+
+> Not an env var: `cosift.json` `crawler.auto_sitemap` (a config field) controls
+> whether auto-sitemap discovery runs at all. A stale code comment references
+> "`COSIFT_AUTO_SITEMAP=false`", but no such env var is read — only
+> `COSIFT_AUTO_SITEMAP_CONCURRENCY` (above) exists.
+
+---
+
+## Pebble Store
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_PEBBLE_CACHE_MB` | int (MB) | `128` | Pebble block-cache size in MB; must be `> 0`. | `internal/store/pebble.go:153` |
+| `COSIFT_PEBBLE_MEMTABLE_MB` | int (MB) | `32` | Pebble memtable size in MB; must be `> 0`. | `internal/store/pebble.go:154` |
+| `COSIFT_PEBBLE_MEMTABLES` | int | `2` | Memtable count; `MemTableStopWritesThreshold = value + 2`. Must be `> 0`. | `internal/store/pebble.go:155` |
+| `COSIFT_PEBBLE_SYNC` | bool (`"false"` disables) | unset → `Sync` (fsync each commit) | Set to `"false"` to use `NoSync` writes (skips per-commit fsync — faster crawls, drops durability vs OS crash; WAL still written so process-crash durability holds). | `internal/store/pebble.go:176` |
+
+---
+
+## Profiling
+
+All profiling knobs are no-ops unless `COSIFT_PPROF_ADDR` is set.
+
+| Variable | Type | Default | Effect | Where read |
+|---|---|---|---|---|
+| `COSIFT_PPROF_ADDR` | string (addr) | unset → pprof server off | When set (e.g. `127.0.0.1:6060`), starts a loopback pprof HTTP server. | `serve_setup.go:627` |
+| `COSIFT_MUTEX_PROFILE_FRACTION` | int | unset → off (`0`) | `runtime.SetMutexProfileFraction(n)` — sample 1-in-`n` lock waits. Only applied when pprof is on and `n > 0`. | `serve_setup.go:633` |
+| `COSIFT_BLOCK_PROFILE_RATE` | int | unset → off (`0`) | `runtime.SetBlockProfileRate(n)`. Only applied when pprof is on and `n > 0`. | `serve_setup.go:639` |
+| `COSIFT_CHECKPOINT_DIR` | string (path) | unset → `/tmp` | Base directory for `/admin/checkpoint` Pebble checkpoints (a `cosift-ckpt-<nanos>` subdir is created under it). | `cmd/cosift/serve_admin.go:100` |
+
+---
+
+## Excluded (test-only, not real config)
+
+These appear **only** in `*_test.go` files (verified: no non-test references)
+and are not runtime configuration:
+`COSIFT_DOTENV_BARE`, `COSIFT_DOTENV_SINGLE`, `COSIFT_DOTENV_DOUBLE`,
+`COSIFT_DOTENV_EXISTS`, `COSIFT_DOTENV_TEST_KEY_1`,
+`COSIFT_FIRSTENV_A`, `COSIFT_FIRSTENV_B`, `COSIFT_FIRSTENV_C`.
diff --git a/docs/HANDOVER.md b/docs/HANDOVER.md
new file mode 100644
index 0000000..deeb7d6
--- /dev/null
+++ b/docs/HANDOVER.md
@@ -0,0 +1,401 @@
+# cosift — Operational Handover Runbook
+
+**Audience:** the incoming operations agency (the "agency") and the principal (system owner).
+**Scope:** giving the agency operational control of the production cosift system while keeping
+real user queries and responses confidential from the agency.
+**Status of this document:** authoritative handover runbook. Read it end to end before touching prod.
+
+---
+
+## 0. The handover model (read first)
+
+This is the decided model. Everything below implements it.
+
+- **The agency GETS full SSH** to the production GH200 box (host `192.222.56.72`, public name
+  `cosift.pilotprotocol.network`). They operate it: deploy, debug, restart, run admin/offline
+  tooling, manage the harvesters. Box shell is real root-equivalent control of the corpus and service.
+- **Real user queries and responses must stay confidential from the agency.** The resolution has
+  two parts:
+  1. **No on-box persistence of raw queries.** The on-box query log is disabled or redacted, the
+     feedback log is off or redacted, retention is bounded, and no ingress component (Caddy) is
+     allowed to log query-bearing URLs. This removes passive/accidental exposure: an operator who
+     pokes around the box finds no historical record of what users asked.
+  2. **Query observability moves off-box to a principal-controlled passthrough.** A thin reverse
+     proxy that the principal owns fronts all requests, terminates ingress, and is the only place
+     `{query, ts, latency, endpoint}` is recorded — for the principal, not the agency.
+
+### Residual risk — state this plainly to the agency and the principal
+
+The two measures above stop **passive and accidental** exposure. They do **not** stop a determined
+malicious operator. A root operator on the box can:
+
+- attach to the live process or sniff loopback (`127.0.0.1:7777`) traffic and read plaintext
+  queries in flight (`tcpdump -i lo`, `strace`/`bpftrace` on the process, an `LD_PRELOAD` shim, etc.);
+- wiretap or replace the local LLM / embed calls the service makes;
+- add their own logging drop-in and restart the service.
+
+The GH200 still receives **plaintext queries** because it has to search them. Technical hardening
+(no logs, localhost-only admin, redaction) raises the cost and removes the easy/accidental paths,
+but the gap against an active malicious root operator is **closed by contract and trust, not by
+technical means.** Put this in writing in the operating agreement. Do not represent the technical
+controls as making queries cryptographically invisible to the operator — they do not.
+
+---
+
+## 1. Secret-ownership matrix
+
+Three ownership classes:
+
+- **principal-only** — only the principal holds the value. Never enters CI, never enters a handover
+  tree, never handed to the agency. (The agency can still *operate* the box; they just don't get the
+  literal secret in their own control plane.)
+- **runtime-on-box** — lives on the box for the service/harvesters to read at runtime. The agency,
+  having SSH, can read these; that is accepted. Keep them out of git and CI.
+- **agency** — the agency owns/rotates it as part of operating the system.
+
+| Secret | Owner | Location | Rotation |
+|---|---|---|---|
+| **OpenAI API key** | principal-only **+** runtime-on-box | **MUST be rotated at handover.** Currently sits in `cosift/.env` (`OPENAI=sk-proj-…`) — that value is compromised by being in the working tree and must be revoked. Move the new key to a systemd `EnvironmentFile` at `/etc/cosift/cosift.env`, owner `root:cosift`, mode `0640`, referenced from `cosift-serve.service`. **Never** in CI, **never** in any handover/git tree, **never** in `.env`. | Rotate now (at handover). Rotate again on any suspected exposure. Principal issues the key; the box reads it. |
+| **`cluster.peer_auth_token`** | runtime-on-box | `/home/ubuntu/cosift.json` (`cluster.peer_auth_token`). Gates **all** `/admin/*` and `/admin/query-log` etc. on the production `pebble-serve` path. **Currently EMPTY → admin gate is skipped.** See §3. | Set at handover; rotate on operator turnover. Same value must be provisioned to the harvesters (§3). |
+| **`server.admin_token`** | runtime-on-box | `/home/ubuntu/cosift.json` (`server.admin_token`). **Note:** this gates only the legacy `serve` path (`internal/server/http.go`). It does **NOT** gate the production `pebble-serve` path — that path uses `cluster.peer_auth_token`. Don't rely on `admin_token` for prod auth. | Low priority; keep set for the legacy path but treat `peer_auth_token` as the real prod gate. |
+| **`crawler.remote_fetcher_token`** | runtime-on-box | `/home/ubuntu/cosift.json` (`crawler.remote_fetcher_token`). Shared Bearer between cosift and any remote fetcher. | On operator turnover. |
+| **`rerank.api_key` / `COHERE_API_KEY` / `VOYAGE_API_KEY`** | runtime-on-box | `rerank.api_key` in `/home/ubuntu/cosift.json`, **or** env `COHERE_API_KEY` / `VOYAGE_API_KEY` (config falls back to env — see `internal/config/config.go` `APIKey`). Put env values in `/etc/cosift/cosift.env` alongside the OpenAI key. | On provider rotation or operator turnover. |
+| **`cosift.pem` (SSH key to the GH200)** | principal-only | Local at `~/Downloads/cosift.pem` (chmod 600). **Never to CI.** CI uses **pull-based** deploy (§4), so the deploy pipeline never needs this key. The agency gets their **own** SSH access (separate key / authorized_keys entry) — do not hand them this exact key; provision per-operator keys so access can be revoked individually. | Rotate when an operator leaves. Per-operator keys make revocation a one-line `authorized_keys` edit. |
+| **ed25519 `cosift-app` publisher key** | principal-only | Principal's machine / secret store. App publishing (the pilot app-store artifact) is **independent of backend deploy** — the agency operating the backend does **not** need this. | Principal-controlled; rotate independently of backend secrets. |
+| **`CANARY_DISPATCH_TOKEN`** | principal-only → replace before repo handover | GitHub repo secret, consumed by `.github/workflows/notify-canary.yml`. **Currently a personal PAT.** Before handing the repo to the agency, **replace it with a scoped app token** (fine-grained, repo-scoped, dispatch-only) so handing over the repo doesn't hand over a personal PAT. | Replace at repo handover; rotate on operator turnover. |
+
+**General rule:** the only secrets that ever reach git or CI are *none of the above*. CI builds and
+signs; it does not hold the SSH key, the OpenAI key, or any runtime secret.
+
+---
+
+## 2. Confidentiality / no-persistence cutover (do this AT handover)
+
+Concrete steps to take on the box before the agency gets the keys.
+
+### 2.1 Disable (or redact) on-box raw query logging
+
+The query log is controlled by the `COSIFT_QUERY_LOG` env var. On prod it is enabled via a systemd
+drop-in. Two acceptable resolutions — **pick one** (open decision, see §7):
+
+**Option A — disable entirely (simplest, recommended for clean confidentiality):**
+
+```bash
+# Remove the drop-in that sets COSIFT_QUERY_LOG.
+sudo rm /etc/systemd/system/cosift-serve.service.d/zz-query-log.conf
+sudo systemctl daemon-reload
+sudo systemctl restart cosift-serve     # 4–5 min HNSW reload — see §4
+```
+
+With `COSIFT_QUERY_LOG` unset, `s.qlogFile == nil` and `qlog()` is a no-op (see
+`cmd/cosift/querylog.go`). `/admin/query-log` then returns `501 query log disabled`. The feedback
+log keys off the same var (`cmd/cosift/feedback.go`), so it goes dark too.
+
+**Option B — keep observability but redact (if the principal wants on-box latency/empty-rate signal):**
+
+Edit `cmd/cosift/querylog.go` so the persisted record cannot reconstruct a user's query or identity:
+
+- hash or truncate the `Q` field (e.g. store `sha256(q)[:8]` or just the length, not the text);
+- drop the `Caller` field entirely (it is `X-Forwarded-For` / `RemoteAddr` today — that is the
+  user's IP).
+
+This keeps `{ts, ep, status, ms, bytes}` for health/perf while removing the query text and caller IP.
+Rebuild and deploy via §4.
+
+> **Note:** Option B still leaves the *hashed* query on the box, which is a confirmation oracle for a
+> guessed query. Prefer Option A unless the principal specifically needs on-box query observability —
+> and remember the principal passthrough (§5) is the *intended* home for query observability anyway.
+
+### 2.2 Feedback log
+
+`COSIFT_FEEDBACK_LOG` (or the default beside the query log) records feedback rows. Ensure it is
+**off** (unset) or redacted the same way as §2.1. Under Option A it is already off because the
+feedback path is gated on `COSIFT_QUERY_LOG` being set.
+
+### 2.3 Bound retention on anything that remains
+
+The code has **no retention/rotation** for the query/feedback logs — `writeQueryLog` only appends
+(`cmd/cosift/querylog.go`). If any log remains (Option B, or operational logs), impose retention at
+the OS level so nothing accumulates indefinitely:
+
+```bash
+# /etc/logrotate.d/cosift
+/var/log/cosift/*.log /home/ubuntu/cosift-data/*.jsonl {
+  daily
+  rotate 3
+  compress
+  missingok
+  notifempty
+  copytruncate
+}
+```
+
+Tune `rotate`/frequency to the principal's retention policy. Under Option A there is nothing to rotate
+here, but keep the rule in place as a backstop.
+
+### 2.4 Embed-cache is a timing / confirmation oracle — decide
+
+The embedder caches vectors to disk as `sha256(model \x00 text)` named `.vec` files
+(`internal/embed/client.go`). If query embeddings share the cache directory with doc embeddings, the
+**presence of a `.vec` file whose name matches `sha256(model\x00<guessed-query>)`** confirms that
+query was issued, and file mtime leaks *when*. This is a confirmation oracle, not a plaintext leak.
+
+Decision (open — §7):
+
+- **(a) Separate caches:** point query-time embedding at a non-persistent / `tmpfs` cache (or disable
+  caching on the query path) so query embeds leave no on-disk fingerprint; keep the persistent cache
+  for doc/eval embeds only.
+- **(b) Accept it:** if the corpus is huge and queries are short/generic, an attacker must already
+  *guess* the exact query to confirm it. Document the residual oracle and move on.
+
+Default recommendation: separate, if cheap to wire; otherwise accept-and-document.
+
+### 2.5 Forbid a Caddy `log` directive
+
+`GET /search?q=…` and `/answer?q=…` carry the **query in the URL**. A Caddy `log` directive writes
+request URLs to disk — that is a raw query log by another name.
+
+- **FORBID** any `log { ... }` block in the production Caddyfile.
+- The repo's `scripts/Caddyfile.tls` contains a `log { output file … }` block — **do not deploy that
+  variant as-is.** Use the no-log `scripts/Caddyfile` shape (or strip the `log` block) for prod.
+- This applies to any front proxy, including the principal passthrough (§5): there, query logging is
+  *intended* and lives with the principal — but it must not be duplicated on the box.
+
+### 2.6 Already done — public `/admin/*` leak closed
+
+The public exposure of `/admin/*` and `/debug/*` has **already been closed** on the production Caddy
+front via a `handle` block that returns **404 for `/admin/*` and `/debug/*`** from the public
+internet. Record this as **DONE** (it was the URGENT fix). Verify it still holds after any Caddy
+change:
+
+```bash
+curl -s -o /dev/null -w '%{http_code}\n' https://cosift.pilotprotocol.network/admin/query-log   # expect 404
+curl -s -o /dev/null -w '%{http_code}\n' https://cosift.pilotprotocol.network/debug/pprof/        # expect 404
+```
+
+> The repo Caddyfiles (`scripts/Caddyfile*`) predate this block — the live prod Caddyfile on the box
+> is the source of truth for the 404 handles. If you regenerate the Caddyfile from the repo, re-add
+> the `/admin/*` + `/debug/*` 404 handles **and** keep the no-`log` rule (§2.5).
+
+---
+
+## 3. Admin auth SOP
+
+**Current state:** `cluster.peer_auth_token` is **EMPTY** in `/home/ubuntu/cosift.json`. Every admin
+handler on the `pebble-serve` path guards with `if want := s.cluster.PeerAuthToken; want != ""` —
+**empty means the gate is skipped.** Public exposure is blocked by Caddy (§2.6), but locally
+everything is open. Set the token so admin actions require a Bearer even from localhost.
+
+### Procedure
+
+1. **Generate a token** and set it in the config:
+
+   ```bash
+   TOKEN=$(openssl rand -hex 32)
+   # Edit /home/ubuntu/cosift.json: set cluster.peer_auth_token = "$TOKEN"
+   ```
+
+2. **Provision the SAME token to the harvesters.** The harvesters already read a `TOKEN` env var and
+   send `Authorization: Bearer $TOKEN` to the admin endpoints. Put the token in a shared
+   `EnvironmentFile` and reference it from the harvester units:
+
+   ```ini
+   # /etc/cosift/harvester.env   (root:cosift 0640)
+   TOKEN=<the same hex token>
+   ```
+   ```ini
+   # in each deploy/systemd/cosift-*-harvest.service (and *-refresh.service):
+   [Service]
+   EnvironmentFile=/etc/cosift/harvester.env
+   ```
+
+3. **Fix `deploy/harvesters/cosift-compact.sh`.** It calls `POST /admin/hnsw-compact` **without** a
+   token:
+
+   ```bash
+   RESULT=$(curl -s --max-time 1800 -X POST http://127.0.0.1:7777/admin/hnsw-compact)
+   ```
+
+   Once the gate is on, that call gets `401`. Patch it to send the Bearer:
+
+   ```bash
+   RESULT=$(curl -s --max-time 1800 -X POST \
+     -H "Authorization: Bearer ${TOKEN}" \
+     http://127.0.0.1:7777/admin/hnsw-compact)
+   ```
+
+   (and ensure the compact timer unit also pulls `EnvironmentFile=/etc/cosift/harvester.env`).
+
+4. **Restart cosift** to pick up the new config:
+
+   ```bash
+   sudo systemctl restart cosift-serve
+   # 4–5 min HNSW reload (43.9M nodes). Poll until ready:
+   until curl -sf http://127.0.0.1:7777/healthz >/dev/null; do sleep 5; done
+   ```
+
+5. **Keep `/admin/*` and `/debug/pprof` localhost-only.** Caddy already 404s them publicly (§2.6).
+   The Bearer token is defense-in-depth for the loopback surface; the public block is the primary
+   control. Never expose either to the public internet.
+
+---
+
+## 4. Deploy model — signed, pull-based, key-out-of-CI
+
+The deploy pipeline is built so the SSH key never enters CI.
+
+```
+ push to main ─► CI (GitHub Actions, .github/workflows/deploy.yml)
+                   • GOOS=linux GOARCH=arm64 go build
+                   • sign artifact: sha256 + minisign
+                   • publish as a GitHub Release
+                              │
+                              ▼
+ GH200 box-side self-update timer (deploy/systemd + deploy/scripts)
+                   • polls Releases for a new signed artifact
+                   • verifies sha256 + minisign signature
+                   • atomic swap into /home/ubuntu/cosift (keeps cosift.prev)
+                   • restart cosift-serve, health-gate /healthz (up to 10 min for HNSW reload)
+                   • auto-rollback to cosift.prev if /healthz never goes green
+```
+
+**Why pull-based:** the box *pulls* a verified artifact rather than CI *pushing* over SSH. CI holds
+no SSH key and no runtime secret — it only builds and signs. The minisign verify key is public; the
+**signing** key is the principal's (or a CI secret scoped to signing only), never the deploy path's.
+
+**File locations (target layout):**
+
+- `.github/workflows/deploy.yml` — build + sign + publish Release.
+- `deploy/scripts/` — the self-update script (pull, verify sha256+minisign, atomic swap, health-gate,
+  rollback).
+- `deploy/systemd/` — the self-update `.timer` + `.service` units, plus the harvester units already
+  present here.
+
+> **Status note:** today the canonical manual deploy is the build/scp/swap procedure in the GH200
+> reference (build arm64 → `scp` to `/tmp/cosift-new` → `mv` into place → restart). The pull-based
+> pipeline above is the **handover target** so the agency operates without ever holding `cosift.pem`
+> in CI. If `deploy.yml` / `deploy/scripts` are not yet wired when you read this, wire them as part
+> of cutover; until then, deploys go through the agency's own per-operator SSH key, not CI.
+
+### Every binary deploy is a HARD 4–5 minute outage
+
+Single box, synchronous HNSW load (43.9M nodes) on every restart. There is **no rolling deploy** —
+the service is down for the duration of the HNSW reload (4–5 min, allow up to 10 for the health gate).
+**Schedule and communicate every deploy.** Health-gate before declaring success:
+
+```bash
+until curl -sf http://127.0.0.1:7777/healthz >/dev/null; do sleep 5; done   # {"status":"ok"} when ready
+```
+
+---
+
+## 5. Principal passthrough design
+
+A thin, **principal-controlled** logging reverse proxy fronts **all** requests. This is where query
+observability and feedback live for the principal — it **replaces** the on-box query log removed in §2.
+
+### What it does
+
+- Terminates public ingress (TLS, the `cosift.pilotprotocol.network` hostname).
+- Logs, for the principal only, a minimal record per request: `{query, ts, latency, endpoint}`
+  (plus status / response size if wanted). This is the principal's observability substrate.
+- Forwards the request to the GH200 (`127.0.0.1:7777` via a private tunnel, or the box's private
+  address) and streams the response back.
+- Owns retention of *its* log per the principal's policy — this log lives on a host the **agency does
+  not control.**
+
+### Hosting options (trade-offs) — the host is an OPEN DECISION for the principal (§7)
+
+| Option | What it is | Pros | Cons |
+|---|---|---|---|
+| **(a) Small principal VPS** running Caddy / nginx / a tiny Go proxy | A cheap box the principal owns, in front of the GH200 | Full control of logging + retention; arbitrary logic; no third party | Principal runs another box; a hop of latency; needs its own TLS + uptime |
+| **(b) Cloudflare Worker** in front | The site is **already behind Cloudflare** — add a Worker that logs `{query, ts, latency, endpoint}` to principal-owned storage (KV / R2 / Logpush) | Zero new infra; rides existing Cloudflare ingress; global edge | Logging logic constrained to the Worker model; observability data sits in Cloudflare (principal-owned, but third-party-hosted) |
+| **(c) Local Mac pilot-daemon path** for overlay traffic | For requests arriving over the pilot overlay, the principal's local pilot daemon is already the entry point and can log there | Reuses an existing principal-controlled hop; natural for overlay/app-store traffic | Only covers overlay traffic, not public-web HTTPS hits; the Mac must be up |
+
+A combination is reasonable: (b) for public web traffic (it's already the front), (c) for overlay
+traffic, with (a) as the fallback if the principal wants logging logic Cloudflare can't express.
+
+### Hard limit — the passthrough relocates observability, it does NOT hide queries from the box
+
+The GH200 **still receives plaintext queries** — it has to, to search them. The passthrough moves
+the *observability record* to a principal-controlled host and removes it from the box; it does
+**not** prevent the box from seeing each query in plaintext at request time. That is exactly the
+residual-risk point in §0: confidentiality from the agency rests on (no on-box persistence) +
+(observability off-box) + (contract/trust for the live-traffic gap). Don't oversell the passthrough.
+
+---
+
+## 6. Control-surface reference
+
+What "the agency gets SSH" actually grants.
+
+### Public endpoints (fronted by Caddy / the passthrough)
+
+`/search` · `/answer` · `/research` · `/find` · `/stats` · `/healthz` · `/metrics` · `/feedback`
+
+These are the user-facing API (`cmd/cosift/pebble_serve.go`). Note `/search?q=` and `/answer?q=`
+carry queries in the URL — the reason §2.5 forbids any front-proxy `log` directive on the box.
+
+### Admin endpoints (24, localhost-only, Bearer-gated by `peer_auth_token`)
+
+All under `/admin/*` on `pebble-serve`, e.g.:
+`crawl-enqueue`, `allow-domain`, `frontier-purge-host`, `frontier-clear`, `frontier-demote-host`,
+`frontier-purge-stale-inflight`, `rss-import`, `crawl-now`, `wet-import`, `wet-import-bulk`,
+`site-pack`, `site-submit`, `embed-backfill`, `host-backfill`, `eval-quick`, `hnsw-compact`,
+`sitemap-import`, `recrawl-sitemap`, `pq-train`, `pq-encode`, `checkpoint`, `query-log`, `feedback`,
+`domains-audit`. Plus `/debug/pprof/*`. **Public access to all of these is 404'd by Caddy** (§2.6);
+local access is gated by `peer_auth_token` once §3 is done.
+
+### Offline subcommands bypass HTTP auth entirely
+
+`purge-domain`, `purge-adult`, `frontier-clear`, `gc`, `reembed` (and similar) are **CLI
+subcommands** in `cmd/cosift/main.go` that open the Pebble store **directly on disk**
+(`openPebbleOrFriendlyErr`). They **do not** go through the HTTP server, so **no token gates them** —
+they need only **disk access** to the data dir (`/home/ubuntu/cosift-data/pebble`).
+
+**Implication:** **box shell == full corpus-mutation power.** Anyone with SSH and disk access can
+purge domains, wipe the frontier, garbage-collect, or re-embed the entire corpus with zero auth.
+This is the core of what "agency gets SSH" grants — it is total operational control of the corpus,
+not just the HTTP surface. Treat SSH access as the real trust boundary, and gate it per-operator
+(§1) so it can be revoked individually.
+
+> Pebble is single-writer: offline subcommands that mutate the store **cannot** run while
+> `cosift-serve` holds the lock. Stop the service first (incurring the §4 outage), or use the
+> equivalent admin endpoint against the live process where one exists (e.g. `host-backfill`).
+
+---
+
+## 7. Open decisions for the principal (decide before cutover)
+
+1. **Query-log resolution (§2.1):** Option A (disable entirely) vs Option B (redact: hash/truncate
+   `q`, drop `Caller`). Recommendation: A, since the passthrough (§5) is the intended observability home.
+2. **Embed-cache oracle (§2.4):** separate query-embeds from doc-embeds (tmpfs / disable on query
+   path) vs accept-and-document the confirmation oracle.
+3. **Passthrough host (§5):** (a) principal VPS, (b) Cloudflare Worker, (c) Mac pilot-daemon — or a
+   combination. This determines where the principal's query observability physically lives.
+4. **Per-operator SSH provisioning (§1):** confirm the agency gets per-operator keys (revocable
+   individually), not a shared copy of `cosift.pem`.
+
+---
+
+## 8. Cutover checklist (one pass, in order)
+
+- [ ] **Rotate the OpenAI key**; revoke the old `sk-proj-…` from `.env`; write the new one to
+      `/etc/cosift/cosift.env` (`root:cosift 0640`); point `cosift-serve.service` at it; remove the
+      key from `.env`.
+- [ ] Move `COHERE_API_KEY` / `VOYAGE_API_KEY` into `/etc/cosift/cosift.env` too.
+- [ ] **Resolve the query log** (§2.1, Option A or B) + ensure feedback log off/redacted (§2.2).
+- [ ] Install the `logrotate` backstop (§2.3).
+- [ ] Decide + apply the embed-cache choice (§2.4).
+- [ ] Confirm **no `log` directive** in the prod Caddyfile (§2.5).
+- [ ] Verify `/admin/*` and `/debug/*` still **404 publicly** (§2.6).
+- [ ] **Set `cluster.peer_auth_token`** in `cosift.json`; provision the same token to harvesters via
+      `EnvironmentFile`; **patch `cosift-compact.sh`** to send the Bearer (§3).
+- [ ] Stand up the **principal passthrough** on the chosen host; point it at the GH200; confirm it
+      logs `{query, ts, latency, endpoint}` to principal-owned storage (§5).
+- [ ] Replace `CANARY_DISPATCH_TOKEN` (personal PAT → scoped app token) **before** repo handover (§1).
+- [ ] Provision **per-operator SSH keys** for the agency; do **not** hand over `cosift.pem`; keep the
+      `cosift-app` publisher key and `cosift.pem` principal-only (§1).
+- [ ] Wire / verify the **pull-based deploy pipeline** so the agency never holds the SSH key in CI (§4).
+- [ ] Restart `cosift-serve`, health-gate `/healthz` (4–5 min HNSW reload), confirm green (§3–§4).
+- [ ] Put the **residual-risk statement (§0)** in the operating agreement in writing.
diff --git a/docs/PLAN-contrib-network.md b/docs/PLAN-contrib-network.md
new file mode 100644
index 0000000..ef24660
--- /dev/null
+++ b/docs/PLAN-contrib-network.md
@@ -0,0 +1,438 @@
+# PLAN — Cosift Open Contribution Network
+
+Status: DRAFT for review · Owner: teodor · Last updated: 2026-06-12
+
+Two coupled initiatives, one shared trust substrate:
+
+1. **Publisher path** — developers index their documentation; the index entry stays
+   active only while their Pilot app holds a live lease (heartbeats), solving the
+   LLM-knowledge-cutoff problem for new dev tools and driving Pilot adoption.
+2. **Open frontier** — third-party peers claim crawl/embed work from the frontier
+   and submit signed results; contributions are quarantined, verified by sampling,
+   attributed, and revocable.
+
+Everything below is grounded in current code. Line numbers verified 2026-06-12
+against branch state; numbers marked `≈` came from exploration agents and should
+be re-confirmed at implementation time (symbol names are authoritative).
+
+---
+
+## 0. Definitions
+
+| Term | Meaning |
+|---|---|
+| **Principal** | An ed25519 pubkey registered with cosift. Roles: `publisher`, `contributor`, or both. The key of an appstore app instance (`$APP/identity.json`) or any Pilot identity. |
+| **Collection** | A claimed domain/path-prefix owned by a publisher principal. Carries lease state. |
+| **Lane** | Frontier priority class. `0=submitted` (publisher docs), `1=refresh`, `2=discovered`, `3=bulk`. |
+| **Claim** | A leased unit of work (fetch one URL, or embed N server-defined chunks) handed to a contributor, with TTL. |
+| **Envelope** | The canonical signed message wrapping every authenticated request (Appendix A). |
+| **Quarantine** | Staging area for contributed results; nothing enters the main index without passing verification. |
+| **Tier** | Per-principal reputation level; determines sampling rate, quotas, and lane access. |
+
+## 0.1 Security invariants (must hold at every phase)
+
+- **I1** — No write reaches even quarantine without a valid envelope signature from a registered, non-banned principal referencing a live claim.
+- **I2** — No contributed content reaches the main index without passing the verification pipeline for its kind.
+- **I3** — Every promoted doc/vector is attributable: `doc → (principal, sig, ts)` is stored and survives restarts.
+- **I4** — Banning a principal makes retroactive purge of all their promoted contributions possible and bounded (one index walk).
+- **I5** — A brand-new principal is sampled at 100%; sampling rate only decreases through verified-good history (Sybil identities gain zero leverage).
+- **I6** — Embed jobs are always server-defined text. We never accept a vector for text the server did not hand out.
+- **I7** — The submission/work auth surface never reuses `Server.AdminToken` or `Cluster.PeerAuthToken`.
+- **I8** — Suspended collections and banned principals are filtered at the single retrieval choke point shared by `/search`, `/query`, `/answer`, `/research`.
+- **I9** — All contribution-path traffic (`/pub/*`, `/work/*`) is transport-bound to Pilot: the only network-reachable surface is an overlay listener. The HTTP server's contribution paths are bound to a loopback/UDS the operator's daemon owns; they are not routed by the public reverse proxy. The envelope's signing key (WS0.1) MUST equal the Pilot peer pubkey that authenticated the overlay tunnel — mismatch = reject. This makes "the data came through Pilot" cryptographically verifiable, not just operationally assumed.
+
+---
+
+## WS0 — Envelope + principal registry (foundation; everything depends on this)
+
+### WS0.1 Canonical signed envelope
+- **Where (new):** `internal/principal/envelope.go` (+ `envelope_test.go`).
+- **Reuse:** `crypto.Verify()` / key codecs from web4 — `pilot-protocol/common/crypto/identity.go` (`Identity.Sign` ≈:32, `Verify` ≈:38, base64 key codecs). Import or vendor; decide module path at implementation (cosift currently has no dependency on `common`).
+- **What:** Define `Envelope{PrincipalPub, Kind, ClaimID, BodySHA256, ModelID, Seq, Sig}` with a deterministic canonical byte serialization (length-prefixed fields, fixed order — not raw JSON). `Kind ∈ {register, pub_claim, pub_submit, heartbeat, work_claim, work_submit, work_renew, retire}`. `Seq` is per-principal strictly monotonic.
+- **Acceptance:**
+  - Round-trip test: sign → verify passes; any single-byte mutation of any field fails verification.
+  - Replay test: same envelope presented twice → second rejected (`seq` not advanced).
+  - Cross-impl fixture: a JSON test vector checked in so the worker app (WS10) can verify its signing against the same bytes.
+  - Fuzz test on the canonical decoder (`go test -fuzz=FuzzEnvelopeDecode -fuzztime=30s` clean).
+
+### WS0.2 Principal registry
+- **Where (new):** `internal/principal/registry.go`; persistence in WS1.1.
+- **What:** `Register(pub, roles)`, `Get(pub)`, `Bump(pub, outcome)` (verified-ok / verified-fail / mismatch counters), `Ban(pub, reason)`, seq tracking, per-role quotas (URLs/day, bytes/day, max in-flight claims).
+- **Acceptance:**
+  - Unit tests for quota windows (day rollover), seq monotonicity, ban idempotency.
+  - Banned principal: every envelope kind rejected with a distinct error code (`403 problem+json, type=principal_banned`).
+
+### WS0.3 HTTP auth middleware
+- **Where:** `cmd/cosift/pebble_serve.go` — new `wrapSigned(handler)` alongside the existing route table (route registration block at `:474-532`). Mounted only on `/pub/*` and `/work/*`.
+- **What:** Parse envelope from `X-Cosift-Envelope` header (base64) covering the request body hash; verify; load principal; enforce quota; stash principal in request context.
+- **Acceptance:**
+  - Table-driven handler tests: no envelope / bad sig / stale seq / banned / over-quota → 401/403/429 respectively with problem+json bodies; happy path passes principal to handler.
+  - `Server.AdminToken` and `Cluster.PeerAuthToken` are never consulted on these routes (assert via test that requests carrying only those tokens are rejected) — invariant **I7**.
+
+---
+
+## WS1 — Store schema (Pebble families, attribution, doc provenance)
+
+Key-family conventions live at `internal/store/pebble.go:12-21` (schema comment
+block) — extend that comment with every new family below.
+
+### WS1.1 New key families
+- **Where:** `internal/store/pebble.go` (+ split into `pebble_contrib.go` if >300 lines).
+- **What:**
+  - `'P' + pubkey` → gob(PrincipalRecord) — registry persistence for WS0.2.
+  - `'C' + collectionID` → gob(CollectionRecord{Owner, Prefix, State, LeaseSeq, LastRenewal, VerifiedAt, Quotas}); secondary `'C' + 'p' + prefix` → collectionID for prefix-claim lookup.
+  - `'S' + ulid` → gob(StagedContribution) — quarantine, TTL'd.
+  - `'A' + pubkey + docID` → empty (principal→docs walk for purge); `'A' + 'd' + docID` → {pubkey, sigSHA, ts} (doc→provenance).
+  - `'W' + claimID` → gob(WorkLease{Kind, Payload, Principal, Expiry, Attempts}).
+- **Acceptance:**
+  - CRUD unit tests per family; `cosift pebble-compact` (cmd/cosift/pebble_compact.go) runs clean over a store containing all new families.
+  - Snapshot/restore: create records → checkpoint (`POST /admin/checkpoint`, route at `:518`) → reopen from checkpoint → records intact.
+
+### WS1.2 Document provenance + docMeta versioning
+- **Where:** `Document` struct `internal/store/store.go:30-66` (add `CollectionID uint32`, `Principal []byte` — empty for operator-crawled); docMeta packed blob written in `UpsertDocument` `internal/store/pebble.go:250-319` (write at ≈:296); all docMeta readers (grep `GetDocMeta`).
+- **What:** Add a leading version byte to the docMeta packed value (`v2 = version, collection_id, url, title`). Reader accepts v1 (no version byte ⇒ collection 0) and v2. No eager migration — lazy upgrade on next write of each doc.
+- **Acceptance:**
+  - Mixed-store test: store with v1 and v2 entries; reads return correct collection for both; search enrichment loop unaffected.
+  - On the GH200 corpus (~9M-host scale): `/stats` doc count identical before/after deploy; zero `docMeta decode` errors in logs over 24h.
+
+### WS1.3 Purge-by-principal / purge-by-collection
+- **Where (new):** `internal/store/purge.go`; deletion paths must cover: doc keys (`'d','u','i','h'` families per pebble.go:12-21), postings (`'p','t','l','g'` — see `IndexDocument` ≈:1033-1100), HNSW passages (delete path near `internal/index/hnsw.go:445`), PQ codes (`'q'` family, `internal/index/pq_persist.go`), attribution rows.
+- **What:** `PurgePrincipal(pub)` and `PurgeCollection(id)` — walk `'A'`/collection index, delete everything, emit count. Rate-limited background job (not request-path).
+- **Acceptance:**
+  - E2E test: index 100 docs from principal X and 100 from Y → purge X → `/search` returns only Y's docs for matching queries; HNSW node count drops accordingly; `/stats` reflects new count; re-running purge is a no-op (idempotent).
+
+---
+
+## WS2 — Frontier lanes + work leases
+
+### WS2.1 Lane byte in frontier keyspace
+- **Where:** `frontierStatusIndexKey` `internal/store/pebble.go:371` (key becomes `'f' + sub + lane + host + 0x00 + url`); `frontierEntry` struct `:415` (add `Lane uint8`); `PushFrontier` `:1187` (new signature or `PushFrontierLane(ctx, url, depth, lane, priority)`); status constants `:404-409` unchanged.
+- **What:** All four lanes; existing callers map to lanes — crawler discovery → lane 2; RSS/sitemap refresh (`internal/crawler/rss.go`, `sitemap.go`) → lane 1; WET/bulk handlers (`pebble_serve.go:2643, :2771`) → lane 3; publisher submissions (WS3) → lane 0.
+- **Migration:** one-time startup scan: re-key existing `'f'+sub+host…` entries to lane 2. Gate behind a meta marker (`'m' + "frontier_lane_migrated"`) so it runs once.
+- **Acceptance:**
+  - `go test ./internal/store/...` green including new lane round-trip tests.
+  - Migration test: build store with old-format keys → open → `/queue` totals (queued/in_flight/done/errored) identical pre/post; all entries report lane 2.
+  - On GH200: migration completes < 10 min for current frontier size; log line reports counts.
+
+### WS2.2 Lane-weighted claiming
+- **Where:** `ClaimFrontier` `internal/store/pebble.go:1230`; per-lane cursor replaces single `frontier_cursor` meta (load ≈:199-201).
+- **What:** Weighted drain (default 50/30/15/5, configurable WS7) via deterministic weighted round-robin over non-empty lanes; host-fairness logic preserved *within* a lane; empty lanes donate their share.
+- **Acceptance:**
+  - Statistical test: seed 4 lanes × 1k URLs, claim 1k → per-lane counts within ±10% of weights; with lane 0 empty, its share redistributes.
+  - Existing crawler e2e (`main_e2e_test.go`, `bench_crawl_test.go`) still green.
+
+### WS2.3 Work leases (external claims)
+- **Where (new):** `internal/store/lease_work.go` using `'W'` family (WS1.1); re-queue uses `frontierEntry.Attempts` (`pebble.go:415`).
+- **What:** `ClaimWork(principal, kinds, lanes, max)` → atomically pop frontier entries (fetch jobs) or embed-pending chunks (WS5) into `'W'` leases with TTL (default 10 min fetch / 5 min embed); `RenewLease`; expiry sweeper returns work to frontier with `Attempts++`; per-principal in-flight cap (default 32).
+- **Lane policy:** lane 0 fetch jobs only to tier ≥ `trusted` principals (or server-fetched) — publisher-owned content has higher poisoning stakes. Lanes 1–3 open per tier table (Appendix C).
+- **Acceptance:**
+  - Lease expiry test (injected clock): claimed-not-submitted work returns to frontier, attempts incremented, claimable by another principal.
+  - Starvation test: principal claiming and dropping repeatedly hits in-flight cap; frontier drains via others.
+  - Double-submit on same claimID → second rejected (409).
+
+---
+
+## WS3 — Publisher path (collections, verification, leases)
+
+### WS3.1 Collection claims + domain verification
+- **Where (new):** `internal/collections/` (records via WS1.1 `'C'`); HTTP in `pebble_serve.go` (`POST /pub/claim`, `GET /pub/verify-status`).
+- **What:** Claim = `{prefix}` where prefix is `host[/path-prefix]` (**OPEN DECISION D2**). Verification worker fetches `https://<host>/.well-known/cosift-verify.txt` expecting the challenge token (DNS TXT `cosift-verify=<token>` fallback). First verified claim on a prefix wins; overlapping claims rejected (longest-prefix rule for nesting). Re-verify every 30 days; failure → collection `unverified` (docs stay, submissions blocked).
+- **Acceptance:**
+  - E2E with `httptest` origin: claim → serve token → verified within one worker tick; wrong/missing token → stays pending with reason; second principal claiming same prefix → 409.
+  - Re-verification failure flips state and blocks `/pub/submit` (403) but does not suspend serving.
+
+### WS3.2 Lease state machine + heartbeats
+- **Where (new):** `internal/collections/lease.go`; sweeper goroutine started where the server wires background loops in `runPebbleServe` (near crawler wiring `pebble_serve.go:758-766`).
+- **What:** `active → grace(72h) → suspended(7d) → purged(60d)`; renewal at any pre-purge state reactivates instantly. Heartbeat = envelope kind `heartbeat`; **nonce-chained**: each heartbeat response carries `next_nonce`; the next heartbeat must sign it (**OPEN DECISION D1** — recommended, no inbound push needed since appstore apps can't receive unsolicited messages).
+- **In-memory gate state:** suspended-collection set rebuilt from `'C'` at startup, mutated by sweeper, consumed by WS6.
+- **Acceptance:**
+  - Clock-injected unit tests for every transition incl. reactivation from grace and suspended.
+  - Heartbeat replay (old nonce) rejected; out-of-order seq rejected.
+  - E2E: submit docs → indexed → stop heartbeats → advance clock 72h → docs rank-decayed (grace); +7d → absent from `/search`; renew → present again without re-crawl; +60d → purged via WS1.3 (verify HNSW/postings shrink).
+
+### WS3.3 Publisher submission
+- **Where:** `pebble_serve.go` new `POST /pub/submit` (urls | sitemap_url | `replace_prefix`); pushes lane 0 via WS2.1; sitemap path reuses `internal/crawler/sitemap.go` streaming seeder.
+- **What:** Per-collection quotas (pages, bytes/day). `replace_prefix` semantics: on successful index of the new set, docs under the prefix not in the new set are purged (supersede; fixes version pollution) (**OPEN DECISION D3** — replace vs coexist; replace recommended).
+- **Acceptance:**
+  - E2E: submit 10 URLs → all crawled from lane 0 ahead of seeded lane-2 noise → docs carry `CollectionID`; over-quota submit → 429.
+  - `replace_prefix` test: v1 set indexed → submit v2 with replace → only v2 retrievable; doc count correct.
+
+---
+
+## WS4 — Open frontier: fetch contributions (quarantine → verify → promote)
+
+### WS4.1 Work + submit API
+- **Where:** `pebble_serve.go` new `POST /work/claim`, `POST /work/submit`, `POST /work/renew` (signed, WS0.3); claims from WS2.3.
+- **What:** Fetch-job payload to contributor: `{claim_id, url, robots_ok: true, politeness_deadline, max_body: 5MB, accepted_content_types}` — mirrors server-side gates (content-type allowlist `internal/crawler/crawler.go` ≈:1131; body cap ≈:1139-1145; robots checked **server-side before handing out**, reusing `internal/crawler/robots.go`). Submission: `{claim_id, final_url, http_status, fetched_at, raw_sha256, extracted: {title, text, lang, published_at, …}}` in an envelope. Lands in `'S'` quarantine only.
+- **Acceptance:**
+  - Submit without claim / expired claim / wrong principal → 4xx, nothing staged.
+  - Robots-disallowed URLs never appear in handed-out claims (test with fixture robots.txt).
+  - Oversized text / disallowed content-type rejected at intake (mirror of crawler gates).
+
+### WS4.2 Content verification pipeline
+- **Where (new):** `internal/verify/` — `content.go`, `scheduler.go`, `simhash.go`.
+- **Reuse:** re-fetch via crawler fetch path incl. `remote_fetcher.go` CF-worker transport (`internal/crawler/remote_fetcher.go:37,68`); extraction/normalization from `internal/crawler/parse.go` (drop-tags ≈:228-242, boilerplate tokens ≈:244-285, `collapseWhitespace` ≈:601-616) so both sides compare like-for-like.
+- **What (new):** 64-bit simhash over word shingles (no near-dup machinery exists today — ContentSHA is exact-only, see `crawler.go` ≈:674-767). Verdicts: `match` (hamming ≤ 6), `partial` (≤ 14), `mismatch`. Sampling per principal tier (Appendix C); `partial` is neutral (page churn, geo-variance); repeated `mismatch` pattern → tier drop / ban via WS0.2.
+- **Promotion gate chain:** schema checks → sampled re-fetch compare → `authority.Score(host)` floor (reject < 0.2 unless lane 0; `internal/authority/authority.go`) → optional judge pass (`internal/judge/judge.go`, fail-open as in `/answer` use at `pebble_serve.go` ≈:5402-5421) → exact-dup check (ContentSHA) → `UpsertDocument` + `IndexDocument` with provenance (WS1.2/WS1.3 fields) → reputation bump.
+- **Acceptance:**
+  - Red-team e2e (new `zz_contrib_redteam_test.go`): honest submissions promote; tampered text (entity swap / injected spam paragraph) at 100% sampling → caught, staged-rejected, principal counters reflect it; after ban, principal's previously-promoted docs purged (I4).
+  - Churn tolerance: same page with rotated ad block / timestamp → `match` or `partial`, never `mismatch` (fixture pair tests for simhash thresholds).
+  - Quarantine TTL: unverified entries older than 14 days dropped; `'S'` family size bounded in soak test.
+  - Throughput: verification worker sustains ≥ 5 re-fetches/s without disturbing crawl politeness (shares hostGate `internal/crawler/gate.go`).
+
+### WS4.3 Reputation, tiers, sampling
+- **Where (new):** `internal/reputation/` (uses WS0.2 counters; tier table Appendix C).
+- **What:** Tier function over verified history; emits sampling rate (1.0 → 0.05 floor), per-tier quotas, lane access. All transitions logged + metric'd.
+- **Acceptance:**
+  - Property test: expected accepted-bad-before-ban ≈ (1−r)/r honored by simulation within tolerance; r=1.0 for fresh principals (I5).
+  - Tier promotion requires ≥ N verified-good (default 200) AND age ≥ 7 days; demotion immediate on mismatch pattern.
+
+---
+
+## WS5 — Embed contributions (server-defined text only — I6)
+
+### WS5.1 Index-level model pin (blob-sha, not name)
+- **Production reality** (verified 2026-06-12 on GH200): embedder is **`nomic-embed-text-v1.5`** (137M params, 768-dim, F16 GGUF, `nomic-bert` family, ~274 MB). Served via 8 ollama replicas (ports 11435-11442); cosift `embeddings.urls` round-robins (`internal/embed/client.go` `RoundRobinEmbedder`). On-disk blob: `sha256-970aa74c0a90ef7482477cf803618e776e173c007bf957f635f1015bfcfef0e6`.
+- **Where:** HNSW persistence meta blob (`internal/index/hnsw_persist.go` meta ≈:28-34, magic `HSW1`) — add `model_family` (string), `model_blob_sha256` (32 bytes), `dim` (already implicit). Surfaced in `/stats` (`buildStatsBody` `pebble_serve.go:3084`) and `/healthz`. The blob sha is the strongest pin — name `nomic-embed-text:latest` can drift; the blob bytes can't.
+- **Server-side blob verification:** at startup, hit each `embeddings.urls` endpoint with a known fixture text, compare returned vector against a checked-in fixture vector (cosine ≥ 0.9999); mismatch → refuse to start with mode `dense_enabled`. Fixture lives at `testdata/embed_fixture.json` (text + expected 768-float vector). Fixes a latent operator-side gap (Pebble path currently stores no model identity at all; only legacy SQLite did).
+- **Acceptance:**
+  - Open index with mismatched configured embedder → startup warning + dense disabled + `/healthz` reports `degraded`.
+  - Matching blob → normal; snapshot round-trip preserves pin.
+  - Fixture cosine check runs against each of the 8 replicas at startup; any replica failing fixture removed from rotation.
+
+### WS5.2 Embed jobs + recompute verification
+- **Where:** job source = the embed-backfill scan (docs lacking vectors — `handleEmbedBackfill` `pebble_serve.go:2383`, chunker `internal/index/chunk.go` 320/64 words, truncation `truncateForEmbedLite` ≈:2485); claims via WS2.3; intake via `/work/submit`; insertion via `AddPassageBatch` `internal/index/hnsw.go:471` (normalization in-place `internal/index/vector.go` ≈:275-287).
+- **What:** Claim payload: `{claim_id, model_family, model_blob_sha256, dim, chunks: [{doc_id, offset, length, text}]}` (server sends the exact text — contributor never chooses it — **I6**). Submission: vectors (dim-checked, finite-checked, then unit-normalized server-side). Verification: recompute a sample with local embedder; **calibrated cosine threshold ≥ 0.9999** (same-blob F16 cross-machine determinism in practice; calibrate on GH200 vs Jetson during WS15.3b — refuse to lower if measurement diverges). Batch fails if any sampled vector fails. Contributors whose reported `model_blob_sha256` ≠ index pin: claim refused at `/work/claim`, never handed work.
+- **Contention framing:** the embed bottleneck is not raw watts (~3 GB GPU, 0-30% util on the H100) — it's **vLLM contention on the same H100 during /answer**. Offloading embeds gives `/answer` latency headroom and unblocks moving to a bigger embedder later (e.g., nomic-embed-text → bge-large) without regressing answer p95.
+- **Acceptance:**
+  - E2E with mock embedder: honest vectors promote into HNSW (searchable via `/search?mode=dense`); perturbed vector (cosine 0.99 vs recompute, below 0.9999 threshold) at sampling=1.0 → batch rejected, reputation hit.
+  - Wrong `model_blob_sha256` / wrong dim / NaN → rejected at `/work/claim` or `/work/submit` (400), never staged.
+  - Dogfood: Jetson node runs worker app (WS10) bundled embedder against staging box; ≥ 10k chunks contributed, sampled-verified, zero false rejects at calibrated threshold.
+  - Contention metric: with contributors active during a synthetic `/answer` load test (10 rps), p95 `/answer` latency drops ≥ 15% vs same load with local embed-backfill running.
+
+### WS5.3 Bundled embed runtime (contributor side, NO ollama dependency)
+- **Where (new, in `pilot-protocol/cosift-node/` — see WS10):** `embed/` package wrapping a llama.cpp sidecar.
+- **What:** The worker app bundle ships:
+  - `bin/embed-runtime` per arch — llama.cpp built in `embedding` mode (server, unix-socket transport), ~5-10 MB per arch. MIT license, redistributable.
+  - `models/nomic-embed-text-v1.5.f16.gguf` — the exact blob (`sha256:970aa74c…`) the GH200 serves. ~274 MB. Apache 2.0 license, redistributable.
+  - `embed_fixture.json` — same fixture used in WS5.1, lets the app self-test on first start.
+- **App lifecycle:** on `node.worker_start`, app spawns embed-runtime with `--model models/…gguf --uds $APP/embed.sock --embedding --pooling mean`; on shutdown kills the child. Health-check by running the fixture text through the sidecar and asserting cosine ≥ 0.9999 vs the fixture vector — refuses to enter the worker loop if it fails (would mean tampered weights or wrong build).
+- **Why a sidecar, not cgo into the Go app:** keeps the Go binary cross-compile-clean (no cgo cross), keeps llama.cpp updates independent of the app, lets the sandbox kill the child without restarting the parent. Standard pattern; well-trodden.
+- **Acceptance:**
+  - Install on linux/arm64 (Jetson) and darwin/arm64 → embed-runtime spawns within 5 s; fixture self-test passes; `node.worker_stats` reports `runtime_ready: true, model_blob_sha256: 970aa74c…`.
+  - Tampered model file (one byte flipped) → fixture cosine fails → worker refuses to start; structured error returned to operator via `node.worker_start`.
+  - Cross-runtime fidelity test: same 200 fixture texts embedded by (a) GH200 ollama and (b) bundled llama.cpp → all pairs cosine ≥ 0.9999. Run in CI to detect any llama.cpp version that drifts from ollama-served output.
+  - No external dependency required: `pilotctl appstore install io.pilot.cosift-node && node.worker_start` works on a freshly-imaged machine with no ollama / no python.
+
+---
+
+## WS6 — Retrieval gating (grace decay, suspension, bans)
+
+- **Where:** single choke point — post-`retrieve()` filter/enrichment loop shared by all retrieval handlers (`pebble_serve.go` ≈:3700-3750, `retrievalFilters` at `:6742`); decay precedent `decayMultiplier`/`applyTimeDecay` ≈:4657-4677.
+- **What:** Check docMeta v2 `collection_id` (WS1.2) against in-memory suspended set (WS3.2) → drop; collection in grace → multiply score by configurable factor (default 0.5), reusing the multiplicative-decay pattern; doc principal banned (rare transient window before purge completes) → drop.
+- **Acceptance:**
+  - Unit: hits from suspended collections absent across `/search`, `/query`, `/answer`, `/research` (one test per handler — they share the loop but assert anyway, I8).
+  - Grace decay changes ordering deterministically in a fixture corpus.
+  - Overhead: enrichment loop p95 unchanged within noise on a 1k-hit benchmark (`bench_compare_test.go` pattern).
+
+---
+
+## WS7 — Config
+
+- **Where:** `internal/config/config.go` — new top-level `Contrib` section beside `Cluster` (≈:57-88) / `Server` (≈:91-103); example file `cosift.json.example`; doc `docs/CLI-FLAGS.md`.
+- **What:**
+  ```jsonc
+  "contrib": {
+    "enabled": false,            // master kill-switch (default OFF)
+    "publishers_enabled": false,
+    "workers_enabled": false,
+    "lane_weights": [50,30,15,5],
+    "lease_ttl_fetch_s": 600, "lease_ttl_embed_s": 300,
+    "grace_h": 72, "suspend_d": 7, "purge_d": 60,
+    "sampling_floor": 0.05, "tier_promote_min_ok": 200,
+    "quotas": { "fetch_per_day": 2000, "embed_chunks_per_day": 50000, "pub_pages_per_day": 1000 },
+    "simhash_match_hamming": 6, "simhash_partial_hamming": 14,
+    "embed_verify_cosine": 0.99,
+    "multi_attestation": false   // OPEN DECISION D4
+  }
+  ```
+- **Acceptance:** defaults compile to a fully inert server (no new routes mounted when `enabled=false` — route-table test); every field has an env override following the existing `COSIFT_*` pattern; `doctor` (zz_doctor_runner_test.go surface) validates ranges.
+
+## WS8 — Observability
+
+- **Where:** `handleMetrics` `pebble_serve.go:3267`, `handleStats` `:3020`, `handleQueue` `:2894`; `assets/openapi.json` + swagger assets.
+- **What:** Prometheus: `cosift_lane_depth{lane}`, `cosift_staging_depth`, `cosift_verify_total{kind,verdict}`, `cosift_principals{tier}`, `cosift_collections{state}`, `cosift_work_claims_active`, `cosift_embed_external_chunks_total`, `cosift_purged_docs_total{cause}`. `/queue` gains per-lane breakdown + open-claim counts (contributor storefront — deliberately public). `/stats` gains `collections` + `contrib` blocks. OpenAPI documents `/pub/*`, `/work/*`, envelope header, problem types.
+- **Acceptance:** metrics appear with zero values when feature off; soak test scrapes parse with `promtool check metrics`; openapi.json passes `swagger-cli validate` (or the repo's existing check); `/queue` p95 unchanged (it's public + hot).
+
+## WS9 — Overlay transport (Pilot-only for contribution traffic — I9)
+
+This is not a "expose it on the overlay too" workstream — it is the only transport for contributions.
+
+- **Where:** new `cmd/cosift/pilot_listener.go`. Web4 primitives: daemon IPC `CmdBind` (`web4/pkg/daemon/ipc.go` ≈:26-75; remember the `/tmp/pilot.sock` cleanup gotcha from cluster memory); gateway `Map` (`gateway/gateway.go` ≈:33-78) is the fallback if the IPC path is blocked.
+- **What — three coupled changes:**
+  1. **Pilot listener with verified peer identity.** Cosift opens an overlay listener via `CmdBind` on virtual port **7700**. Each accepted stream's remote pubkey comes from the Pilot tunnel handshake (cryptographically authenticated by Pilot itself — see `pkg/daemon/ports.go` ≈:357-371 listener `AcceptCh`). The listener wraps each stream as an in-process `http.Handler` invocation with the remote pubkey stamped into request context under a sentinel key `pilotPeerPubKey`.
+  2. **Envelope ↔ tunnel identity binding.** The signed envelope middleware (WS0.3) reads `pilotPeerPubKey` from context and asserts `envelope.PrincipalPub == ctx.PilotPeerPubKey`. Mismatch → 401, principal counter bumped (this is a Sybil-ish indicator). For the HTTP/loopback dev path, this check is bypassed only when the request originates from `127.0.0.1` AND a `Contrib.AllowHTTPDev` config flag is set (default false).
+  3. **Public HTTP surface stripped of contribution routes.** `/pub/*` and `/work/*` are removed from the Caddy reverse-proxy config; they bind on cosift's HTTP server but are 404 to any non-overlay-non-loopback caller. The Caddy site config (`/etc/caddy/Caddyfile` on GH200) gains an explicit `respond /pub/* 404` and `respond /work/* 404` ahead of the `reverse_proxy` directive. (`encode gzip` stays out — known SSE killer from prior memory.)
+- **What overlay consumers see:** `/pub/*`, `/work/*`, `/queue`, `/healthz` (read-only). `/admin/*` is **forbidden via overlay** (route allowlist enforced at the listener wrapper, not just at handler-time).
+- **Acceptance:**
+  - From a second Pilot node with a fresh pubkey K: `net.dial` → `/work/claim` with envelope signed by K → succeeds end-to-end over overlay.
+  - From the same node: `/work/claim` with envelope signed by K' ≠ K → 401 `envelope_peer_mismatch`, counter on K bumps.
+  - From the public internet via Caddy: `/work/claim` → 404 (route stripped); `/pub/submit` → 404.
+  - From loopback with `Contrib.AllowHTTPDev=false`: same → 404. With flag on (dev only): bypasses tunnel-identity check, still requires valid envelope.
+  - `/admin/*` over overlay → 403 (allowlist refuses).
+  - GH200 daemon restart re-binds within 10 s (systemd retry loop, IPC reconnect); listener test covers daemon-death-during-stream cleanup.
+  - `docs/API.md` documents the overlay-required posture and the dev-flag escape hatch.
+
+## WS10 — `io.pilot.cosift-node` appstore app (publisher + worker; self-contained)
+
+- **Where (new repo/dir):** `pilot-protocol/cosift-node/` modeled on `pilot-protocol/wallet` (manifest schema `app-store/pkg/manifest/manifest.go` ≈:23-155; IPC dispatcher `app-store/pkg/ipc/server.go` ≈:22-95; envelope `pkg/ipc/envelope.go` ≈:33-41). Bundles the embed runtime (WS5.3) — **no external ollama / python / model-download steps for the user**.
+- **What:**
+  - Manifest: id `io.pilot.cosift-node`, `exposes`: `node.help`, `node.claim_prefix`, `node.submit_docs`, `node.status`, `node.retire`, `node.worker_start`, `node.worker_stop`, `node.worker_stats`, `node.embed_runtime_status`. Grants: `net.dial` (overlay to cosift broker), fs on `$APP/`, spawn-child for the bundled embed sidecar. Per-app identity `$APP/identity.json` (supervisor passes `--identity`) is **both** the appstore-app key and the cosift principal key — single key throughout. **OPEN DECISION D5**: one app two roles (recommended) vs two apps.
+  - **All broker traffic goes over Pilot (I9).** The app uses `net.dial` to virtual port 7700 on the broker's overlay address; **never** dials the broker's public HTTPS surface. Configuration carries `broker_overlay_addr`, not a URL.
+  - Publisher role: registers principal, claims prefix, drives `/pub/*`, runs heartbeat loop (6h, nonce-chained). Uninstall ⇒ identity destroyed ⇒ heartbeats stop ⇒ lease lapses — the "active only while installed" semantics enforced by key lifecycle.
+  - Worker role: loop = claim → either (a) fetch (own egress; honor handed-out politeness deadline + robots/UA fields) or (b) embed via bundled sidecar (WS5.3) — claim refused server-side if app's reported `model_blob_sha256` doesn't match index pin → sign → submit. Honors `worker_stop`; backs off on 429.
+  - First-run sequence: identity load-or-create → spawn embed sidecar → run fixture self-test (WS5.3 acceptance) → register principal with broker (overlay) → publish `node.help` schema → wait for IPC. If sidecar self-test fails, app stays alive for `node.help` / `node.status` but returns `runtime_ready: false` and refuses worker_start.
+- **Acceptance:**
+  - `node.help` returns full method/param schema; advertises `embed_runtime: { family: nomic-embed-text-v1.5, blob_sha256: 970aa74c…, dim: 768, runtime: llama.cpp }`.
+  - Clean-machine install (no ollama, no python): `pilotctl appstore install io.pilot.cosift-node` → spawn → ready → `node.worker_start` produces signed-and-submitted embed contributions within 60 s.
+  - Publisher path: `node.claim_prefix` + well-known token on a test origin → verified via overlay → docs live; `pilotctl appstore uninstall` → (clock-advanced on broker) suspension. Reinstall → new key ⇒ re-claim required and succeeds via still-present well-known token.
+  - Identity binding (I9): tamper with envelope to use a different pubkey from the overlay tunnel → broker rejects with `envelope_peer_mismatch`; counter incremented on tunnel's pubkey, not on forged envelope.
+  - Worker soak: 24h on Jetson, ≥ 99% of claims either submitted or expired cleanly (no leaks), zero signature failures; embed sidecar memory steady (< 1.5 GB RSS), no leaks.
+  - Signing cross-check uses WS0.1 fixture vectors.
+  - All broker calls visible in pilot tunnel logs; if `net.dial` to overlay fails (daemon down), app surfaces structured error and retries with backoff (does NOT silently fall back to HTTPS).
+
+## WS11 — Packaging & catalogue (chunky bundle, deliberately)
+
+- **Where:** bundles per `pilot-protocol/catalog/README.md` (tar.gz: `manifest.json` + `bin/<binary>` + extra files; sha256 pinned manifest→binary, catalogue→bundle); catalogue entry in `pilot-protocol/web4/catalogue/catalogue.json`.
+- **Bundle contents (per arch):**
+  - `manifest.json` (~2 KB)
+  - `bin/cosift-node` Go binary (~25 MB)
+  - `bin/embed-runtime` llama.cpp embedding server (~8 MB)
+  - `models/nomic-embed-text-v1.5.f16.gguf` (~274 MB; Apache 2.0)
+  - `testdata/embed_fixture.json` (~30 KB)
+  - **Total per arch: ~307 MB compressed (~310 MB uncompressed).**
+- **Why F16 and not quantized:** matches the server blob byte-for-byte, enabling cosine ≥ 0.9999 verification (WS5.2). Q5_K_M would save ~180 MB per bundle but lower the threshold to ~0.998, weakening the rejection signal. Worth offering Q5_K_M as a second variant later (manifest `variants` list) if download size hurts adoption.
+- **Manifest extension:** add a `data` array alongside `binary`, each entry `{path, sha256, size}` — supervisor verifies every entry's sha256 on each spawn (`app-store/INTEGRATION.md` ≈:120-137 already verifies the binary blob; extend to `data` entries). **OPEN DECISION D8:** require appstore manifest schema extension (cross-repo PR) or stash the GGUF inside a `.tar` blob whose sha256 the supervisor checks as a single file. *Recommended: extend manifest, cleaner long term.*
+- **What:** Per-arch builds (linux/amd64, linux/arm64, darwin/arm64); CI job producing bundles + checksums; staged rollout (staging catalogue first if one exists, else version-gate). Bundles published to GitHub Releases or GCS (decide at impl); bundle_url + bundle_sha256 inserted into `catalogue.json`.
+- **Acceptance:**
+  - `pilotctl appstore install io.pilot.cosift-node` on macOS arm64 + Jetson (linux/arm64) downloads + verifies + installs in < 60 s on a 100 Mbps link; spawns; `ready`.
+  - Supervisor sha256 re-verification on restart covers binary + model file; tampered model → refuses to spawn (mirrors WS5.3 fixture check, but earlier).
+  - Bundle size displayed clearly in catalogue + install confirmation.
+  - License manifest: NOTICE file in bundle lists nomic-embed (Apache-2.0) + llama.cpp (MIT) attributions.
+
+## WS11a — Multi-broker forward-compatibility (federation runway)
+
+Stated requirement: "the actual broker needs to live on pilot, or we should have multiple brokers orchestrating the distribution and ingest of things." Single broker now, multi-broker later — but Phase 1-5 must not paint us into a corner.
+
+- **What stays single in v1:** the broker (one cosift instance on GH200) is the only frontier authority, the only `'P'`/`'C'`/`'A'` ledger, the only verifier. This is fine for launch and removes 80% of the federation complexity.
+- **What must be portable from day one (so v2 federation is additive, not a rewrite):**
+  - **Principal records (`'P'` family, WS1.1):** signed by the principal themselves on registration (`register` envelope kind in WS0.1). Other brokers must be able to import a principal's signed registration and verify it standalone — no broker-specific secret in the record.
+  - **Collection claims (WS3.1):** the well-known token contains the broker's pubkey as part of the challenge, but the verified-domain *fact* is portable — a second broker can re-verify against the same well-known endpoint and accept the same claim.
+  - **Attribution rows (`'A'`, WS1.1):** include the broker's pubkey alongside the principal's signature. A doc promoted by broker B1 carries an attribution row signed by B1; broker B2 importing the doc keeps the trail intact.
+  - **Reputation deltas:** counters are local-broker state in v1, but expose a signed-export endpoint (`GET /pub/reputation-export?since=…` envelope-authed) so v2 brokers can opt to import. Don't ship the import side in v1.
+- **Discovery:** v1 advertises a single overlay address. v2: brokers advertise themselves on a Pilot directory (likely `list-agents` keyword `cosift-broker`), workers/publishers discover and pick a broker by latency/affinity. The app's `broker_overlay_addr` config field (WS10) becomes `broker_overlay_addrs` (plural) — design the schema as plural from v1 to avoid migration.
+- **Frontier ownership in multi-broker:** reuse the existing sharding code (`internal/config/cluster.go` `ShardOf` FNV-32 hashing) — each broker owns a hash range; cross-broker forwarding via the existing `ForwardFn` seam (`internal/crawler/crawler.go:316`); claims handed only by the owning broker.
+- **Conflict resolution between brokers:** out of scope for this plan. The portability decisions above keep the door open without committing.
+- **Acceptance (forward-compat sanity checks, not features):**
+  - Principal record round-trip test: register on broker A → export the signed registration → verify in broker B's verification logic against pubkey only (no broker A secret involved).
+  - Collection-claim portability test: well-known token includes broker A's pubkey → second broker B (with different pubkey, mock origin) issues its own challenge token → both verify independently without contradiction.
+  - App config schema: `broker_overlay_addrs: [string]` parses with a single-element list in v1; tests assert behavior identical to a hypothetical scalar `broker_overlay_addr`.
+
+## WS12 — MCP shim (consumer reach; unchanged scope)
+
+- **Where (new):** `pilot-protocol/cosift-mcp/` — stdio MCP server wrapping `pilotctl appstore call io.pilot.cosift cosift.search|cosift.contents` (+ `cosift.answer` optional). Published to npm for copy-paste Claude Code config.
+- **What:** Tools `cosift_search`, `cosift_contents`, `cosift_answer`; missing pilotctl/daemon → actionable error ("install Pilot: …") — transport is overlay-only by design (adoption requirement).
+- **Acceptance:** `claude mcp add` flow works on a machine with Pilot (search returns results inside Claude Code); on a machine without Pilot the tool error explains the install path; README covers both.
+
+## WS13 — Docs
+
+- **Where:** `docs/API.md` (envelope, `/pub/*`, `/work/*`, error catalogue), `docs/CONTRIB-NETWORK.md` (operator guide: tiers, sampling, ban/purge runbook), `README.md` (one paragraph + links), `assets/openapi.json` (WS8), publisher-facing quickstart in cosift-node repo.
+- **Acceptance:** a publisher can go claim→verified→indexed using only the quickstart (validated by a clean-room run); runbook covers: ban a principal, purge, raise quotas, tune sampling, read verification metrics.
+
+## WS14 — Test & eval matrix (cross-cutting)
+
+| Layer | What | Gate |
+|---|---|---|
+| Unit | envelope, registry, lanes, leases (clock-injected), simhash, tier function | `go test ./... -parallel 4` green (repo norm) |
+| E2E | publisher lifecycle; fetch contribution happy/red-team; embed contribution; purge | new `zz_contrib_*_test.go` suites green |
+| Migration | frontier lane re-key; docMeta v1/v2 | counts invariant tests (WS1.2/WS2.1) |
+| Soak (staging box) | 24h worker + verifier; staging/lease sweepers | no leaks: `'S'`/`'W'` bounded, goroutines flat |
+| Red-team | poisoning at each tier; Sybil burst (100 fresh keys); replay; claim-hoarding | all caught; (1−r)/r property holds |
+| Quality eval | existing eval harness (`answer_eval_test.go`, eval-baseline-v2.json) before/after contributions enabled | pass-rate non-regression (±1pt) |
+| Perf | retrieval p95 with gating on; `/queue`; claim path 100 rps | within noise vs baseline |
+
+## WS15 — Rollout (GH200) & migration order
+
+1. Deploy Phase 1 binary, `contrib.enabled=false` → **acceptance:** byte-identical behavior on eval suite + 24h error-log clean; lane migration ran once (log marker).
+2. Enable `publishers_enabled` with allowlist of 1 (your own test principal) → run WS3 acceptance live.
+3. Enable `workers_enabled` allowlisted (Jetson fleet principals) → WS4/WS5 acceptance live; calibrate `embed_verify_cosine` + simhash thresholds with real traffic.
+4. Open registration (no allowlist) + publish app to catalogue + announce.
+5. Each step reversible by config flag; **rollback:** flag off → routes unmount, sweepers stop; data families are inert when disabled. Caddyfile untouched throughout (no `encode gzip` — known SSE killer). Snapshots (`snapshot.sh`, hourly timer) already cover new families since they live in the same Pebble dir — verify restore in step 1.
+
+---
+
+## Phase → work-item map
+
+| Phase | Items | Exit criterion |
+|---|---|---|
+| 1 Foundation | WS0.*, WS1.*, WS2.1–2.2, WS6, WS7, WS8 (metrics only) | deployed flag-off on GH200, WS15.1 green |
+| 2 Publisher | WS2.3 (pub-only), WS3.*, WS13 (API.md), WS15.2 | own docs collection live + lease lifecycle proven on box |
+| 3 Open frontier (fetch) | WS4.*, WS8 (/queue lanes), WS15.3a | Jetson worker contributing crawl, red-team suite green in CI |
+| 4 Embed jobs | WS5.*, WS15.3b | measurable embed offload, calibrated thresholds |
+| 5 Apps + overlay | WS9 (overlay-only enforcement), WS10 (incl. bundled embed runtime), WS11, WS11a (portability hooks only) | install→claim→serve works from a clean Mac via catalogue with no external deps; I9 holds in adversarial tests |
+| 6 Flywheel | WS12, analytics extension of WS8, credits (**D6**), reassess **D10** for federation | MCP in npm; first external publisher onboarded |
+
+## Open decisions (blockers marked ⛔ for their phase)
+
+- **D1** ⛔P2 — Nonce-chained heartbeats (no inbound push). *Recommended: yes.*
+- **D2** ⛔P2 — Claim granularity: `host[/path-prefix]`, longest-prefix nesting. *Recommended: yes.*
+- **D3** ⛔P2 — Publisher docs replace operator-crawled docs under claimed prefix. *Recommended: replace.*
+- **D4** P3 — Multi-attestation (2 independent contributors agree) for low-tier sensitive lanes. *Recommended: config-flag, default off.*
+- **D5** ⛔P5 — One app `io.pilot.cosift-node` with both roles. *Recommended: one.*
+- **D6** P6 — Credits economy (work → lane-0 quota). *Recommended: defer; quotas suffice at launch.*
+- **D7** ⛔P1 — Dependency route for `crypto.Verify`: import `pilot-protocol/common` vs vendor the ~40-line ed25519 wrapper. *Recommended: vendor (keeps cosift module dependency-light).*
+- **D8** ⛔P5 — Appstore manifest extension for multi-file bundles (WS11). Extend `manifest.json` schema with a `data` array (each entry sha256-pinned, supervisor-verified per spawn) vs stash the model + binaries inside one `.tar` blob whose single sha256 is in the existing `binary` field. *Recommended: extend manifest — cleaner long-term, and the model file is the first instance of a class of payloads (PQ codebooks, fixtures) that will recur.*
+- **D9** P5 — Embed model quantization for the worker bundle: F16 (~274 MB, matches server byte-for-byte, cosine ≥ 0.9999) vs Q5_K_M (~95 MB, cosine ≈ 0.998). *Recommended: F16 in v1; offer Q5_K_M as a manifest `variants` option later if download size is a real adoption blocker.*
+- **D10** P6 — Multi-broker federation kickoff: stay single-broker indefinitely vs commit to v2 work (WS11a portability hooks become user-facing). *Recommended: ship v1 single-broker; reassess after 90 days based on broker load and external interest. WS11a deliverables keep the door open regardless.*
+
+## Top risks
+
+| Risk | Mitigation |
+|---|---|
+| docMeta/frontier migration corrupts hot store | version-byte + run-once marker + checkpoint before deploy + count-invariant acceptance (WS1.2/WS2.1) |
+| Content-churn false mismatches punish honest workers | graded verdicts, `partial` neutral, per-principal patterns not single-doc slashing (WS4.2) |
+| Embed cosine threshold wrong → mass false rejects | calibrate on GH200-vs-Jetson real noise before freezing (WS5.2 acceptance) |
+| New auth surface bug = open write path | I1–I8 invariant tests in CI; routes unmounted unless enabled; allowlist phases (WS15) |
+| Verification cost swamps box | sampling floors, verifier shares hostGate politeness, staging TTL bounds queue (WS4.2) |
+| Branch drift during long build (external process switches cosift checkout) | per repo memory: verify `git branch --show-current` before every commit |
+
+---
+
+## Appendix A — Envelope canonical bytes
+
+```
+ver(1) ‖ kind(1) ‖ pub(32) ‖ seq(8,be) ‖ claim_id(16; zero for register/claim kinds)
+       ‖ body_sha256(32) ‖ model_id_len(2,be) ‖ model_id ‖ sig(64) over all preceding bytes
+```
+Transport: base64 in `X-Cosift-Envelope`; body hash binds the HTTP body. Server state per principal: `last_seq`, `pending_nonce` (heartbeats).
+
+## Appendix B — State machines
+
+```
+Collection: claimed → verified → active ⇄ grace(72h) → suspended(7d) → purged(60d)
+                         │ re-verify fail → unverified (serving continues, writes blocked)
+Principal:  registered → tier{new→basic→trusted} ; mismatch-pattern → banned → purged
+WorkLease:  open → claimed(TTL) → submitted → verified|rejected ; TTL expiry → open(attempts++)
+Staged:     received → sampled? → promoted | rejected ; age>14d → dropped
+```
+
+## Appendix C — Tier table (initial values; config-tunable)
+
+| Tier | Entry condition | Sampling | Fetch/day | Embed chunks/day | Lanes |
+|---|---|---|---|---|---|
+| new | registration | 1.00 | 200 | 5,000 | 2,3 |
+| basic | ≥200 ok ∧ ≥7d ∧ 0 mismatches/30d | 0.25 | 2,000 | 50,000 | 1,2,3 |
+| trusted | ≥2,000 ok ∧ ≥30d ∧ operator approve | 0.05 | 10,000 | 250,000 | 0,1,2,3 |
+| banned | mismatch pattern / operator | — | 0 | 0 | — |
+
+Expected un-caught bad submissions per burned identity ≈ (1−r)/r: new=0, basic=3, trusted=19 — and purge (I4) reverses survivors.
diff --git a/docs/cosift-diagrams.pdf b/docs/cosift-diagrams.pdf
new file mode 100644
index 0000000..ad844b1
Binary files /dev/null and b/docs/cosift-diagrams.pdf differ
diff --git a/internal/config/config.go b/internal/config/config.go
index 09ef8db..ebf2ad2 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -22,9 +22,6 @@ type Config struct {
 
 	Crawler Crawler `json:"crawler"`
 
-	// Federation: which external retrieval sources to consult. Each is optional.
-	Federation Federation `json:"federation"`
-
 	// Embeddings: address of an HTTP embedding service. Empty = vector search disabled.
 	Embeddings Embeddings `json:"embeddings"`
 
@@ -287,19 +284,33 @@ type Crawler struct {
 	// the offline `cosift purge-adult` sweep cleans content already
 	// indexed before this was enabled.
 	FilterAdult bool `json:"filter_adult,omitempty"`
-}
-
-// Federation configures upstream search backends used as no-key
-// fallbacks when the local corpus is empty or partial.
-type Federation struct {
-	// DuckDuckGoHTML enables the no-key DDG HTML proxy fallback for /search.
-	DuckDuckGoHTML bool `json:"duckduckgo_html"`
 
-	// SearXNGURL points at a self-hosted SearXNG instance, if any.
-	SearXNGURL string `json:"searxng_url"`
-
-	// Pilot enables the cosift→Pilot specialist router (requires pilotctl).
-	Pilot bool `json:"pilot"`
+	// AuthorityPriority, when true, uses domain-authority signals (Tranco
+	// rank, Majestic TrustFlow, embedded whitelist) to set the frontier
+	// priority for every discovered link. High-authority hosts
+	// (arxiv.org, Wikipedia, MDN, …) receive priority ~0.8-1.0; unknown
+	// blogs fall back to 0.5. The priority feeds ClaimFrontier's ORDER BY
+	// so the worker pool naturally crawls the best-known domains first.
+	//
+	// Requires at least one of AuthorityTrancoCSV / AuthorityMajesticCSV
+	// to be populated, or relies on the embedded trusted-host whitelist
+	// alone (still useful for a known-good bootstrap set).
+	//
+	// Default false preserves uniform-priority (0.5) behavior.
+	AuthorityPriority bool `json:"authority_priority,omitempty"`
+
+	// AuthorityTrancoCSV is the local path to the Tranco top-1M CSV file.
+	// Download: https://tranco-list.eu/ (updated weekly, ~7 MB).
+	// File format: "rank,hostname" per line, no header. Rank 1 = most popular.
+	// Only read at startup; the crawler does not hot-reload it.
+	// Ignored when AuthorityPriority=false.
+	AuthorityTrancoCSV string `json:"authority_tranco_csv,omitempty"`
+
+	// AuthorityMajesticCSV is the local path to the Majestic Million CSV.
+	// Download: https://majestic.com/reports/majestic-million (~35 MB).
+	// Columns 2 (Domain) and 9 (TrustFlow 0–100) are used; header is auto-detected.
+	// Ignored when AuthorityPriority=false.
+	AuthorityMajesticCSV string `json:"authority_majestic_csv,omitempty"`
 }
 
 // Embeddings configures the dense-vector backend: model, dimension,
@@ -449,7 +460,6 @@ func Default() *Config {
 			MaxDepth:       2,
 			RespectRobots:  true,
 		},
-		Federation: Federation{},
 		Embeddings: Embeddings{},
 	}
 }
diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go
index 751c093..c9709ec 100644
--- a/internal/crawler/crawler.go
+++ b/internal/crawler/crawler.go
@@ -69,7 +69,7 @@ type Crawler struct {
 	// an int read on the hot path; not atomic — diagnostic only).
 	zombieDebugLogged int
 
-// per-host error-rate tracking via sync.Map + atomic
+	// per-host error-rate tracking via sync.Map + atomic
 	// counters. With 512 workers we cannot afford a single write lock
 	// on every claim's completion — that bottlenecked and cost
 	// ~25% throughput. sync.Map.LoadOrStore lets us avoid the lock on
@@ -87,11 +87,11 @@ type Crawler struct {
 	// keeps memory predictable; non-blocking send means a slow embedder
 	// can't stall the crawl (dropped jobs land in embed-backfill later,
 	// which the operator runs anyway).
-	embedQ         chan *embedJob
-	embedQueued    atomic.Int64
-	embedDropped   atomic.Int64
-	embedDone      atomic.Int64
-	embedFailed    atomic.Int64
+	embedQ       chan *embedJob
+	embedQueued  atomic.Int64
+	embedDropped atomic.Int64
+	embedDone    atomic.Int64
+	embedFailed  atomic.Int64
 
 	// URLs whose frontier transition happened inside processClaimed via
 	// WriteCrawlResult. The worker checks this set before its own
@@ -103,6 +103,85 @@ type Crawler struct {
 	// path (enqueueLinks) consults this set so we don't keep
 	// re-enqueuing the same dead URLs the sweeper just purged.
 	autoBlocked sync.Map // host (string) → struct{}
+
+	// Dynamic allowlist: runtime-promoted domains that allowedDomain accepts
+	// IN ADDITION to the static cfg.IncludeDomains. Lets curated sources
+	// (HN/Reddit harvesters) grow the crawlable set organically — a domain
+	// that recurs on HN's front page is promoted via AddAllowedDomain and
+	// persisted to dynDomainsFile so it survives restart. Empty file path
+	// (COSIFT_DYNAMIC_DOMAINS_FILE unset) disables the feature.
+	dynDomains     sync.Map // eTLD+1 / host (string) → struct{}
+	dynDomainsFile string
+	dynDomainsMu   sync.Mutex // serializes file appends
+
+	// authority provides per-host quality scores used to set frontier
+	// priority on discovered links. nil = uniform 0.5 priority (pre behavior).
+	// Populated via WithAuthority when cfg.AuthorityPriority=true.
+	authority authorityScorer
+}
+
+// LoadDynamicDomains reads previously-promoted domains from dynDomainsFile into
+// the in-memory set. Called at startup. Missing file is fine (no promotions yet).
+func (c *Crawler) LoadDynamicDomains() {
+	if c.dynDomainsFile == "" {
+		return
+	}
+	b, err := os.ReadFile(c.dynDomainsFile)
+	if err != nil {
+		return
+	}
+	n := 0
+	for _, line := range strings.Split(string(b), "\n") {
+		d := strings.TrimSpace(strings.ToLower(line))
+		if d == "" || strings.HasPrefix(d, "#") {
+			continue
+		}
+		c.dynDomains.Store(d, struct{}{})
+		n++
+	}
+	if n > 0 {
+		log.Printf("crawler: loaded %d dynamically-allowed domains from %s", n, c.dynDomainsFile)
+	}
+}
+
+// AddAllowedDomain promotes a domain into the dynamic allowlist (idempotent) and
+// persists it. Safe for concurrent callers. No-op if the feature is disabled.
+func (c *Crawler) AddAllowedDomain(domain string) error {
+	domain = strings.TrimSpace(strings.ToLower(domain))
+	if domain == "" {
+		return nil
+	}
+	if _, exists := c.dynDomains.LoadOrStore(domain, struct{}{}); exists {
+		return nil // already promoted
+	}
+	if c.dynDomainsFile == "" {
+		return nil // in-memory only (feature persistence disabled)
+	}
+	c.dynDomainsMu.Lock()
+	defer c.dynDomainsMu.Unlock()
+	f, err := os.OpenFile(c.dynDomainsFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.WriteString(domain + "\n")
+	log.Printf("crawler: promoted domain to dynamic allowlist: %s", domain)
+	return err
+}
+
+// dynamicAllowed reports whether host matches any promoted domain
+// (exact or dot-boundary subdomain).
+func (c *Crawler) dynamicAllowed(host string) bool {
+	allowed := false
+	c.dynDomains.Range(func(k, _ any) bool {
+		d := k.(string)
+		if host == d || strings.HasSuffix(host, "."+d) {
+			allowed = true
+			return false // stop
+		}
+		return true
+	})
+	return allowed
 }
 
 // markCompletedInline records that the URL's frontier transition
@@ -255,17 +334,20 @@ func newBare(cfg config.Crawler) *Crawler {
 			sitemapCap = n
 		}
 	}
-	return &Crawler{
-		cfg:        cfg,
-		http:       httpClient,
-		robots:     robots,
-		sitemapSem: make(chan struct{}, sitemapCap),
+	c := &Crawler{
+		cfg:            cfg,
+		http:           httpClient,
+		robots:         robots,
+		sitemapSem:     make(chan struct{}, sitemapCap),
+		dynDomainsFile: os.Getenv("COSIFT_DYNAMIC_DOMAINS_FILE"),
 	}
+	c.LoadDynamicDomains()
+	return c
 }
 
 // maybeAutoSitemap kicks off a background sitemap discovery the first
 // time we see a host. Subsequent URLs from the same host return
-// immediately. Disabled when COSIFT_AUTO_SITEMAP=false.
+// immediately. Gated by the crawler.auto_sitemap config field (cosift.json).
 func (c *Crawler) maybeAutoSitemap(ctx context.Context, u *url.URL) {
 	if u == nil || u.Host == "" {
 		return
@@ -373,6 +455,56 @@ func (c *Crawler) WithRouter(route RouteFn, forward ForwardFn) *Crawler {
 	return c
 }
 
+// authorityScorer is a narrow interface so the crawler package does not import
+// internal/authority directly — avoids a potential dep-graph cycle and keeps
+// the package boundary clean. internal/authority.Scorer satisfies this.
+type authorityScorer interface {
+	Score(host string) float64
+}
+
+// WithAuthority attaches a domain-authority scorer. When set, links discovered
+// during crawl are pushed to the frontier with a priority derived from the
+// host's authority score instead of the flat 0.5 default. Chaining-style builder.
+//
+// The scorer is read-only after this call (Score is called on the hot path);
+// callers must finish loading Tranco/Majestic data BEFORE calling WithAuthority.
+func (c *Crawler) WithAuthority(s authorityScorer) *Crawler {
+	c.authority = s
+	return c
+}
+
+// linkPriority returns the frontier priority to assign to a discovered link at
+// the given crawl depth. When authority scoring is enabled, high-authority
+// domains (Tranco top-1k, Majestic high-TF, embedded whitelist) receive
+// priority up to 1.0; unknown domains fall back to 0.5.
+//
+// Depth decay keeps shallow-found links from high-authority domains ahead of
+// deep-found links from any domain:
+//
+//	priority = authority_score / (1 + 0.25 * depth)
+//
+// At depth 0 (seed-equivalent): no decay — top-authority host stays at 1.0.
+// At depth 4: divisor = 2.0, so even a top-authority host is at 0.5, equal to
+// an unknown host at depth 0. Deep discoveries from weak domains converge to
+// ~0.25, staying below any shallow discovery.
+//
+// Floor of 0.05 ensures even low-authority deep links are eventually processed
+// when the frontier drains.
+func (c *Crawler) linkPriority(host string, depth int) float64 {
+	base := 0.5
+	if c.authority != nil {
+		base = c.authority.Score(host)
+	}
+	if depth <= 0 {
+		return base
+	}
+	p := base / (1.0 + 0.25*float64(depth))
+	if p < 0.05 {
+		return 0.05
+	}
+	return p
+}
+
 // Seed pushes a URL onto the persistent frontier at depth 0.
 //
 // `INSERT OR IGNORE` semantics: if the URL is already in the frontier (queued,
@@ -492,6 +624,7 @@ func (c *Crawler) Run(ctx context.Context) error {
 		embedWG.Wait()
 		log.Printf("crawler: embed pool drained — queued=%d done=%d failed=%d dropped=%d",
 			c.embedQueued.Load(), c.embedDone.Load(), c.embedFailed.Load(), c.embedDropped.Load())
+		c.embedQ = nil // nil so FetchAndIndexNow doesn't send to closed channel between restarts
 	}
 	return nil
 }
@@ -769,9 +902,9 @@ func (c *Crawler) runHostSweep(
 	demoter HostFrontierDemoter,
 ) {
 	type hostJudgement struct {
-		host    string
-		dead    bool
-		attempts int32
+		host      string
+		dead      bool
+		attempts  int32
 		successes int32
 	}
 	var verdicts []hostJudgement
@@ -1386,7 +1519,12 @@ func (c *Crawler) enqueueLinks(ctx context.Context, links []string, depth int) {
 			}
 		}
 		// PushFrontier is INSERT OR IGNORE — dedup is at the persistent layer.
-		_ = c.store.PushFrontier(ctx, cand.canon, depth, 0.5)
+		// Priority is authority-weighted when a scorer is attached; otherwise
+		// falls back to flat 0.5 (pre-authority behavior). INSERT OR IGNORE means
+		// we never downgrade the priority of a URL already in the queue — first
+		// writer wins, which is fine since seeds (depth=0, priority≥base) always
+		// arrive before link-discovered copies (depth≥1).
+		_ = c.store.PushFrontier(ctx, cand.canon, depth, c.linkPriority(cand.host, depth))
 		if hostCap > 0 {
 			queuedPerHost[cand.host]++
 		}
@@ -1421,6 +1559,10 @@ func (c *Crawler) allowedDomain(rawURL string) bool {
 			return true
 		}
 	}
+	// Runtime-promoted domains (HN/Reddit harvesters grow this organically).
+	if c.dynamicAllowed(h) {
+		return true
+	}
 	return false
 }
 
diff --git a/internal/crawler/dynamic_allowlist_test.go b/internal/crawler/dynamic_allowlist_test.go
new file mode 100644
index 0000000..a0f7ce4
--- /dev/null
+++ b/internal/crawler/dynamic_allowlist_test.go
@@ -0,0 +1,57 @@
+package crawler
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pilot-protocol/cosift/internal/config"
+)
+
+// TestDynamicAllowlist verifies that AddAllowedDomain promotes a domain so
+// allowedDomain accepts it (and its subdomains), alongside static IncludeDomains,
+// and that promotions persist to the file + survive a reload.
+func TestDynamicAllowlist(t *testing.T) {
+	dynFile := filepath.Join(t.TempDir(), "dyn-domains.txt")
+	t.Setenv("COSIFT_DYNAMIC_DOMAINS_FILE", dynFile)
+
+	cfg := config.Default().Crawler
+	cfg.RespectRobots = false
+	cfg.IncludeDomains = []string{"arxiv.org"} // static allowlist (non-empty)
+	c := newBare(cfg)
+
+	// Static domain allowed; unknown domain rejected.
+	if !c.allowedDomain("https://arxiv.org/abs/1234") {
+		t.Error("static allowlisted arxiv.org should be allowed")
+	}
+	if c.allowedDomain("https://example.com/x") {
+		t.Error("example.com should be rejected before promotion")
+	}
+
+	// Promote example.com.
+	if err := c.AddAllowedDomain("example.com"); err != nil {
+		t.Fatalf("AddAllowedDomain: %v", err)
+	}
+	if !c.allowedDomain("https://example.com/x") {
+		t.Error("example.com should be allowed after promotion")
+	}
+	if !c.allowedDomain("https://blog.example.com/y") {
+		t.Error("subdomain of promoted domain should be allowed (dot-boundary)")
+	}
+	// Adversarial: evilexample.com must NOT match promoted example.com.
+	if c.allowedDomain("https://evilexample.com/z") {
+		t.Error("evilexample.com must not match promoted example.com")
+	}
+
+	// Persisted to file.
+	b, err := os.ReadFile(dynFile)
+	if err != nil || len(b) == 0 {
+		t.Fatalf("promotion not persisted: %v", err)
+	}
+
+	// A fresh crawler loading the same file inherits the promotion.
+	c2 := newBare(cfg)
+	if !c2.allowedDomain("https://example.com/x") {
+		t.Error("reloaded crawler should inherit promoted domain from file")
+	}
+}
diff --git a/internal/crawler/store_iface.go b/internal/crawler/store_iface.go
index 6b637a6..d73189e 100644
--- a/internal/crawler/store_iface.go
+++ b/internal/crawler/store_iface.go
@@ -61,6 +61,15 @@ type LexicalIndexer interface {
 	IndexDocument(ctx context.Context, docID int64, title, text string) error
 }
 
+// BulkLexicalIndexer is the optional fast path for bulk ingest (WET): index
+// BM25 postings WITHOUT the host-partition write, halving write amplification
+// for one-off web hosts that site= queries don't target. *index.PebbleBM25
+// implements it; backends that don't are transparently handled by the caller
+// falling back to IndexDocument.
+type BulkLexicalIndexer interface {
+	IndexDocumentBulk(ctx context.Context, docID int64, title, text string) error
+}
+
 // HostFrontierPurger is the optional surface the in-crawler host sweeper
 // uses to drain dead hosts. Pebble satisfies it; the SQLite legacy path
 // doesn't need it (no auto-sweeper there).
diff --git a/internal/crawler/wet.go b/internal/crawler/wet.go
index 92261f5..c6537e1 100644
--- a/internal/crawler/wet.go
+++ b/internal/crawler/wet.go
@@ -273,7 +273,15 @@ func (c *Crawler) indexWetRecord(ctx context.Context, rec *WetRecord, lexicalOnl
 	if err != nil {
 		return err
 	}
-	if err := c.idx.IndexDocument(ctx, id, title, text); err != nil {
+	// Bulk WET ingest pulls millions of one-off web hosts that site= queries
+	// never target, so use the host-partition-skipping fast path when the
+	// backend offers it — halves posting writes and prevents the import from
+	// saturating IO. Falls back to the normal write if unsupported.
+	if bulk, ok := c.idx.(BulkLexicalIndexer); ok {
+		if err := bulk.IndexDocumentBulk(ctx, id, title, text); err != nil {
+			return err
+		}
+	} else if err := c.idx.IndexDocument(ctx, id, title, text); err != nil {
 		return err
 	}
 	if lexicalOnly {
diff --git a/internal/embed/chat.go b/internal/embed/chat.go
index bcea21b..d20514f 100644
--- a/internal/embed/chat.go
+++ b/internal/embed/chat.go
@@ -100,12 +100,15 @@ func chatMaxTokens() int {
 	return 2048
 }
 
-// chatTemplateKwargs is the per-call extra args dict vLLM forwards into the
-// model's Jinja chat template. Today the only consumer is qwen3/qwen3.5
-// `enable_thinking` toggle — emitting "Thinking Process:" prose mid-answer
-// is a quality + latency regression for /answer. Disable by default; iter
-// 477e. Operators wanting to inspect reasoning can flip COSIFT_THINKING=1.
-func chatTemplateKwargs() map[string]interface{} {
+// chatTemplateKwargs returns the per-call extra args dict vLLM forwards into
+// the model's Jinja chat template. Only sent for Qwen3/Qwen3.5 models —
+// OpenAI and other backends reject unknown fields. Disables the thinking mode
+// by default (quality + latency regression for /answer); COSIFT_THINKING=1
+// re-enables it.
+func chatTemplateKwargs(model string) map[string]interface{} {
+	if !strings.Contains(strings.ToLower(model), "qwen3") {
+		return nil
+	}
 	if os.Getenv("COSIFT_THINKING") == "1" {
 		return nil
 	}
@@ -159,7 +162,7 @@ func (c *OpenAIChatClient) Chat(ctx context.Context, msgs []ChatMsg) (string, er
 	if len(msgs) == 0 {
 		return "", errors.New("chat: no messages")
 	}
-	body, err := json.Marshal(chatReq{Model: c.model, Messages: msgs, Temperature: 0, MaxTokens: chatMaxTokens(), ChatTemplateKwargs: chatTemplateKwargs()})
+	body, err := json.Marshal(chatReq{Model: c.model, Messages: msgs, Temperature: 0, MaxTokens: chatMaxTokens(), ChatTemplateKwargs: chatTemplateKwargs(c.model)})
 	if err != nil {
 		return "", err
 	}
@@ -219,7 +222,7 @@ func (c *OpenAIChatClient) ChatStream(ctx context.Context, msgs []ChatMsg, onChu
 	if len(msgs) == 0 {
 		return "", errors.New("chat: no messages")
 	}
-	body, err := json.Marshal(chatStreamReq{Model: c.model, Messages: msgs, Temperature: 0, Stream: true, MaxTokens: chatMaxTokens(), ChatTemplateKwargs: chatTemplateKwargs()})
+	body, err := json.Marshal(chatStreamReq{Model: c.model, Messages: msgs, Temperature: 0, Stream: true, MaxTokens: chatMaxTokens(), ChatTemplateKwargs: chatTemplateKwargs(c.model)})
 	if err != nil {
 		return "", err
 	}
diff --git a/internal/index/bm25.go b/internal/index/bm25.go
index b41fe34..67319af 100644
--- a/internal/index/bm25.go
+++ b/internal/index/bm25.go
@@ -47,16 +47,6 @@ type Hit struct {
 	URL   string
 	Title string
 	Score float64
-
-	// Snippet is a short window of the document body centered on the earliest
-	// query-term match, populated for top-k hits when text is available.
-	// Empty when computation was skipped (text not indexed, or no terms hit).
-	// replaces BM25 hits' generic body-prefix excerpt with a
-	// query-aware passage. Dense/hybrid hits keep their existing per-passage
-	// Highlight (computed from the embedded passage span); this fills the
-	// equivalent gap for BM25-only hits.
-	Snippet       string
-	SnippetOffset int
 }
 
 // NewBM25 returns a BM25 index backed by the SQLite store. Parameters
diff --git a/internal/index/hnsw.go b/internal/index/hnsw.go
index b38a6ad..2513135 100644
--- a/internal/index/hnsw.go
+++ b/internal/index/hnsw.go
@@ -185,7 +185,7 @@ func (h *HNSW) PQStatus() PQStatus {
 //     over reconstructed (uncompressed) approximation. For unit-normalized
 //     vectors this is monotonic with cosine distance, so HNSW pruning
 //     thresholds stay coherent.
-//   - Raw branch: -dot(q, h.nodes[idx].vec). Identical to the
+//   - Raw branch: -Dot(q, h.nodes[idx].vec). Identical to the
 //     baseline.
 //
 // pqTable is the M*K-element lookup precomputed once per search via
@@ -209,7 +209,7 @@ func (h *HNSW) distanceToNode(q []float32, pqTable []float32, idx int) float64 {
 	if len(h.nodes[idx].vec) == 0 {
 		return math.MaxFloat64 // skip zombie nodes
 	}
-	return -float64(dot(q, h.nodes[idx].vec))
+	return -float64(Dot(q, h.nodes[idx].vec))
 }
 
 // BruteForceTopK performs an exact O(N) scan over every valid node and
@@ -240,7 +240,7 @@ func (h *HNSW) BruteForceTopK(query []float32, k int) []VectorHit {
 		if len(h.nodes[i].vec) == 0 {
 			continue // zombie / partial-persisted
 		}
-		s := dot(q, h.nodes[i].vec)
+		s := Dot(q, h.nodes[i].vec)
 		url := h.nodes[i].url
 		cur, ok := bestByURL[url]
 		if !ok || s > cur.score {
@@ -806,7 +806,7 @@ func (h *HNSW) addBackLink(neighbor, newIdx, lvl int) {
 		if nb < 0 || nb >= len(h.nodes) {
 			continue // drop oob neighbor entries silently
 		}
-		cands = append(cands, candEntry{idx: nb, dist: -dot(nbVec, h.nodes[nb].vec)})
+		cands = append(cands, candEntry{idx: nb, dist: -Dot(nbVec, h.nodes[nb].vec)})
 	}
 	sort.Slice(cands, func(a, b int) bool { return cands[a].dist < cands[b].dist })
 	if len(cands) > mCap {
diff --git a/internal/index/hnsw_writer.go b/internal/index/hnsw_writer.go
deleted file mode 100644
index 53d59e5..0000000
--- a/internal/index/hnsw_writer.go
+++ /dev/null
@@ -1,105 +0,0 @@
-// HNSW write bridge for the Pebble-backed crawler path.
-//
-// The PassageWriter interface lets the crawler stream passage
-// vectors to whatever backend the operator wires. On SQLite, *store.Store
-// satisfies it directly (writes go to the passages table). On Pebble there
-// was no implementation — vector indexing during crawl was silently
-// skipped, which made the Pebble path BM25-only.
-//
-// HNSWWriter bridges the gap: passages flow into an in-memory HNSW graph
-// and get periodically flushed to the PebbleStore via HNSW.Persist.
-// Cheap per-passage cost (one graph insert + one
-// GetDocMeta lookup); the heavy on-disk write is amortized across
-// many inserts.
-
-package index
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	"github.com/pilot-protocol/cosift/internal/store"
-)
-
-// HNSWWriter satisfies the crawler's PassageWriter interface against a
-// PebbleStore + HNSW pair. Passage URLs/titles are resolved from the
-// 'i' family (cheap, no full Document gob decode).
-//
-// Persistence is checkpointed every PersistEvery passages. Setting
-// PersistEvery to 0 (the default if you don't configure it) means
-// "never auto-persist" — callers can call Persist manually at safe
-// points (e.g. crawler shutdown).
-type HNSWWriter struct {
-	hnsw         *HNSW
-	store        *store.PebbleStore
-	persistEvery int
-
-	mu             sync.Mutex
-	sinceLastFlush int
-}
-
-// NewHNSWWriter returns a PassageWriter that adds each incoming passage
-// to the HNSW graph and persists to the PebbleStore every persistEvery
-// passages. persistEvery=0 disables auto-persist (caller drives it via
-// HNSWWriter.Flush).
-func NewHNSWWriter(h *HNSW, ps *store.PebbleStore, persistEvery int) *HNSWWriter {
-	return &HNSWWriter{hnsw: h, store: ps, persistEvery: persistEvery}
-}
-
-// UpsertPassage satisfies crawler.PassageWriter. Resolves the doc's URL +
-// title via PebbleStore.GetDocMeta, adds the passage to HNSW, and
-// triggers a checkpoint Persist when the threshold is reached.
-//
-// store.Passage carries Embedding + DocID + Offset/Length + Model;
-// HNSW.AddPassage needs URL + title for hit-display.
-func (w *HNSWWriter) UpsertPassage(ctx context.Context, p *store.Passage) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	url, title, ok, err := w.store.GetDocMeta(ctx, p.DocID)
-	if err != nil {
-		return fmt.Errorf("lookup doc %d: %w", p.DocID, err)
-	}
-	if !ok {
-		// Doc was deleted between Upsert and Passage write. Skip silently —
-		// no point indexing a vector pointing at nothing.
-		return nil
-	}
-
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.hnsw.AddPassage(url, title, p.Offset, p.Length, p.Embedding)
-	w.sinceLastFlush++
-	if w.persistEvery > 0 && w.sinceLastFlush >= w.persistEvery {
-		w.sinceLastFlush = 0
-		// Persist holds the HNSW RLock; we release our own writer mutex
-		// briefly first? Actually no — w.mu protects sinceLastFlush, not
-		// HNSW. HNSW.Persist takes its own RLock independent of ours.
-		if err := w.hnsw.Persist(ctx, w.store); err != nil {
-			return fmt.Errorf("hnsw persist: %w", err)
-		}
-	}
-	return nil
-}
-
-// MarkURLInvalid zeros out every HNSW node whose url matches. Returns
-// the count zeroed. Lets the crawler tell us "this URL's prior passages
-// are stale" before pushing the new chunk batch. Satisfies the optional
-// crawler.URLInvalidator interface.
-func (w *HNSWWriter) MarkURLInvalid(ctx context.Context, url string) (int, error) {
-	if err := ctx.Err(); err != nil {
-		return 0, err
-	}
-	return w.hnsw.MarkURLPassagesInvalid(url), nil
-}
-
-// Flush forces an immediate Persist of the current HNSW state. Crawler
-// callers should invoke this at shutdown so the last partial batch of
-// in-memory inserts isn't lost on next startup.
-func (w *HNSWWriter) Flush(ctx context.Context) error {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.sinceLastFlush = 0
-	return w.hnsw.Persist(ctx, w.store)
-}
diff --git a/internal/index/hnsw_writer_test.go b/internal/index/hnsw_writer_test.go
deleted file mode 100644
index 56a0922..0000000
--- a/internal/index/hnsw_writer_test.go
+++ /dev/null
@@ -1,126 +0,0 @@
-package index
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/pilot-protocol/cosift/internal/store"
-)
-
-// TestHNSWWriterBridgesPassagesIntoGraph — The PassageWriter
-// satisfies the crawler interface and routes incoming passages into the
-// HNSW graph + the PebbleStore.
-func TestHNSWWriterBridgesPassagesIntoGraph(t *testing.T) {
-	dir := filepath.Join(t.TempDir(), "pebble")
-	ps, err := store.OpenPebble(dir)
-	if err != nil {
-		t.Fatalf("OpenPebble: %v", err)
-	}
-	defer ps.Close()
-
-	// Need a Document in the store so GetDocMeta resolves URL+title.
-	ctx := context.Background()
-	id, err := ps.UpsertDocument(ctx, &store.Document{
-		URL: "https://x/doc1", Title: "Doc One", FetchedAt: time.Now(),
-	})
-	if err != nil {
-		t.Fatalf("upsert: %v", err)
-	}
-
-	h := NewHNSW(4)
-	w := NewHNSWWriter(h, ps, 0) // no auto-flush; test drives Flush manually
-
-	vec := []float32{0.7, 0.1, 0.1, 0.7}
-	if err := w.UpsertPassage(ctx, &store.Passage{
-		DocID: id, Offset: 0, Length: 50, Model: "test", Embedding: vec,
-	}); err != nil {
-		t.Fatalf("upsert passage: %v", err)
-	}
-
-	// HNSW now holds one entry; search returns the right URL.
-	hits := h.Search(ctx, vec, 5)
-	if len(hits) != 1 {
-		t.Fatalf("want 1 hit, got %d", len(hits))
-	}
-	if hits[0].URL != "https://x/doc1" {
-		t.Errorf("hit URL: want https://x/doc1, got %s", hits[0].URL)
-	}
-	if hits[0].Title != "Doc One" {
-		t.Errorf("hit Title: got %q", hits[0].Title)
-	}
-
-	// Persist + reload — confirm the bridge produces a recoverable graph.
-	if err := w.Flush(ctx); err != nil {
-		t.Fatalf("flush: %v", err)
-	}
-	loaded, ok, err := LoadHNSW(ctx, ps)
-	if err != nil || !ok {
-		t.Fatalf("load: ok=%v err=%v", ok, err)
-	}
-	loadedHits := loaded.Search(ctx, vec, 5)
-	if len(loadedHits) != 1 || loadedHits[0].URL != "https://x/doc1" {
-		t.Errorf("post-reload hits: %+v", loadedHits)
-	}
-}
-
-// TestHNSWWriterAutoFlushOnThreshold — when persistEvery is set, the
-// writer triggers Persist automatically after N inserts. Locks in the
-// auto-checkpoint contract so callers don't have to poll Flush manually.
-func TestHNSWWriterAutoFlushOnThreshold(t *testing.T) {
-	dir := filepath.Join(t.TempDir(), "pebble")
-	ps, err := store.OpenPebble(dir)
-	if err != nil {
-		t.Fatalf("OpenPebble: %v", err)
-	}
-	defer ps.Close()
-
-	ctx := context.Background()
-	// Three docs to write three passages for.
-	for i := 1; i <= 3; i++ {
-		_, err := ps.UpsertDocument(ctx, &store.Document{
-			URL:       "https://x/" + string(rune('a'+i-1)),
-			Title:     "doc",
-			FetchedAt: time.Now(),
-		})
-		if err != nil {
-			t.Fatalf("upsert %d: %v", i, err)
-		}
-	}
-
-	h := NewHNSW(4)
-	w := NewHNSWWriter(h, ps, 2) // auto-persist every 2 passages
-
-	vec := []float32{1, 0, 0, 0}
-	for i := int64(1); i <= 3; i++ {
-		if err := w.UpsertPassage(ctx, &store.Passage{
-			DocID: i, Offset: 0, Length: 50, Model: "test", Embedding: vec,
-		}); err != nil {
-			t.Fatalf("passage %d: %v", i, err)
-		}
-	}
-
-	// After 3 passages and persistEvery=2, exactly one auto-persist
-	// happened (after the 2nd). The 3rd is in memory but not yet on disk —
-	// load should see 2 nodes, not 3.
-	loaded, ok, err := LoadHNSW(ctx, ps)
-	if err != nil || !ok {
-		t.Fatalf("load: ok=%v err=%v", ok, err)
-	}
-	// Loaded must have at least the auto-flushed batch; in-memory writer
-	// has all three. After explicit Flush, persisted count catches up.
-	if loaded.Len() < 2 {
-		t.Errorf("auto-flush at threshold 2: want loaded ≥2 nodes, got %d", loaded.Len())
-	}
-	if err := w.Flush(ctx); err != nil {
-		t.Fatalf("manual flush: %v", err)
-	}
-	loaded2, ok, err := LoadHNSW(ctx, ps)
-	if err != nil || !ok {
-		t.Fatalf("re-load: ok=%v err=%v", ok, err)
-	}
-	if loaded2.Len() != 3 {
-		t.Errorf("post-flush count: want 3, got %d", loaded2.Len())
-	}
-}
diff --git a/internal/index/host_partition_test.go b/internal/index/host_partition_test.go
new file mode 100644
index 0000000..e1caf26
--- /dev/null
+++ b/internal/index/host_partition_test.go
@@ -0,0 +1,159 @@
+package index
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// indexHostDocs upserts + indexes docs across two hosts. When the
+// COSIFT_HOST_PARTITION env is set by the caller before this runs, the 'P'
+// family is populated on index; otherwise the caller backfills.
+func indexHostDocs(t *testing.T, ps *store.PebbleStore, idx *PebbleBM25) {
+	t.Helper()
+	ctx := context.Background()
+	docs := []struct {
+		url, domain, title, text string
+	}{
+		{"https://site-a.com/install", "site-a.com", "Install guide", "install the daemon and run the cli tool to register"},
+		{"https://site-a.com/trust", "site-a.com", "Trust model", "trust model overlay network peers handshake encryption"},
+		{"https://site-a.com/cooking", "site-a.com", "Pasta recipe", "boil water salt pasta drain sauce al dente"},
+		{"https://other-b.com/install", "other-b.com", "Install other", "install the daemon and run the cli tool to register agents"},
+		{"https://other-b.com/trust", "other-b.com", "Trust other", "trust model overlay network peers handshake encryption setup"},
+	}
+	for _, d := range docs {
+		id, err := ps.UpsertDocument(ctx, &store.Document{
+			URL: d.url, Domain: d.domain, Title: d.title, Text: d.text, FetchedAt: time.Now(),
+		})
+		if err != nil {
+			t.Fatalf("upsert: %v", err)
+		}
+		if err := idx.IndexDocument(ctx, id, d.title, d.text); err != nil {
+			t.Fatalf("index: %v", err)
+		}
+	}
+}
+
+// TestSearchInHostScopesToHost: with the partition written at index time,
+// SearchInHost returns ONLY docs from the requested host, and its result set
+// equals Search filtered to that host.
+func TestSearchInHostScopesToHost(t *testing.T) {
+	t.Setenv("COSIFT_HOST_PARTITION", "1")
+	ps, idx := newPebbleBM25(t)
+	ctx := context.Background()
+	indexHostDocs(t, ps, idx)
+
+	hits, err := idx.SearchInHost(ctx, "install daemon cli register", "site-a.com", 10)
+	if err != nil {
+		t.Fatalf("SearchInHost: %v", err)
+	}
+	if len(hits) == 0 {
+		t.Fatal("no hits from SearchInHost")
+	}
+	for _, h := range hits {
+		if !contains(h.URL, "site-a.com") {
+			t.Errorf("SearchInHost leaked a non-site-a host: %s", h.URL)
+		}
+	}
+	// Top hit should be the install page, not the trust or cooking page.
+	if !contains(hits[0].URL, "/install") {
+		t.Errorf("top hit: got %s, want a /install page", hits[0].URL)
+	}
+}
+
+// TestSearchInHostEmptyPartition: a host with no partition returns the
+// sentinel so callers can fall back.
+func TestSearchInHostEmptyPartition(t *testing.T) {
+	t.Setenv("COSIFT_HOST_PARTITION", "1")
+	ps, idx := newPebbleBM25(t)
+	indexHostDocs(t, ps, idx)
+	_, err := idx.SearchInHost(context.Background(), "anything", "never-indexed.example", 10)
+	if !errors.Is(err, ErrHostPartitionEmpty) {
+		t.Fatalf("want ErrHostPartitionEmpty, got %v", err)
+	}
+}
+
+// TestBackfillHostPostings: index WITHOUT the write flag (no 'P' family),
+// then backfill, then SearchInHost works and matches the at-index-time path.
+func TestBackfillHostPostings(t *testing.T) {
+	ps, idx := newPebbleBM25(t) // flag NOT set → no partition on index
+	ctx := context.Background()
+	indexHostDocs(t, ps, idx)
+
+	// Before backfill: partition is empty.
+	if _, err := idx.SearchInHost(ctx, "install", "site-a.com", 10); !errors.Is(err, ErrHostPartitionEmpty) {
+		t.Fatalf("pre-backfill want ErrHostPartitionEmpty, got %v", err)
+	}
+
+	written, err := ps.BackfillHostPostings(ctx, "", nil)
+	if err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	if written == 0 {
+		t.Fatal("backfill wrote 0 postings")
+	}
+
+	hits, err := idx.SearchInHost(ctx, "install daemon cli register", "site-a.com", 10)
+	if err != nil {
+		t.Fatalf("post-backfill SearchInHost: %v", err)
+	}
+	if len(hits) == 0 {
+		t.Fatal("post-backfill: no hits")
+	}
+	for _, h := range hits {
+		if !contains(h.URL, "site-a.com") {
+			t.Errorf("post-backfill leaked host: %s", h.URL)
+		}
+	}
+}
+
+// TestIndexDocumentBulkSkipsHostPartition: even with the partition write flag
+// ON, docs indexed via IndexDocumentBulk must NOT land in the 'P' family, so
+// SearchInHost can't find them (they fall back to the boost path). This is the
+// WET-ingest write-amplification fix.
+func TestIndexDocumentBulkSkipsHostPartition(t *testing.T) {
+	t.Setenv("COSIFT_HOST_PARTITION", "1")
+	ps, idx := newPebbleBM25(t)
+	ctx := context.Background()
+
+	// Normal index → partitioned.
+	id1, err := ps.UpsertDocument(ctx, &store.Document{
+		URL: "https://normal.com/p", Domain: "normal.com", Title: "Normal", Text: "alpha beta gamma", FetchedAt: time.Now(),
+	})
+	if err != nil {
+		t.Fatalf("upsert normal: %v", err)
+	}
+	if err := idx.IndexDocument(ctx, id1, "Normal", "alpha beta gamma"); err != nil {
+		t.Fatalf("index normal: %v", err)
+	}
+	// Bulk index → must NOT be partitioned.
+	id2, err := ps.UpsertDocument(ctx, &store.Document{
+		URL: "https://bulk.com/p", Domain: "bulk.com", Title: "Bulk", Text: "alpha beta gamma", FetchedAt: time.Now(),
+	})
+	if err != nil {
+		t.Fatalf("upsert bulk: %v", err)
+	}
+	if err := idx.IndexDocumentBulk(ctx, id2, "Bulk", "alpha beta gamma"); err != nil {
+		t.Fatalf("index bulk: %v", err)
+	}
+
+	// normal.com has a partition (found via SearchInHost).
+	hits, err := idx.SearchInHost(ctx, "alpha beta gamma", "normal.com", 10)
+	if err != nil {
+		t.Fatalf("SearchInHost normal: %v", err)
+	}
+	if len(hits) == 0 {
+		t.Error("normal.com should be in the host partition but SearchInHost found nothing")
+	}
+	// bulk.com has NO partition → SearchInHost returns the empty sentinel.
+	_, err = idx.SearchInHost(ctx, "alpha beta gamma", "bulk.com", 10)
+	if !errors.Is(err, ErrHostPartitionEmpty) {
+		t.Errorf("bulk.com should NOT be partitioned; want ErrHostPartitionEmpty, got %v", err)
+	}
+}
+
+func contains(s, sub string) bool { return strings.Contains(s, sub) }
diff --git a/internal/index/pebble_bm25.go b/internal/index/pebble_bm25.go
index cf618d6..9772d45 100644
--- a/internal/index/pebble_bm25.go
+++ b/internal/index/pebble_bm25.go
@@ -13,6 +13,7 @@ package index
 
 import (
 	"context"
+	"errors"
 	"math"
 	"net/url"
 	"os"
@@ -53,6 +54,12 @@ type PebbleBM25 struct {
 	// in the long tail without requiring a separate ranking pass. Nil
 	// = passthrough (multiplier = 1.0 for every hit).
 	authority *authority.Scorer
+
+	// boostIDs maps docID → score multiplier. Applied post-scoring so
+	// site-filtered queries can surface small-site docs that BM25 ranks
+	// below top-k globally. Created via WithBoost (returns a shallow copy);
+	// nil = no boost.
+	boostIDs map[int64]float64
 }
 
 // NewPebbleBM25 returns a search-only handle over the given PebbleStore.
@@ -70,6 +77,16 @@ func (b *PebbleBM25) WithAuthority(a *authority.Scorer) *PebbleBM25 {
 	return b
 }
 
+// WithBoost returns a shallow copy of b with the given docID→multiplier map
+// applied post-scoring. Use this for site= queries: enumerate the site's
+// docIDs, pass a 50× multiplier, and the site's docs will always appear in
+// top-k even when they rank outside the global top-k on raw BM25 score.
+func (b *PebbleBM25) WithBoost(ids map[int64]float64) *PebbleBM25 {
+	c := *b
+	c.boostIDs = ids
+	return &c
+}
+
 // WithBM25Params overrides the default k1 / b for this instance. Values ≤0
 // are ignored (keep the package-constant default for that knob). Chainable.
 func (b *PebbleBM25) WithBM25Params(k1, blen float64) *PebbleBM25 {
@@ -97,6 +114,12 @@ func (b *PebbleBM25) IndexDocument(ctx context.Context, docID int64, title, text
 	return b.store.IndexDocument(ctx, docID, title, text, Tokenize, TitleBoost)
 }
 
+// IndexDocumentBulk is IndexDocument without the host-partition write — the
+// fast path for bulk WET ingest. See PebbleStore.IndexDocumentBulk.
+func (b *PebbleBM25) IndexDocumentBulk(ctx context.Context, docID int64, title, text string) error {
+	return b.store.IndexDocumentBulk(ctx, docID, title, text, Tokenize, TitleBoost)
+}
+
 // Search returns the top-k hits for q from the Pebble-backed index.
 // Algorithm matches BM25.Search verbatim (k1, b, title-boost, phrase filter);
 // only the storage backend differs.
@@ -186,6 +209,21 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error)
 		}
 	}
 	for i, c := range active {
+		// Pre-scan MaxScore check: decide whether term i (and everything
+		// after it, since active is sorted descending by IDF) can still push
+		// an unseen doc into top-K BEFORE paying to scan term i's postings.
+		// remainingMax here is the sum of max contributions of terms i..end
+		// (idf*(k1+1) each). A doc not yet in `scores` can gain at most
+		// remainingMax from the un-scanned terms; if that's below the current
+		// K-th best score, scanning i..end can only reorder within top-K —
+		// which the reranker fixes — so stop. This catches the common-term
+		// last-term full scan the old post-scan i==len-1 guard always paid.
+		if maxScoreEnabled && i > 0 && len(scores) >= k {
+			theta := kthLargest(scores, k)
+			if remainingMax < theta {
+				break
+			}
+		}
 		err := b.store.IteratePostings(ctx, c.info.ID, func(p store.PostingEntry) bool {
 			// docLen is inline in the posting value — no separate
 			// GetDocLen call. Removed ~25k Pebble Gets per query at N=10k.
@@ -202,17 +240,36 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error)
 		if err != nil {
 			return nil, err
 		}
-		if !maxScoreEnabled || i == len(active)-1 || len(scores) < k {
-			continue
+		// Consume term i's max contribution now that it's been scanned, so
+		// remainingMax entering the next iteration covers only un-scanned terms.
+		if maxScoreEnabled {
+			remainingMax -= c.idf * (b.k1 + 1.0)
 		}
-		remainingMax -= c.idf * (b.k1 + 1.0)
-		// Find current K-th best score (threshold for top-K membership).
-		theta := kthLargest(scores, k)
-		// No unseen doc can accumulate more than remainingMax. If that's
-		// below the current threshold, remaining terms can only shuffle
-		// in-top-K ranking — which the reranker fixes — so stop.
-		if remainingMax < theta {
-			break
+	}
+
+	// Apply per-doc boosts (e.g. site= queries) before sorting. Boosted docs
+	// already present in `scores` (i.e. they matched ≥1 query term) get the
+	// full multiplier. Boosted docs with zero term overlap are absent from
+	// `scores`; for small boost sets we seed them with a tiny base score
+	// (boostSeedBase) before multiplying so they still enter the candidate
+	// pool — landing at boostSeedBase*mult (e.g. 0.05), below any genuine
+	// single-term BM25 match but enough for the reranker to judge them. This
+	// also closes the MaxScore early-termination gap: a boosted doc whose only
+	// matching posting list was skipped still gets seeded and surfaced.
+	//
+	// Seeding is gated by boostSeedMaxIDs: large boost sets (big sites) would
+	// inject a GetDocMeta per zero-overlap doc and flood the pool, so above
+	// the threshold we keep the old present-only behavior.
+	const (
+		boostSeedBase   = 0.001
+		boostSeedMaxIDs = 256
+	)
+	seedZeroOverlap := len(b.boostIDs) > 0 && len(b.boostIDs) <= boostSeedMaxIDs
+	for id, mult := range b.boostIDs {
+		if cur, ok := scores[id]; ok {
+			scores[id] = cur * mult
+		} else if seedZeroOverlap {
+			scores[id] = boostSeedBase * mult
 		}
 	}
 
@@ -258,6 +315,116 @@ func (b *PebbleBM25) Search(ctx context.Context, q string, k int) ([]Hit, error)
 	return hits, nil
 }
 
+// ErrHostPartitionEmpty signals that the host has no 'P'-family partition
+// (never indexed under COSIFT_HOST_PARTITION, or backfill pending). Callers
+// fall back to the global Search + post-filter path.
+var ErrHostPartitionEmpty = errors.New("pebble-bm25: host partition empty")
+
+// SearchInHost is Search scoped to a single host via the 'P' posting
+// partition: it scans only that host's posting lists (O(site_docs)) instead of
+// the global lists (O(corpus)). IDF and avgDocLen stay GLOBAL — a host-local
+// IDF would over-weight terms common across the web but rare on the site, and
+// no per-host DocFreq exists. Returns ErrHostPartitionEmpty when the host has
+// no partition so the caller can fall back. MaxScore early-termination is
+// omitted: host lists are tiny, so a full lossless scan is cheap.
+func (b *PebbleBM25) SearchInHost(ctx context.Context, q, host string, k int) ([]Hit, error) {
+	hostID, ok, err := b.store.GetHostID(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, ErrHostPartitionEmpty
+	}
+
+	searchQ, phrases := parsePhrases(q)
+	tokens := Tokenize(searchQ)
+	if len(tokens) == 0 {
+		return nil, nil
+	}
+	totalDocs, avgDocLen, err := b.corpusStats(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if totalDocs == 0 {
+		return nil, nil
+	}
+	if avgDocLen < 1 {
+		avgDocLen = 1
+	}
+
+	scores := make(map[int64]float64)
+	minIDF := bm25MinIDF()
+	type termPost struct {
+		idf  float64
+		info store.TermInfo
+	}
+	uniqTerms := dedupeStrings(tokens)
+	candidates := make([]termPost, 0, len(uniqTerms))
+	for _, term := range uniqTerms {
+		info, ok, err := b.store.GetTermInfo(ctx, term)
+		if err != nil {
+			return nil, err
+		}
+		if !ok || info.DocFreq == 0 {
+			continue
+		}
+		idf := math.Log(1.0 + (float64(totalDocs)-float64(info.DocFreq)+0.5)/(float64(info.DocFreq)+0.5))
+		candidates = append(candidates, termPost{idf: idf, info: info})
+	}
+	if len(candidates) == 0 {
+		return nil, nil
+	}
+	sort.Slice(candidates, func(i, j int) bool { return candidates[i].idf > candidates[j].idf })
+	active := candidates[:0]
+	for _, c := range candidates {
+		if c.idf >= minIDF {
+			active = append(active, c)
+		}
+	}
+	if len(active) == 0 {
+		active = candidates[:1]
+	}
+
+	for _, c := range active {
+		idf := c.idf
+		if err := b.store.IterateHostPostings(ctx, hostID, c.info.ID, func(p store.PostingEntry) bool {
+			docLen := p.DocLen
+			if docLen <= 0 {
+				docLen = 1
+			}
+			tfF := float64(p.TF)
+			lenNorm := 1.0 - b.b + b.b*(float64(docLen)/avgDocLen)
+			scores[p.DocID] += idf * ((tfF * (b.k1 + 1.0)) / (tfF + b.k1*lenNorm))
+			return true
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	// Hit resolution mirrors Search: cheap 'i' side-blob, authority multiplier,
+	// phrase filter, sort, top-k. No boostIDs path — the partition already
+	// guarantees host membership, so the 50× site boost is unnecessary here.
+	hits := make([]Hit, 0, len(scores))
+	for docID, sc := range scores {
+		url, title, ok, err := b.store.GetDocMeta(ctx, docID)
+		if err != nil || !ok || url == "" {
+			continue
+		}
+		if b.authority != nil {
+			sc *= b.authority.Multiplier(hostFromURL(url))
+		}
+		hits = append(hits, Hit{DocID: docID, URL: url, Title: title, Score: sc})
+	}
+	sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score })
+
+	if len(phrases) > 0 {
+		hits = b.filterByPhrasesPebble(ctx, hits, phrases, k)
+	} else if k > 0 && len(hits) > k {
+		hits = hits[:k]
+	}
+	return hits, nil
+}
+
 // corpusStats returns (indexedDocs, avgDocLen). Reads the
 // running (sum_doc_len, indexed_docs) counters maintained by
 // PebbleStore.IndexDocument. O(1) per query; replaces the per-query
diff --git a/internal/index/vector.go b/internal/index/vector.go
index 3e3dbaf..5f01c47 100644
--- a/internal/index/vector.go
+++ b/internal/index/vector.go
@@ -96,7 +96,7 @@ func (v *VectorIndex) Search(_ context.Context, query []float32, k int) []Vector
 	}
 	bestByURL := make(map[string]entry, len(v.docs))
 	for i := range v.docs {
-		s := dot(q, v.docs[i].vec)
+		s := Dot(q, v.docs[i].vec)
 		url := v.docs[i].url
 		cur, ok := bestByURL[url]
 		if !ok || s > cur.score {
@@ -229,7 +229,7 @@ func (v *VectorIndex) SearchMMR(ctx context.Context, query []float32, k int, lam
 		maxSimToSel = maxSimToSel[:last]
 		// Update maxSimToSel for the now-shorter remaining slice using picked.vec.
 		for i := range remaining {
-			sim := dot(picked.vec, remaining[i].vec)
+			sim := Dot(picked.vec, remaining[i].vec)
 			if sim > maxSimToSel[i] {
 				maxSimToSel[i] = sim
 			}
@@ -256,12 +256,15 @@ func (v *VectorIndex) Len() int {
 	return len(v.docs)
 }
 
-func dot(a, b []float32) float32 {
+// Dot returns the dot product of two equal-length float32 slices, or 0 on
+// length mismatch. For unit-normalized vectors (as HNSW stores them) this is
+// equivalent to cosine similarity. Hand-unrolled by 4 — modest speed up
+// without going SIMD.
+func Dot(a, b []float32) float32 {
 	if len(a) != len(b) {
 		return 0
 	}
 	var s float32
-	// Hand-unrolled by 4 — modest speed up without going SIMD.
 	i := 0
 	for ; i <= len(a)-4; i += 4 {
 		s += a[i]*b[i] + a[i+1]*b[i+1] + a[i+2]*b[i+2] + a[i+3]*b[i+3]
diff --git a/internal/server/http.go b/internal/server/http.go
index e4f4e5c..86fe674 100644
--- a/internal/server/http.go
+++ b/internal/server/http.go
@@ -959,11 +959,10 @@ func (s *Server) handleAdminReembed(w http.ResponseWriter, r *http.Request) {
 
 // SearchResponse is the on-the-wire shape of /search results.
 type SearchResponse struct {
-	Query string         `json:"query"`
-	K     int            `json:"k"`
-	Hits  []SearchHit    `json:"hits"`
-	Took  string         `json:"took"`
-	Meta  map[string]any `json:"meta,omitempty"`
+	Query string      `json:"query"`
+	K     int         `json:"k"`
+	Hits  []SearchHit `json:"hits"`
+	Took  string      `json:"took"`
 }
 
 // SearchHit is one result row in /search responses: URL, title, fused score,
diff --git a/internal/sla/sla.go b/internal/sla/sla.go
index c30247f..59d07f8 100644
--- a/internal/sla/sla.go
+++ b/internal/sla/sla.go
@@ -26,7 +26,6 @@ package sla
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"os"
 	"sort"
 	"sync"
@@ -51,15 +50,14 @@ type Sample struct {
 
 // Violation is what the evaluator records when a target is breached.
 type Violation struct {
-	Endpoint   string        `json:"endpoint"`
-	At         time.Time     `json:"at"`
-	Metric     string        `json:"metric"`   // "p95", "p99", "error_rate"
-	Observed   float64       `json:"observed"` // ms for latency, ratio for error_rate
-	Target     float64       `json:"target"`
-	WindowSec  int           `json:"window_sec"`
-	SampleSize int           `json:"sample_size"`
-	Severity   string        `json:"severity"` // "warn" / "critical"
-	Latency    time.Duration `json:"-"`
+	Endpoint   string    `json:"endpoint"`
+	At         time.Time `json:"at"`
+	Metric     string    `json:"metric"`   // "p95", "p99", "error_rate"
+	Observed   float64   `json:"observed"` // ms for latency, ratio for error_rate
+	Target     float64   `json:"target"`
+	WindowSec  int       `json:"window_sec"`
+	SampleSize int       `json:"sample_size"`
+	Severity   string    `json:"severity"` // "warn" / "critical"
 }
 
 // Monitor is the in-process SLA tracker.
@@ -124,9 +122,7 @@ func New(targets []Target, window time.Duration, logPath string) (*Monitor, erro
 		m.buckets[t.Endpoint] = newRingBuffer(1024)
 	}
 	if logPath != "" {
-		if err := m.rotateLog(); err != nil {
-			return nil, fmt.Errorf("sla: rotate log: %w", err)
-		}
+		m.rotateLog()
 	}
 	return m, nil
 }
@@ -308,15 +304,14 @@ func (m *Monitor) persist(v Violation) {
 
 // rotateLog moves any pre-existing log aside with a timestamp suffix
 // so the live log file stays bounded.
-func (m *Monitor) rotateLog() error {
+func (m *Monitor) rotateLog() {
 	if m.logPath == "" {
-		return nil
+		return
 	}
 	if _, err := os.Stat(m.logPath); err == nil {
 		ts := time.Now().Format("20060102T150405")
 		_ = os.Rename(m.logPath, m.logPath+"."+ts)
 	}
-	return nil
 }
 
 // Snapshot returns the latest evaluation state.
diff --git a/internal/store/pebble.go b/internal/store/pebble.go
index b758bb4..e41d602 100644
--- a/internal/store/pebble.go
+++ b/internal/store/pebble.go
@@ -107,6 +107,13 @@ const (
 	//                       reverse index from docID to its term IDs, so
 	//                       re-indexing the same doc can delete orphaned postings
 	//                       for terms that vanished from the new content.
+	famHostTerm byte = 'T' // 'T' + host-string → uint64-be(hostID)
+	//                          stable per-host id, allocated like termID.
+	famHostPosting byte = 'P' // 'P' + hostID(8) + termID(8) + docID(8) → (tf, docLen)
+	//                          host-partitioned postings: a site= query scans
+	//                          only one host's lists (O(site_docs)) instead of
+	//                          the global 'p' family (O(corpus)). Written
+	//                          alongside 'p' when COSIFT_HOST_PARTITION=1.
 	famFrontier byte = 'f' // 'f' + 'u' + url → packed frontier entry
 	//                       Status byte +
 	//                       depth + priority + enqueued_at + attempts + host
@@ -443,20 +450,19 @@ func frontierStatusIndexKeyLane(sub, lane byte, host, url string) []byte {
 	return k
 }
 
-// frontierStatusIndexHostLane extracts (host, lane) from a lane-format
-// secondary index key. Returns ("", 0) if the key shape is wrong.
-func frontierStatusIndexHostLane(key []byte) (host string, lane byte) {
+// frontierStatusIndexHostLane extracts the host from a lane-format secondary
+// index key. Returns "" if the key shape is wrong.
+func frontierStatusIndexHostLane(key []byte) (host string) {
 	if len(key) < 4 || key[0] != famFrontier {
-		return "", 0
+		return ""
 	}
-	lane = key[2]
 	rest := key[3:]
 	for i, b := range rest {
 		if b == 0x00 {
-			return string(rest[:i]), lane
+			return string(rest[:i])
 		}
 	}
-	return "", 0
+	return ""
 }
 
 // frontierLanePrefix is the lower bound for an iteration scoped to one
@@ -796,6 +802,40 @@ func postingPrefixUpper(termID int64) []byte {
 	return k
 }
 
+func hostTermKey(host string) []byte {
+	k := make([]byte, 1+len(host))
+	k[0] = famHostTerm
+	copy(k[1:], host)
+	return k
+}
+
+func hostPostingKey(hostID, termID, docID int64) []byte {
+	k := make([]byte, 1+8+8+8)
+	k[0] = famHostPosting
+	binary.BigEndian.PutUint64(k[1:], uint64(hostID))
+	binary.BigEndian.PutUint64(k[9:], uint64(termID))
+	binary.BigEndian.PutUint64(k[17:], uint64(docID))
+	return k
+}
+
+func hostPostingPrefix(hostID, termID int64) []byte {
+	k := make([]byte, 1+8+8)
+	k[0] = famHostPosting
+	binary.BigEndian.PutUint64(k[1:], uint64(hostID))
+	binary.BigEndian.PutUint64(k[9:], uint64(termID))
+	return k
+}
+
+// hostPostingPrefixUpper is the exclusive upper bound for a (hostID, termID)
+// scan: same hostID, termID+1.
+func hostPostingPrefixUpper(hostID, termID int64) []byte {
+	k := make([]byte, 1+8+8)
+	k[0] = famHostPosting
+	binary.BigEndian.PutUint64(k[1:], uint64(hostID))
+	binary.BigEndian.PutUint64(k[9:], uint64(termID+1))
+	return k
+}
+
 func docLenKey(docID int64) []byte {
 	k := make([]byte, 1+8)
 	k[0] = famDocLen
@@ -1123,6 +1163,20 @@ type TermInfo struct {
 // owns the postings layout. Title tokens contribute the title
 // boost (3x TF). doc_len records raw token count for BM25 length norm.
 func (p *PebbleStore) IndexDocument(ctx context.Context, docID int64, title, text string, tokenize func(string) []string, titleBoost int) error {
+	return p.indexDocumentImpl(ctx, docID, title, text, tokenize, titleBoost, true)
+}
+
+// IndexDocumentBulk is IndexDocument WITHOUT the host-partition ('P' family)
+// write, for bulk ingest paths (e.g. CommonCrawl WET) that index millions of
+// one-off web hosts site= queries never target. Skipping the partition halves
+// the per-doc posting writes — the doubling otherwise saturates IO and starves
+// the box during large imports. Those hosts still resolve via the 50× boost
+// fallback if ever site-scoped.
+func (p *PebbleStore) IndexDocumentBulk(ctx context.Context, docID int64, title, text string, tokenize func(string) []string, titleBoost int) error {
+	return p.indexDocumentImpl(ctx, docID, title, text, tokenize, titleBoost, false)
+}
+
+func (p *PebbleStore) indexDocumentImpl(ctx context.Context, docID int64, title, text string, tokenize func(string) []string, titleBoost int, writeHostPart bool) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -1210,6 +1264,19 @@ func (p *PebbleStore) IndexDocument(ctx context.Context, docID int64, title, tex
 	}
 	newSet := make(map[int64]struct{}, len(tf))
 
+	// Host-partition write (opt-in). IndexDocument lacks the domain in its
+	// signature, so when enabled we read it from the doc record (cheap Get,
+	// non-hot path) and mirror every posting into the 'P' family keyed by
+	// hostID. hostID==0 (flag off, empty domain, or doc missing) skips it.
+	var hostID int64
+	if writeHostPart && HostPartitionEnabled() {
+		if d, derr := p.GetDocByID(ctx, docID); derr == nil && d != nil && d.Domain != "" {
+			if hid, herr := p.getOrAllocHostIDLocked(d.Domain); herr == nil {
+				hostID = hid
+			}
+		}
+	}
+
 	for term, freq := range tf {
 		info, ok, err := p.getTermInfoLocked(term)
 		if err != nil {
@@ -1236,6 +1303,11 @@ func (p *PebbleStore) IndexDocument(ctx context.Context, docID int64, title, tex
 		if err := batch.Set(postingKey(info.ID, docID), pvBuf, nil); err != nil {
 			return err
 		}
+		if hostID != 0 {
+			if err := batch.Set(hostPostingKey(hostID, info.ID, docID), pvBuf, nil); err != nil {
+				return err
+			}
+		}
 	}
 
 	// delete postings for terms in oldSet \ newSet (vanished from
@@ -1250,6 +1322,11 @@ func (p *PebbleStore) IndexDocument(ctx context.Context, docID int64, title, tex
 		if err := batch.Delete(postingKey(oldID, docID), nil); err != nil {
 			return err
 		}
+		if hostID != 0 {
+			if err := batch.Delete(hostPostingKey(hostID, oldID, docID), nil); err != nil {
+				return err
+			}
+		}
 	}
 
 	// Write the new term-ID set for this doc.
@@ -1414,6 +1491,16 @@ func (p *PebbleStore) WriteCrawlResult(
 		}
 		newSet := make(map[int64]struct{}, len(tf))
 
+		// Host-partition write (opt-in): d.Domain is in hand here, so mirror
+		// every posting into the 'P' family keyed by hostID. hostID==0 (flag
+		// off or empty domain) skips it.
+		var hostID int64
+		if HostPartitionEnabled() && d.Domain != "" {
+			if hid, herr := p.getOrAllocHostIDLocked(d.Domain); herr == nil {
+				hostID = hid
+			}
+		}
+
 		for term, freq := range tf {
 			info, ok, err := p.getTermInfoLocked(term)
 			if err != nil {
@@ -1435,6 +1522,11 @@ func (p *PebbleStore) WriteCrawlResult(
 			if err := batch.Set(postingKey(info.ID, id), pvBuf, nil); err != nil {
 				return 0, err
 			}
+			if hostID != 0 {
+				if err := batch.Set(hostPostingKey(hostID, info.ID, id), pvBuf, nil); err != nil {
+					return 0, err
+				}
+			}
 		}
 		for oldID := range oldSet {
 			if _, stillPresent := newSet[oldID]; stillPresent {
@@ -1443,6 +1535,11 @@ func (p *PebbleStore) WriteCrawlResult(
 			if err := batch.Delete(postingKey(oldID, id), nil); err != nil {
 				return 0, err
 			}
+			if hostID != 0 {
+				if err := batch.Delete(hostPostingKey(hostID, oldID, id), nil); err != nil {
+					return 0, err
+				}
+			}
 		}
 		newIDs := make([]int64, 0, len(newSet))
 		for tid := range newSet {
@@ -1799,8 +1896,7 @@ func decodeFrontierIndexHost(key []byte) string {
 		return ""
 	}
 	if key[2] < laneCount {
-		h, _ := frontierStatusIndexHostLane(key)
-		return h
+		return frontierStatusIndexHostLane(key)
 	}
 	return frontierStatusIndexHost(key)
 }
@@ -1828,7 +1924,7 @@ func (p *PebbleStore) scanLaneForFreeHost(lane byte, inflightHosts map[string]st
 
 	scan := func(start func() bool) (found bool) {
 		for valid := start(); valid; valid = qIt.Next() {
-			h, _ := frontierStatusIndexHostLane(qIt.Key())
+			h := frontierStatusIndexHostLane(qIt.Key())
 			if h == "" {
 				continue
 			}
@@ -2229,6 +2325,48 @@ func (p *PebbleStore) IterDocsLite(ctx context.Context, fn func(docID int64, url
 	return nil
 }
 
+// IterHostDocIDs iterates over all docIDs whose stored Domain equals host
+// (exact match, case-folded). Uses the famHost prefix index — O(site_docs)
+// rather than a full corpus scan. Caller returns false from fn to stop early.
+func (p *PebbleStore) IterHostDocIDs(ctx context.Context, host string, fn func(docID int64) bool) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	host = strings.ToLower(strings.TrimSpace(host))
+	if host == "" {
+		return nil
+	}
+	// Key layout: [famHost][host bytes][0x00][docID uint64-be]
+	lower := make([]byte, 1+len(host)+1)
+	lower[0] = famHost
+	copy(lower[1:], host)
+	lower[len(lower)-1] = 0x00
+	upper := make([]byte, 1+len(host)+1)
+	upper[0] = famHost
+	copy(upper[1:], host)
+	upper[len(upper)-1] = 0x01 // one past 0x00 → terminates the range
+	it, err := p.db.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper})
+	if err != nil {
+		return err
+	}
+	defer it.Close()
+	prefixLen := len(lower)
+	for valid := it.First(); valid; valid = it.Next() {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		key := it.Key()
+		if len(key) < prefixLen+8 {
+			continue
+		}
+		id := int64(binary.BigEndian.Uint64(key[prefixLen:]))
+		if !fn(id) {
+			return nil
+		}
+	}
+	return nil
+}
+
 // IterDocMeta is like IterDocsLite but also yields the document title from
 // the cheap 'i' side-blob (no full gob decode). Used by content sweeps
 // (e.g. purge-adult) that classify on URL + title across the whole corpus.
@@ -2895,7 +3033,7 @@ func (p *PebbleStore) PurgeStaleInFlight(ctx context.Context) (int, error) {
 		}
 		var host, url string
 		if k[2] < laneCount {
-			host, _ = frontierStatusIndexHostLane(k)
+			host = frontierStatusIndexHostLane(k)
 			urlOffset := 3 + len(host) + 1
 			if urlOffset > len(k) {
 				continue
@@ -3425,6 +3563,319 @@ func (p *PebbleStore) nextTermID() int64 {
 	return next
 }
 
+// HostPartitionEnabled reports whether host-partitioned postings should be
+// written on index. Gated by COSIFT_HOST_PARTITION=1 so the ~2x posting
+// write-amplification is opt-in until the backfill has run.
+func HostPartitionEnabled() bool {
+	return os.Getenv("COSIFT_HOST_PARTITION") == "1"
+}
+
+// getOrAllocHostIDLocked returns a stable hostID for host, allocating one on
+// first sight. Called under p.mu; the counter Set mirrors nextTermID's pattern
+// (persist immediately, race held off by p.mu). Returns 0 for empty host.
+func (p *PebbleStore) getOrAllocHostIDLocked(host string) (int64, error) {
+	if host == "" {
+		return 0, nil
+	}
+	val, closer, err := p.db.Get(hostTermKey(host))
+	if err == nil {
+		var id int64
+		if len(val) == 8 {
+			id = int64(binary.BigEndian.Uint64(val))
+		}
+		_ = closer.Close()
+		if id != 0 {
+			return id, nil
+		}
+	} else if !errors.Is(err, pebble.ErrNotFound) {
+		return 0, err
+	}
+	// Allocate.
+	cur, c2, err := p.db.Get(metaKey("next_host_id"))
+	var current int64
+	if err == nil {
+		if len(cur) == 8 {
+			current = int64(binary.BigEndian.Uint64(cur))
+		}
+		_ = c2.Close()
+	} else if !errors.Is(err, pebble.ErrNotFound) {
+		return 0, err
+	}
+	next := current + 1
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, uint64(next))
+	if err := p.db.Set(metaKey("next_host_id"), buf, nil); err != nil {
+		return 0, err
+	}
+	if err := p.db.Set(hostTermKey(host), buf, nil); err != nil {
+		return 0, err
+	}
+	return next, nil
+}
+
+// GetHostID resolves an existing hostID. ok=false means the host has no
+// partition (never indexed under COSIFT_HOST_PARTITION, or backfill pending) —
+// callers fall back to the global posting scan.
+func (p *PebbleStore) GetHostID(ctx context.Context, host string) (int64, bool, error) {
+	if host == "" {
+		return 0, false, nil
+	}
+	val, closer, err := p.db.Get(hostTermKey(host))
+	if errors.Is(err, pebble.ErrNotFound) {
+		return 0, false, nil
+	}
+	if err != nil {
+		return 0, false, err
+	}
+	defer closer.Close()
+	if len(val) != 8 {
+		return 0, false, nil
+	}
+	return int64(binary.BigEndian.Uint64(val)), true, nil
+}
+
+// BackfillHostPostings rebuilds the 'P' host-partition family from existing
+// data, for corpora indexed before COSIFT_HOST_PARTITION was enabled. No
+// re-tokenize, no text decode — a pure key reshuffle reusing 'h', 'g' and 'p'.
+// It only writes the new 'P'/'T' families (and the 'next_host_id' counter),
+// never touching 'p'/'d'/'v'. Run it with the COSIFT_HOST_PARTITION WRITE flag
+// OFF so concurrent crawl indexing can't race hostID allocation.
+//
+// onlyHost == "": full-corpus single-pass — build a docID→hostID map from one
+// 'h' scan, then stream the global 'p' family once routing each posting to its
+// host. onlyHost != "": fast targeted path — for just that host's docs, walk
+// the 'g' reverse index (docID→termIDs) and copy each posting. The targeted
+// path is O(host_docs × terms_per_doc) — near-instant for a small site.
+//
+// progress, if non-nil, is called periodically with postingsWritten. Returns
+// the number of host postings written.
+func (p *PebbleStore) BackfillHostPostings(ctx context.Context, onlyHost string, progress func(written int64)) (int64, error) {
+	onlyHost = strings.ToLower(strings.TrimSpace(onlyHost))
+	if onlyHost != "" {
+		return p.backfillHostPostingsTargeted(ctx, onlyHost, progress)
+	}
+	return p.backfillHostPostingsFull(ctx, progress)
+}
+
+// backfillHostPostingsTargeted handles a single host via the 'g' reverse index.
+func (p *PebbleStore) backfillHostPostingsTargeted(ctx context.Context, host string, progress func(int64)) (int64, error) {
+	hostID, err := p.getOrAllocHostIDLocked(host)
+	if err != nil {
+		return 0, err
+	}
+	if hostID == 0 {
+		return 0, nil
+	}
+	// Collect this host's docIDs (exact match on the doc's stored domain;
+	// IterHostDocIDs uses the 'h' index which is keyed by d.Domain).
+	var docIDs []int64
+	if err := p.IterHostDocIDs(ctx, host, func(id int64) bool {
+		docIDs = append(docIDs, id)
+		return true
+	}); err != nil {
+		return 0, err
+	}
+	batch := p.db.NewBatch()
+	defer func() { _ = batch.Close() }()
+	var written, pending int64
+	for _, docID := range docIDs {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		termIDs, err := p.readDocTermsLocked(docID)
+		if err != nil {
+			return written, err
+		}
+		for _, termID := range termIDs {
+			val, closer, gerr := p.db.Get(postingKey(termID, docID))
+			if gerr != nil {
+				continue // posting absent (shouldn't happen) — skip
+			}
+			if len(val) == 16 {
+				vcopy := make([]byte, 16)
+				copy(vcopy, val)
+				if err := batch.Set(hostPostingKey(hostID, termID, docID), vcopy, nil); err != nil {
+					_ = closer.Close()
+					return written, err
+				}
+				written++
+				pending++
+			}
+			_ = closer.Close()
+			if pending >= 10000 {
+				if err := batch.Commit(p.writeOpts); err != nil {
+					return written, err
+				}
+				_ = batch.Close()
+				batch = p.db.NewBatch()
+				pending = 0
+				if progress != nil {
+					progress(written)
+				}
+			}
+		}
+	}
+	if pending > 0 {
+		if err := batch.Commit(p.writeOpts); err != nil {
+			return written, err
+		}
+	}
+	if progress != nil {
+		progress(written)
+	}
+	return written, nil
+}
+
+// backfillHostPostingsFull is the full-corpus single-pass variant.
+func (p *PebbleStore) backfillHostPostingsFull(ctx context.Context, progress func(written int64)) (int64, error) {
+	// Phase 1: docID → hostID, allocating a stable hostID per distinct host.
+	docToHost := make(map[int64]int64, 1<<20)
+	hostToID := make(map[string]int64, 1<<16)
+	{
+		it, err := p.db.NewIter(&pebble.IterOptions{
+			LowerBound: []byte{famHost},
+			UpperBound: []byte{famHost + 1},
+		})
+		if err != nil {
+			return 0, err
+		}
+		for valid := it.First(); valid; valid = it.Next() {
+			if err := ctx.Err(); err != nil {
+				_ = it.Close()
+				return 0, err
+			}
+			key := it.Key()
+			if len(key) < 1+1+8 { // 'h' + host(≥0) + 0x00 + docID(8) — need ≥10
+				continue
+			}
+			docID := int64(binary.BigEndian.Uint64(key[len(key)-8:]))
+			host := string(key[1 : len(key)-9]) // drop trailing 0x00 + docID
+			if host == "" {
+				continue
+			}
+			hid, ok := hostToID[host]
+			if !ok {
+				hid, err = p.getOrAllocHostIDLocked(host)
+				if err != nil {
+					_ = it.Close()
+					return 0, err
+				}
+				hostToID[host] = hid
+			}
+			docToHost[docID] = hid
+		}
+		if err := it.Close(); err != nil {
+			return 0, err
+		}
+	}
+
+	// Phase 2: stream the global 'p' family; mirror each posting into 'P'.
+	it, err := p.db.NewIter(&pebble.IterOptions{
+		LowerBound: []byte{famPosting},
+		UpperBound: []byte{famPosting + 1},
+	})
+	if err != nil {
+		return 0, err
+	}
+	defer it.Close()
+	batch := p.db.NewBatch()
+	defer func() { _ = batch.Close() }()
+	const batchFlush = 10000
+	var written, pending int64
+	for valid := it.First(); valid; valid = it.Next() {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		key := it.Key()
+		if len(key) != 17 { // 'p' + termID(8) + docID(8)
+			continue
+		}
+		termID := int64(binary.BigEndian.Uint64(key[1:9]))
+		docID := int64(binary.BigEndian.Uint64(key[9:17]))
+		hid, ok := docToHost[docID]
+		if !ok {
+			continue // doc has no host index entry — skip
+		}
+		val, err := it.ValueAndErr()
+		if err != nil {
+			return written, err
+		}
+		if len(val) != 16 {
+			continue
+		}
+		vcopy := make([]byte, 16)
+		copy(vcopy, val)
+		if err := batch.Set(hostPostingKey(hid, termID, docID), vcopy, nil); err != nil {
+			return written, err
+		}
+		written++
+		pending++
+		if pending >= batchFlush {
+			if err := batch.Commit(p.writeOpts); err != nil {
+				return written, err
+			}
+			_ = batch.Close()
+			batch = p.db.NewBatch()
+			pending = 0
+			if progress != nil {
+				progress(written)
+			}
+		}
+	}
+	if pending > 0 {
+		if err := batch.Commit(p.writeOpts); err != nil {
+			return written, err
+		}
+	}
+	if progress != nil {
+		progress(written)
+	}
+	return written, nil
+}
+
+// IterateHostPostings scans the host-partitioned posting list for one
+// (hostID, termID). Mirrors IteratePostings but over the 'P' family, so the
+// scan is bounded by the host's docs that contain termID — not the global
+// posting list. Caller returns false from fn to stop early.
+func (p *PebbleStore) IterateHostPostings(ctx context.Context, hostID, termID int64, fn func(PostingEntry) bool) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	it, err := p.db.NewIter(&pebble.IterOptions{
+		LowerBound: hostPostingPrefix(hostID, termID),
+		UpperBound: hostPostingPrefixUpper(hostID, termID),
+	})
+	if err != nil {
+		return err
+	}
+	defer it.Close()
+	for valid := it.First(); valid; valid = it.Next() {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		key := it.Key()
+		if len(key) != 25 { // 'P' + hostID(8) + termID(8) + docID(8)
+			continue
+		}
+		val, err := it.ValueAndErr()
+		if err != nil {
+			return err
+		}
+		if len(val) != 16 {
+			continue
+		}
+		entry := PostingEntry{
+			DocID:  int64(binary.BigEndian.Uint64(key[17:])),
+			TF:     int64(binary.BigEndian.Uint64(val[0:8])),
+			DocLen: int64(binary.BigEndian.Uint64(val[8:16])),
+		}
+		if !fn(entry) {
+			return nil
+		}
+	}
+	return nil
+}
+
 func packTermInfo(t TermInfo) []byte {
 	b := make([]byte, 16)
 	binary.BigEndian.PutUint64(b[0:8], uint64(t.ID))
diff --git a/internal/store/store.go b/internal/store/store.go
index 236721f..f75a5c4 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -1084,6 +1084,29 @@ func (s *Store) GCErroredFrontier(ctx context.Context, minAttempts int) (int64,
 	return res.RowsAffected()
 }
 
+// TrimDoneFrontier deletes completed (status='done') frontier rows that were
+// enqueued more than maxAge ago. Long-lived crawls accumulate millions of done
+// rows that serve no purpose — dedup is handled by INSERT OR IGNORE so a done
+// row is only useful for the IsKnownURL fast-path, which already falls through
+// to the documents table. Trimming done rows reclaims disk space and keeps
+// ClaimFrontier's ORDER BY scan fast.
+//
+// Rows younger than maxAge are kept because a short-TTL re-crawl (refresh-due
+// at 24 h) may re-enqueue them; dropping them would lose the dedup guard
+// and cause redundant fetches. Pass maxAge ≥ your refresh interval
+// (e.g. 7*24*time.Hour for a weekly refresh cycle).
+//
+// Returns the number of rows deleted.
+func (s *Store) TrimDoneFrontier(ctx context.Context, maxAge time.Duration) (int64, error) {
+	cutoff := time.Now().Add(-maxAge).Unix()
+	res, err := s.db.ExecContext(ctx,
+		`DELETE FROM frontier WHERE status = 'done' AND enqueued_at < ?;`, cutoff)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
 // Vacuum reclaims disk space after large deletes. Cheap on small DBs, can
 // take minutes on multi-GB ones — caller's responsibility to schedule.
 func (s *Store) Vacuum(ctx context.Context) error {
@@ -1391,7 +1414,6 @@ func (s *Store) RecrawlURL(ctx context.Context, url string) error {
 
 // Passage is one (doc_id, offset, length, model, vec) row.
 type Passage struct {
-	ID        int64
 	DocID     int64
 	Offset    int
 	Length    int
diff --git a/scripts/cosift-crawl.service b/scripts/cosift-crawl.service
deleted file mode 100644
index 5ef3af0..0000000
--- a/scripts/cosift-crawl.service
+++ /dev/null
@@ -1,22 +0,0 @@
-[Unit]
-Description=cosift crawl (oneshot — stops cosift-serve, crawls, restarts serve)
-# Both services want the Pebble writer lock; declare mutual exclusion so
-# starting one cleanly stops the other.
-Conflicts=cosift-serve.service
-
-[Service]
-Type=oneshot
-User=ubuntu
-Group=ubuntu
-WorkingDirectory=/home/ubuntu
-Environment=COSIFT_HNSW_CHECKPOINT_SEC=60
-# Run for up to 30 minutes per cycle. Periodic HNSW checkpoint means we
-# keep all vectors even if SIGTERM cuts us short.
-ExecStart=/home/ubuntu/cosift -config /home/ubuntu/cosift.json crawl -backend pebble -seeds-file /home/ubuntu/seeds.txt -duration 30m
-ExecStartPost=/bin/systemctl start cosift-serve.service
-TimeoutStartSec=2400
-StandardOutput=journal
-StandardError=journal
-
-[Install]
-WantedBy=multi-user.target
diff --git a/scripts/cosift-crawl.timer b/scripts/cosift-crawl.timer
deleted file mode 100644
index 13dfab4..0000000
--- a/scripts/cosift-crawl.timer
+++ /dev/null
@@ -1,14 +0,0 @@
-[Unit]
-Description=cosift crawl — cycle every 6h
-# Triggers cosift-crawl.service, which stops cosift-serve.service via the
-# Conflicts= directive, runs the crawl, then ExecStartPost restarts serve.
-
-[Timer]
-# 4 cycles per day. Crawl runs 30 min, serve runs the other ~85 min before
-# the next trigger. Tune to taste.
-OnCalendar=*-*-* 00,06,12,18:00:00
-Persistent=true
-Unit=cosift-crawl.service
-
-[Install]
-WantedBy=timers.target
diff --git a/scripts/cosift-serve.service b/scripts/cosift-serve.service
deleted file mode 100644
index 9b107ab..0000000
--- a/scripts/cosift-serve.service
+++ /dev/null
@@ -1,30 +0,0 @@
-[Unit]
-Description=cosift pebble-serve (with in-process continuous crawler)
-After=network-online.target ollama.service
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=ubuntu
-Group=ubuntu
-WorkingDirectory=/home/ubuntu
-Environment=COSIFT_LOAD_HNSW=true
-Environment=COSIFT_RATELIMIT_RPM=240
-Environment=COSIFT_RATELIMIT_BURST=80
-Environment=COSIFT_RATELIMIT_WHITELIST=104.28.216.88,127.0.0.1
-Environment=COSIFT_PPROF_ADDR=127.0.0.1:6060
-Environment=COSIFT_MUTEX_PROFILE_FRACTION=100
-Environment=COSIFT_DISABLE_PQ=true
-Environment=COSIFT_HNSW_EF_SEARCH=200
-Environment=COSIFT_CRAWL_EMBED_CONCURRENCY=16
-Environment=COSIFT_PEBBLE_CACHE_MB=32768
-ExecStart=/home/ubuntu/cosift -config /home/ubuntu/cosift.json pebble-serve -dir /home/ubuntu/cosift-data/pebble -addr 127.0.0.1:7777 -crawl-seeds-file /home/ubuntu/seeds.txt -crawl-checkpoint 60s
-Restart=on-failure
-RestartSec=5
-LimitNOFILE=131072
-TimeoutStopSec=30
-MemoryMax=262G
-MemoryHigh=240G
-
-[Install]
-WantedBy=multi-user.target
diff --git a/scripts/cosift-snapshot.service b/scripts/cosift-snapshot.service
deleted file mode 100644
index 2f86294..0000000
--- a/scripts/cosift-snapshot.service
+++ /dev/null
@@ -1,19 +0,0 @@
-[Unit]
-Description=cosift snapshot — tarball pebble + config, upload to GCS
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=oneshot
-User=ubuntu
-Group=ubuntu
-WorkingDirectory=/home/ubuntu
-Environment=COSIFT_ADMIN_URL=http://127.0.0.1:7777
-Environment=COSIFT_CONFIG=/home/ubuntu/cosift.json
-Environment=COSIFT_GCS_BUCKET=gs://pilot-cosift-index
-Environment=COSIFT_KEEP=14
-# COSIFT_ADMIN_TOKEN — set via /etc/cosift/snapshot.env if cluster.peer_auth_token is configured
-EnvironmentFile=-/etc/cosift/snapshot.env
-ExecStart=/usr/bin/bash /home/ubuntu/snapshot.sh
-StandardOutput=journal
-StandardError=journal
diff --git a/scripts/cosift-snapshot.timer b/scripts/cosift-snapshot.timer
deleted file mode 100644
index 834d86d..0000000
--- a/scripts/cosift-snapshot.timer
+++ /dev/null
@@ -1,13 +0,0 @@
-[Unit]
-Description=cosift snapshot — every 4h
-# Triggers cosift-snapshot.service. 4-hour cadence × KEEP=14 = ~56 hours
-# of recovery history. Tune by changing OnCalendar + COSIFT_KEEP env.
-
-[Timer]
-OnCalendar=*-*-* 00,04,08,12,16,20:00:00
-Persistent=true
-RandomizedDelaySec=60
-Unit=cosift-snapshot.service
-
-[Install]
-WantedBy=timers.target
diff --git a/scripts/ollama-replica.service.tmpl b/scripts/ollama-replica.service.tmpl
deleted file mode 100644
index b01cc72..0000000
--- a/scripts/ollama-replica.service.tmpl
+++ /dev/null
@@ -1,18 +0,0 @@
-[Unit]
-Description=Ollama Service (replica 1, port 11435)
-After=network-online.target
-
-[Service]
-ExecStart=/usr/local/bin/ollama serve
-User=ollama
-Group=ollama
-Restart=always
-RestartSec=3
-Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-Environment="OLLAMA_HOST=127.0.0.1:11435"
-Environment="OLLAMA_NUM_PARALLEL=32"
-Environment="OLLAMA_KEEP_ALIVE=24h"
-Environment="HOME=/usr/share/ollama"
-
-[Install]
-WantedBy=default.target
diff --git a/scripts/ollama.service b/scripts/ollama.service
deleted file mode 100644
index 875e97a..0000000
--- a/scripts/ollama.service
+++ /dev/null
@@ -1,16 +0,0 @@
-[Unit]
-Description=Ollama Service
-After=network-online.target
-
-[Service]
-ExecStart=/usr/local/bin/ollama serve
-Environment="OLLAMA_NUM_PARALLEL=10"
-Environment="OLLAMA_KEEP_ALIVE=24h"
-User=ollama
-Group=ollama
-Restart=always
-RestartSec=3
-Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin"
-
-[Install]
-WantedBy=default.target
diff --git a/testdata/eval/queries-realworld.json b/testdata/eval/queries-realworld.json
new file mode 100644
index 0000000..907987f
--- /dev/null
+++ b/testdata/eval/queries-realworld.json
@@ -0,0 +1,36 @@
+{
+  "name": "realworld-intents",
+  "description": "Representative real-user queries spanning the intents cosift must serve — especially known-item/resource lookup (the failure mode the LPR feedback exposed), which the prior eval sets never tested. relevant[] left empty: this set is for the LLM-judge answer-eval (answer quality/grounding), not recall metrics. Edit/extend freely; this is the definition-of-useful artifact.",
+  "queries": [
+    { "text": "license plate detection model for NVIDIA Jetson aarch64 with OCR", "relevant": [], "intent": "resource" },
+    { "text": "rust async postgres connection pool crate", "relevant": [], "intent": "resource" },
+    { "text": "python library to extract tables from PDF", "relevant": [], "intent": "resource" },
+    { "text": "streaming low-latency speech-to-text model", "relevant": [], "intent": "resource" },
+    { "text": "embeddable vector database for on-device RAG", "relevant": [], "intent": "resource" },
+    { "text": "go library for robust HTML parsing and scraping", "relevant": [], "intent": "resource" },
+    { "text": "open source OCR that handles handwriting", "relevant": [], "intent": "resource" },
+    { "text": "small language model that runs on a raspberry pi", "relevant": [], "intent": "resource" },
+
+    { "text": "how to quantize a model to INT8 with TensorRT on Jetson", "relevant": [], "intent": "howto" },
+    { "text": "how to set up OAuth for the Reddit API", "relevant": [], "intent": "howto" },
+    { "text": "how to do retrieval-augmented generation with citations", "relevant": [], "intent": "howto" },
+    { "text": "how to serve an LLM with vLLM behind an API", "relevant": [], "intent": "howto" },
+
+    { "text": "polars vs pandas for large dataframe pipelines", "relevant": [], "intent": "comparison" },
+    { "text": "qwen vs llama for local inference quality and speed", "relevant": [], "intent": "comparison" },
+    { "text": "BM25 vs dense retrieval vs hybrid for search", "relevant": [], "intent": "comparison" },
+    { "text": "gRPC vs REST for internal microservices", "relevant": [], "intent": "comparison" },
+
+    { "text": "how does Raft achieve distributed consensus", "relevant": [], "intent": "explain" },
+    { "text": "what is HNSW and why is it fast for vector search", "relevant": [], "intent": "explain" },
+    { "text": "how does NAT traversal work for peer-to-peer connections", "relevant": [], "intent": "explain" },
+    { "text": "explain the transformer attention mechanism", "relevant": [], "intent": "explain" },
+
+    { "text": "recent advances in small language models 2026", "relevant": [], "intent": "fresh" },
+    { "text": "latest approaches to long-context LLM inference", "relevant": [], "intent": "fresh" },
+    { "text": "state of the art in real-time object detection", "relevant": [], "intent": "fresh" },
+
+    { "text": "build an edge ANPR security camera system end to end", "relevant": [], "intent": "multifaceted" },
+    { "text": "design a self-hosted semantic search engine with reranking", "relevant": [], "intent": "multifaceted" }
+  ]
+}