Add Gemini reranking for video search results

README.md

@@ -154,15 +154,20 @@ No confident match found (best score: 0.28). Show results anyway? [y/N]:
 
 With `--no-trim`, low-confidence results are shown with a note instead of a prompt.
 
-Options: `--results N`, `--output-dir DIR`, `--no-trim` to skip auto-trimming, `--threshold 0.5` to adjust the confidence cutoff, `--save-top N` to save the top N clips instead of just the best match, `--dedupe` to drop results too similar to a higher-ranked pick (prevents near-duplicate chunks of the same event from filling the list). Backend and model are auto-detected from the index — pass `--backend` or `--model` only to override.
+Options: `--results N`, `--output-dir DIR`, `--no-trim` to skip auto-trimming, `--threshold 0.5` to adjust the confidence cutoff, `--save-top N` to save the top N clips instead of just the best match, `--dedupe` to drop results too similar to a higher-ranked pick (prevents near-duplicate chunks of the same event from filling the list), and `--rerank` to ask Gemini Flash to re-rank the returned candidates before trimming. Backend and model are auto-detected from the index — pass `--backend` or `--model` only to override.
 
 ```bash
 # Save top 5 clips, dropping near-duplicates
 sentrysearch search "red truck" --save-top 5 --dedupe 0.9
+
+# Re-rank the top 10 embedding matches with Gemini Flash before trimming
+sentrysearch search "pedestrian crossing behind the car" --rerank --results 10
 ```
 
 The `--dedupe` value is a cosine similarity ceiling (0–1). Any result whose similarity to an already-kept higher-ranked result exceeds this value is dropped. Lower values are stricter: `0.8` requires results to be very distinct, `0.95` only removes near-identical chunks. `0.9` is a good default.
 
+`--rerank` extracts each returned candidate clip, sends it to Gemini 2.5 Flash with the query, and sorts likely visual matches ahead of embedding-only results. If reranking cannot run or a candidate cannot be scored, SentrySearch keeps the embedding-ranked results instead of failing the search.
+
 ### Search by image
 
 Use a reference image as the query — useful for "find clips that look like this" when describing the scene in words is awkward (a screenshot of a specific car, a reference frame from another video, etc.).
@@ -176,7 +181,7 @@ $ sentrysearch img ~/Downloads/image.jpg
 Saved clip: ./match_2026-03-12_10-44-17-left_repeater_00m00s-00m30s.mp4
 ```
 
-The image is embedded into the same vector space as the indexed video chunks and ranked by cosine similarity. All `search` flags are supported (`--results`, `--threshold`, `--save-top`, `--dedupe`, `--overlay`, `--no-trim`, `--backend`, `--model`).
+The image is embedded into the same vector space as the indexed video chunks and ranked by cosine similarity. Image search supports `--results`, `--threshold`, `--save-top`, `--dedupe`, `--overlay`, `--no-trim`, `--backend`, and `--model`.
 
 Supported formats: JPG, PNG, WEBP, GIF, HEIC/HEIF on the Gemini backend; the local backend additionally accepts anything PIL can decode (BMP, TIFF, etc.).
 

sentrysearch/cli.py

@@ -91,6 +91,13 @@ def _cache_last_search(
         )
 
 
+def _strip_private_result_fields(results: list[dict]) -> list[dict]:
+    return [
+        {key: value for key, value in result.items() if not key.startswith("_")}
+        for result in results
+    ]
+
+
 def _open_file(path: str) -> None:
     """Open a file with the system's default application."""
     try:
@@ -651,8 +658,10 @@ def index(directory, chunk_duration, overlap, preprocess, target_resolution,
 @click.option("--dedupe", "dedupe_threshold", default=None, type=float,
               help="Drop results whose cosine similarity to a higher-ranked "
                    "result exceeds this (e.g. 0.9).")
+@click.option("--rerank", is_flag=True,
+              help="Use Gemini Flash to rerank candidates before trimming.")
 @click.option("--verbose", is_flag=True, help="Show debug info.")
-def search(query, n_results, output_dir, trim, save_top, threshold, overlay, backend, model, dashscope_model, quantize, dedupe_threshold, verbose):
+def search(query, n_results, output_dir, trim, save_top, threshold, overlay, backend, model, dashscope_model, quantize, dedupe_threshold, rerank, verbose):
     """Search indexed footage with a natural language QUERY."""
     from .embedder import get_embedder, reset_embedder
     from .local_embedder import normalize_model_key
@@ -728,16 +737,59 @@ def search(query, n_results, output_dir, trim, save_top, threshold, overlay, bac
 
         results = search_footage(query, store, n_results=n_results, verbose=verbose,
                                  dedupe_threshold=dedupe_threshold)
-        _cache_last_search(results, query=query)
-        _present_results(results, threshold, trim, save_top, output_dir, overlay, verbose)
+        rerank_enabled = rerank
+        if rerank and results:
+            from .gemini_embedder import GeminiAPIKeyError
+            from .gemini_reranker import GeminiReranker
+            from .reranker import rerank_results
+
+            try:
+                reranker = GeminiReranker()
+            except GeminiAPIKeyError:
+                click.secho(
+                    "--rerank skipped: GEMINI_API_KEY is not set; "
+                    "showing embedding results.",
+                    fg="yellow",
+                    err=True,
+                )
+                rerank_enabled = False
+            else:
+                import tempfile
+
+                with tempfile.TemporaryDirectory(
+                    prefix="sentrysearch_rerank_",
+                ) as candidate_dir:
+                    results = rerank_results(
+                        query, results, reranker,
+                        candidate_dir=candidate_dir,
+                        verbose=verbose,
+                    )
+                    _cache_last_search(
+                        _strip_private_result_fields(results), query=query,
+                    )
+                    _present_results(
+                        results, threshold, trim, save_top, output_dir,
+                        overlay, verbose, rerank_enabled=rerank_enabled,
+                    )
+                    return
+        _cache_last_search(
+            _strip_private_result_fields(results), query=query,
+        )
+        _present_results(
+            results, threshold, trim, save_top, output_dir, overlay, verbose,
+            rerank_enabled=rerank_enabled,
+        )
 
     except Exception as e:
         _handle_error(e)
     finally:
         reset_embedder()
 
 
-def _present_results(results, threshold, trim, save_top, output_dir, overlay, verbose):
+def _present_results(
+    results, threshold, trim, save_top, output_dir, overlay, verbose, *,
+    rerank_enabled=False,
+):
     if not results:
         click.echo(
             "No results found.\n\n"
@@ -748,12 +800,30 @@ def _present_results(results, threshold, trim, save_top, output_dir, overlay, ve
         )
         return
 
-    best_score = results[0]["similarity_score"]
-    low_confidence = best_score < threshold
+    best = results[0]
+    best_score = best["similarity_score"]
+    rerank_match = best.get("rerank_match")
+    rerank_confidence = best.get("rerank_confidence")
+    has_rerank_score = (
+        rerank_enabled
+        and isinstance(rerank_match, bool)
+        and not isinstance(rerank_confidence, bool)
+        and isinstance(rerank_confidence, (int, float))
+    )
+    if has_rerank_score:
+        low_confidence = not rerank_match
+        low_confidence_detail = (
+            f"VLM match={str(rerank_match).lower()}, "
+            f"rerank confidence: {rerank_confidence:.2f}, "
+            f"embedding score: {best_score:.2f}"
+        )
+    else:
+        low_confidence = best_score < threshold
+        low_confidence_detail = f"best score: {best_score:.2f}"
 
     if low_confidence and not trim:
         click.secho(
-            f"(low confidence — best score: {best_score:.2f})",
+            f"(low confidence — {low_confidence_detail})",
             fg="yellow",
             err=True,
         )
@@ -772,7 +842,7 @@ def _present_results(results, threshold, trim, save_top, output_dir, overlay, ve
     if should_trim:
         if low_confidence:
             if not click.confirm(
-                f"No confident match found (best score: {best_score:.2f}). "
+                f"No confident match found ({low_confidence_detail}). "
                 "Show results anyway?",
                 default=False,
             ):

sentrysearch/gemini_reranker.py

@@ -0,0 +1,106 @@
+"""Gemini Flash VLM reranker."""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+
+from dotenv import load_dotenv
+
+from .gemini_embedder import (
+    GeminiAPIKeyError,
+    GeminiEmbedder,
+    _RateLimiter,
+    _retry,
+)
+from .reranker import RerankScore, parse_rerank_response
+
+load_dotenv()
+
+RERANK_MODEL = "gemini-2.5-flash"
+
+_RERANK_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "rerank_match": {"type": "boolean"},
+        "rerank_confidence": {
+            "type": "number",
+            "minimum": 0.0,
+            "maximum": 1.0,
+        },
+    },
+    "required": ["rerank_match", "rerank_confidence"],
+}
+
+
+def _prompt(query: str) -> str:
+    return (
+        "You are reranking video search candidates.\n"
+        "Look at the clip and decide whether it visually matches this search "
+        f"query: {query!r}\n\n"
+        "Return JSON only with this exact shape:\n"
+        '{"rerank_match": true, "rerank_confidence": 0.0}\n'
+        "Use rerank_match=true only when the clip contains the requested event. "
+        "Use rerank_confidence as your confidence from 0.0 to 1.0."
+    )
+
+
+class GeminiReranker:
+    """Gemini Flash reranker for candidate video clips."""
+
+    def __init__(self):
+        from google import genai
+
+        api_key = os.environ.get("GEMINI_API_KEY")
+        if not api_key:
+            raise GeminiAPIKeyError(
+                "GEMINI_API_KEY is not set.\n\n"
+                "Run: sentrysearch init\n\n"
+                "Or set it manually:\n"
+                "  export GEMINI_API_KEY=your-key"
+            )
+        self._client = genai.Client(api_key=api_key)
+        self._limiter = _RateLimiter()
+
+    def score(
+        self,
+        query: str,
+        clip_path: str,
+        *,
+        verbose: bool = False,
+    ) -> RerankScore | None:
+        """Return a validated rerank score, or None for unparsable model output."""
+        from google.genai import types
+
+        video_part = GeminiEmbedder._make_video_part(clip_path, types)
+        prompt_part = types.Part(text=_prompt(query))
+        config = types.GenerateContentConfig(
+            response_mime_type="application/json",
+            response_json_schema=_RERANK_SCHEMA,
+            temperature=0.0,
+        )
+
+        self._limiter.wait()
+        t0 = time.monotonic()
+        response = _retry(
+            lambda: self._client.models.generate_content(
+                model=RERANK_MODEL,
+                contents=types.Content(parts=[video_part, prompt_part]),
+                config=config,
+            )
+        )
+        elapsed = time.monotonic() - t0
+
+        score = parse_rerank_response(getattr(response, "text", None))
+        if verbose:
+            status = "fallback" if score is None else (
+                f"match={score.rerank_match}, "
+                f"confidence={score.rerank_confidence:.2f}"
+            )
+            print(
+                f"  [verbose] rerank {RERANK_MODEL}: {status}, "
+                f"api_time={elapsed:.2f}s",
+                file=sys.stderr,
+            )
+        return score

sentrysearch/reranker.py

@@ -0,0 +1,106 @@
+"""VLM reranking helpers for search results."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import sys
+from dataclasses import dataclass
+
+from .trimmer import trim_clip
+
+
+@dataclass(frozen=True)
+class RerankScore:
+    """Validated VLM rerank response."""
+
+    rerank_match: bool
+    rerank_confidence: float
+
+
+def parse_rerank_response(text: str) -> RerankScore | None:
+    """Parse and validate the Gemini rerank JSON response.
+
+    Invalid model output returns ``None`` so callers can fall back to the
+    candidate's embedding rank.
+    """
+    try:
+        data = json.loads(text)
+    except (json.JSONDecodeError, TypeError):
+        return None
+
+    if not isinstance(data, dict):
+        return None
+
+    rerank_match = data.get("rerank_match")
+    if not isinstance(rerank_match, bool):
+        return None
+
+    rerank_confidence = data.get("rerank_confidence")
+    if isinstance(rerank_confidence, bool) or not isinstance(
+        rerank_confidence, (int, float),
+    ):
+        return None
+
+    rerank_confidence = float(rerank_confidence)
+    if not math.isfinite(rerank_confidence):
+        return None
+    if rerank_confidence < 0.0 or rerank_confidence > 1.0:
+        return None
+
+    return RerankScore(
+        rerank_match=rerank_match,
+        rerank_confidence=rerank_confidence,
+    )
+
+
+def _sort_key(item: tuple[dict, int, RerankScore | None]) -> tuple:
+    _result, original_rank, score = item
+    if score is None:
+        return (1, original_rank)
+    if score.rerank_match:
+        return (0, -score.rerank_confidence, original_rank)
+    return (2, original_rank)
+
+
+def rerank_results(
+    query: str,
+    results: list[dict],
+    reranker,
+    *,
+    candidate_dir: str,
+    verbose: bool = False,
+) -> list[dict]:
+    """Extract candidate clips, score them with *reranker*, and reorder results."""
+    if not results:
+        return []
+
+    scored: list[tuple[dict, int, RerankScore | None]] = []
+    for original_rank, result in enumerate(results):
+        reranked = dict(result)
+        score = None
+        clip_path = os.path.join(candidate_dir, f"candidate_{original_rank:03d}.mp4")
+        try:
+            clip_path = trim_clip(
+                result["source_file"],
+                result["start_time"],
+                result["end_time"],
+                clip_path,
+            )
+            reranked["_rerank_clip_path"] = clip_path
+            score = reranker.score(query, clip_path, verbose=verbose)
+        except Exception as exc:
+            if verbose:
+                print(
+                    f"  [verbose] rerank candidate #{original_rank + 1} "
+                    f"fallback: {exc}",
+                    file=sys.stderr,
+                )
+        if score is not None:
+            reranked["rerank_match"] = score.rerank_match
+            reranked["rerank_confidence"] = score.rerank_confidence
+        scored.append((reranked, original_rank, score))
+
+    scored.sort(key=_sort_key)
+    return [result for result, _rank, _score in scored]