From f53e5129318e44ab4dbf6d0c04f06d8a110371ba Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Fri, 10 Jul 2026 18:43:34 +0300 Subject: [PATCH] fix: make importance and recency score boosts opt-in --- ...0-relevance-score-controls-weighted-rrf.md | 69 ++++++++++++++ src/brainlayer/search_repo.py | 6 ++ tests/test_hybrid_search.py | 90 +++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 docs/plans/2026-07-10-relevance-score-controls-weighted-rrf.md diff --git a/docs/plans/2026-07-10-relevance-score-controls-weighted-rrf.md b/docs/plans/2026-07-10-relevance-score-controls-weighted-rrf.md new file mode 100644 index 00000000..26519462 --- /dev/null +++ b/docs/plans/2026-07-10-relevance-score-controls-weighted-rrf.md @@ -0,0 +1,69 @@ +# Relevance Score Controls and Weighted RRF Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Finish the two settled relevance wins by exposing score boosts as opt-in environment controls and making warm hybrid search use configurable two-leg weighted RRF. + +**Architecture:** Keep the existing explicit rerank kwargs and add small call-time environment helpers so current callers remain compatible while process-wide opt-ins work without module reloads. Keep fusion in `search_repo.py`, but isolate alpha parsing and deterministic two-leg RRF math so vector and FTS ranks are the only fusion inputs; the MCP layer continues to expose honest FTS fallback metadata when embeddings are unavailable. + +**Tech Stack:** Python 3.11+, pytest, APSW/FTS5, Ruff, GitHub CLI. + +--- + +### Task 1: Add opt-in environment controls for importance and recency boosts + +**Files:** +- Modify: `src/brainlayer/search_repo.py` +- Test: `tests/test_hybrid_search.py` + +**Step 1: Write the failing tests** + +Add focused tests proving that unset environment controls leave equal base scores, `BRAINLAYER_SCORE_IMPORTANCE_BOOST=1` enables only the importance multiplier, `BRAINLAYER_SCORE_RECENCY_DECAY=1` enables only recency decay, and explicit kwargs still enable their existing behavior. + +**Step 2: Run tests to verify RED** + +Run: `pytest -q tests/test_hybrid_search.py -k 'score_boost or default_does_not_apply_recency'` + +Expected: the environment-controlled tests fail because `hybrid_search` does not yet read either setting. + +**Step 3: Write the minimal implementation** + +Add a strict boolean environment helper, combine each environment control with its existing explicit kwarg at the start of `hybrid_search`, and use the effective values for the cache key and boost gates. + +**Step 4: Run tests to verify GREEN** + +Run the focused tests, then `pytest -q tests/test_hybrid_search.py tests/test_search_quality.py tests/test_agent_profiles.py tests/test_consumer_scoping.py`. + +**Step 5: Verify and publish PR 1** + +Run the brief-required Ruff commands and permitted pytest suite, commit only PR-1 files, push with `BRAINLAYER_PREPUSH_SCOPE=changed-only`, open the specified ready-for-review PR, and request Codex/Cursor/Bugbot review. + +### Task 2: Make warm hybrid fusion a configurable two-leg weighted RRF + +**Files:** +- Modify: `src/brainlayer/search_repo.py` +- Modify: `src/brainlayer/mcp/search_handler.py` only if the existing fallback response lacks the required explicit degraded marker +- Test: `tests/test_hybrid_search.py` +- Test: `tests/test_search_handler.py` + +**Step 1: Write the failing tests** + +Add deterministic rank-fixture tests for alpha `0.25`, a valid `BRAINLAYER_RRF_ALPHA` override, invalid/non-finite fallback to `0.25`, no trigram contribution, and MCP FTS-only fallback with explicit degraded honesty. + +**Step 2: Run tests to verify RED** + +Run: `pytest -q tests/test_hybrid_search.py tests/test_search_handler.py -k 'weighted_rrf or trigram or fts_fallback'` + +Expected: math/override/trigram tests fail because fusion is inline, alpha is constant, and trigram is still a scored leg; the degraded marker test fails if the current MCP payload exposes only `search_mode` and `fallback_reason`. + +**Step 3: Write the minimal implementation** + +Add a bounded finite alpha parser and a pure two-leg weighted-RRF helper. Fuse only semantic and FTS candidate ranks, retain trigram diagnostics only if needed outside fusion, and add `degraded: true` to fallback payloads without changing hybrid success payloads. + +**Step 4: Run tests to verify GREEN** + +Run the focused tests, then all permitted search/MCP tests and the full suite excluding `tests/test_vector_store.py` and `tests/test_engine.py`. + +**Step 5: Verify and publish PR 2** + +Branch from PR 1, run Ruff and permitted pytest verification, commit only PR-2 files, push with the scoped pre-push gate, open the specified stacked PR, request reviewers, and record both URLs/evidence in `docs.local/handoffs/BL-A-REPORT.md`. diff --git a/src/brainlayer/search_repo.py b/src/brainlayer/search_repo.py index cc454393..881382e6 100644 --- a/src/brainlayer/search_repo.py +++ b/src/brainlayer/search_repo.py @@ -121,6 +121,10 @@ _hybrid_cache: "OrderedDict[tuple, tuple[dict, float]]" = OrderedDict() +def _env_flag_enabled(name: str) -> bool: + return os.environ.get(name, "").strip().casefold() in {"1", "true", "yes", "on"} + + def _has_recency_intent(query_text: str) -> bool: """Return true when recency words appear as terms, not substrings.""" return bool(_RECENCY_SINGLE_TERM_RE.search(query_text) or _RECENCY_THIS_WEEK_RE.search(query_text)) @@ -1764,6 +1768,8 @@ def hybrid_search( Cache is module-level LRU (128 entries) with defensive copy-on-read. """ + recency_rerank = recency_rerank or _env_flag_enabled("BRAINLAYER_SCORE_RECENCY_DECAY") + importance_rerank = importance_rerank or _env_flag_enabled("BRAINLAYER_SCORE_IMPORTANCE_BOOST") project_filter = _effective_project_filter(project_filter, consumer_scope) source_filter = _effective_source_filter(source_filter, consumer_scope) include_checkpoints = _effective_include_checkpoints(include_checkpoints, consumer_scope) diff --git a/tests/test_hybrid_search.py b/tests/test_hybrid_search.py index 10d2c4db..2d2005bb 100644 --- a/tests/test_hybrid_search.py +++ b/tests/test_hybrid_search.py @@ -232,6 +232,8 @@ def test_hybrid_search_weighted_rrf_uses_vector_alpha(self, store, monkeypatch): assert results["ids"][0] == ["lexical-only", "vector-only"] def test_hybrid_search_default_does_not_apply_recency_or_importance_boost(self, store, monkeypatch): + monkeypatch.delenv("BRAINLAYER_SCORE_IMPORTANCE_BOOST", raising=False) + monkeypatch.delenv("BRAINLAYER_SCORE_RECENCY_DECAY", raising=False) monkeypatch.setattr(search_repo, "RRF_VECTOR_ALPHA", 0.5) query_embedding = _embed("neutral rerank evidence") _insert_chunk( @@ -268,7 +270,95 @@ def capture_scores(scored, *, n_results): assert captured_scores["old-important"] == pytest.approx(captured_scores["fresh-low"]) + def test_hybrid_search_importance_score_boost_env_opt_in(self, store, monkeypatch): + monkeypatch.setenv("BRAINLAYER_SCORE_IMPORTANCE_BOOST", "1") + monkeypatch.delenv("BRAINLAYER_SCORE_RECENCY_DECAY", raising=False) + monkeypatch.setattr(search_repo, "RRF_VECTOR_ALPHA", 0.5) + query_embedding = _embed("importance env rerank evidence") + _insert_chunk( + store, + chunk_id="high-importance", + content="importance env rerank evidence padded exact hit with extra words", + embedding=query_embedding, + importance=10.0, + created_at="2020-01-01T00:00:00Z", + ) + _insert_chunk( + store, + chunk_id="low-importance", + content="importance env rerank evidence", + embedding=[value + 0.00001 for value in query_embedding], + importance=0.0, + created_at="2020-01-01T00:00:00Z", + ) + store.build_binary_index() + monkeypatch.setattr(store, "_trigram_fts_available", False) + captured_scores: dict[str, float] = {} + + def capture_scores(scored, *, n_results): + captured_scores.update({chunk_id: score for score, chunk_id, *_rest in scored}) + return scored + + monkeypatch.setattr(store, "_mmr_rerank_scored_results", capture_scores) + + store.hybrid_search( + query_embedding=query_embedding, + query_text="importance env rerank evidence", + n_results=2, + ) + + base_rrf = 0.5 / 60.0 + 0.5 / 61.0 + assert captured_scores["high-importance"] == pytest.approx(base_rrf * 1.5) + assert captured_scores["low-importance"] == pytest.approx(base_rrf) + + def test_hybrid_search_recency_score_boost_env_opt_in(self, store, monkeypatch): + monkeypatch.delenv("BRAINLAYER_SCORE_IMPORTANCE_BOOST", raising=False) + monkeypatch.setenv("BRAINLAYER_SCORE_RECENCY_DECAY", "1") + monkeypatch.setattr(search_repo, "RRF_VECTOR_ALPHA", 0.5) + query_embedding = _embed("recency env rerank evidence") + _insert_chunk( + store, + chunk_id="old-result", + content="recency env rerank evidence padded exact hit with extra words", + embedding=query_embedding, + importance=5.0, + created_at="2020-01-01T00:00:00Z", + ) + _insert_chunk( + store, + chunk_id="fresh-result", + content="recency env rerank evidence", + embedding=[value + 0.00001 for value in query_embedding], + importance=5.0, + created_at="2999-01-01T00:00:00Z", + ) + store.build_binary_index() + monkeypatch.setattr(store, "_trigram_fts_available", False) + captured_scores: dict[str, float] = {} + + def capture_scores(scored, *, n_results): + captured_scores.update({chunk_id: score for score, chunk_id, *_rest in scored}) + return scored + + monkeypatch.setattr(store, "_mmr_rerank_scored_results", capture_scores) + + store.hybrid_search( + query_embedding=query_embedding, + query_text="recency env rerank evidence", + n_results=2, + ) + + base_rrf = 0.5 / 60.0 + 0.5 / 61.0 + old_age_days = ( + datetime.now(timezone.utc) - datetime.fromisoformat("2020-01-01T00:00:00+00:00") + ).total_seconds() / 86400 + old_recency_boost = 0.7 + 0.3 * math.exp(-0.023 * old_age_days) + assert captured_scores["old-result"] == pytest.approx(base_rrf * old_recency_boost) + assert captured_scores["fresh-result"] == pytest.approx(base_rrf) + def test_hybrid_search_opt_in_recency_and_importance_rerank_change_scores(self, store, monkeypatch): + monkeypatch.delenv("BRAINLAYER_SCORE_IMPORTANCE_BOOST", raising=False) + monkeypatch.delenv("BRAINLAYER_SCORE_RECENCY_DECAY", raising=False) monkeypatch.setattr(search_repo, "RRF_VECTOR_ALPHA", 0.5) query_embedding = _embed("opt in rerank evidence") _insert_chunk(