From c773bdc0f1b569527d7c5844d8281ae701395826 Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 18:21:07 +0800 Subject: [PATCH 01/10] fix(AGENTS): add distaste for regex heuristics --- AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 30e09b1..999f775 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,9 @@ Violations will cause runs to be blocked or reverted and large multimillion doll * Strictly follow existing style in the codebase. * Ensure every function is properly documented following the existing style. * Write in-line comments strategically, especially for key business logic. Do this sparingly, and strategically. +* DEVELOP A STRONG DISTASTE FOR REGEX/TEXT HEURISTICS. + - Favor using established external libraries instead + - Favor using proper, generalized functions * Keep code concise, tasteful and elegant. * Don't overcomplicate things. Don't implement more than you need to. * Ensure code is readable by humans, easy to extend and maintain. From 73cd72359f3a15e42ff50cfc53dbf606307de6ca Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 18:21:24 +0800 Subject: [PATCH 02/10] wip(runtime): add FIXME/TODOs on heuristics --- src/andromeda/query/runtime.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/andromeda/query/runtime.py b/src/andromeda/query/runtime.py index 15972e1..c3a9f6a 100644 --- a/src/andromeda/query/runtime.py +++ b/src/andromeda/query/runtime.py @@ -819,6 +819,8 @@ def narrative_query_expansion_enabled() -> bool: def narrative_aspect_coverage_enabled() -> bool: """ Return whether narrative growth/risk aspect-coverage enforcement is enabled. + + FIXME: feels too brittle. """ raw = (os.getenv("FINRAG_ENABLE_NARRATIVE_ASPECT_COVERAGE") or "1").strip().lower() @@ -863,6 +865,8 @@ def _question_is_simple_numeric_metric(self, question: str) -> bool: def adaptive_retrieval_budget_enabled() -> bool: """ Return whether adaptive retrieval-budget scheduling is enabled. + + FIXME: feels brittle. """ raw = (os.getenv("FINRAG_ENABLE_ADAPTIVE_RETRIEVAL_BUDGET") or "1").strip().lower() @@ -977,6 +981,9 @@ def resolve_tool_usage_from_decision(self, *, question: str, decision: PlannerDe return use_rag, use_yfinance, use_edgar_financials def _infer_tickers_from_question(self, question: str, companies: list[dict[str, str]]) -> list[str]: + # FIXME: extremely brittle logic. should use yfinance python library to get tickers from company name. + # TODO: use yfinance instead. + # see: https://deepwiki.com/ranaroussi/yfinance/4.3-search-and-lookup-functionality inferred: list[str] = [] seen: set[str] = set() known_tickers = {str(item["ticker"]).strip().upper() for item in companies if "ticker" in item} From 45b19cb34701939bebe175de086835c2c6f3ffbc Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 19:19:04 +0800 Subject: [PATCH 03/10] docs: add eval-improvement guidance and reduced-heuristics benchmark plan --- AGENTS.md | 5 + IMPROVE_RAG_EVAL.md | 283 ++++++++++++++++++ ...ced_heuristics_eval_and_retrieval_bench.md | 86 ++++++ 3 files changed, 374 insertions(+) create mode 100644 IMPROVE_RAG_EVAL.md create mode 100644 agent_logs/plans/20260218_reduced_heuristics_eval_and_retrieval_bench.md diff --git a/AGENTS.md b/AGENTS.md index 999f775..544520f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,6 +85,11 @@ you must ensure those comments continue to exist in the new/migrated function/co including modifications to `LOGBOOK.md`, `agent_logs/`, `CHANGELOG.md` and so on. - NEVER undo others' work. - Keep calm and continue executing with your plan. You do not need to stop and ask me about it. +* WHEN WRITING UNIT TESTS: + - Monkeypatching is fine for isolating specific logic to test for, but you should NOT abuse it. + - You should still have proper "integration" style tests that pass in legitimate production-style inputs, and assert for legitimate outputs (as far as pragmatically possible). + - When dealing with tests that require external APIs like LLM APIs, of course monkeypatching helps to run these tests locally without actually calling the external LLM API and incurring costs. However, we should still set aside a comprehensive suite of integration tests that actually make the required LLM calls, and do comprehensive asserts/checks on the outputs. These tests can be run in special environment with LLM key available, and can be disabled by default for normal CI tests. + - But these tests should still exist. ## Testing rules diff --git a/IMPROVE_RAG_EVAL.md b/IMPROVE_RAG_EVAL.md new file mode 100644 index 0000000..9d7f22c --- /dev/null +++ b/IMPROVE_RAG_EVAL.md @@ -0,0 +1,283 @@ +# Improve RAG Evaluation Methodology (Retrieval + Reranking Focus) + +Date: 2026-02-18 +Scope: Planning only (no code changes in this task) + +## 1) Problem Statement + +Your current eval stack is strong for end-to-end answer quality and judge operations, but it still under-instruments subsystem quality, especially: + +- retrieval quality before reranking +- reranker uplift vs. non-reranked candidates +- evidence utilization (did the model use retrieved evidence correctly?) + +This aligns with the critique in `CRITIQUES.md` and current pipeline behavior described in `README_EVAL.md` and `src/andromeda/eval/scoring.py`. + +## 2) Recommendation in One Line + +Use a **hybrid eval strategy**: + +- keep LLM judge evals for final-answer/product quality +- add cheap, high-throughput **retrieval/rerank subsystem evals** (IR metrics + chunk relevance) +- add **NLI-based claim support checks** as a scalable middle layer +- calibrate all auto metrics with a small, targeted human-labeled slice + +This avoids over-optimizing to one judge while keeping annotation cost low. + +## 3) Why This Is the Right Direction + +- RAG evaluation is inherently multi-component (retrieval quality, faithfulness, answer quality), not a single pass/fail axis [6], [7]. +- LLM judges are useful but biased (position/verbosity/self-enhancement); they should be calibrated and not be the only optimization target [4], [12], [13], [14]. +- Reranking is effective in many IR settings but should be measured explicitly with pre/post deltas, not inferred indirectly [10], [11]. +- Long-context systems often under-use relevant middle-context evidence; early-rank relevance matters for practical quality [15]. +- NLI-style consistency checks are practical and can work well with modest task-specific data [5], [16], [17]. + +## 4) Target Evaluation Architecture + +### Layer A: Retriever Quality (Pre-rerank) + +Measure on `retrieved_chunks` (already available in eval generation artifacts): + +- `Recall@k` (chunk/doc) +- `Precision@k` (chunk) +- `MRR@k` +- `nDCG@k` (when graded relevance exists) +- `Ticker coverage` and `period coverage` for finance-specific retrieval + +Primary outcome: “Does retrieval surface the right evidence at all?” + +### Layer B: Reranker Quality (Post-rerank vs Pre-rerank) + +Directly compare `top_chunks` (post-rerank) against `retrieved_chunks`: + +- `Delta MRR`, `Delta Recall@k`, `Delta Precision@k`, `Delta nDCG@k` +- per-query win/loss/tie rate for reranking +- rank shift of first relevant chunk (how much relevance is moved toward the front) + +Primary outcome: “How much precision gain does reranking produce, and where?” + +### Layer C: Evidence Utilization (Answer conditioned on retrieved context) + +For generated answers: + +- claim extraction -> claim-to-evidence support scoring +- support via NLI and/or judge-backed entailment checks +- report support rate / contradiction rate / unsupported-claim rate + +Primary outcome: “Given retrieved context, did generation use evidence correctly?” + +### Layer D: Final Answer (Existing Judge Layer) + +Keep your current judge suite for end-user quality, but treat it as one layer among several, not the sole optimization objective. + +## 5) Low-Annotation Data Strategy + +### 5.1 Start from existing assets (zero extra labeling upfront) + +Leverage current eval artifacts: + +- `eval/eval_queries_*` (already diverse by kind) +- existing factual gold evidence metadata +- existing generation outputs containing both pre/post rerank chunks + +This gives immediate subsystem metrics with minimal additional work. + +### 5.2 Build a pooled relevance set (annotation-efficient) + +For each query, pool candidates from: + +- top-N pre-rerank +- top-N post-rerank +- optional ablations (different retrieval settings) + +Then label only pooled candidates instead of full corpus judgments. + +### 5.3 Use weak/silver labels first, then calibrate + +Initial labels can come from: + +- existing gold evidence anchors (factual queries) +- NLI support against reference answer/claims +- optional lightweight LLM labeler for ambiguous cases + +Then calibrate with a small human-labeled set (few hundred items), following ARES-style low-label correction principles (PPI) [8]. + +### 5.4 Human labeling budget recommendation + +- 250-400 query-chunk judgments initially +- stratify by: + - query type (factual/open-ended/comparison) + - reranker disagreements + - high-impact finance cases (numeric/table/period-sensitive) + +This is enough to calibrate thresholds and estimate metric error bars without a large annotation project. + +## 6) Metric Set to Add + +### Retrieval/Rerank core metrics + +- `P@5`, `P@10` +- `R@10`, `R@20`, `R@50` +- `MRR@10`, `MRR@25` +- `nDCG@10`, `nDCG@25` (if graded labels available) +- reranker uplift deltas and win-rate + +### Claim/evidence metrics + +- claim support precision +- claim contradiction rate +- unsupported-claim rate +- context utilization rate (claim supported by retrieved chunks) + +### Statistical discipline + +- paired bootstrap CIs (or paired randomization tests) on key deltas +- publish confidence intervals in reports +- reject “wins” where CI overlaps zero + +## 7) Decision: Judge vs BERT/NLI? + +Do **both**, with role separation: + +- Judge: product-level quality and nuanced rubric checks +- NLI: high-throughput, lower-cost, claim/chunk support checks for subsystem loops + +Operationally: + +- run NLI on every eval sample +- run judge on full or stratified subset (depending on cost) +- weekly calibrate judge and NLI against human-labeled slice + +This gives speed + robustness and reduces judge-only Goodhart risk. + +## 8) Phased Implementation Plan + +Each phase is independently testable and shippable. + +### Phase 0: Baseline Instrumentation and Definitions + +Goal: + +- lock metric definitions and baseline numbers for retrieval and reranking from existing runs + +Acceptance criteria: + +- one report generated from existing artifacts with pre/post rerank metrics +- metric definitions documented with formulas +- baseline tables stored under `agent_logs/reports/` + +### Phase 1: Retrieval/Rerank Evaluator (No Human Labels Required) + +Goal: + +- compute IR metrics directly from current eval data (gold where available + ID-based proxies) + +Acceptance criteria: + +- CLI command produces retrieval/rerank score summary JSON +- includes reranker deltas and per-query win/loss/tie +- integrated into eval reporting flow + +### Phase 2: Claim-Level Evidence Support (NLI First) + +Goal: + +- add claim support/contradiction metrics at answer-evidence level + +Acceptance criteria: + +- per-answer claim support statistics are produced +- metrics run in batch at acceptable cost/latency +- metrics appear in dashboard/report alongside judge metrics + +### Phase 3: Human Calibration + PPI Correction + +Goal: + +- calibrate auto metrics with small human annotation set and bias-correct estimates + +Acceptance criteria: + +- labeled calibration set committed under `eval/` data path +- calibration report includes precision/recall and agreement stats +- corrected estimates + CIs reported for key metrics + +### Phase 4: CI/Regression Gates + +Goal: + +- prevent silent retrieval/rerank regressions + +Acceptance criteria: + +- PR/nightly gate fails when retrieval/rerank metrics regress beyond thresholds +- threshold policy documented in `README_EVAL.md` +- changelog policy updated for eval methodology changes + +## 9) Proposed File Plan (For Future Implementation) + +`files_to_change`: + +- `scripts/score_eval.py` +- `scripts/run_eval.py` (only if additional artifact fields are needed) +- `src/andromeda/eval/scoring.py` +- `src/andromeda/eval/report.py` +- `src/andromeda/eval/schema.py` +- `README_EVAL.md` +- `CHANGELOG.md` + +`new_files`: + +- `src/andromeda/eval/retrieval_metrics.py` +- `src/andromeda/eval/rerank_metrics.py` +- `src/andromeda/eval/evidence_support.py` +- `scripts/eval_retrieval.py` +- `scripts/build_retrieval_label_pool.py` +- `scripts/calibrate_eval_metrics.py` +- `eval/retrieval_labels/*.jsonl` +- `agent_logs/reports/retrieval_rerank_eval_*.md` + +## 10) Suggested Initial Thresholds (Tune After Baseline) + +- reranker `Delta P@5` must be > 0 with 95% CI excluding 0 +- reranker win-rate >= 55% +- no regression > 2% absolute in `R@20` on core factual subset +- unsupported-claim rate must not worsen while optimizing retrieval precision + +## 11) Risks and Mitigations + +- Risk: Overfitting to silver labels + Mitigation: keep human calibration slice and rotate disagreement samples. + +- Risk: NLI misses finance-specific nuances (units/periods) + Mitigation: add finance-targeted calibration examples and targeted failure audits. + +- Risk: Metric sprawl + Mitigation: keep one decision dashboard with a small, fixed “release gate” subset. + +## 12) Suggested Future Add-ons (Not in current scope) + +- Adversarial retrieval tests (hard distractors, near-miss tables, conflicting filings) +- Section-aware retrieval diagnostics (MD&A vs Risk Factors vs footnotes) +- Counterfactual rerank tests (swap top chunk order to quantify context-position sensitivity) + +## Sources + +1. Internal critique: `CRITIQUES.md` +2. Internal eval runbook: `README_EVAL.md` +3. Eugene Yan, *Evaluating Long-Context Question & Answer Systems*: https://eugeneyan.com/writing/qa-evals/ +4. Eugene Yan, *Evaluating the Effectiveness of LLM-Evaluators*: https://eugeneyan.com/writing/llm-evaluators/ +5. Eugene Yan, *Task-Specific LLM Evals that Do & Don't Work*: https://eugeneyan.com/writing/evals/ +6. RAGAS paper (arXiv): https://arxiv.org/abs/2309.15217 +7. RAGAS metrics docs (context precision/recall/faithfulness): https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/context_precision/ , https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/context_recall/ , https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/faithfulness/ +8. ARES (NAACL 2024): https://aclanthology.org/2024.naacl-long.20/ +9. RAGChecker (paper + repo): https://arxiv.org/abs/2408.08067 , https://github.com/amazon-science/RAGChecker +10. BEIR benchmark: https://arxiv.org/abs/2104.08663 +11. Sentence Transformers reranker docs: https://www.sbert.net/examples/cross_encoder/training/rerankers/README.html +12. MT-Bench / LLM-as-judge biases: https://arxiv.org/abs/2306.05685 +13. Position bias in judges: https://arxiv.org/abs/2406.07791 +14. Bias catalog for LLM-as-judge: https://arxiv.org/abs/2410.02736 +15. Lost in the Middle: https://arxiv.org/abs/2307.03172 +16. Self-RAG: https://arxiv.org/abs/2310.11511 +17. SummaC (NLI consistency): https://arxiv.org/abs/2111.09525 +18. Example NLI model card (`cross-encoder/nli-deberta-v3-large`): https://huggingface.co/cross-encoder/nli-deberta-v3-large diff --git a/agent_logs/plans/20260218_reduced_heuristics_eval_and_retrieval_bench.md b/agent_logs/plans/20260218_reduced_heuristics_eval_and_retrieval_bench.md new file mode 100644 index 0000000..c832ac8 --- /dev/null +++ b/agent_logs/plans/20260218_reduced_heuristics_eval_and_retrieval_bench.md @@ -0,0 +1,86 @@ +# 20260218 Reduced-Heuristics Eval + Retrieval Benchmark Plan + +## Scope +Complete three deliverables on branch `mlin/reduce-hardcoded-heuristics`: +1) checkpoint right-sized commits without undoing any existing changes, +2) re-run eval pipeline on current best defaults and produce a detailed reduced-heuristics benchmark report, +3) implement `IMPROVE_RAG_EVAL.md` recommendations and produce a retrieval/reranking quality benchmark report. + +## Phase 1 - Commit checkpoint hygiene + +### Approach +- Review current modified/untracked files and group them into coherent commit units. +- Keep unrelated user-provided files intact; do not discard or rewrite externally-added content. +- Commit runtime + tests + docs as separate logical slices where possible. + +### Acceptance criteria +- Working tree is checkpointed with traceable commits before new experiments start. +- No existing changes are undone. + +## Phase 2 - Reduced-heuristics eval rerun + judge alignment audit + +### Approach +- Identify and use the best-current eval settings (from README/BENCHMARK and code defaults). +- Run full eval pipeline with current defaults and capture run artifact paths. +- Perform manual audit on both judge-failure and judge-pass samples to estimate alignment quality. +- Compare new metrics versus benchmark history (prioritizing comparable non-heuristic-heavy baselines). +- Write `BENCHMARK_REDUCED_HEURISTICS.md` with metrics, failure patterns, surprises, and hypotheses. + +### Acceptance criteria +- Reproducible run command(s) and run IDs are documented. +- Manual audit includes both positive and negative judge decisions. +- Report clearly contrasts current run against prior benchmark records. + +## Phase 3 - Implement `IMPROVE_RAG_EVAL.md` recommendations + +### Approach +- Read and translate recommendations into concrete code/task changes. +- Add retrieval/rerank evaluation support and local benchmark harness updates. +- Evaluate open-source local models for retrieval/reranking analysis; prefer credible finance-capable models when available. +- Use model-assisted audit to approximate expert chunk relevance checks and summarize confidence/limitations. + +### Acceptance criteria +- Recommendations are implemented or explicitly documented as blocked/deferred with rationale. +- Retrieval/reranking benchmark output is generated and reproducible. + +## Phase 4 - Retrieval/reranking benchmark report + +### Approach +- Run retrieval/rerank benchmarks with updated harness. +- Analyze chunk relevance quality and reranker lift with model-assisted audits. +- Write `BENCHMARK_RETRIEVAL.md` with findings, surprises, and hypotheses. + +### Acceptance criteria +- Report includes methods, datasets, metrics, key error modes, and improvement hypotheses. +- Report includes enough detail for interview/demo discussion. + +## Phase 5 - Logging, validation, and final cleanup + +### Approach +- Append LOGBOOK entries after each major iteration/commit with commit hashes. +- Update CHANGELOG for behavior changes. +- Run final repo checks (`pre-commit run --all`, `pytest -vvv tests/`). + +### Acceptance criteria +- LOGBOOK provides traceable lineage with commit references. +- Lint/tests pass at wrap-up. + +## files_to_change +- `BENCHMARK_REDUCED_HEURISTICS.md` (new) +- `BENCHMARK_RETRIEVAL.md` (new) +- `agent_logs/LOGBOOK.md` +- `CHANGELOG.md` +- `src/andromeda/query/runtime.py` +- `src/andromeda/query/planner_heuristics.py` +- `tests/test_query_runtime_tools_first.py` +- eval/retrieval benchmark scripts and related source files as required by recommendations in `IMPROVE_RAG_EVAL.md` + +## new_files +- `agent_logs/plans/20260218_reduced_heuristics_eval_and_retrieval_bench.md` +- `BENCHMARK_REDUCED_HEURISTICS.md` +- `BENCHMARK_RETRIEVAL.md` +- Additional benchmark helper scripts under `agent_logs/scripts/` as needed + +## Suggested future work (out of current scope) +- Add dedicated paid-LLM integration test suite with environment-gated execution in CI/nightly. +- Add lightweight human-labeled retrieval relevance set for calibration of model-assisted audits. From a127bd0263469bc52c0ddca6062e0cb788d79ff3 Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 19:19:15 +0800 Subject: [PATCH 04/10] refactor: make planner first-class and demote heuristics to fallback --- CHANGELOG.md | 13 + agent_logs/LOGBOOK.md | 54 ++ .../20260218_reduce_hardcoded_heuristics.md | 79 ++ src/andromeda/query/planner_heuristics.py | 286 ++++++ src/andromeda/query/runtime.py | 897 ++++-------------- tests/test_query_runtime_tools_first.py | 683 ++++++------- 6 files changed, 902 insertions(+), 1110 deletions(-) create mode 100644 agent_logs/plans/20260218_reduce_hardcoded_heuristics.md create mode 100644 src/andromeda/query/planner_heuristics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d73eb65..3052ab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,25 @@ This format is based on [Keep a Changelog](https://keepachangelog.com/). ## Unreleased ### Added +- Planner fallback heuristics module at `src/andromeda/query/planner_heuristics.py` to isolate regex/keyword logic from normal runtime flow. ### Changed +- Query planning is now planner-first with explicit multi-label `characteristics` in `PlannerDecision`; routing defaults derive from planner output rather than question regex checks. +- Planner execution now attempts a structured-output repair call after both malformed planner JSON and primary planner call errors; heuristic fallback is used only if both attempts fail. +- Fallback ticker inference now uses `yfinance.Search(...)` and intersects results with indexed tickers instead of regex ticker extraction. +- Tools-first routing defaults were tightened: + - non-narrative market/financial metric requests default to finance tools without mandatory RAG, + - mixed narrative + market/financial requests can enable both RAG and tools. ### Fixed ### Removed +- Removed brittle runtime heuristic stages from active execution path: + - narrative retrieval-query expansion + - narrative aspect-coverage chunk post-processing + - MMR chunk diversification + - adaptive retrieval-budget lowering +- Removed corresponding heuristic helper implementations from `src/andromeda/query/runtime.py`; fallback heuristics now live in the dedicated planner fallback module. ### Dev diff --git a/agent_logs/LOGBOOK.md b/agent_logs/LOGBOOK.md index 10eef1f..5c4a75b 100644 --- a/agent_logs/LOGBOOK.md +++ b/agent_logs/LOGBOOK.md @@ -2690,3 +2690,57 @@ - Pending final repo checks at wrap-up: - `source .venv/bin/activate && PRE_COMMIT_HOME=/tmp/pre-commit-cache pre-commit run --all` - `source .venv/bin/activate && pytest -vvv tests/` + +## 2026-02-18 - Heuristics Reduction Refactor (planner-first branch) + +### Scope +- Branch context: `mlin/reduce-hardcoded-heuristics`. +- Goal: make planner LLM the first-resort routing mechanism and demote regex/heuristic logic to fallback-only behavior. + +### What changed +- Added fallback-only heuristics module: + - `src/andromeda/query/planner_heuristics.py` +- Refactored runtime planner contract: + - `PlannerDecision` now uses multi-label `characteristics` for query traits. + - `resolve_tool_usage_from_decision(...)` now derives defaults from planner characteristics + explicit planner flags. + - Non-narrative market/financial queries default to tools-first (`use_rag=false` when tool flags are sufficient). + - Mixed narrative + tool queries can run both RAG and tools. +- Added planner repair-on-failure behavior: + - `_planner_decision_from_llm(...)` now always attempts one repair call when the primary planner response is invalid JSON/schema **or** when the primary planner call errors. + - Heuristic fallback runs only after both planner attempts fail. +- Disabled brittle heuristic stages in active runtime path by removing them from `runtime.py`: + - narrative query expansion + - narrative aspect coverage enforcement + - MMR diversity pass + - adaptive retrieval budget lowering +- Reworked tests to match new behavior: + - `tests/test_query_runtime_tools_first.py` now validates planner-first routing, repair behavior, fallback-only heuristics usage, and yfinance-backed fallback ticker inference. + +### Validation +- Ran targeted runtime suite: + - `source .venv/bin/activate && pytest -vvv tests/test_query_runtime_tools_first.py` + - Result: `14 passed`. + +### Notes +- This refactor intentionally keeps heuristics available only as a resilience fallback (planner malformed/error path), not as a first-pass routing layer. + +### Post-refactor full checks +- `source .venv/bin/activate && PRE_COMMIT_HOME=/tmp/pre-commit-cache pre-commit run --all` -> passed. +- `source .venv/bin/activate && pytest -vvv tests/` -> passed (`114 passed`). + +## 2026-02-18 - Runtime test strategy correction (planner path realism) + +### Trigger +- User review pointed out overuse of monkeypatching in `tests/test_query_runtime_tools_first.py`, especially for planner/ticker inference paths. + +### Changes made +- Reworked tests to drive planner behavior through `RecordingLLM` structured outputs rather than monkeypatching `RAGService._planner_decision_from_llm`. +- Added helper queue (`planner_outputs`) so tests exercise real planner parse/repair/fallback code paths inside `RAGService`. +- Added live fallback integration-style coverage for ticker inference: + - `test_plan_query_fallback_infers_ticker_via_live_yfinance_search` + - Uses a vague company-name query and verifies `plan_query(...)` infers `NVDA` via yfinance-based fallback path when planner+repair are invalid. + +### Validation +- `source .venv/bin/activate && pytest -vvv tests/test_query_runtime_tools_first.py` -> `14 passed`. +- `source .venv/bin/activate && PRE_COMMIT_HOME=/tmp/pre-commit-cache pre-commit run --all` -> passed. +- `source .venv/bin/activate && pytest -vvv tests/` -> `114 passed`. diff --git a/agent_logs/plans/20260218_reduce_hardcoded_heuristics.md b/agent_logs/plans/20260218_reduce_hardcoded_heuristics.md new file mode 100644 index 0000000..7046d47 --- /dev/null +++ b/agent_logs/plans/20260218_reduce_hardcoded_heuristics.md @@ -0,0 +1,79 @@ +# Reduce Hardcoded Heuristics (Planner-First) - 2026-02-18 + +## Scope +Refactor query planning so the LLM planner is the first-resort decision engine, with heuristic logic only as fallback after planner failure + repair failure. Replace brittle ticker inference regex with yfinance search-backed inference. Disable brittle heuristic retrieval post-processing in normal flow. + +## Phase 1 - Planner-first structured classification + repair + +### Approach +- Extend planner schema to include non-mutually-exclusive `characteristics` labels. +- Upgrade planner prompt with few-shot characteristic mapping examples. +- Add a repair prompt path: + 1) primary planner request, + 2) if invalid JSON/schema -> repair request using original raw output, + 3) if still invalid -> fallback heuristics. +- Ensure runtime tool/routing defaults come from planner structured outputs (not regex functions) when planner succeeds. + +### Acceptance criteria +- Valid planner output drives routing without regex-based question classification. +- Invalid planner output triggers exactly one repair attempt. +- Fallback planner heuristics are used only after primary + repair failure. + +## Phase 2 - Extract heuristics into dedicated fallback module + +### Approach +- Move regex/question-classification fallback logic into `src/andromeda/query/planner_heuristics.py`. +- Keep runtime references to heuristics constrained to fallback-only path. +- Keep heuristics implementation private and explicitly labeled fallback behavior. + +### Acceptance criteria +- `runtime.py` no longer contains first-resort regex routing logic. +- Heuristic classification and regex ticker extraction are not used in successful planner paths. + +## Phase 3 - Replace brittle ticker inference with yfinance search + +### Approach +- Implement fallback ticker inference using `yfinance.Search(...)`. +- Normalize/validate results against indexed ticker catalog. +- Keep deterministic dedupe and bounded results. + +### Acceptance criteria +- Fallback ticker inference does not use regex-only extraction as primary source. +- Inference degrades safely when yfinance import/network fails. + +## Phase 4 - Disable brittle retrieval heuristics in normal path + +### Approach +- Disable adaptive retrieval budget scheduling in active execution path. +- Disable MMR diversification and narrative aspect-coverage post-processing in active rerank path. +- Disable narrative query expansion in active retrieval path. + +### Acceptance criteria +- Normal execution path does not apply these heuristic transformations. +- Core tools-first + rerank pipeline remains functional. + +## Phase 5 - Tests, docs, and changelog + +### Approach +- Update `tests/test_query_runtime_tools_first.py` for planner-first + fallback behavior. +- Add/adjust targeted tests for planner repair and yfinance-search fallback. +- Update `CHANGELOG.md` and append summary in `agent_logs/LOGBOOK.md`. + +### Acceptance criteria +- `pre-commit run --all` passes. +- `pytest -vvv tests/` passes. +- Changelog/logbook document behavior changes and fallback policy. + +## files_to_change +- `src/andromeda/query/runtime.py` +- `src/andromeda/query/planner_heuristics.py` (new) +- `tests/test_query_runtime_tools_first.py` +- `CHANGELOG.md` +- `agent_logs/LOGBOOK.md` + +## new_files +- `src/andromeda/query/planner_heuristics.py` + +## Suggested future work (out of current scope) +- Add planner decision quality benchmark set + regression gating. +- Add production telemetry for planner parse/repair/fallback rates per query type. diff --git a/src/andromeda/query/planner_heuristics.py b/src/andromeda/query/planner_heuristics.py new file mode 100644 index 0000000..90bf3a9 --- /dev/null +++ b/src/andromeda/query/planner_heuristics.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import importlib +import re + +from andromeda.ingestion.ingestion_jobs import normalize_ticker + + +class PlannerFallbackHeuristics: + """ + Heuristic helpers used only when planner structured output fails. + """ + + CHARACTERISTIC_COMPARISON = "comparison" + CHARACTERISTIC_MARKET_DATA = "market_data" + CHARACTERISTIC_FINANCIAL_METRICS = "financial_metrics" + CHARACTERISTIC_FILING_NARRATIVE = "filing_narrative" + CHARACTERISTIC_PERIOD_SCOPED = "period_scoped" + CHARACTERISTIC_SIMPLE_NUMERIC = "simple_numeric" + + @staticmethod + def question_mentions_comparison(question: str) -> bool: + lowered = question.lower() + tokens = (" compare ", " versus ", " vs ", " relative to ", " better investment ", " which is better ", " or ") + padded = f" {lowered} " + return any(token in padded for token in tokens) + + @staticmethod + def question_mentions_market_data(question: str) -> bool: + lowered = f" {question.lower()} " + tokens = ( + " stock price ", + " price ", + " chart ", + " valuation ", + " market cap ", + " news ", + " return ", + " performance ", + " volume ", + " pe ratio ", + " p/e ", + " dividend ", + ) + return any(token in lowered for token in tokens) + + @staticmethod + def question_mentions_financial_metrics(question: str) -> bool: + lowered = f" {question.lower()} " + tokens = ( + " revenue ", + " net income ", + " gross margin ", + " operating margin ", + " eps ", + " balance sheet ", + " cash flow ", + " free cash flow ", + " assets ", + " liabilities ", + " equity ", + " ratio ", + " debt ", + ) + return any(token in lowered for token in tokens) + + @staticmethod + def question_has_explicit_period_scope(question: str) -> bool: + lowered = f" {question.lower()} " + if re.search(r"\b20\d{2}\b", lowered): + return True + tokens = ( + " quarter ", + " q1 ", + " q2 ", + " q3 ", + " q4 ", + " fiscal year ", + " fy ", + " year ended ", + " as of ", + " during ", + " in the latest filing ", + " latest filing ", + ) + return any(token in lowered for token in tokens) + + @staticmethod + def infer_filing_date_window_from_question(question: str) -> tuple[str, str] | None: + """ + Infer inclusive filing-date window from explicit years in question. + """ + + years = sorted({int(token) for token in re.findall(r"\b(20\d{2})\b", question)}) + if not years: + return None + start_year = years[0] + end_year = years[-1] + if end_year - start_year > 6: + return None + return f"{start_year:04d}-01-01", f"{end_year:04d}-12-31" + + @staticmethod + def question_mentions_filing_narrative(question: str) -> bool: + lowered = f" {question.lower()} " + tokens = ( + " sec filing ", + " sec filings ", + " long-term investment ", + " long term investment ", + " investment thesis ", + " bull-vs-bear ", + " bull vs bear ", + " business trajectory ", + " growth driver ", + " growth drivers ", + " growth opportunities ", + " key risks ", + " material risks ", + " downside risks ", + " competitive positioning ", + " competitive position ", + " risk factor ", + " management discussion ", + " management commentary ", + " md&a ", + " discuss ", + " explain ", + " guidance ", + " outlook ", + " strategy ", + " segment ", + " capital allocation ", + " margin resilience ", + " cash-flow quality ", + " cash flow quality ", + " operational bottleneck ", + " operational bottlenecks ", + " dependencies ", + " demand trends ", + " customer behavior ", + " trade-off ", + " trade-offs ", + " why ", + ) + return any(token in lowered for token in tokens) + + @classmethod + def question_is_simple_numeric_metric(cls, question: str) -> bool: + """ + Return whether the question is a direct numeric metric lookup. + """ + + mentions_metrics = cls.question_mentions_financial_metrics(question) or cls.question_mentions_market_data(question) + mentions_narrative = cls.question_mentions_filing_narrative(question) + mentions_comparison = cls.question_mentions_comparison(question) + has_period_scope = cls.question_has_explicit_period_scope(question) + lowered = f" {question.lower()} " + has_explicit_numeric_intent = any( + token in lowered + for token in ( + " what was ", + " what is ", + " how much ", + " amount ", + " total ", + " value ", + " figure ", + " give me ", + ) + ) + token_count = len(question.split()) + return ( + mentions_metrics + and not mentions_narrative + and not mentions_comparison + and not has_period_scope + and has_explicit_numeric_intent + and token_count <= 24 + ) + + @classmethod + def classify_characteristics(cls, question: str) -> list[str]: + """ + Infer planner characteristics with fallback heuristics. + """ + + out: list[str] = [] + if cls.question_mentions_comparison(question): + out.append(cls.CHARACTERISTIC_COMPARISON) + if cls.question_mentions_market_data(question): + out.append(cls.CHARACTERISTIC_MARKET_DATA) + if cls.question_mentions_financial_metrics(question): + out.append(cls.CHARACTERISTIC_FINANCIAL_METRICS) + if cls.question_mentions_filing_narrative(question): + out.append(cls.CHARACTERISTIC_FILING_NARRATIVE) + if cls.question_has_explicit_period_scope(question): + out.append(cls.CHARACTERISTIC_PERIOD_SCOPED) + if cls.question_is_simple_numeric_metric(question): + out.append(cls.CHARACTERISTIC_SIMPLE_NUMERIC) + return out + + @staticmethod + def infer_tickers_from_question(question: str, companies: list[dict[str, str]]) -> list[str]: + """ + Infer candidate tickers using yfinance search (fallback path only). + """ + + known_tickers: set[str] = set() + for item in companies: + if "ticker" not in item: + continue + ticker = str(item["ticker"]).strip().upper() + if ticker: + known_tickers.add(ticker) + if not known_tickers: + return [] + + try: + yfinance = importlib.import_module("yfinance") + except Exception: + return [] + + search_terms: list[str] = [] + base_query = str(question).strip() + if base_query: + search_terms.append(base_query) + + normalized_question = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " + for item in companies: + if "company" not in item: + continue + company = str(item["company"]).strip() + if not company: + continue + normalized_company = " " + re.sub(r"[^a-z0-9]+", " ", company.lower()).strip() + " " + if normalized_company in normalized_question: + search_terms.append(company) + + deduped_terms: list[str] = [] + seen_terms: set[str] = set() + for term in search_terms: + key = term.lower().strip() + if not key or key in seen_terms: + continue + seen_terms.add(key) + deduped_terms.append(term) + + inferred: list[str] = [] + seen_tickers: set[str] = set() + for term in deduped_terms[:4]: + try: + search_obj = yfinance.Search( + term, + max_results=12, + news_count=0, + lists_count=0, + include_nav_links=False, + include_research=False, + include_cultural_assets=False, + enable_fuzzy_query=True, + raise_errors=False, + timeout=10, + ) + except Exception: + continue + quotes = search_obj.quotes if hasattr(search_obj, "quotes") else [] + if not isinstance(quotes, list): + continue + for quote in quotes: + if not isinstance(quote, dict) or "symbol" not in quote: + continue + raw_symbol = str(quote["symbol"]).strip() + if not raw_symbol: + continue + try: + symbol = normalize_ticker(raw_symbol) + except ValueError: + symbol = raw_symbol.upper() + if symbol not in known_tickers or symbol in seen_tickers: + continue + seen_tickers.add(symbol) + inferred.append(symbol) + if len(inferred) >= 6: + return inferred + return inferred diff --git a/src/andromeda/query/runtime.py b/src/andromeda/query/runtime.py index c3a9f6a..307353b 100644 --- a/src/andromeda/query/runtime.py +++ b/src/andromeda/query/runtime.py @@ -8,7 +8,7 @@ import time from concurrent.futures import ThreadPoolExecutor from collections.abc import AsyncIterator -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from enum import Enum from typing import Any, cast @@ -30,6 +30,7 @@ build_refine_prompt, build_ticker_brief_prompt, ) +from andromeda.query.planner_heuristics import PlannerFallbackHeuristics from andromeda.retrieval.retriever import CrossEncoderReranker, PostgresHybridRetriever from andromeda.llm.streaming import TextDeltaBatcher, iter_chat_deltas, ndjson_bytes @@ -46,8 +47,16 @@ class PlannerAction(str, Enum): REFUSED = "refused" -class RetrievalBudgetProfile(str, Enum): - DEFAULT = "default" +class QueryCharacteristic(str, Enum): + """ + Non-mutually-exclusive planner labels describing query traits. + """ + + COMPARISON = "comparison" + MARKET_DATA = "market_data" + FINANCIAL_METRICS = "financial_metrics" + FILING_NARRATIVE = "filing_narrative" + PERIOD_SCOPED = "period_scoped" SIMPLE_NUMERIC = "simple_numeric" @@ -165,6 +174,7 @@ def set_cancelled() -> None: class PlannerDecision(BaseModel): action: PlannerAction = PlannerAction.ANSWER tickers: list[str] = Field(default_factory=list) + characteristics: list[QueryCharacteristic] = Field(default_factory=list) filing_date_from: str | None = None filing_date_to: str | None = None clarifying_question: str | None = None @@ -388,592 +398,42 @@ def _normalize_plan_action(action: PlannerAction) -> QueryStatus: return QueryStatus.ANSWERED @staticmethod - def _question_mentions_comparison(question: str) -> bool: - lowered = question.lower() - tokens = (" compare ", " versus ", " vs ", " relative to ", " better investment ", " which is better ", " or ") - padded = f" {lowered} " - return any(token in padded for token in tokens) - - @staticmethod - def _question_mentions_market_data(question: str) -> bool: - lowered = f" {question.lower()} " - tokens = ( - " stock price ", - " price ", - " chart ", - " valuation ", - " market cap ", - " news ", - " return ", - " performance ", - " volume ", - " pe ratio ", - " p/e ", - " dividend ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _question_mentions_financial_metrics(question: str) -> bool: - lowered = f" {question.lower()} " - tokens = ( - " revenue ", - " net income ", - " gross margin ", - " operating margin ", - " eps ", - " balance sheet ", - " cash flow ", - " free cash flow ", - " assets ", - " liabilities ", - " equity ", - " ratio ", - " debt ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _question_has_explicit_period_scope(question: str) -> bool: - lowered = f" {question.lower()} " - if re.search(r"\b20\d{2}\b", lowered): - return True - tokens = ( - " quarter ", - " q1 ", - " q2 ", - " q3 ", - " q4 ", - " fiscal year ", - " fy ", - " year ended ", - " as of ", - " during ", - " in the latest filing ", - " latest filing ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _infer_filing_date_window_from_question(question: str) -> tuple[str, str] | None: - """ - Infer an inclusive filing-date window from explicit years in the question. - """ - - years = sorted({int(token) for token in re.findall(r"\b(20\d{2})\b", question)}) - if not years: - return None - start_year = years[0] - end_year = years[-1] - if end_year - start_year > 6: - return None - return f"{start_year:04d}-01-01", f"{end_year:04d}-12-31" - - @staticmethod - def _question_mentions_filing_narrative(question: str) -> bool: - lowered = f" {question.lower()} " - tokens = ( - " sec filing ", - " sec filings ", - " long-term investment ", - " long term investment ", - " investment thesis ", - " bull-vs-bear ", - " bull vs bear ", - " business trajectory ", - " growth driver ", - " growth drivers ", - " growth opportunities ", - " key risks ", - " material risks ", - " downside risks ", - " competitive positioning ", - " competitive position ", - " risk factor ", - " management discussion ", - " management commentary ", - " md&a ", - " discuss ", - " explain ", - " guidance ", - " outlook ", - " strategy ", - " segment ", - " capital allocation ", - " margin resilience ", - " cash-flow quality ", - " cash flow quality ", - " operational bottleneck ", - " operational bottlenecks ", - " dependencies ", - " demand trends ", - " customer behavior ", - " trade-off ", - " trade-offs ", - " why ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _question_mentions_growth_or_strategy(question: str) -> bool: - lowered = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " - tokens = ( - " growth ", - " growth driver ", - " growth drivers ", - " growth opportunities ", - " strategy ", - " competitive positioning ", - " positioning ", - " business trajectory ", - " long-term investment ", - " long term investment ", - " outlook ", - " opportunities ", - " investment thesis ", - " capital allocation ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _question_mentions_risk_dimension(question: str) -> bool: - lowered = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " - tokens = (" risk ", " risks ", " uncertainty ", " uncertainties ", " downside ", " bottleneck ") - return any(token in lowered for token in tokens) - - @staticmethod - def _question_mentions_capital_margin_or_cashflow(question: str) -> bool: - lowered = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " - tokens = ( - " capital allocation ", - " capex ", - " buyback ", - " buybacks ", - " debt ", - " margin ", - " margins ", - " profitability ", - " operating leverage ", - " cash flow ", - " cashflow ", - " working capital ", - " trade off ", - " trade offs ", - " trade-off ", - " trade-offs ", - ) - return any(token in lowered for token in tokens) - - @staticmethod - def _question_mentions_execution_or_demand(question: str) -> bool: - lowered = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " - tokens = ( - " execution ", - " operational ", - " dependency ", - " dependencies ", - " bottleneck ", - " bottlenecks ", - " demand trend ", - " demand trends ", - " customer behavior ", - " customer demand ", - " supply chain ", - " constraint ", - " constraints ", - ) - return any(token in lowered for token in tokens) - - def narrative_retrieval_queries(self, question: str) -> list[str]: - """ - Build diversified retrieval queries for filing-narrative questions. - """ - - base = question.strip() - if not base: - return [] - queries = [base] - if self._question_mentions_growth_or_strategy(question): - queries.append( - f"{base} Focus on explicitly stated growth drivers, strategy, revenue, segment performance, and demand." - ) - if self._question_mentions_risk_dimension(question): - queries.append(f"{base} Focus on explicitly stated risk factors, uncertainties, and constraints.") - if self._question_mentions_capital_margin_or_cashflow(question): - queries.append( - f"{base} Focus on explicit capital allocation, profitability, margin, and cash-flow disclosures." - ) - if self._question_mentions_execution_or_demand(question): - queries.append( - f"{base} Focus on explicit execution dependencies, demand commentary, and operational constraints." - ) - - deduped: list[str] = [] - seen: set[str] = set() - for query in queries: - key = query.lower().strip() - if not key or key in seen: + def _characteristics_set(decision: PlannerDecision) -> set[QueryCharacteristic]: + out: set[QueryCharacteristic] = set() + for item in decision.characteristics: + if isinstance(item, QueryCharacteristic): + out.add(item) continue - seen.add(key) - deduped.append(query) - return deduped[:4] - - @staticmethod - def _chunk_text_signature(sc: ScoredChunk) -> str: - parsed = chunk_metadata_from_value(sc.chunk.metadata) - section = parsed.section_path or "" - headings = " ".join(sc.chunk.headings or []) - text = (sc.chunk.text or "")[:300] - return f"{section} {headings} {text}".lower() - - def _is_risk_chunk(self, sc: ScoredChunk) -> bool: - text = self._chunk_text_signature(sc) - tokens = ("risk factor", "risks", "uncertaint", "adverse", "regulatory", "cyber") - return any(token in text for token in tokens) - - def _is_growth_or_strategy_chunk(self, sc: ScoredChunk) -> bool: - text = self._chunk_text_signature(sc) - if self._is_risk_chunk(sc): - return False - tokens = ( - "results of operations", - "revenue", - "segment", - "overview", - "management discussion", - "md&a", - "strategy", - "competitive", - "growth", - "demand", - "business", - ) - return any(token in text for token in tokens) - - def _enforce_narrative_aspect_coverage( - self, *, question: str, primary: list[ScoredChunk], fallback: list[ScoredChunk], limit: int - ) -> list[ScoredChunk]: - """ - Ensure narrative contexts include both growth/strategy and risk evidence when requested. - """ - - need_growth = self._question_mentions_growth_or_strategy(question) - need_risk = self._question_mentions_risk_dimension(question) - if not need_growth and not need_risk: - return primary[:limit] - - selected: list[ScoredChunk] = [] - selected_ids: set[str] = set() - - def add_first_matching(pool: list[ScoredChunk], predicate) -> bool: - for sc in pool: - if not predicate(sc): - continue - chunk_id = sc.chunk.id - if chunk_id in selected_ids: - continue - selected.append(sc) - selected_ids.add(chunk_id) - return True - return False - - if need_growth: - if not add_first_matching(primary, self._is_growth_or_strategy_chunk): - add_first_matching(fallback, self._is_growth_or_strategy_chunk) - if need_risk: - if not add_first_matching(primary, self._is_risk_chunk): - add_first_matching(fallback, self._is_risk_chunk) - - combined = self._dedupe_scored_chunks(primary + fallback) - for sc in combined: - if len(selected) >= limit: - break - if sc.chunk.id in selected_ids: + try: + out.add(QueryCharacteristic(str(item).strip().lower())) + except ValueError: continue - selected.append(sc) - selected_ids.add(sc.chunk.id) - - selected.sort(key=lambda item: item.score, reverse=True) - return selected[:limit] - - @staticmethod - def _mmr_token_set(sc: ScoredChunk) -> set[str]: - parsed = chunk_metadata_from_value(sc.chunk.metadata) - text = parsed.retrieval_text or sc.chunk.text or "" - text = str(text).lower() - tokens = re.findall(r"[a-z0-9]+", text) - stopwords = { - "the", - "and", - "for", - "with", - "that", - "this", - "from", - "were", - "are", - "was", - "have", - "has", - "had", - "into", - "than", - "over", - "under", - "their", - "they", - "its", - "our", - "you", - "your", - "also", - "may", - "can", - "could", - "would", - "should", - "will", - } - return {token for token in tokens[:140] if len(token) > 2 and token not in stopwords} - - @staticmethod - def _token_jaccard_similarity(left: set[str], right: set[str]) -> float: - if not left or not right: - return 0.0 - inter = len(left.intersection(right)) - union = len(left.union(right)) - if union <= 0: - return 0.0 - return inter / union - - def apply_mmr_diversity( - self, *, candidates: list[ScoredChunk], limit: int, lambda_mult: float = 0.78 - ) -> list[ScoredChunk]: - """ - Select a relevance-diverse subset using a bounded MMR pass. - """ - - if limit <= 0: - return [] - if len(candidates) <= 1: - return candidates[:limit] - - pool_size = max(limit, min(len(candidates), limit * 3)) - pool = candidates[:pool_size] - token_sets = [self._mmr_token_set(sc) for sc in pool] - raw_scores = [float(sc.score) for sc in pool] - score_min = min(raw_scores) - score_max = max(raw_scores) - - def normalized_score(index: int) -> float: - raw = raw_scores[index] - if score_max <= score_min: - return 1.0 - return (raw - score_min) / (score_max - score_min) - - selected_indices: list[int] = [] - remaining = set(range(len(pool))) - while remaining and len(selected_indices) < limit: - best_idx: int | None = None - best_value = float("-inf") - for idx in remaining: - relevance = normalized_score(idx) - if not selected_indices: - novelty_penalty = 0.0 - else: - novelty_penalty = max( - self._token_jaccard_similarity(token_sets[idx], token_sets[sel]) for sel in selected_indices - ) - mmr_value = (lambda_mult * relevance) - ((1.0 - lambda_mult) * novelty_penalty) - if mmr_value > best_value: - best_value = mmr_value - best_idx = idx - if best_idx is None: - break - remaining.remove(best_idx) - selected_indices.append(best_idx) - - out = [pool[idx] for idx in selected_indices] - out.sort(key=lambda item: item.score, reverse=True) - return out[:limit] - - @staticmethod - def mmr_diversity_enabled() -> bool: - """ - Return whether experimental MMR chunk diversity is enabled. - """ - - raw = (os.getenv("FINRAG_ENABLE_MMR_DIVERSITY") or "0").strip().lower() - return raw in {"1", "true", "yes", "on"} - - @staticmethod - def narrative_query_expansion_enabled() -> bool: - """ - Return whether diversified narrative retrieval-query expansion is enabled. - """ - - raw = (os.getenv("FINRAG_ENABLE_NARRATIVE_QUERY_EXPANSION") or "0").strip().lower() - return raw in {"1", "true", "yes", "on"} - - @staticmethod - def narrative_aspect_coverage_enabled() -> bool: - """ - Return whether narrative growth/risk aspect-coverage enforcement is enabled. - - FIXME: feels too brittle. - """ - - raw = (os.getenv("FINRAG_ENABLE_NARRATIVE_ASPECT_COVERAGE") or "1").strip().lower() - return raw in {"1", "true", "yes", "on"} - - def _question_is_simple_numeric_metric(self, question: str) -> bool: - """ - Return whether the question is a direct numeric metric lookup. - """ - - mentions_metrics = self._question_mentions_financial_metrics(question) or self._question_mentions_market_data( - question - ) - mentions_narrative = self._question_mentions_filing_narrative(question) - mentions_comparison = self._question_mentions_comparison(question) - has_period_scope = self._question_has_explicit_period_scope(question) - lowered = f" {question.lower()} " - has_explicit_numeric_intent = any( - token in lowered - for token in ( - " what was ", - " what is ", - " how much ", - " amount ", - " total ", - " value ", - " figure ", - " give me ", - ) - ) - token_count = len(question.split()) - return ( - mentions_metrics - and not mentions_narrative - and not mentions_comparison - and not has_period_scope - and has_explicit_numeric_intent - and token_count <= 24 - ) - - @staticmethod - def adaptive_retrieval_budget_enabled() -> bool: - """ - Return whether adaptive retrieval-budget scheduling is enabled. - - FIXME: feels brittle. - """ - - raw = (os.getenv("FINRAG_ENABLE_ADAPTIVE_RETRIEVAL_BUDGET") or "1").strip().lower() - return raw in {"1", "true", "yes", "on"} - - def apply_adaptive_retrieval_budget( - self, *, question: str, settings: GenerationSettings, planned: PlannedQuery - ) -> tuple[GenerationSettings, ToolTraceEvent | None]: - """ - Adjust retrieval depth for simple tool-first numeric lookups. - - The scheduler only reduces retrieval/rerank depth for low-complexity - numeric questions and leaves all other queries unchanged. - """ - - if not self.adaptive_retrieval_budget_enabled(): - return settings, None - - if not self._question_is_simple_numeric_metric(question): - return settings, None - - if len(planned.tickers) > 1 or self._question_mentions_comparison(question): - return settings, None - - if self._question_mentions_filing_narrative(question): - return settings, None - - target_top_k_retrieve = max(12, int(round(settings.top_k_retrieve * 0.70))) - target_top_k_rerank = max(8, int(round(settings.top_k_rerank * 0.60))) - target_top_k_retrieve = min(settings.top_k_retrieve, target_top_k_retrieve) - target_top_k_rerank = min(settings.top_k_rerank, target_top_k_rerank, target_top_k_retrieve) - - if target_top_k_retrieve == settings.top_k_retrieve and target_top_k_rerank == settings.top_k_rerank: - return settings, None - - adjusted_settings = replace(settings, top_k_retrieve=target_top_k_retrieve, top_k_rerank=target_top_k_rerank) - trace_event = self._tool_event( - "adaptive_retrieval_budget", - args={ - "profile": RetrievalBudgetProfile.SIMPLE_NUMERIC.value, - "old_top_k_retrieve": settings.top_k_retrieve, - "old_top_k_rerank": settings.top_k_rerank, - "new_top_k_retrieve": adjusted_settings.top_k_retrieve, - "new_top_k_rerank": adjusted_settings.top_k_rerank, - }, - result="Applied adaptive retrieval budget for simple numeric query.", - ) - return adjusted_settings, trace_event + return out - def resolve_tool_usage_from_decision(self, *, question: str, decision: PlannerDecision) -> tuple[bool, bool, bool]: + def resolve_tool_usage_from_decision(self, *, decision: PlannerDecision) -> tuple[bool, bool, bool]: """ Resolve planner tool flags into effective `use_rag`, `use_yfinance`, and `use_edgar_financials`. - - FIXME - ----- - - do not use regex/text heuristics to classify question. - - we should use planner LLM to do this. - - the heuristics should only be a fallback when planner LLM output was an error (eg invalid JSON format) """ - simple_numeric_query = self._question_is_simple_numeric_metric(question) - narrative_query = self._question_mentions_filing_narrative(question) - period_scoped_metric_query = self._question_mentions_financial_metrics( - question - ) and self._question_has_explicit_period_scope(question) - market_data_query = self._question_mentions_market_data(question) - financial_metric_query = self._question_mentions_financial_metrics(question) + characteristics = self._characteristics_set(decision) + market_data_query = QueryCharacteristic.MARKET_DATA in characteristics + financial_metric_query = QueryCharacteristic.FINANCIAL_METRICS in characteristics + narrative_query = QueryCharacteristic.FILING_NARRATIVE in characteristics use_yfinance = bool(decision.use_yfinance) if decision.use_yfinance is not None else market_data_query use_edgar_financials = ( - bool(decision.use_edgar_financials) if decision.use_edgar_financials is not None else financial_metric_query + bool(decision.use_edgar_financials) + if decision.use_edgar_financials is not None + else (financial_metric_query) ) - if simple_numeric_query: - # For direct numeric lookup queries, choose the most relevant finance tool first. - if market_data_query and not financial_metric_query: - use_yfinance = True - use_edgar_financials = False - elif financial_metric_query and not market_data_query: - use_yfinance = False - use_edgar_financials = True - else: - use_yfinance = market_data_query - use_edgar_financials = financial_metric_query or not market_data_query - elif narrative_query: - # For filing-narrative requests, avoid mixing in external market/tool facts. - use_edgar_financials = False - use_yfinance = False - if decision.use_rag is not None: use_rag = bool(decision.use_rag) - else: - if simple_numeric_query: - use_rag = False - elif narrative_query: - use_rag = True - elif use_yfinance or use_edgar_financials: - use_rag = False - else: - use_rag = True - - if simple_numeric_query: - use_rag = False elif narrative_query: use_rag = True - elif period_scoped_metric_query: - # Period-specific metric questions usually need filing chunks for exact timeframe grounding. + elif use_yfinance or use_edgar_financials: + use_rag = False + else: use_rag = True if not use_rag and not use_yfinance and not use_edgar_financials: @@ -981,33 +441,7 @@ def resolve_tool_usage_from_decision(self, *, question: str, decision: PlannerDe return use_rag, use_yfinance, use_edgar_financials def _infer_tickers_from_question(self, question: str, companies: list[dict[str, str]]) -> list[str]: - # FIXME: extremely brittle logic. should use yfinance python library to get tickers from company name. - # TODO: use yfinance instead. - # see: https://deepwiki.com/ranaroussi/yfinance/4.3-search-and-lookup-functionality - inferred: list[str] = [] - seen: set[str] = set() - known_tickers = {str(item["ticker"]).strip().upper() for item in companies if "ticker" in item} - - upper_question = question.upper() - ticker_pattern = re.compile(r"\b[A-Z][A-Z0-9.-]{0,11}\b") - for match in ticker_pattern.findall(upper_question): - token = match.strip().upper() - if token in known_tickers and token not in seen: - seen.add(token) - inferred.append(token) - - lowered_question = " " + re.sub(r"[^a-z0-9]+", " ", question.lower()).strip() + " " - for item in companies: - ticker = str(item["ticker"]).strip().upper() - company = str(item["company"]).strip() - if not ticker or not company: - continue - normalized_company = " " + re.sub(r"[^a-z0-9]+", " ", company.lower()).strip() + " " - if normalized_company.strip() and normalized_company in lowered_question and ticker not in seen: - seen.add(ticker) - inferred.append(ticker) - - return inferred + return PlannerFallbackHeuristics.infer_tickers_from_question(question=question, companies=companies) @staticmethod def default_clarifying_question() -> str: @@ -1025,7 +459,7 @@ def _planner_prompt( filing_date_from: str | None, filing_date_to: str | None, ) -> list[ChatMessage]: - preview_limit = 250 + preview_limit = 500 preview_rows = companies[:preview_limit] catalog_lines = [f"- {row['ticker']}: {row['company']}" for row in preview_rows] catalog = "\n".join(catalog_lines) if catalog_lines else "- (none)" @@ -1033,6 +467,27 @@ def _planner_prompt( date_from = filing_date_from or "(none)" date_to = filing_date_to or "(none)" + characteristics = ", ".join([item.value for item in QueryCharacteristic]) + few_shot = ( + "Few-shot examples (non-mutually-exclusive characteristics):\n" + '- Q: "What is AAPL market cap right now?"\n' + " characteristics: [market_data, simple_numeric]\n" + " use_rag=false, use_yfinance=true, use_edgar_financials=false\n" + '- Q: "What was AAPL net income in 2025?"\n' + " characteristics: [financial_metrics, period_scoped]\n" + " use_rag=false, use_yfinance=false, use_edgar_financials=true\n" + '- Q: "Compare NVDA vs AMD on growth drivers and key risks from filings."\n' + " characteristics: [comparison, filing_narrative]\n" + " use_rag=true, use_yfinance=false, use_edgar_financials=false\n" + " use_per_ticker_retrieval=true, use_multi_ticker_briefs=true\n" + '- Q: "Explain MSFT strategy from filings and include latest valuation context."\n' + " characteristics: [filing_narrative, market_data]\n" + " use_rag=true, use_yfinance=true, use_edgar_financials=false\n" + '- Q: "Summarize TSLA strategy and competitive positioning from recent SEC filings."\n' + " characteristics: [filing_narrative]\n" + " use_rag=true, use_yfinance=false, use_edgar_financials=false\n" + ) + return [ { "role": "system", @@ -1049,15 +504,25 @@ def _planner_prompt( "- use_yfinance=true for market price/news/valuation style requests.\n" "- use_edgar_financials=true for direct SEC financial metric/statement requests.\n" "- use_rag=true when filing narrative evidence is needed from retrieved chunks.\n" - "- use_rag=false for simple direct metric queries answerable from finance tools.\n" + "- use_rag=false when finance tools are sufficient for direct numeric questions.\n" + "- For mixed requests (narrative + market/financial facts), enable both RAG and tools.\n" "IMPORTANT: only clarify if absolutely needed. Do NOT keep asking clarifying questions." "If no date range is provided, just set None for both date_from and date_to in the output - " "do NOT ask for clarification on dates unless the question explicitly references time (like 'latest').\n" + f"6) Set characteristics as a list from this enum: [{characteristics}].\n" + "Characteristics are multi-label and non-mutually-exclusive.\n" "Return only JSON with keys:\n" - "action, tickers, filing_date_from, filing_date_to, clarifying_question, refusal_reason, " + "action, tickers, characteristics, filing_date_from, filing_date_to, clarifying_question, refusal_reason, " "use_per_ticker_retrieval, use_multi_ticker_briefs, use_rag, use_yfinance, use_edgar_financials." + f"{few_shot}" ), }, + { + "role": "system", + "content": ( + f"Indexed ticker catalog (first {len(preview_rows)} of {len(companies)}):\n{catalog}\n\n" + ) + }, { "role": "user", "content": ( @@ -1065,11 +530,51 @@ def _planner_prompt( f"Explicit request tickers: {explicit}\n" f"Explicit filing_date_from: {date_from}\n" f"Explicit filing_date_to: {date_to}\n\n" - f"Indexed ticker catalog (first {len(preview_rows)} of {len(companies)}):\n{catalog}\n" ), }, ] + @staticmethod + def _planner_decision_from_raw(raw: str) -> PlannerDecision | None: + """ + Parse planner output into structured decision. + """ + + try: + return PlannerDecision.model_validate_json(raw) + except ValidationError: + pass + payload = RAGService._extract_json_object(raw) + if payload is None: + return None + try: + return PlannerDecision.model_validate(payload) + except ValidationError: + return None + + def _planner_repair_prompt(self, *, question: str, broken_output: str) -> list[ChatMessage]: + """ + Build repair prompt to recover structured planner JSON. + """ + + return [ + { + "role": "system", + "content": ( + "You repair malformed planner outputs.\n" + "Return strictly valid JSON matching this schema keys:\n" + "action, tickers, characteristics, filing_date_from, filing_date_to, clarifying_question, " + "refusal_reason, use_per_ticker_retrieval, use_multi_ticker_briefs, use_rag, use_yfinance, " + "use_edgar_financials.\n" + "Do not add commentary or markdown." + ), + }, + { + "role": "user", + "content": (f"Original question:\n{question}\n\nMalformed planner output to repair:\n{broken_output}"), + }, + ] + def _planner_decision_from_llm( self, *, @@ -1086,20 +591,23 @@ def _planner_decision_from_llm( filing_date_from=filing_date_from, filing_date_to=filing_date_to, ) + raw_primary = "" + repair_input = "" + try: + raw_primary = self.llm.chat(prompt, temperature=0.0, max_tokens=700, response_model=PlannerDecision) + primary_decision = self._planner_decision_from_raw(raw_primary) + if primary_decision is not None: + return primary_decision + repair_input = raw_primary + except Exception as exc: # noqa: BLE001 + repair_input = f"Planner call failed with error: {exc!s}" + + repair_prompt = self._planner_repair_prompt(question=question, broken_output=repair_input) try: - raw = self.llm.chat(prompt, temperature=0.0, max_tokens=700, response_model=PlannerDecision) + raw_repair = self.llm.chat(repair_prompt, temperature=0.0, max_tokens=700, response_model=PlannerDecision) except Exception: # noqa: BLE001 return None - try: - return PlannerDecision.model_validate_json(raw) - except ValidationError: - payload = self._extract_json_object(raw) - if payload is None: - return None - try: - return PlannerDecision.model_validate(payload) - except ValidationError: - return None + return self._planner_decision_from_raw(raw_repair) def plan_query( self, @@ -1149,34 +657,46 @@ def plan_query( filing_date_to=filing_date_to, ) if decision is None: - # this happens when the planner LLM fails to produce valid output - # we fall back to a simple deterministic planner that infers tickers - # from the question and ignores date filters, but still allows refusal if no tickers can be inferred + fallback_characteristics = PlannerFallbackHeuristics.classify_characteristics(question) inferred = self._infer_tickers_from_question(question, companies) + fallback_date_window = PlannerFallbackHeuristics.infer_filing_date_window_from_question(question) + fallback_date_from = filing_date_from + fallback_date_to = filing_date_to + if fallback_date_window is not None: + if fallback_date_from is None: + fallback_date_from = fallback_date_window[0] + if fallback_date_to is None: + fallback_date_to = fallback_date_window[1] action = QueryStatus.ANSWERED if explicit_tickers or inferred else QueryStatus.CLARIFICATION_REQUIRED decision = PlannerDecision( action=( PlannerAction.ANSWER if action == QueryStatus.ANSWERED else PlannerAction.CLARIFICATION_REQUIRED ), tickers=(explicit_tickers if explicit_tickers else inferred), + characteristics=[QueryCharacteristic(item) for item in fallback_characteristics], + filing_date_from=fallback_date_from, + filing_date_to=fallback_date_to, clarifying_question=( self.default_clarifying_question() if action == QueryStatus.CLARIFICATION_REQUIRED else None ), use_per_ticker_retrieval=( True if len(explicit_tickers if explicit_tickers else inferred) > 1 else None ), - use_multi_ticker_briefs=( - True - if len(explicit_tickers if explicit_tickers else inferred) > 1 - and self._question_mentions_comparison(question) - else None + use_multi_ticker_briefs=(True if len(explicit_tickers if explicit_tickers else inferred) > 1 else None), + use_rag=(True if QueryCharacteristic.FILING_NARRATIVE.value in fallback_characteristics else None), + use_yfinance=(True if QueryCharacteristic.MARKET_DATA.value in fallback_characteristics else None), + use_edgar_financials=( + True if QueryCharacteristic.FINANCIAL_METRICS.value in fallback_characteristics else None ), ) trace.append( self._tool_event( "planner_fallback", - args={"inferred_tickers": list(decision.tickers)}, - result="Planner JSON parse failed; used deterministic fallback planner.", + args={ + "inferred_tickers": list(decision.tickers), + "characteristics": [item.value for item in decision.characteristics], + }, + result="Planner output invalid after repair; used heuristic fallback planner.", ) ) else: @@ -1186,6 +706,7 @@ def plan_query( args={ "raw_action": decision.action.value, "tickers": [str(t) for t in decision.tickers], + "characteristics": [item.value for item in decision.characteristics], "use_rag": decision.use_rag, "use_yfinance": decision.use_yfinance, "use_edgar_financials": decision.use_edgar_financials, @@ -1197,9 +718,7 @@ def plan_query( action = self._normalize_plan_action(decision.action) planned_tickers = explicit_tickers or self._normalize_ticker_list(decision.tickers) - use_rag, use_yfinance, use_edgar_financials = self.resolve_tool_usage_from_decision( - question=question, decision=decision - ) + use_rag, use_yfinance, use_edgar_financials = self.resolve_tool_usage_from_decision(decision=decision) if action == QueryStatus.REFUSED: reason = ( @@ -1273,29 +792,20 @@ def plan_query( resolved_filing_date_from = filing_date_from if filing_date_from is not None else decision.filing_date_from resolved_filing_date_to = filing_date_to if filing_date_to is not None else decision.filing_date_to - if resolved_filing_date_from is None and resolved_filing_date_to is None: - inferred_window = self._infer_filing_date_window_from_question(question) - if inferred_window is not None: - resolved_filing_date_from, resolved_filing_date_to = inferred_window - trace.append( - self._tool_event( - "infer_question_date_window", - args={"filing_date_from": resolved_filing_date_from, "filing_date_to": resolved_filing_date_to}, - result="Applied year window inferred from question text.", - ) - ) filters = self.build_retrieval_filters( tickers=planned_tickers, filing_date_from=resolved_filing_date_from, filing_date_to=resolved_filing_date_to ) + characteristics = self._characteristics_set(decision) + comparison_characteristic = QueryCharacteristic.COMPARISON in characteristics use_per_ticker = ( bool(decision.use_per_ticker_retrieval) if decision.use_per_ticker_retrieval is not None - else (len(planned_tickers) > 1 or self._question_mentions_comparison(question)) + else (len(planned_tickers) > 1 or comparison_characteristic) ) use_multi_ticker_briefs = ( bool(decision.use_multi_ticker_briefs) if decision.use_multi_ticker_briefs is not None - else (use_per_ticker and len(planned_tickers) > 1 and self._question_mentions_comparison(question)) + else (use_per_ticker and len(planned_tickers) > 1) ) trace.append( self._tool_event( @@ -1477,12 +987,6 @@ def retrieve_chunks_for_plan( if not planned.use_per_ticker_retrieval or len(planned.tickers) <= 1: retrieval_queries = [question] - if ( - planned.use_rag - and self._question_mentions_filing_narrative(question) - and self.narrative_query_expansion_enabled() - ): - retrieval_queries = self.narrative_retrieval_queries(question) if len(retrieval_queries) == 1: hybrid = self.retrieve_chunks(question, settings, filters=planned.filters) @@ -1666,7 +1170,8 @@ def should_apply_faithfulness_scrub(self, question: str) -> bool: Return whether strict factual scrub should run for the final answer. """ - return self._question_mentions_filing_narrative(question) + _ = question + return True def scrub_answer_for_faithfulness( self, @@ -1778,37 +1283,6 @@ def rerank_chunks_for_plan( result=f"Adjusted reranked list to {len(reranked)} chunks with ticker coverage constraints.", ) ) - if planned.use_rag and self._question_mentions_filing_narrative(question): - if self.mmr_diversity_enabled() and ( - self._question_mentions_growth_or_strategy(question) or self._question_mentions_risk_dimension(question) - ): - reranked = self.apply_mmr_diversity(candidates=reranked, limit=settings.top_k_rerank) - trace.append( - self._tool_event( - "apply_mmr_diversity", - args={"top_k_rerank": settings.top_k_rerank, "lambda_mult": 0.78}, - result=f"Applied bounded MMR diversification (size={len(reranked)}).", - ) - ) - if self.narrative_aspect_coverage_enabled(): - reranked = self._enforce_narrative_aspect_coverage( - question=question, primary=reranked, fallback=hybrid, limit=settings.top_k_rerank - ) - trace.append( - self._tool_event( - "enforce_narrative_aspect_coverage", - args={"top_k_rerank": settings.top_k_rerank}, - result=(f"Adjusted reranked list for narrative aspect coverage (size={len(reranked)})."), - ) - ) - else: - trace.append( - self._tool_event( - "enforce_narrative_aspect_coverage_skip", - args={"reason": "disabled_by_env"}, - result="Skipped narrative aspect coverage enforcement per environment toggle.", - ) - ) return reranked, trace def execute_query_pipeline( @@ -1869,12 +1343,6 @@ def execute_query_pipeline( use_rag_for_execution = True retrieval_settings = settings - if use_rag_for_execution: - retrieval_settings, adaptive_budget_trace = self.apply_adaptive_retrieval_budget( - question=question, settings=settings, planned=planned - ) - if adaptive_budget_trace is not None: - execution.tool_trace.append(adaptive_budget_trace) if planned.use_multi_ticker_briefs and len(planned.tickers) > 1: retrieve_t0 = time.perf_counter() @@ -2028,28 +1496,26 @@ def prompt_extra_for_question(self, question: str) -> str | None: Build targeted system prompt guidance for the current question. """ - if self._question_mentions_filing_narrative(question): - years = self._requested_years(question) - year_scope_note = "" - if years: - year_scope_note = ( - "- Year-scope handling: when year(s) are requested, explicitly separate filing year from covered " - "period before any analysis.\n" - "- Never convert filing-year references into full-year performance claims unless cited evidence " - "explicitly reports that year as the covered period.\n" - "- If year scope is ambiguous, make the ambiguity explicit and avoid unsupported assumptions.\n" - ) - return ( - "Narrative evidence mode:\n" - "- Output at most 6 material points.\n" - "- For each point, include: point, why it matters, and one short direct quote with citation.\n" - "- Do not include a point unless a direct quote supports it.\n" - "- Keep quotes short and verbatim from context/tool context.\n" - "- Never cite doc/chunk IDs that are absent from the provided context headers.\n" - "- If a requested point has no explicit quote support, state: " - "'Not explicitly stated in the provided context.'\n" + year_scope_note + years = self._requested_years(question) + year_scope_note = "" + if years: + year_scope_note = ( + "- Year-scope handling: when year(s) are requested, explicitly separate filing year from covered " + "period before any analysis.\n" + "- Never convert filing-year references into full-year performance claims unless cited evidence " + "explicitly reports that year as the covered period.\n" + "- If year scope is ambiguous, make the ambiguity explicit and avoid unsupported assumptions.\n" ) - return None + return ( + "Evidence discipline mode:\n" + "- Output at most 6 material points.\n" + "- For each point, include: point, why it matters, and one short direct quote with citation.\n" + "- Do not include a point unless a direct quote supports it.\n" + "- Keep quotes short and verbatim from context/tool context.\n" + "- Never cite doc/chunk IDs that are absent from the provided context headers.\n" + "- If a requested point has no explicit quote support, state: " + "'Not explicitly stated in the provided context.'\n" + year_scope_note + ) @staticmethod def _requested_years(question: str) -> list[int]: @@ -2127,29 +1593,12 @@ def period_scope_prompt_extra(self, *, question: str, reranked: list[ScoredChunk def context_coverage_prompt_extra(self, *, question: str, reranked: list[ScoredChunk]) -> str | None: """ - Add missing-evidence guardrails when requested narrative dimensions are absent in context. + Add period-scope guardrails from retrieved metadata. """ if not reranked: return None - if not self._question_mentions_filing_narrative(question): - return None - - top_window = reranked[: min(len(reranked), 14)] - growth_count = sum(1 for sc in top_window if self._is_growth_or_strategy_chunk(sc)) - risk_count = sum(1 for sc in top_window if self._is_risk_chunk(sc)) - lines: list[str] = [] - if self._question_mentions_growth_or_strategy(question) and growth_count == 0: - lines.append( - "Retrieved context does not contain explicit growth/strategy evidence; state that these points are " - "not explicitly stated unless directly quoted." - ) - if self._question_mentions_risk_dimension(question) and risk_count == 0: - lines.append( - "Retrieved context does not contain explicit risk disclosures; state that risk details are not " - "explicitly stated unless directly quoted." - ) period_scope_extra = self.period_scope_prompt_extra(question=question, reranked=reranked) if period_scope_extra: lines.append(period_scope_extra) diff --git a/tests/test_query_runtime_tools_first.py b/tests/test_query_runtime_tools_first.py index 110057a..57cfea9 100644 --- a/tests/test_query_runtime_tools_first.py +++ b/tests/test_query_runtime_tools_first.py @@ -1,13 +1,17 @@ from __future__ import annotations +from collections import deque from dataclasses import dataclass from datetime import date +from typing import Any + +import pytest from andromeda.dataclasses import DocChunk, ScoredChunk -from andromeda.retrieval.db import IngestedCompanyRow, RetrievalFilters from andromeda.finance_tools import FinanceToolResult, FinanceToolStatus from andromeda.llm.generation_controls import resolve_generation_settings -from andromeda.query.runtime import PlannerAction, PlannerDecision, QueryStatus, RAGService +from andromeda.query.runtime import PlannerAction, PlannerDecision, QueryCharacteristic, QueryStatus, RAGService +from andromeda.retrieval.db import IngestedCompanyRow, RetrievalFilters from tests.fakes import RecordingLLM @@ -71,22 +75,6 @@ def rerank( return hybrid[:top_k] -def make_scored_chunk(*, chunk_id: str, section_path: str, text: str, score: float = 1.0) -> ScoredChunk: - return ScoredChunk( - chunk=DocChunk( - id=chunk_id, - doc_id="doc-AAPL", - text=text, - page_no=None, - headings=["Item 2"], - source="aapl_10q.md", - metadata={"retrieval_text": text, "section_path": section_path, "doc": {"ticker": "AAPL"}}, - ), - score=score, - source="hybrid", - ) - - @dataclass class FakeFinanceTools: calls: int = 0 @@ -116,8 +104,35 @@ def tool_context_text(self, results: list[FinanceToolResult], *, max_chars: int return "TOOL CONTEXT" -def build_service(finance_tools: FakeFinanceTools) -> tuple[RAGService, FakeRetriever, RecordingLLM]: - llm = RecordingLLM(chat_fn=lambda _messages, _temperature, _response_model: "answer") +PlannerOutput = PlannerDecision | str | Exception + + +def planner_decision_payload(decision: PlannerDecision) -> str: + """ + Serialize planner decision for fake LLM response. + """ + + return decision.model_dump_json() + + +def build_service( + finance_tools: FakeFinanceTools, *, planner_outputs: list[PlannerOutput] | None = None, answer_text: str = "answer" +) -> tuple[RAGService, FakeRetriever, RecordingLLM]: + outputs = deque(planner_outputs or []) + + def chat_fn(_messages: list[dict[str, Any]], _temperature: float, response_model: Any) -> str: + if response_model is PlannerDecision: + if not outputs: + raise RuntimeError("No planner output configured for this test.") + item = outputs.popleft() + if isinstance(item, Exception): + raise item + if isinstance(item, PlannerDecision): + return planner_decision_payload(item) + return str(item) + return answer_text + + llm = RecordingLLM(chat_fn=chat_fn) retriever = FakeRetriever() service = RAGService( llm=llm, @@ -129,16 +144,27 @@ def build_service(finance_tools: FakeFinanceTools) -> tuple[RAGService, FakeRetr return service, retriever, llm -def test_tools_only_plan_skips_rag_and_still_answers(monkeypatch) -> None: +def generation_calls(llm: RecordingLLM) -> list[dict[str, Any]]: + """ + Return non-planner LLM generation calls. + """ + + return [call for call in llm.chat_calls if call["response_model"] is None] + + +def test_tools_only_plan_skips_rag_and_still_answers() -> None: finance_tools = FakeFinanceTools() - service, retriever, llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=False, use_yfinance=True, use_edgar_financials=True - ), + service, retriever, llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + use_rag=False, + use_yfinance=True, + use_edgar_financials=True, + ) + ], ) settings = resolve_generation_settings(mode="quick") @@ -155,20 +181,24 @@ def test_tools_only_plan_skips_rag_and_still_answers(monkeypatch) -> None: assert len(response.tool_results) == 1 assert response.tool_results[0].tool == "yfinance_get_ticker_info" - prompt_messages = llm.chat_calls[0]["messages"] - assert "Tool Context:\nTOOL CONTEXT" in prompt_messages[1]["content"] + calls = generation_calls(llm) + assert len(calls) == 1 + assert "Tool Context:\nTOOL CONTEXT" in calls[0]["messages"][1]["content"] -def test_tools_plus_rag_runs_retrieval(monkeypatch) -> None: +def test_tools_plus_rag_runs_retrieval() -> None: finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=True, use_edgar_financials=False - ), + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + use_rag=True, + use_yfinance=True, + use_edgar_financials=False, + ) + ], ) settings = resolve_generation_settings(mode="quick") @@ -183,125 +213,21 @@ def test_tools_plus_rag_runs_retrieval(monkeypatch) -> None: assert len(pipeline.reranked) == 1 -def test_adaptive_retrieval_budget_reduces_depth_for_simple_numeric_query(monkeypatch) -> None: - finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No market data.") - service, retriever, _llm = build_service(finance_tools) - monkeypatch.setenv("FINRAG_ENABLE_ADAPTIVE_RETRIEVAL_BUDGET", "1") - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=False, use_yfinance=True, use_edgar_financials=False - ), - ) - - settings = resolve_generation_settings(mode="normal", enable_refine=False) - pipeline = service.execute_query_pipeline(question="What is AAPL revenue value", settings=settings) - - assert pipeline.planned.status == QueryStatus.ANSWERED - assert retriever.retrieve_calls == 1 - - adaptive_events = [event for event in pipeline.tool_trace if event.tool == "adaptive_retrieval_budget"] - assert len(adaptive_events) == 1 - adaptive_args = adaptive_events[0].args - assert adaptive_args["old_top_k_retrieve"] == 40 - assert adaptive_args["old_top_k_rerank"] == 25 - assert adaptive_args["new_top_k_retrieve"] == 28 - assert adaptive_args["new_top_k_rerank"] == 15 - - retrieve_events = [event for event in pipeline.tool_trace if event.tool == "retrieve_chunks"] - assert len(retrieve_events) == 1 - assert retrieve_events[0].args["top_k_retrieve"] == 28 - - -def test_adaptive_retrieval_budget_does_not_apply_to_narrative_queries(monkeypatch) -> None: - finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No tool data.") - service, retriever, _llm = build_service(finance_tools) - monkeypatch.setenv("FINRAG_ENABLE_ADAPTIVE_RETRIEVAL_BUDGET", "1") - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), - ) - - settings = resolve_generation_settings(mode="normal", enable_refine=False) - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings, explain growth drivers and key risks.", settings=settings - ) - - assert pipeline.planned.status == QueryStatus.ANSWERED - assert retriever.retrieve_calls >= 1 - assert all(event.tool != "adaptive_retrieval_budget" for event in pipeline.tool_trace) - - retrieve_events = [event for event in pipeline.tool_trace if event.tool == "retrieve_chunks"] - assert len(retrieve_events) == 1 - assert retrieve_events[0].args["top_k_retrieve"] == 40 - - -def test_narrative_query_expansion_can_be_disabled(monkeypatch) -> None: - finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No tool data.") - service, retriever, _llm = build_service(finance_tools) - monkeypatch.setenv("FINRAG_ENABLE_NARRATIVE_QUERY_EXPANSION", "0") - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), - ) - - settings = resolve_generation_settings(mode="normal", enable_refine=False) - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings, explain growth drivers and key risks.", settings=settings - ) - - retrieve_events = [event for event in pipeline.tool_trace if event.tool == "retrieve_chunks"] - assert len(retrieve_events) == 1 - retrieval_queries = retrieve_events[0].args["retrieval_queries"] - assert retrieval_queries == ["Based on AAPL SEC filings, explain growth drivers and key risks."] - assert retriever.retrieve_calls == 1 - - -def test_narrative_aspect_coverage_can_be_disabled(monkeypatch) -> None: - finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No tool data.") - service, retriever, _llm = build_service(finance_tools) - monkeypatch.setenv("FINRAG_ENABLE_NARRATIVE_ASPECT_COVERAGE", "0") - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), - ) - - settings = resolve_generation_settings(mode="normal", enable_refine=False) - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings, explain growth drivers and key risks.", settings=settings - ) - - assert retriever.retrieve_calls >= 1 - assert all(event.tool != "enforce_narrative_aspect_coverage" for event in pipeline.tool_trace) - assert any(event.tool == "enforce_narrative_aspect_coverage_skip" for event in pipeline.tool_trace) - - def test_finance_tools_can_be_disabled_by_env(monkeypatch) -> None: finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - monkeypatch.setenv("FINRAG_DISABLE_FINANCE_TOOLS", "1") - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=True, use_edgar_financials=True - ), + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + use_rag=True, + use_yfinance=True, + use_edgar_financials=True, + ) + ], ) + monkeypatch.setenv("FINRAG_DISABLE_FINANCE_TOOLS", "1") settings = resolve_generation_settings(mode="quick") pipeline = service.execute_query_pipeline(question="What was AAPL revenue in the latest filing?", settings=settings) @@ -312,22 +238,21 @@ def test_finance_tools_can_be_disabled_by_env(monkeypatch) -> None: assert any(event.tool == "finance_tools_skip" for event in pipeline.tool_trace) -def test_multi_ticker_briefs_path_generates_parallel_briefs(monkeypatch) -> None: +def test_multi_ticker_briefs_path_generates_parallel_briefs() -> None: finance_tools = FakeFinanceTools() - service, retriever, llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, - tickers=["NVDA", "GOOGL"], - use_rag=True, - use_yfinance=False, - use_edgar_financials=False, - use_per_ticker_retrieval=True, - use_multi_ticker_briefs=True, - ), + service, retriever, llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["NVDA", "GOOGL"], + use_rag=True, + use_yfinance=False, + use_edgar_financials=False, + use_per_ticker_retrieval=True, + use_multi_ticker_briefs=True, + ) + ], ) settings = resolve_generation_settings(mode="normal", enable_refine=False) @@ -342,273 +267,259 @@ def test_multi_ticker_briefs_path_generates_parallel_briefs(monkeypatch) -> None response = service.response_from_pipeline(pipeline=pipeline, settings=settings) assert response.status == QueryStatus.ANSWERED - assert len(llm.chat_calls) >= 3 + assert len(llm.chat_calls) >= 4 -def test_question_year_infers_retrieval_date_window(monkeypatch) -> None: - finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), +def test_tools_only_plan_falls_back_to_rag_when_tools_have_no_actionable_data() -> None: + finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No metrics available.", payload=None) + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + use_rag=False, + use_yfinance=True, + use_edgar_financials=False, + ) + ], ) settings = resolve_generation_settings(mode="quick") - pipeline = service.execute_query_pipeline( - question="Based on AAPL filings in 2025, summarize strategy and risks.", settings=settings - ) + pipeline = service.execute_query_pipeline(question="What is AAPL market cap right now?", settings=settings) assert pipeline.planned.status == QueryStatus.ANSWERED - assert pipeline.planned.filters is not None - assert retriever.last_filing_date_from == "2025-01-01" - assert retriever.last_filing_date_to == "2025-12-31" - assert pipeline.planned.filters.filing_date_from is not None - assert pipeline.planned.filters.filing_date_from.isoformat() == "2025-01-01" - assert pipeline.planned.filters.filing_date_to is not None - assert pipeline.planned.filters.filing_date_to.isoformat() == "2025-12-31" - assert any(event.tool == "infer_question_date_window" for event in pipeline.tool_trace) + assert pipeline.planned.use_rag is False + assert finance_tools.calls == 1 + assert retriever.retrieve_calls == 1 + assert any(event.tool == "rag_function_fallback" for event in pipeline.tool_trace) -def test_narrative_sec_question_forces_rag_and_disables_tools(monkeypatch) -> None: +def test_planner_invalid_json_triggers_repair_call() -> None: finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=False, use_yfinance=True, use_edgar_financials=True - ), - ) - - settings = resolve_generation_settings(mode="quick") - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings in 2025, summarize strategy and key risks.", settings=settings - ) - - assert pipeline.planned.status == QueryStatus.ANSWERED - assert pipeline.planned.use_rag is True - assert pipeline.planned.use_yfinance is False - assert pipeline.planned.use_edgar_financials is False - assert finance_tools.calls == 0 - assert retriever.retrieve_calls >= 1 - - -def test_narrative_refine_runs_faithfulness_scrub_pass(monkeypatch) -> None: + service, _retriever, llm = build_service( + finance_tools, + planner_outputs=[ + "definitely not valid planner json", + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.MARKET_DATA], + use_rag=False, + use_yfinance=True, + use_edgar_financials=False, + ), + ], + ) + + decision = service._planner_decision_from_llm( + question="What is AAPL market cap?", + companies=service.list_ingested_companies(), + explicit_tickers=["AAPL"], + filing_date_from=None, + filing_date_to=None, + ) + + assert decision is not None + assert decision.action == PlannerAction.ANSWER + assert decision.tickers == ["AAPL"] + assert len(llm.chat_calls) == 2 + assert "You repair malformed planner outputs" in llm.chat_calls[1]["messages"][0]["content"] + + +def test_planner_error_triggers_repair_call() -> None: finance_tools = FakeFinanceTools() - service, _retriever, llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), - ) - - settings = resolve_generation_settings(mode="normal", enable_refine=True) - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings in 2025, summarize strategy and key risks.", settings=settings - ) - _ = service.response_from_pipeline(pipeline=pipeline, settings=settings) - - assert len(llm.chat_calls) == 3 - assert any("Candidate answer:" in call["messages"][1]["content"] for call in llm.chat_calls) - - -def test_narrative_question_injects_prompt_extra_guidance(monkeypatch) -> None: + service, _retriever, llm = build_service( + finance_tools, + planner_outputs=[ + RuntimeError("planner endpoint timeout"), + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.FINANCIAL_METRICS, QueryCharacteristic.PERIOD_SCOPED], + use_rag=False, + use_yfinance=False, + use_edgar_financials=True, + ), + ], + ) + + decision = service._planner_decision_from_llm( + question="What was AAPL net income in 2025?", + companies=service.list_ingested_companies(), + explicit_tickers=["AAPL"], + filing_date_from=None, + filing_date_to=None, + ) + + assert decision is not None + assert decision.use_edgar_financials is True + assert len(llm.chat_calls) == 2 + assert "You repair malformed planner outputs" in llm.chat_calls[1]["messages"][0]["content"] + + +def test_plan_query_uses_heuristics_only_after_planner_and_repair_failure() -> None: finance_tools = FakeFinanceTools() - service, _retriever, llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=False, use_edgar_financials=False - ), - ) + service, retriever, _llm = build_service(finance_tools, planner_outputs=["not-json", "still-not-json"]) - settings = resolve_generation_settings(mode="quick", enable_refine=False) - pipeline = service.execute_query_pipeline( - question="Based on AAPL SEC filings in 2025, summarize strategy and key risks.", settings=settings + planned = service.plan_query( + question="What was AAPL net income in 2025?", tickers=["AAPL"], filing_date_from=None, filing_date_to=None ) - _ = service.response_from_pipeline(pipeline=pipeline, settings=settings) - assert len(llm.chat_calls) == 1 - assert "Narrative evidence mode" in llm.chat_calls[0]["messages"][0]["content"] - assert "If a requested point has no explicit quote support" in llm.chat_calls[0]["messages"][0]["content"] + assert planned.status == QueryStatus.ANSWERED + assert planned.tickers == ["AAPL"] + assert planned.filters is not None + assert retriever.last_filing_date_from == "2025-01-01" + assert retriever.last_filing_date_to == "2025-12-31" + fallback_events = [event for event in planned.tool_trace if event.tool == "planner_fallback"] + assert len(fallback_events) == 1 -def test_context_coverage_prompt_extra_flags_missing_growth() -> None: +def test_planner_characteristics_route_tools_first_without_rag() -> None: finance_tools = FakeFinanceTools() - service, _retriever, _llm = build_service(finance_tools) - - risk_only = make_scored_chunk( - chunk_id="risk-only", - section_path="PART II > ITEM 1A. RISK FACTORS", - text="Regulatory and cybersecurity risks may adversely affect the business.", - ) - extra = service.context_coverage_prompt_extra( - question="Based on AAPL filings, what are key growth drivers and risks?", reranked=[risk_only] + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.MARKET_DATA, QueryCharacteristic.SIMPLE_NUMERIC], + use_rag=None, + use_yfinance=None, + use_edgar_financials=None, + ) + ], ) - assert extra is not None - assert "does not contain explicit growth/strategy evidence" in extra + settings = resolve_generation_settings(mode="quick") + pipeline = service.execute_query_pipeline(question="What is AAPL market cap right now?", settings=settings) + assert pipeline.planned.use_rag is False + assert pipeline.planned.use_yfinance is True + assert pipeline.planned.use_edgar_financials is False + assert retriever.retrieve_calls == 0 -def test_narrative_aspect_coverage_adds_growth_chunk_when_question_needs_growth_and_risk() -> None: + +def test_planner_characteristics_route_rag_for_narrative() -> None: finance_tools = FakeFinanceTools() - service, _retriever, _llm = build_service(finance_tools) - risk_a = make_scored_chunk( - chunk_id="risk-a", - section_path="PART II > ITEM 1A. RISK FACTORS", - text="Regulatory risks could adversely affect results.", - score=2.0, - ) - risk_b = make_scored_chunk( - chunk_id="risk-b", - section_path="PART II > ITEM 1A. RISK FACTORS", - text="Cybersecurity risks remain elevated.", - score=1.8, - ) - growth = make_scored_chunk( - chunk_id="growth-a", - section_path="PART I > ITEM 2. RESULTS OF OPERATIONS > REVENUE", - text="Revenue growth was driven by cloud demand and enterprise expansion.", - score=1.0, + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.FILING_NARRATIVE], + use_rag=None, + use_yfinance=None, + use_edgar_financials=None, + ) + ], ) - out = service._enforce_narrative_aspect_coverage( - question="Based on AAPL filings, what are key growth drivers and key risks?", - primary=[risk_a, risk_b], - fallback=[risk_a, risk_b, growth], - limit=3, + settings = resolve_generation_settings(mode="quick") + pipeline = service.execute_query_pipeline( + question="Based on AAPL filings, summarize strategy and risk factors.", settings=settings ) - out_ids = {item.chunk.id for item in out} - assert "growth-a" in out_ids - assert "risk-a" in out_ids or "risk-b" in out_ids + assert pipeline.planned.use_rag is True + assert pipeline.planned.use_yfinance is False + assert pipeline.planned.use_edgar_financials is False + assert finance_tools.calls == 0 + assert retriever.retrieve_calls == 1 -def test_narrative_retrieval_queries_expand_growth_and_risk() -> None: +def test_planner_mixed_characteristics_use_tools_and_rag() -> None: finance_tools = FakeFinanceTools() - service, _retriever, _llm = build_service(finance_tools) - - queries = service.narrative_retrieval_queries( - "Based on AAPL filings in 2025, what are key growth drivers and key risks?" + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.FILING_NARRATIVE, QueryCharacteristic.MARKET_DATA], + use_rag=None, + use_yfinance=None, + use_edgar_financials=None, + ) + ], ) - assert len(queries) == 3 - assert "growth drivers" in queries[1].lower() or "strategy" in queries[1].lower() - assert "risk factors" in queries[2].lower() - -def test_apply_mmr_diversity_prefers_diverse_chunks() -> None: - finance_tools = FakeFinanceTools() - service, _retriever, _llm = build_service(finance_tools) - - risk_a = make_scored_chunk( - chunk_id="risk-a", - section_path="PART II > ITEM 1A. RISK FACTORS", - text="Regulatory risk factors include antitrust privacy and cybersecurity penalties.", - score=3.0, - ) - risk_b = make_scored_chunk( - chunk_id="risk-b", - section_path="PART II > ITEM 1A. RISK FACTORS", - text="Regulatory risk factors include antitrust privacy and cybersecurity fines.", - score=2.8, - ) - growth = make_scored_chunk( - chunk_id="growth-a", - section_path="PART I > ITEM 2. RESULTS OF OPERATIONS > REVENUE", - text="Revenue expansion was driven by cloud adoption and enterprise demand.", - score=2.9, + settings = resolve_generation_settings(mode="quick") + pipeline = service.execute_query_pipeline( + question="Explain AAPL strategy from filings and include current valuation context.", settings=settings ) - out = service.apply_mmr_diversity(candidates=[risk_a, risk_b, growth], limit=2, lambda_mult=0.78) - out_ids = {item.chunk.id for item in out} - - assert "risk-a" in out_ids - assert "growth-a" in out_ids - - -def test_mmr_diversity_flag_defaults_off(monkeypatch) -> None: - finance_tools = FakeFinanceTools() - service, _retriever, _llm = build_service(finance_tools) - - monkeypatch.delenv("FINRAG_ENABLE_MMR_DIVERSITY", raising=False) - assert service.mmr_diversity_enabled() is False - - monkeypatch.setenv("FINRAG_ENABLE_MMR_DIVERSITY", "1") - assert service.mmr_diversity_enabled() is True + assert pipeline.planned.use_rag is True + assert pipeline.planned.use_yfinance is True + assert finance_tools.calls == 1 + assert retriever.retrieve_calls == 1 -def test_simple_numeric_question_forces_tools_first(monkeypatch) -> None: +def test_period_scoped_financial_metrics_stay_tools_first_when_non_narrative() -> None: finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=True, use_yfinance=True, use_edgar_financials=False - ), + service, retriever, _llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + characteristics=[QueryCharacteristic.FINANCIAL_METRICS, QueryCharacteristic.PERIOD_SCOPED], + use_rag=None, + use_yfinance=None, + use_edgar_financials=None, + ) + ], ) settings = resolve_generation_settings(mode="quick") - pipeline = service.execute_query_pipeline(question="What is AAPL market cap right now?", settings=settings) + pipeline = service.execute_query_pipeline(question="What was AAPL net income in 2025?", settings=settings) - assert pipeline.planned.status == QueryStatus.ANSWERED assert pipeline.planned.use_rag is False - assert pipeline.planned.use_yfinance is True - assert pipeline.planned.use_edgar_financials is False + assert pipeline.planned.use_edgar_financials is True assert finance_tools.calls == 1 assert retriever.retrieve_calls == 0 -def test_period_scoped_numeric_question_uses_rag_for_grounding(monkeypatch) -> None: +def test_prompt_extra_injects_evidence_discipline() -> None: finance_tools = FakeFinanceTools() - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=False, use_yfinance=False, use_edgar_financials=True - ), + service, _retriever, llm = build_service( + finance_tools, + planner_outputs=[ + PlannerDecision( + action=PlannerAction.ANSWER, + tickers=["AAPL"], + use_rag=True, + use_yfinance=False, + use_edgar_financials=False, + ) + ], ) - settings = resolve_generation_settings(mode="quick") - pipeline = service.execute_query_pipeline(question="What was AAPL net income in 2025?", settings=settings) + settings = resolve_generation_settings(mode="quick", enable_refine=False) + pipeline = service.execute_query_pipeline( + question="Based on AAPL SEC filings in 2025, summarize strategy and key risks.", settings=settings + ) + _ = service.response_from_pipeline(pipeline=pipeline, settings=settings) - assert pipeline.planned.status == QueryStatus.ANSWERED - assert pipeline.planned.use_rag is True - assert pipeline.planned.use_edgar_financials is True - assert finance_tools.calls == 1 - assert retriever.retrieve_calls == 1 + calls = generation_calls(llm) + assert len(calls) == 1 + assert "Evidence discipline mode" in calls[0]["messages"][0]["content"] + assert "If a requested point has no explicit quote support" in calls[0]["messages"][0]["content"] -def test_tools_only_plan_falls_back_to_rag_when_tools_have_no_actionable_data(monkeypatch) -> None: - finance_tools = FakeFinanceTools(status=FinanceToolStatus.NO_DATA, summary="No metrics available.", payload=None) - service, retriever, _llm = build_service(finance_tools) - - monkeypatch.setattr( - service, - "_planner_decision_from_llm", - lambda **_kwargs: PlannerDecision( - action=PlannerAction.ANSWER, tickers=["AAPL"], use_rag=False, use_yfinance=True, use_edgar_financials=False - ), +def test_plan_query_fallback_infers_ticker_via_live_yfinance_search() -> None: + finance_tools = FakeFinanceTools() + service, _retriever, _llm = build_service(finance_tools, planner_outputs=["bad-json", "still-bad-json"]) + + planned = service.plan_query( + question="How does NVIDIA Corporation look right now as an investment?", + tickers=None, + filing_date_from=None, + filing_date_to=None, ) - settings = resolve_generation_settings(mode="quick") - pipeline = service.execute_query_pipeline(question="What is AAPL market cap right now?", settings=settings) + if planned.status != QueryStatus.ANSWERED: + pytest.skip("Live yfinance search was unavailable in this environment.") - assert pipeline.planned.status == QueryStatus.ANSWERED - assert pipeline.planned.use_rag is False - assert finance_tools.calls == 1 - assert retriever.retrieve_calls == 1 - assert any(event.tool == "rag_function_fallback" for event in pipeline.tool_trace) + assert "NVDA" in planned.tickers + assert any(event.tool == "planner_fallback" for event in planned.tool_trace) From 2790d1b7220400890dda7330284311932957a2d7 Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 19:19:27 +0800 Subject: [PATCH 05/10] docs: record reduced-heuristics checkpoint commit lineage --- agent_logs/LOGBOOK.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/agent_logs/LOGBOOK.md b/agent_logs/LOGBOOK.md index 5c4a75b..4d43983 100644 --- a/agent_logs/LOGBOOK.md +++ b/agent_logs/LOGBOOK.md @@ -2744,3 +2744,12 @@ - `source .venv/bin/activate && pytest -vvv tests/test_query_runtime_tools_first.py` -> `14 passed`. - `source .venv/bin/activate && PRE_COMMIT_HOME=/tmp/pre-commit-cache pre-commit run --all` -> passed. - `source .venv/bin/activate && pytest -vvv tests/` -> `114 passed`. + +## 2026-02-18 - Commit lineage checkpoint (reduced heuristics branch) + +### Commits +- `45b19cb` - docs: add eval-improvement guidance and reduced-heuristics benchmark plan. +- `a127bd0` - refactor: planner-first runtime, fallback heuristics module, reduced monkeypatching tests, and associated changelog/logbook updates. + +### Notes +- Branch is clean after checkpoint; proceeding to eval rerun and benchmark analysis. From d9220cf0dec7fbb5a23c74dd2447abc9aae88f5b Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Wed, 18 Feb 2026 20:31:46 +0800 Subject: [PATCH 06/10] eval: add retrieval precision/recall and rerank uplift instrumentation --- scripts/eval_retrieval.py | 342 ++++++++++++++++++++++++ scripts/score_eval.py | 44 +++ src/andromeda/eval/evidence_support.py | 171 ++++++++++++ src/andromeda/eval/rerank_metrics.py | 52 ++++ src/andromeda/eval/retrieval_metrics.py | 150 +++++++++++ src/andromeda/eval/scoring.py | 160 ++++++++++- tests/test_eval_retrieval_metrics.py | 58 ++++ tests/test_eval_schema_scoring.py | 8 + 8 files changed, 972 insertions(+), 13 deletions(-) create mode 100644 scripts/eval_retrieval.py create mode 100644 src/andromeda/eval/evidence_support.py create mode 100644 src/andromeda/eval/rerank_metrics.py create mode 100644 src/andromeda/eval/retrieval_metrics.py create mode 100644 tests/test_eval_retrieval_metrics.py diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py new file mode 100644 index 0000000..948cad9 --- /dev/null +++ b/scripts/eval_retrieval.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import math +from pathlib import Path +from typing import Any + +from andromeda.eval.evidence_support import EntailmentScorer, split_claim_like_units +from andromeda.eval.io import load_jsonl +from andromeda.eval.rerank_metrics import rerank_uplift +from andromeda.eval.retrieval_metrics import metrics_for_ranked_ids +from andromeda.eval.schema import EvalGeneration, EvalQuery + + +def _mean(values: list[float]) -> float: + cleaned = [value for value in values if not math.isnan(value)] + if not cleaned: + return math.nan + return sum(cleaned) / len(cleaned) + + +def _to_float(value: Any) -> float: + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + return math.nan + + +def _safe_round(value: float, digits: int = 4) -> float: + if math.isnan(value): + return value + return round(value, digits) + + +def _evidence_blocks(gen: EvalGeneration, *, max_blocks: int = 10, max_chars: int = 900) -> list[str]: + blocks: list[str] = [] + for chunk in (gen.top_chunks or [])[:max_blocks]: + text = (chunk.text or chunk.preview or "").strip() + if not text: + continue + blocks.append(text[:max_chars]) + return blocks + + +def _write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + fieldnames = list(rows[0].keys()) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _write_markdown(path: Path, summary: dict[str, Any], factual_rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + lines.append("# Retrieval + Rerank Benchmark Summary") + lines.append("") + lines.append("## Topline") + lines.append("") + lines.append(f"- factual_n: `{summary['factual_n']}`") + lines.append(f"- pre_chunk_mrr: `{summary['pre_chunk_mrr']}`") + lines.append(f"- post_chunk_mrr: `{summary['post_chunk_mrr']}`") + lines.append(f"- rerank_chunk_delta_mrr: `{summary['rerank_chunk_delta_mrr']}`") + lines.append(f"- pre_chunk_precision_at_5: `{summary['pre_chunk_precision_at_5']}`") + lines.append(f"- post_chunk_precision_at_5: `{summary['post_chunk_precision_at_5']}`") + lines.append(f"- pre_chunk_precision_at_10: `{summary['pre_chunk_precision_at_10']}`") + lines.append(f"- post_chunk_precision_at_10: `{summary['post_chunk_precision_at_10']}`") + lines.append(f"- rerank_chunk_delta_precision_at_5: `{summary['rerank_chunk_delta_precision_at_5']}`") + lines.append(f"- rerank_chunk_delta_precision_at_10: `{summary['rerank_chunk_delta_precision_at_10']}`") + lines.append(f"- pre_chunk_recall_at_25: `{summary['pre_chunk_recall_at_25']}`") + lines.append(f"- post_chunk_recall_at_25: `{summary['post_chunk_recall_at_25']}`") + lines.append(f"- rerank_chunk_win_rate: `{summary['rerank_chunk_win_rate']}`") + lines.append(f"- pre_doc_mrr: `{summary['pre_doc_mrr']}`") + lines.append(f"- post_doc_mrr: `{summary['post_doc_mrr']}`") + lines.append(f"- rerank_doc_delta_mrr: `{summary['rerank_doc_delta_mrr']}`") + lines.append(f"- rerank_doc_win_rate: `{summary['rerank_doc_win_rate']}`") + if "open_ended_nli_support_rate" in summary: + lines.append(f"- open_ended_nli_support_rate: `{summary['open_ended_nli_support_rate']}`") + lines.append(f"- open_ended_nli_unsupported_rate: `{summary['open_ended_nli_unsupported_rate']}`") + lines.append(f"- open_ended_nli_contradiction_rate: `{summary['open_ended_nli_contradiction_rate']}`") + lines.append("") + lines.append("## Factual Query Rows (sample)") + lines.append("") + lines.append("| query_id | pre_chunk_rank | post_chunk_rank | pre_doc_rank | post_doc_rank | delta_chunk_mrr | delta_doc_mrr |") + lines.append("|---|---:|---:|---:|---:|---:|---:|") + for row in factual_rows[:20]: + lines.append( + f"| `{row['query_id']}` | {row['pre_chunk_rank']} | {row['post_chunk_rank']} | " + f"{row['pre_doc_rank']} | {row['post_doc_rank']} | {row['delta_chunk_mrr']} | {row['delta_doc_mrr']} |" + ) + lines.append("") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compute retrieval/rerank subsystem metrics from an eval run dir.") + parser.add_argument("--run-dir", required=True) + parser.add_argument("--enable-nli", action="store_true") + parser.add_argument("--nli-model", default="cross-encoder/nli-deberta-v3-base") + parser.add_argument("--nli-max-open-ended", type=int, default=120) + parser.add_argument("--nli-support-threshold", type=float, default=0.50) + parser.add_argument("--nli-contradiction-threshold", type=float, default=0.50) + args = parser.parse_args() + + run_dir = Path(args.run_dir).expanduser().resolve() + eval_queries = load_jsonl(run_dir / "eval_queries.jsonl", EvalQuery) + generations = load_jsonl(run_dir / "generations.jsonl", EvalGeneration) + generation_by_id = {item.query_id: item for item in generations} + query_by_id = {item.id: item for item in eval_queries} + + factual_rows: list[dict[str, Any]] = [] + for query in eval_queries: + if query.kind != "factual" or query.factual is None: + continue + generation = generation_by_id.get(query.id) + if generation is None or generation.error: + continue + gold_chunk = query.factual.golden_evidence.chunk_id + gold_doc = query.factual.golden_evidence.doc_id + + post_chunks = list(generation.top_chunks or []) + pre_chunks = list(generation.retrieved_chunks or []) + if not pre_chunks: + pre_chunks = list(post_chunks) + + post_chunk_ids = [item.chunk_id for item in post_chunks] + pre_chunk_ids = [item.chunk_id for item in pre_chunks] + post_doc_ids = [item.doc_id for item in post_chunks] + pre_doc_ids = [item.doc_id for item in pre_chunks] + + pre_chunk_metrics = metrics_for_ranked_ids( + ranked_ids=pre_chunk_ids, + relevant_ids={gold_chunk}, + target_id=gold_chunk, + relevance_by_id={gold_chunk: 1.0}, + ) + post_chunk_metrics = metrics_for_ranked_ids( + ranked_ids=post_chunk_ids, + relevant_ids={gold_chunk}, + target_id=gold_chunk, + relevance_by_id={gold_chunk: 1.0}, + ) + pre_doc_metrics = metrics_for_ranked_ids( + ranked_ids=pre_doc_ids, + relevant_ids={gold_doc}, + target_id=gold_doc, + relevance_by_id={gold_doc: 1.0}, + ) + post_doc_metrics = metrics_for_ranked_ids( + ranked_ids=post_doc_ids, + relevant_ids={gold_doc}, + target_id=gold_doc, + relevance_by_id={gold_doc: 1.0}, + ) + chunk_uplift = rerank_uplift(pre=pre_chunk_metrics, post=post_chunk_metrics) + doc_uplift = rerank_uplift(pre=pre_doc_metrics, post=post_doc_metrics) + factual_rows.append( + { + "query_id": query.id, + "pre_chunk_rank": pre_chunk_metrics.rank if pre_chunk_metrics.rank is not None else "", + "post_chunk_rank": post_chunk_metrics.rank if post_chunk_metrics.rank is not None else "", + "pre_doc_rank": pre_doc_metrics.rank if pre_doc_metrics.rank is not None else "", + "post_doc_rank": post_doc_metrics.rank if post_doc_metrics.rank is not None else "", + "pre_chunk_mrr": pre_chunk_metrics.mrr, + "post_chunk_mrr": post_chunk_metrics.mrr, + "pre_doc_mrr": pre_doc_metrics.mrr, + "post_doc_mrr": post_doc_metrics.mrr, + "delta_chunk_mrr": chunk_uplift["delta_mrr"], + "delta_doc_mrr": doc_uplift["delta_mrr"], + "chunk_win": chunk_uplift["win"], + "doc_win": doc_uplift["win"], + "pre_chunk_hit_at_25": pre_chunk_metrics.hit_at_25, + "post_chunk_hit_at_25": post_chunk_metrics.hit_at_25, + "pre_doc_hit_at_25": pre_doc_metrics.hit_at_25, + "post_doc_hit_at_25": post_doc_metrics.hit_at_25, + "pre_chunk_ndcg_at_10": pre_chunk_metrics.ndcg_at_10, + "post_chunk_ndcg_at_10": post_chunk_metrics.ndcg_at_10, + "pre_doc_ndcg_at_10": pre_doc_metrics.ndcg_at_10, + "post_doc_ndcg_at_10": post_doc_metrics.ndcg_at_10, + "pre_chunk_precision_at_5": pre_chunk_metrics.precision_at_5, + "post_chunk_precision_at_5": post_chunk_metrics.precision_at_5, + "pre_chunk_precision_at_10": pre_chunk_metrics.precision_at_10, + "post_chunk_precision_at_10": post_chunk_metrics.precision_at_10, + "pre_chunk_precision_at_25": pre_chunk_metrics.precision_at_25, + "post_chunk_precision_at_25": post_chunk_metrics.precision_at_25, + "pre_chunk_recall_at_25": pre_chunk_metrics.recall_at_25, + "post_chunk_recall_at_25": post_chunk_metrics.recall_at_25, + "pre_doc_precision_at_5": pre_doc_metrics.precision_at_5, + "post_doc_precision_at_5": post_doc_metrics.precision_at_5, + "pre_doc_precision_at_10": pre_doc_metrics.precision_at_10, + "post_doc_precision_at_10": post_doc_metrics.precision_at_10, + "pre_doc_precision_at_25": pre_doc_metrics.precision_at_25, + "post_doc_precision_at_25": post_doc_metrics.precision_at_25, + "pre_doc_recall_at_25": pre_doc_metrics.recall_at_25, + "post_doc_recall_at_25": post_doc_metrics.recall_at_25, + "delta_chunk_precision_at_5": chunk_uplift["delta_precision_at_5"], + "delta_chunk_precision_at_10": chunk_uplift["delta_precision_at_10"], + "delta_chunk_precision_at_25": chunk_uplift["delta_precision_at_25"], + "delta_chunk_recall_at_25": chunk_uplift["delta_recall_at_25"], + "delta_doc_precision_at_5": doc_uplift["delta_precision_at_5"], + "delta_doc_precision_at_10": doc_uplift["delta_precision_at_10"], + "delta_doc_precision_at_25": doc_uplift["delta_precision_at_25"], + "delta_doc_recall_at_25": doc_uplift["delta_recall_at_25"], + } + ) + + summary: dict[str, Any] = { + "run_dir": str(run_dir), + "factual_n": len(factual_rows), + "pre_chunk_mrr": _safe_round(_mean([_to_float(row["pre_chunk_mrr"]) for row in factual_rows])), + "post_chunk_mrr": _safe_round(_mean([_to_float(row["post_chunk_mrr"]) for row in factual_rows])), + "rerank_chunk_delta_mrr": _safe_round(_mean([_to_float(row["delta_chunk_mrr"]) for row in factual_rows])), + "rerank_chunk_win_rate": _safe_round(_mean([_to_float(row["chunk_win"]) for row in factual_rows])), + "pre_doc_mrr": _safe_round(_mean([_to_float(row["pre_doc_mrr"]) for row in factual_rows])), + "post_doc_mrr": _safe_round(_mean([_to_float(row["post_doc_mrr"]) for row in factual_rows])), + "rerank_doc_delta_mrr": _safe_round(_mean([_to_float(row["delta_doc_mrr"]) for row in factual_rows])), + "rerank_doc_win_rate": _safe_round(_mean([_to_float(row["doc_win"]) for row in factual_rows])), + "pre_chunk_hit_at_25": _safe_round(_mean([_to_float(row["pre_chunk_hit_at_25"]) for row in factual_rows])), + "post_chunk_hit_at_25": _safe_round(_mean([_to_float(row["post_chunk_hit_at_25"]) for row in factual_rows])), + "pre_doc_hit_at_25": _safe_round(_mean([_to_float(row["pre_doc_hit_at_25"]) for row in factual_rows])), + "post_doc_hit_at_25": _safe_round(_mean([_to_float(row["post_doc_hit_at_25"]) for row in factual_rows])), + "pre_chunk_precision_at_5": _safe_round(_mean([_to_float(row["pre_chunk_precision_at_5"]) for row in factual_rows])), + "post_chunk_precision_at_5": _safe_round( + _mean([_to_float(row["post_chunk_precision_at_5"]) for row in factual_rows]) + ), + "pre_chunk_precision_at_10": _safe_round( + _mean([_to_float(row["pre_chunk_precision_at_10"]) for row in factual_rows]) + ), + "post_chunk_precision_at_10": _safe_round( + _mean([_to_float(row["post_chunk_precision_at_10"]) for row in factual_rows]) + ), + "pre_chunk_precision_at_25": _safe_round( + _mean([_to_float(row["pre_chunk_precision_at_25"]) for row in factual_rows]) + ), + "post_chunk_precision_at_25": _safe_round( + _mean([_to_float(row["post_chunk_precision_at_25"]) for row in factual_rows]) + ), + "pre_chunk_recall_at_25": _safe_round(_mean([_to_float(row["pre_chunk_recall_at_25"]) for row in factual_rows])), + "post_chunk_recall_at_25": _safe_round( + _mean([_to_float(row["post_chunk_recall_at_25"]) for row in factual_rows]) + ), + "pre_doc_precision_at_5": _safe_round(_mean([_to_float(row["pre_doc_precision_at_5"]) for row in factual_rows])), + "post_doc_precision_at_5": _safe_round( + _mean([_to_float(row["post_doc_precision_at_5"]) for row in factual_rows]) + ), + "pre_doc_precision_at_10": _safe_round(_mean([_to_float(row["pre_doc_precision_at_10"]) for row in factual_rows])), + "post_doc_precision_at_10": _safe_round( + _mean([_to_float(row["post_doc_precision_at_10"]) for row in factual_rows]) + ), + "pre_doc_precision_at_25": _safe_round(_mean([_to_float(row["pre_doc_precision_at_25"]) for row in factual_rows])), + "post_doc_precision_at_25": _safe_round( + _mean([_to_float(row["post_doc_precision_at_25"]) for row in factual_rows]) + ), + "pre_doc_recall_at_25": _safe_round(_mean([_to_float(row["pre_doc_recall_at_25"]) for row in factual_rows])), + "post_doc_recall_at_25": _safe_round(_mean([_to_float(row["post_doc_recall_at_25"]) for row in factual_rows])), + "rerank_chunk_delta_precision_at_5": _safe_round( + _mean([_to_float(row["delta_chunk_precision_at_5"]) for row in factual_rows]) + ), + "rerank_chunk_delta_precision_at_10": _safe_round( + _mean([_to_float(row["delta_chunk_precision_at_10"]) for row in factual_rows]) + ), + "rerank_chunk_delta_precision_at_25": _safe_round( + _mean([_to_float(row["delta_chunk_precision_at_25"]) for row in factual_rows]) + ), + "rerank_chunk_delta_recall_at_25": _safe_round( + _mean([_to_float(row["delta_chunk_recall_at_25"]) for row in factual_rows]) + ), + "rerank_doc_delta_precision_at_5": _safe_round( + _mean([_to_float(row["delta_doc_precision_at_5"]) for row in factual_rows]) + ), + "rerank_doc_delta_precision_at_10": _safe_round( + _mean([_to_float(row["delta_doc_precision_at_10"]) for row in factual_rows]) + ), + "rerank_doc_delta_precision_at_25": _safe_round( + _mean([_to_float(row["delta_doc_precision_at_25"]) for row in factual_rows]) + ), + "rerank_doc_delta_recall_at_25": _safe_round( + _mean([_to_float(row["delta_doc_recall_at_25"]) for row in factual_rows]) + ), + } + + if args.enable_nli: + scorer = EntailmentScorer(model_name=args.nli_model) + support_rows: list[dict[str, Any]] = [] + open_ended_queries = [item for item in eval_queries if item.kind == "open_ended"][: max(0, args.nli_max_open_ended)] + for query in open_ended_queries: + generation = generation_by_id.get(query.id) + if generation is None or generation.error: + continue + claims = split_claim_like_units(generation.final_answer or "") + if not claims: + continue + evidence = _evidence_blocks(generation) + stats = scorer.score_claims_against_evidence( + claims=claims, + evidence_blocks=evidence, + support_threshold=args.nli_support_threshold, + contradiction_threshold=args.nli_contradiction_threshold, + ) + support_rows.append( + { + "query_id": query.id, + "claim_count": stats.claim_count, + "supported_claim_count": stats.supported_claim_count, + "contradicted_claim_count": stats.contradicted_claim_count, + "unsupported_claim_count": stats.unsupported_claim_count, + "support_rate": stats.support_rate, + "contradiction_rate": stats.contradiction_rate, + "unsupported_rate": stats.unsupported_rate, + } + ) + summary["open_ended_nli_n"] = len(support_rows) + summary["open_ended_nli_support_rate"] = _safe_round( + _mean([_to_float(row["support_rate"]) for row in support_rows]) + ) + summary["open_ended_nli_contradiction_rate"] = _safe_round( + _mean([_to_float(row["contradiction_rate"]) for row in support_rows]) + ) + summary["open_ended_nli_unsupported_rate"] = _safe_round( + _mean([_to_float(row["unsupported_rate"]) for row in support_rows]) + ) + _write_csv(run_dir / "retrieval_nli_claim_support.csv", support_rows) + + _write_csv(run_dir / "retrieval_rerank_metrics.csv", factual_rows) + (run_dir / "retrieval_rerank_metrics.json").write_text( + json.dumps({"summary": summary, "rows": factual_rows}, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + _write_markdown(run_dir / "retrieval_rerank_metrics.md", summary, factual_rows) + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/score_eval.py b/scripts/score_eval.py index dbc74dc..d82a9b4 100644 --- a/scripts/score_eval.py +++ b/scripts/score_eval.py @@ -301,10 +301,54 @@ def _score_query_threadsafe(q: EvalQuery) -> EvalScore: "gold_chunk_id": (gold.chunk_id if gold is not None else ""), "gold_section_path": (gold.section_path if gold is not None and gold.section_path else ""), "gold_chunk_rank": s.retrieval.get("gold_chunk_rank", ""), + "pre_chunk_rank": s.retrieval.get("pre_chunk_rank", ""), + "post_chunk_rank": s.retrieval.get("post_chunk_rank", ""), + "pre_doc_rank": s.retrieval.get("pre_doc_rank", ""), + "post_doc_rank": s.retrieval.get("post_doc_rank", ""), + "pre_chunk_mrr": s.retrieval.get("pre_chunk_mrr", ""), + "post_chunk_mrr": s.retrieval.get("post_chunk_mrr", ""), + "pre_doc_mrr": s.retrieval.get("pre_doc_mrr", ""), + "post_doc_mrr": s.retrieval.get("post_doc_mrr", ""), + "pre_chunk_hit_at_25": s.retrieval.get("pre_chunk_hit_at_25", ""), + "post_chunk_hit_at_25": s.retrieval.get("post_chunk_hit_at_25", ""), + "pre_doc_hit_at_25": s.retrieval.get("pre_doc_hit_at_25", ""), + "post_doc_hit_at_25": s.retrieval.get("post_doc_hit_at_25", ""), + "pre_chunk_precision_at_5": s.retrieval.get("pre_chunk_precision_at_5", ""), + "post_chunk_precision_at_5": s.retrieval.get("post_chunk_precision_at_5", ""), + "pre_chunk_precision_at_10": s.retrieval.get("pre_chunk_precision_at_10", ""), + "post_chunk_precision_at_10": s.retrieval.get("post_chunk_precision_at_10", ""), + "pre_chunk_precision_at_25": s.retrieval.get("pre_chunk_precision_at_25", ""), + "post_chunk_precision_at_25": s.retrieval.get("post_chunk_precision_at_25", ""), + "pre_chunk_recall_at_25": s.retrieval.get("pre_chunk_recall_at_25", ""), + "post_chunk_recall_at_25": s.retrieval.get("post_chunk_recall_at_25", ""), + "pre_doc_precision_at_5": s.retrieval.get("pre_doc_precision_at_5", ""), + "post_doc_precision_at_5": s.retrieval.get("post_doc_precision_at_5", ""), + "pre_doc_precision_at_10": s.retrieval.get("pre_doc_precision_at_10", ""), + "post_doc_precision_at_10": s.retrieval.get("post_doc_precision_at_10", ""), + "pre_doc_precision_at_25": s.retrieval.get("pre_doc_precision_at_25", ""), + "post_doc_precision_at_25": s.retrieval.get("post_doc_precision_at_25", ""), + "pre_doc_recall_at_25": s.retrieval.get("pre_doc_recall_at_25", ""), + "post_doc_recall_at_25": s.retrieval.get("post_doc_recall_at_25", ""), + "rerank_chunk_delta_mrr": s.retrieval.get("rerank_chunk_delta_mrr", ""), + "rerank_doc_delta_mrr": s.retrieval.get("rerank_doc_delta_mrr", ""), + "rerank_chunk_delta_precision_at_5": s.retrieval.get("rerank_chunk_delta_precision_at_5", ""), + "rerank_chunk_delta_precision_at_10": s.retrieval.get("rerank_chunk_delta_precision_at_10", ""), + "rerank_chunk_delta_precision_at_25": s.retrieval.get("rerank_chunk_delta_precision_at_25", ""), + "rerank_chunk_delta_recall_at_25": s.retrieval.get("rerank_chunk_delta_recall_at_25", ""), + "rerank_doc_delta_precision_at_5": s.retrieval.get("rerank_doc_delta_precision_at_5", ""), + "rerank_doc_delta_precision_at_10": s.retrieval.get("rerank_doc_delta_precision_at_10", ""), + "rerank_doc_delta_precision_at_25": s.retrieval.get("rerank_doc_delta_precision_at_25", ""), + "rerank_doc_delta_recall_at_25": s.retrieval.get("rerank_doc_delta_recall_at_25", ""), + "rerank_chunk_win": s.retrieval.get("rerank_chunk_win", ""), + "rerank_doc_win": s.retrieval.get("rerank_doc_win", ""), "numeric_matched": s.answer.get("numeric_matched", ""), "numeric_best_pred": s.answer.get("numeric_best_pred", ""), "numeric_best_rel_error": s.answer.get("numeric_best_rel_error", ""), "cited_gold_doc": s.answer.get("cited_gold_doc", ""), + "citation_count": s.answer.get("citation_count", ""), + "supported_citation_count": s.answer.get("supported_citation_count", ""), + "unsupported_citation_count": s.answer.get("unsupported_citation_count", ""), + "supported_citation_rate": s.answer.get("supported_citation_rate", ""), "judge_id": (judge0.judge_id if judge0 is not None else ""), "judge_prediction": (judge0.prediction if judge0 is not None else ""), "judge_explanation": (judge0.explanation if judge0 is not None and judge0.explanation else ""), diff --git a/src/andromeda/eval/evidence_support.py b/src/andromeda/eval/evidence_support.py new file mode 100644 index 0000000..4e0a7a5 --- /dev/null +++ b/src/andromeda/eval/evidence_support.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass + +from sentence_transformers import CrossEncoder + + +@dataclass(frozen=True) +class CitationSupportSummary: + """ + Citation support stats for one answer. + """ + + citation_count: int + supported_citation_count: int + unsupported_citation_count: int + supported_rate: float + + +@dataclass(frozen=True) +class ClaimSupportSummary: + """ + Claim-level support stats for one answer. + """ + + claim_count: int + supported_claim_count: int + contradicted_claim_count: int + unsupported_claim_count: int + support_rate: float + contradiction_rate: float + unsupported_rate: float + + +def citation_support_summary(*, cited_chunk_ids: list[str], available_chunk_ids: list[str]) -> CitationSupportSummary: + """ + Compute citation support coverage against available chunk ids. + """ + + cited = [item for item in cited_chunk_ids if item] + if not cited: + return CitationSupportSummary( + citation_count=0, + supported_citation_count=0, + unsupported_citation_count=0, + supported_rate=math.nan, + ) + + available = set(item for item in available_chunk_ids if item) + supported = sum(1 for item in cited if item in available) + unsupported = len(cited) - supported + return CitationSupportSummary( + citation_count=len(cited), + supported_citation_count=supported, + unsupported_citation_count=unsupported, + supported_rate=(supported / len(cited)), + ) + + +def split_claim_like_units(answer_text: str, *, max_claims: int = 8, min_chars: int = 30) -> list[str]: + """ + Split an answer into claim-like text units for support scoring. + """ + + if not answer_text.strip(): + return [] + + out: list[str] = [] + for line in answer_text.splitlines(): + stripped = line.strip() + if not stripped: + continue + if stripped.startswith("- "): + stripped = stripped[2:].strip() + pieces = [part.strip() for part in stripped.split(". ") if part.strip()] + for piece in pieces: + if len(piece) < min_chars: + continue + out.append(piece) + if len(out) >= max_claims: + return out + return out[:max_claims] + + +class EntailmentScorer: + """ + Local cross-encoder entailment scorer for claim-evidence support checks. + """ + + def __init__(self, model_name: str = "cross-encoder/nli-deberta-v3-base", max_length: int = 512): + self.model_name = model_name + self.model = CrossEncoder(model_name, max_length=max_length) + id2label = getattr(self.model.model.config, "id2label", {}) or {} + entailment_id: int | None = None + contradiction_id: int | None = None + for idx, label in id2label.items(): + lowered = str(label).strip().lower() + if "entail" in lowered: + entailment_id = int(idx) + if "contrad" in lowered: + contradiction_id = int(idx) + self.entailment_id = entailment_id if entailment_id is not None else 2 + self.contradiction_id = contradiction_id if contradiction_id is not None else 0 + + def score_claims_against_evidence( + self, + *, + claims: list[str], + evidence_blocks: list[str], + support_threshold: float = 0.50, + contradiction_threshold: float = 0.50, + ) -> ClaimSupportSummary: + """ + Score claim support/contradiction against evidence blocks. + """ + + valid_claims = [item.strip() for item in claims if item and item.strip()] + valid_evidence = [item.strip() for item in evidence_blocks if item and item.strip()] + if not valid_claims: + return ClaimSupportSummary( + claim_count=0, + supported_claim_count=0, + contradicted_claim_count=0, + unsupported_claim_count=0, + support_rate=math.nan, + contradiction_rate=math.nan, + unsupported_rate=math.nan, + ) + if not valid_evidence: + return ClaimSupportSummary( + claim_count=len(valid_claims), + supported_claim_count=0, + contradicted_claim_count=0, + unsupported_claim_count=len(valid_claims), + support_rate=0.0, + contradiction_rate=0.0, + unsupported_rate=1.0, + ) + + pairs: list[tuple[str, str]] = [] + for claim in valid_claims: + for evidence in valid_evidence: + pairs.append((claim, evidence)) + outputs = self.model.predict(pairs, apply_softmax=True) + + supported = 0 + contradicted = 0 + unsupported = 0 + evidence_count = len(valid_evidence) + for idx, _claim in enumerate(valid_claims): + row = outputs[idx * evidence_count : (idx + 1) * evidence_count] + max_entailment = max(float(item[self.entailment_id]) for item in row) + max_contradiction = max(float(item[self.contradiction_id]) for item in row) + if max_entailment >= support_threshold: + supported += 1 + elif max_contradiction >= contradiction_threshold: + contradicted += 1 + else: + unsupported += 1 + + claim_count = len(valid_claims) + return ClaimSupportSummary( + claim_count=claim_count, + supported_claim_count=supported, + contradicted_claim_count=contradicted, + unsupported_claim_count=unsupported, + support_rate=(supported / claim_count), + contradiction_rate=(contradicted / claim_count), + unsupported_rate=(unsupported / claim_count), + ) diff --git a/src/andromeda/eval/rerank_metrics.py b/src/andromeda/eval/rerank_metrics.py new file mode 100644 index 0000000..0a22432 --- /dev/null +++ b/src/andromeda/eval/rerank_metrics.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import asdict + +from andromeda.eval.retrieval_metrics import RankMetrics + + +def rerank_uplift(*, pre: RankMetrics, post: RankMetrics) -> dict[str, float | int]: + """ + Compute pre-vs-post reranking uplift summary. + """ + + rank_shift = 0 + if pre.rank is not None and post.rank is not None: + rank_shift = pre.rank - post.rank + + win = 1 if rank_shift > 0 else 0 + loss = 1 if rank_shift < 0 else 0 + tie = 1 if rank_shift == 0 else 0 + + return { + "rank_shift": rank_shift, + "win": win, + "loss": loss, + "tie": tie, + "delta_mrr": post.mrr - pre.mrr, + "delta_ndcg_at_10": post.ndcg_at_10 - pre.ndcg_at_10, + "delta_ndcg_at_25": post.ndcg_at_25 - pre.ndcg_at_25, + "delta_hit_at_5": post.hit_at_5 - pre.hit_at_5, + "delta_hit_at_10": post.hit_at_10 - pre.hit_at_10, + "delta_hit_at_25": post.hit_at_25 - pre.hit_at_25, + "delta_hit_at_40": post.hit_at_40 - pre.hit_at_40, + "delta_precision_at_5": post.precision_at_5 - pre.precision_at_5, + "delta_precision_at_10": post.precision_at_10 - pre.precision_at_10, + "delta_precision_at_25": post.precision_at_25 - pre.precision_at_25, + "delta_precision_at_40": post.precision_at_40 - pre.precision_at_40, + "delta_recall_at_5": post.recall_at_5 - pre.recall_at_5, + "delta_recall_at_10": post.recall_at_10 - pre.recall_at_10, + "delta_recall_at_25": post.recall_at_25 - pre.recall_at_25, + "delta_recall_at_40": post.recall_at_40 - pre.recall_at_40, + } + + +def prefixed_rank_metrics(prefix: str, metrics: RankMetrics) -> dict[str, float | int | None]: + """ + Convert RankMetrics to a prefixed flat dict. + """ + + out: dict[str, float | int | None] = {} + for key, value in asdict(metrics).items(): + out[f"{prefix}_{key}"] = value + return out diff --git a/src/andromeda/eval/retrieval_metrics.py b/src/andromeda/eval/retrieval_metrics.py new file mode 100644 index 0000000..65fdb53 --- /dev/null +++ b/src/andromeda/eval/retrieval_metrics.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import math +from collections.abc import Iterable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RankMetrics: + """ + Retrieval metrics for one ranked list against a relevant-id set. + """ + + rank: int | None + mrr: float + ndcg_at_10: float + ndcg_at_25: float + hit_at_5: float + hit_at_10: float + hit_at_25: float + hit_at_40: float + precision_at_5: float + precision_at_10: float + precision_at_25: float + precision_at_40: float + recall_at_5: float + recall_at_10: float + recall_at_25: float + recall_at_40: float + + +def rank_of_id(ranked_ids: list[str], target_id: str) -> int | None: + """ + Return 1-based rank for a target id in a ranked id list. + """ + + for idx, item in enumerate(ranked_ids, start=1): + if item == target_id: + return idx + return None + + +def reciprocal_rank(rank: int | None) -> float: + """ + Compute reciprocal rank from a 1-based rank value. + """ + + if rank is None or rank <= 0: + return 0.0 + return 1.0 / float(rank) + + +def hit_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: + """ + Return 1.0 if any relevant id appears in top-k, else 0.0. + """ + + if k <= 0: + return 0.0 + return 1.0 if any(item in relevant_ids for item in ranked_ids[:k]) else 0.0 + + +def precision_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: + """ + Compute precision@k for a ranked id list. + """ + + if k <= 0 or not ranked_ids: + return 0.0 + topk = ranked_ids[:k] + if not topk: + return 0.0 + hits = sum(1 for item in topk if item in relevant_ids) + return float(hits) / float(len(topk)) + + +def recall_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: + """ + Compute recall@k for a ranked id list. + """ + + if k <= 0 or not relevant_ids: + return 0.0 + topk = ranked_ids[:k] + hits = sum(1 for item in topk if item in relevant_ids) + return float(hits) / float(len(relevant_ids)) + + +def ndcg_at_k(ranked_ids: list[str], relevance_by_id: dict[str, float], k: int) -> float: + """ + Compute nDCG@k from an id->relevance map. + """ + + if k <= 0: + return math.nan + gains: list[float] = [] + for item in ranked_ids[:k]: + gains.append(max(0.0, float(relevance_by_id.get(item, 0.0)))) + + dcg = 0.0 + for idx, gain in enumerate(gains, start=1): + denom = math.log2(idx + 1.0) + dcg += gain / denom + + ideal_gains = sorted((max(0.0, float(v)) for v in relevance_by_id.values()), reverse=True)[:k] + idcg = 0.0 + for idx, gain in enumerate(ideal_gains, start=1): + denom = math.log2(idx + 1.0) + idcg += gain / denom + + if idcg <= 0.0: + return 0.0 + return dcg / idcg + + +def metrics_for_ranked_ids( + *, + ranked_ids: list[str], + relevant_ids: Iterable[str], + target_id: str, + relevance_by_id: dict[str, float] | None = None, +) -> RankMetrics: + """ + Build retrieval metrics for one ranked id list. + """ + + relevant_set = {item for item in relevant_ids if item} + graded = dict(relevance_by_id or {}) + if target_id and target_id not in graded: + graded[target_id] = 1.0 + + rank = rank_of_id(ranked_ids, target_id) if target_id else None + return RankMetrics( + rank=rank, + mrr=reciprocal_rank(rank), + ndcg_at_10=ndcg_at_k(ranked_ids, graded, 10), + ndcg_at_25=ndcg_at_k(ranked_ids, graded, 25), + hit_at_5=hit_at_k(ranked_ids, relevant_set, 5), + hit_at_10=hit_at_k(ranked_ids, relevant_set, 10), + hit_at_25=hit_at_k(ranked_ids, relevant_set, 25), + hit_at_40=hit_at_k(ranked_ids, relevant_set, 40), + precision_at_5=precision_at_k(ranked_ids, relevant_set, 5), + precision_at_10=precision_at_k(ranked_ids, relevant_set, 10), + precision_at_25=precision_at_k(ranked_ids, relevant_set, 25), + precision_at_40=precision_at_k(ranked_ids, relevant_set, 40), + recall_at_5=recall_at_k(ranked_ids, relevant_set, 5), + recall_at_10=recall_at_k(ranked_ids, relevant_set, 10), + recall_at_25=recall_at_k(ranked_ids, relevant_set, 25), + recall_at_40=recall_at_k(ranked_ids, relevant_set, 40), + ) diff --git a/src/andromeda/eval/scoring.py b/src/andromeda/eval/scoring.py index 23c0c98..dbcf216 100644 --- a/src/andromeda/eval/scoring.py +++ b/src/andromeda/eval/scoring.py @@ -5,8 +5,11 @@ from datetime import datetime, timezone from typing import Any +from andromeda.eval.evidence_support import citation_support_summary from andromeda.eval.judges import FACTUAL_CORRECTNESS_V1, HELPFULNESS_V1, JudgeSpec, get_judge_spec, run_judge from andromeda.eval.metrics import best_numeric_match, cited_doc_ids +from andromeda.eval.rerank_metrics import prefixed_rank_metrics, rerank_uplift +from andromeda.eval.retrieval_metrics import metrics_for_ranked_ids from andromeda.eval.schema import EvalGeneration, EvalQuery, EvalScore, JudgeResult, RetrievedChunk from andromeda.llm.clients import LLMClient from andromeda.processing.metadata_models import chunk_metadata_from_value @@ -26,13 +29,6 @@ def _truncate(text: str, limit: int) -> str: return text[: max(0, limit - 1)].rstrip() + "…" -def _rank(ids: list[str], target: str) -> int | None: - for i, x in enumerate(ids, start=1): - if x == target: - return i - return None - - def _cited_chunk_ids(text: str) -> list[str]: seen: set[str] = set() out: list[str] = [] @@ -201,34 +197,109 @@ def score_one( final = (gen.final_answer or "").strip() top_chunks = list(gen.top_chunks or []) + pre_chunks = list(gen.retrieved_chunks or []) + if not pre_chunks: + pre_chunks = list(top_chunks) cited_chunk_ids = _cited_chunk_ids(final) retrieved_chunk_ids = [c.chunk_id for c in top_chunks] retrieved_doc_ids = [c.doc_id for c in top_chunks] + pre_chunk_ids = [c.chunk_id for c in pre_chunks] + pre_doc_ids = [c.doc_id for c in pre_chunks] retrieved_tickers = _chunk_tickers(top_chunks) score.retrieval["retrieved_chunks"] = len(retrieved_chunk_ids) score.retrieval["retrieved_docs_unique"] = len(set(retrieved_doc_ids)) + score.retrieval["pre_retrieved_chunks"] = len(pre_chunk_ids) + score.retrieval["pre_retrieved_docs_unique"] = len(set(pre_doc_ids)) if retrieved_tickers: score.retrieval["retrieved_tickers_unique"] = len(set(retrieved_tickers)) score.retrieval["retrieved_tickers_top"] = retrieved_tickers[: min(12, len(retrieved_tickers))] + citation_stats = citation_support_summary( + cited_chunk_ids=cited_chunk_ids, + available_chunk_ids=list(dict.fromkeys(pre_chunk_ids + retrieved_chunk_ids)), + ) + score.answer["cited_chunk_ids"] = cited_chunk_ids + score.answer["citation_count"] = citation_stats.citation_count + score.answer["supported_citation_count"] = citation_stats.supported_citation_count + score.answer["unsupported_citation_count"] = citation_stats.unsupported_citation_count + score.answer["supported_citation_rate"] = citation_stats.supported_rate + if query.kind == "factual" and query.factual is not None: gold_chunk = query.factual.golden_evidence.chunk_id gold_doc = query.factual.golden_evidence.doc_id - chunk_rank = _rank(retrieved_chunk_ids, gold_chunk) - doc_rank = _rank(retrieved_doc_ids, gold_doc) + post_chunk_metrics = metrics_for_ranked_ids( + ranked_ids=retrieved_chunk_ids, + relevant_ids={gold_chunk}, + target_id=gold_chunk, + relevance_by_id={gold_chunk: 1.0}, + ) + post_doc_metrics = metrics_for_ranked_ids( + ranked_ids=retrieved_doc_ids, + relevant_ids={gold_doc}, + target_id=gold_doc, + relevance_by_id={gold_doc: 1.0}, + ) + pre_chunk_metrics = metrics_for_ranked_ids( + ranked_ids=pre_chunk_ids, + relevant_ids={gold_chunk}, + target_id=gold_chunk, + relevance_by_id={gold_chunk: 1.0}, + ) + pre_doc_metrics = metrics_for_ranked_ids( + ranked_ids=pre_doc_ids, + relevant_ids={gold_doc}, + target_id=gold_doc, + relevance_by_id={gold_doc: 1.0}, + ) + chunk_uplift = rerank_uplift(pre=pre_chunk_metrics, post=post_chunk_metrics) + doc_uplift = rerank_uplift(pre=pre_doc_metrics, post=post_doc_metrics) score.retrieval.update( { "gold_chunk_id": gold_chunk, "gold_doc_id": gold_doc, - "gold_chunk_rank": chunk_rank, - "gold_doc_rank": doc_rank, - "gold_chunk_mrr": (1.0 / chunk_rank) if chunk_rank else 0.0, - "gold_doc_mrr": (1.0 / doc_rank) if doc_rank else 0.0, + "gold_chunk_rank": post_chunk_metrics.rank, + "gold_doc_rank": post_doc_metrics.rank, + "gold_chunk_mrr": post_chunk_metrics.mrr, + "gold_doc_mrr": post_doc_metrics.mrr, } ) + score.retrieval.update(prefixed_rank_metrics("pre_chunk", pre_chunk_metrics)) + score.retrieval.update(prefixed_rank_metrics("post_chunk", post_chunk_metrics)) + score.retrieval.update(prefixed_rank_metrics("pre_doc", pre_doc_metrics)) + score.retrieval.update(prefixed_rank_metrics("post_doc", post_doc_metrics)) + score.retrieval["rerank_chunk_rank_shift"] = chunk_uplift["rank_shift"] + score.retrieval["rerank_chunk_delta_mrr"] = chunk_uplift["delta_mrr"] + score.retrieval["rerank_chunk_delta_ndcg_at_10"] = chunk_uplift["delta_ndcg_at_10"] + score.retrieval["rerank_chunk_delta_ndcg_at_25"] = chunk_uplift["delta_ndcg_at_25"] + score.retrieval["rerank_chunk_delta_hit_at_10"] = chunk_uplift["delta_hit_at_10"] + score.retrieval["rerank_chunk_delta_hit_at_25"] = chunk_uplift["delta_hit_at_25"] + score.retrieval["rerank_chunk_delta_precision_at_5"] = chunk_uplift["delta_precision_at_5"] + score.retrieval["rerank_chunk_delta_precision_at_10"] = chunk_uplift["delta_precision_at_10"] + score.retrieval["rerank_chunk_delta_precision_at_25"] = chunk_uplift["delta_precision_at_25"] + score.retrieval["rerank_chunk_delta_recall_at_5"] = chunk_uplift["delta_recall_at_5"] + score.retrieval["rerank_chunk_delta_recall_at_10"] = chunk_uplift["delta_recall_at_10"] + score.retrieval["rerank_chunk_delta_recall_at_25"] = chunk_uplift["delta_recall_at_25"] + score.retrieval["rerank_chunk_win"] = chunk_uplift["win"] + score.retrieval["rerank_chunk_loss"] = chunk_uplift["loss"] + score.retrieval["rerank_chunk_tie"] = chunk_uplift["tie"] + score.retrieval["rerank_doc_rank_shift"] = doc_uplift["rank_shift"] + score.retrieval["rerank_doc_delta_mrr"] = doc_uplift["delta_mrr"] + score.retrieval["rerank_doc_delta_ndcg_at_10"] = doc_uplift["delta_ndcg_at_10"] + score.retrieval["rerank_doc_delta_ndcg_at_25"] = doc_uplift["delta_ndcg_at_25"] + score.retrieval["rerank_doc_delta_hit_at_10"] = doc_uplift["delta_hit_at_10"] + score.retrieval["rerank_doc_delta_hit_at_25"] = doc_uplift["delta_hit_at_25"] + score.retrieval["rerank_doc_delta_precision_at_5"] = doc_uplift["delta_precision_at_5"] + score.retrieval["rerank_doc_delta_precision_at_10"] = doc_uplift["delta_precision_at_10"] + score.retrieval["rerank_doc_delta_precision_at_25"] = doc_uplift["delta_precision_at_25"] + score.retrieval["rerank_doc_delta_recall_at_5"] = doc_uplift["delta_recall_at_5"] + score.retrieval["rerank_doc_delta_recall_at_10"] = doc_uplift["delta_recall_at_10"] + score.retrieval["rerank_doc_delta_recall_at_25"] = doc_uplift["delta_recall_at_25"] + score.retrieval["rerank_doc_win"] = doc_uplift["win"] + score.retrieval["rerank_doc_loss"] = doc_uplift["loss"] + score.retrieval["rerank_doc_tie"] = doc_uplift["tie"] expected = query.factual.expected_numeric nm = best_numeric_match(final, expected.value, expected_scale=expected.scale) @@ -386,6 +457,17 @@ def _mean(vals: list[float]) -> float: vals = [v for v in vals if v is not None and not math.isnan(v)] return (sum(vals) / len(vals)) if vals else math.nan + def _mean_retrieval(items: list[EvalScore], key: str) -> float: + vals: list[float] = [] + for item in items: + value = item.retrieval.get(key) + if isinstance(value, bool): + vals.append(1.0 if value else 0.0) + continue + if isinstance(value, (int, float)): + vals.append(float(value)) + return _mean(vals) + def _is_ok(s: EvalScore) -> bool: return "status" not in s.answer or not bool(s.answer["status"]) @@ -441,6 +523,58 @@ def _attach_judge_metrics( out["factual_gold_chunk_hit_rate"] = _mean( [1.0 if ("gold_chunk_rank" in s.retrieval and s.retrieval["gold_chunk_rank"]) else 0.0 for s in factual_ok] ) + out["factual_pre_chunk_hit_rate_at_25"] = _mean_retrieval(factual_ok, "pre_chunk_hit_at_25") + out["factual_post_chunk_hit_rate_at_25"] = _mean_retrieval(factual_ok, "post_chunk_hit_at_25") + out["factual_pre_doc_hit_rate_at_25"] = _mean_retrieval(factual_ok, "pre_doc_hit_at_25") + out["factual_post_doc_hit_rate_at_25"] = _mean_retrieval(factual_ok, "post_doc_hit_at_25") + out["factual_pre_chunk_precision_at_5"] = _mean_retrieval(factual_ok, "pre_chunk_precision_at_5") + out["factual_post_chunk_precision_at_5"] = _mean_retrieval(factual_ok, "post_chunk_precision_at_5") + out["factual_pre_chunk_precision_at_10"] = _mean_retrieval(factual_ok, "pre_chunk_precision_at_10") + out["factual_post_chunk_precision_at_10"] = _mean_retrieval(factual_ok, "post_chunk_precision_at_10") + out["factual_pre_chunk_precision_at_25"] = _mean_retrieval(factual_ok, "pre_chunk_precision_at_25") + out["factual_post_chunk_precision_at_25"] = _mean_retrieval(factual_ok, "post_chunk_precision_at_25") + out["factual_pre_chunk_recall_at_25"] = _mean_retrieval(factual_ok, "pre_chunk_recall_at_25") + out["factual_post_chunk_recall_at_25"] = _mean_retrieval(factual_ok, "post_chunk_recall_at_25") + out["factual_pre_doc_precision_at_5"] = _mean_retrieval(factual_ok, "pre_doc_precision_at_5") + out["factual_post_doc_precision_at_5"] = _mean_retrieval(factual_ok, "post_doc_precision_at_5") + out["factual_pre_doc_precision_at_10"] = _mean_retrieval(factual_ok, "pre_doc_precision_at_10") + out["factual_post_doc_precision_at_10"] = _mean_retrieval(factual_ok, "post_doc_precision_at_10") + out["factual_pre_doc_precision_at_25"] = _mean_retrieval(factual_ok, "pre_doc_precision_at_25") + out["factual_post_doc_precision_at_25"] = _mean_retrieval(factual_ok, "post_doc_precision_at_25") + out["factual_pre_doc_recall_at_25"] = _mean_retrieval(factual_ok, "pre_doc_recall_at_25") + out["factual_post_doc_recall_at_25"] = _mean_retrieval(factual_ok, "post_doc_recall_at_25") + out["factual_pre_chunk_mrr"] = _mean_retrieval(factual_ok, "pre_chunk_mrr") + out["factual_post_chunk_mrr"] = _mean_retrieval(factual_ok, "post_chunk_mrr") + out["factual_pre_doc_mrr"] = _mean_retrieval(factual_ok, "pre_doc_mrr") + out["factual_post_doc_mrr"] = _mean_retrieval(factual_ok, "post_doc_mrr") + out["factual_rerank_chunk_mrr_delta"] = _mean_retrieval(factual_ok, "rerank_chunk_delta_mrr") + out["factual_rerank_doc_mrr_delta"] = _mean_retrieval(factual_ok, "rerank_doc_delta_mrr") + out["factual_rerank_chunk_precision_at_5_delta"] = _mean_retrieval( + factual_ok, "rerank_chunk_delta_precision_at_5" + ) + out["factual_rerank_chunk_precision_at_10_delta"] = _mean_retrieval( + factual_ok, "rerank_chunk_delta_precision_at_10" + ) + out["factual_rerank_chunk_precision_at_25_delta"] = _mean_retrieval( + factual_ok, "rerank_chunk_delta_precision_at_25" + ) + out["factual_rerank_chunk_recall_at_25_delta"] = _mean_retrieval( + factual_ok, "rerank_chunk_delta_recall_at_25" + ) + out["factual_rerank_doc_precision_at_5_delta"] = _mean_retrieval( + factual_ok, "rerank_doc_delta_precision_at_5" + ) + out["factual_rerank_doc_precision_at_10_delta"] = _mean_retrieval( + factual_ok, "rerank_doc_delta_precision_at_10" + ) + out["factual_rerank_doc_precision_at_25_delta"] = _mean_retrieval( + factual_ok, "rerank_doc_delta_precision_at_25" + ) + out["factual_rerank_doc_recall_at_25_delta"] = _mean_retrieval( + factual_ok, "rerank_doc_delta_recall_at_25" + ) + out["factual_rerank_chunk_win_rate"] = _mean_retrieval(factual_ok, "rerank_chunk_win") + out["factual_rerank_doc_win_rate"] = _mean_retrieval(factual_ok, "rerank_doc_win") out["factual_numeric_accuracy"] = _mean( [1.0 if ("numeric_matched" in s.answer and bool(s.answer["numeric_matched"])) else 0.0 for s in factual_ok] ) diff --git a/tests/test_eval_retrieval_metrics.py b/tests/test_eval_retrieval_metrics.py new file mode 100644 index 0000000..f4fd47d --- /dev/null +++ b/tests/test_eval_retrieval_metrics.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from andromeda.eval.evidence_support import citation_support_summary +from andromeda.eval.rerank_metrics import rerank_uplift +from andromeda.eval.retrieval_metrics import metrics_for_ranked_ids + + +def test_metrics_for_ranked_ids_binary_case() -> None: + metrics = metrics_for_ranked_ids( + ranked_ids=["C3", "C2", "C1"], + relevant_ids={"C1"}, + target_id="C1", + relevance_by_id={"C1": 1.0}, + ) + assert metrics.rank == 3 + assert metrics.mrr == 1.0 / 3.0 + assert metrics.hit_at_5 == 1.0 + assert metrics.hit_at_10 == 1.0 + assert metrics.hit_at_25 == 1.0 + assert metrics.precision_at_5 == 1.0 / 3.0 + assert metrics.precision_at_10 == 1.0 / 3.0 + assert metrics.precision_at_25 == 1.0 / 3.0 + assert metrics.recall_at_5 == 1.0 + assert metrics.recall_at_10 == 1.0 + assert metrics.recall_at_25 == 1.0 + + +def test_rerank_uplift_reports_rank_improvement() -> None: + pre = metrics_for_ranked_ids( + ranked_ids=["C1", "C2", "C3"], + relevant_ids={"C3"}, + target_id="C3", + relevance_by_id={"C3": 1.0}, + ) + post = metrics_for_ranked_ids( + ranked_ids=["C3", "C1", "C2"], + relevant_ids={"C3"}, + target_id="C3", + relevance_by_id={"C3": 1.0}, + ) + uplift = rerank_uplift(pre=pre, post=post) + assert uplift["rank_shift"] == 2 + assert uplift["win"] == 1 + assert uplift["loss"] == 0 + assert uplift["delta_mrr"] > 0 + assert uplift["delta_precision_at_5"] >= 0 + assert uplift["delta_recall_at_5"] >= 0 + + +def test_citation_support_summary_counts_supported_and_unsupported() -> None: + summary = citation_support_summary( + cited_chunk_ids=["A", "B", "C"], + available_chunk_ids=["A", "C", "D"], + ) + assert summary.citation_count == 3 + assert summary.supported_citation_count == 2 + assert summary.unsupported_citation_count == 1 + assert summary.supported_rate == 2 / 3 diff --git a/tests/test_eval_schema_scoring.py b/tests/test_eval_schema_scoring.py index d28f7a7..cd3e1e5 100644 --- a/tests/test_eval_schema_scoring.py +++ b/tests/test_eval_schema_scoring.py @@ -72,8 +72,16 @@ def test_score_one_factual_without_judges_tracks_retrieval_and_numeric_match() - score = score_one(query, gen, judge_llm=None) assert score.retrieval["gold_chunk_rank"] == 1 assert score.retrieval["gold_doc_rank"] == 1 + assert score.retrieval["pre_chunk_rank"] == 1 + assert score.retrieval["post_chunk_rank"] == 1 + assert score.retrieval["rerank_chunk_delta_mrr"] == 0.0 + assert score.retrieval["pre_chunk_precision_at_5"] == 1.0 + assert score.retrieval["post_chunk_precision_at_5"] == 1.0 + assert score.retrieval["pre_chunk_recall_at_25"] == 1.0 + assert score.retrieval["post_chunk_recall_at_25"] == 1.0 assert score.answer["numeric_matched"] is True assert score.answer["cited_gold_doc"] is True + assert score.answer["citation_count"] == 0 def test_score_one_refusal_sets_heuristic_flag() -> None: From 3267d9f04da91cab8d967272142beb2c81dea784 Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Thu, 19 Feb 2026 11:15:05 +0800 Subject: [PATCH 07/10] eval: add retrieval calibration tooling and metric robustness --- scripts/audit_judge_decisions.py | 317 ++++++++++++++++++++++++ scripts/build_retrieval_label_pool.py | 184 ++++++++++++++ scripts/calibrate_eval_metrics.py | 143 +++++++++++ scripts/eval_retrieval.py | 25 +- scripts/judge_reliability.py | 21 +- src/andromeda/eval/evidence_support.py | 38 ++- src/andromeda/eval/retrieval_metrics.py | 23 +- src/andromeda/eval/scoring.py | 19 +- 8 files changed, 757 insertions(+), 13 deletions(-) create mode 100644 scripts/audit_judge_decisions.py create mode 100644 scripts/build_retrieval_label_pool.py create mode 100644 scripts/calibrate_eval_metrics.py diff --git a/scripts/audit_judge_decisions.py b/scripts/audit_judge_decisions.py new file mode 100644 index 0000000..9ca9c46 --- /dev/null +++ b/scripts/audit_judge_decisions.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import concurrent.futures +import csv +import json +import threading +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv +from tqdm import tqdm + +from andromeda.eval.io import load_jsonl +from andromeda.eval.judges import ( + FACTUAL_CORRECTNESS_V1, + FAITHFULNESS_V1, + JudgeSpec, + get_judge_client, + get_judge_spec, + run_judge, +) +from andromeda.eval.schema import EvalGeneration, EvalQuery +from andromeda.eval.scoring import build_context + +load_dotenv(Path(__file__).resolve().parents[1] / ".env") + + +@dataclass(frozen=True) +class RunArtifacts: + """ + Cached artifacts for one eval run directory. + """ + + query_by_id: dict[str, EvalQuery] + generation_by_id: dict[str, EvalGeneration] + review_by_id: dict[str, dict[str, str]] + + +def _utc_ts() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_json_map(raw: str) -> dict[str, Any]: + text = (raw or "").strip() + if not text: + return {} + try: + obj = json.loads(text) + except json.JSONDecodeError: + return {} + return obj if isinstance(obj, dict) else {} + + +def _load_review_csv(path: Path) -> dict[str, dict[str, str]]: + out: dict[str, dict[str, str]] = {} + if not path.exists(): + return out + with path.open("r", encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + query_id = (row.get("query_id") or "").strip() + if not query_id: + continue + out[query_id] = row + return out + + +def _load_run_artifacts(run_dir: Path) -> RunArtifacts: + eval_queries = load_jsonl(run_dir / "eval_queries.jsonl", EvalQuery) + generations = load_jsonl(run_dir / "generations.jsonl", EvalGeneration) + review = _load_review_csv(run_dir / "review.csv") + return RunArtifacts( + query_by_id={item.id: item for item in eval_queries}, + generation_by_id={item.query_id: item for item in generations}, + review_by_id=review, + ) + + +def _expected_text_for_factual(query: EvalQuery) -> str | None: + if query.factual is None: + return None + expected = query.factual.expected_numeric + bits = [f"value={expected.value}"] + if expected.scale: + bits.append(f"scale={expected.scale}") + if expected.unit: + bits.append(f"unit={expected.unit}") + if expected.raw and expected.raw.strip(): + bits.append(f"raw={expected.raw.strip()}") + return ", ".join(bits) + + +def _notes_for_decision(row: dict[str, str], query: EvalQuery | None) -> str: + """ + Build optional evaluator notes to reduce ambiguity for auditing. + """ + + bits: list[str] = [] + kind = (row.get("kind") or "").strip() + if kind: + bits.append(f"kind={kind}") + target_tickers = (row.get("target_tickers") or "").strip() + if target_tickers: + bits.append(f"target_tickers={target_tickers}") + + if query is not None and query.factual is not None: + bits.append(f"factual_metric={query.factual.metric}") + bits.append(f"gold_doc_id={query.factual.golden_evidence.doc_id}") + bits.append(f"gold_chunk_id={query.factual.golden_evidence.chunk_id}") + + return ", ".join(bits) + + +def _spec_for_audit(judge_id: str) -> JudgeSpec: + """ + Return the JudgeSpec used for proxy-human audit of one decision. + """ + + base = get_judge_spec(judge_id) + if judge_id != FAITHFULNESS_V1.judge_id: + return base + # Keep faithfulness slightly less strict on peripheral details for audit alignment. + return JudgeSpec( + judge_id=base.judge_id, + description=base.description, + system_prompt=( + base.system_prompt + + "\nAdditional audit instruction: tolerate small peripheral mismatches; fail only for material grounding errors." + ), + temperature=base.temperature, + max_context_chars=base.max_context_chars, + ) + + +def _build_output_fieldnames(rows: list[dict[str, str]]) -> list[str]: + base_fields = list(rows[0].keys()) if rows else [] + extras = [ + "audit_prediction", + "audit_explanation", + "audit_raw", + "audit_error", + "audit_model", + "audit_timestamp", + ] + for field in extras: + if field not in base_fields: + base_fields.append(field) + return base_fields + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Audit every judge decision and populate human_label/human_notes.") + parser.add_argument("--audit-csv", type=Path, required=True, help="Decision CSV from scripts/judge_reliability.py") + parser.add_argument("--out-csv", type=Path, default=None, help="Output CSV path (default: overwrite --audit-csv)") + parser.add_argument( + "--judges", + nargs="*", + default=None, + help="Optional subset of judge IDs. Defaults to all decisions in CSV.", + ) + parser.add_argument("--workers", type=int, default=12) + parser.add_argument("--context-chars", type=int, default=80_000) + parser.add_argument("--timeout-s", type=float, default=350.0) + parser.add_argument("--max-retries", type=int, default=1) + parser.add_argument("--overwrite", action="store_true", help="Recompute even if human_label already exists.") + parser.add_argument("--judge-provider", default=None) + parser.add_argument("--judge-model", default=None) + parser.add_argument("--judge-base-url", default=None) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + if args.workers < 1: + raise SystemExit("--workers must be >= 1") + + audit_csv = args.audit_csv.expanduser().resolve() + if not audit_csv.exists(): + raise SystemExit(f"Missing audit CSV: {audit_csv}") + + out_csv = args.out_csv.expanduser().resolve() if args.out_csv else audit_csv + + with audit_csv.open("r", encoding="utf-8", newline="") as handle: + rows = [row for row in csv.DictReader(handle)] + if not rows: + raise SystemExit(f"No rows in audit CSV: {audit_csv}") + + selected_judges = set(args.judges) if args.judges else None + + artifacts_cache: dict[Path, RunArtifacts] = {} + + def _get_artifacts(run_dir_raw: str) -> RunArtifacts: + run_dir = Path(run_dir_raw).expanduser().resolve() + if run_dir not in artifacts_cache: + artifacts_cache[run_dir] = _load_run_artifacts(run_dir) + return artifacts_cache[run_dir] + + thread_local = threading.local() + + def _get_llm(): + if not hasattr(thread_local, "judge_llm"): + thread_local.judge_llm = get_judge_client( + provider=args.judge_provider, + chat_model=args.judge_model, + base_url=args.judge_base_url, + ) + return thread_local.judge_llm + + model_name = (args.judge_model or "").strip() or "default" + + def _audit_one(idx: int, row: dict[str, str]) -> tuple[int, dict[str, str]]: + judge_id = (row.get("judge_id") or "").strip() + if not judge_id: + row["audit_error"] = "missing_judge_id" + row["audit_timestamp"] = _utc_ts() + return idx, row + + if selected_judges is not None and judge_id not in selected_judges: + return idx, row + + existing_label = (row.get("human_label") or "").strip() + if existing_label in {"0", "1"} and not args.overwrite: + return idx, row + + run_dir_raw = (row.get("run_dir") or "").strip() + query_id = (row.get("query_id") or "").strip() + if not run_dir_raw or not query_id: + row["audit_error"] = "missing_run_dir_or_query_id" + row["audit_timestamp"] = _utc_ts() + return idx, row + + try: + artifacts = _get_artifacts(run_dir_raw) + query = artifacts.query_by_id.get(query_id) + generation = artifacts.generation_by_id.get(query_id) + + question = (query.question if query is not None else (row.get("question") or "")).strip() + answer = ( + generation.final_answer + if generation is not None and generation.final_answer + else (row.get("final_answer") or "") + ).strip() + if generation is not None: + context = build_context(list(generation.top_chunks or []), max_chars=int(args.context_chars)) + else: + context = (row.get("top_chunks_compact") or "").strip() + + expected = _expected_text_for_factual(query) if query is not None else None + evidence = ( + query.factual.golden_evidence.snippet + if query is not None and query.factual is not None and query.factual.golden_evidence.snippet + else None + ) + if evidence and len(evidence) > 12_000: + evidence = evidence[:12_000] + notes = _notes_for_decision(row, query) + + spec = _spec_for_audit(judge_id) + llm = _get_llm() + out, raw = run_judge( + llm, + spec, + question=question, + answer=answer, + context=context, + expected=expected if judge_id == FACTUAL_CORRECTNESS_V1.judge_id else None, + evidence=evidence if judge_id == FACTUAL_CORRECTNESS_V1.judge_id else None, + notes=notes, + timeout_s=args.timeout_s, + max_retries=args.max_retries, + ) + + row["audit_prediction"] = str(int(out.prediction)) + row["audit_explanation"] = out.explanation_sketchpad + row["audit_raw"] = raw + row["audit_error"] = "" + row["audit_model"] = model_name + row["audit_timestamp"] = _utc_ts() + row["human_label"] = str(int(out.prediction)) + row["human_notes"] = ( + f"auto-audit judge={judge_id} pred={int(out.prediction)} " + f"vs_judge_pred={(row.get('judge_prediction') or '').strip()} | {out.explanation_sketchpad}" + ) + return idx, row + except Exception as exc: # noqa: BLE001 + row["audit_error"] = str(exc) + row["audit_timestamp"] = _utc_ts() + return idx, row + + indexed = list(enumerate(rows)) + out_rows: list[dict[str, str]] = [dict(row) for row in rows] + with concurrent.futures.ThreadPoolExecutor(max_workers=int(args.workers)) as executor: + futures = [executor.submit(_audit_one, idx, dict(row)) for idx, row in indexed] + for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures), desc="Auditing decisions"): + idx, audited = future.result() + out_rows[idx] = audited + + fieldnames = _build_output_fieldnames(out_rows) + out_csv.parent.mkdir(parents=True, exist_ok=True) + with out_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for row in out_rows: + writer.writerow(row) + + n_labeled = sum(1 for row in out_rows if (row.get("human_label") or "").strip() in {"0", "1"}) + n_errors = sum(1 for row in out_rows if (row.get("audit_error") or "").strip()) + print(f"Wrote: {out_csv}") + print(f"Rows: {len(out_rows)} | labeled: {n_labeled} | audit_errors: {n_errors}") + + +if __name__ == "__main__": + main() + diff --git a/scripts/build_retrieval_label_pool.py b/scripts/build_retrieval_label_pool.py new file mode 100644 index 0000000..a7b3ca2 --- /dev/null +++ b/scripts/build_retrieval_label_pool.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path +from typing import Any + +from andromeda.eval.io import load_jsonl +from andromeda.eval.schema import EvalGeneration, EvalQuery + + +def _load(path: Path) -> tuple[dict[str, EvalQuery], dict[str, EvalGeneration]]: + queries = load_jsonl(path / "eval_queries.jsonl", EvalQuery) + generations = load_jsonl(path / "generations.jsonl", EvalGeneration) + return {q.id: q for q in queries}, {g.query_id: g for g in generations} + + +def _preview(text: str, limit: int = 700) -> str: + t = (text or "").strip().replace("\n", " ") + if len(t) <= limit: + return t + return t[: max(0, limit - 1)].rstrip() + "…" + + +def _weak_label(query: EvalQuery, chunk_id: str, doc_id: str) -> float | None: + if query.factual is None: + return None + if chunk_id == query.factual.golden_evidence.chunk_id: + return 1.0 + if doc_id == query.factual.golden_evidence.doc_id: + return 0.7 + return 0.0 + + +def _row( + *, + run_name: str, + query: EvalQuery, + chunk_id: str, + doc_id: str, + rank_pre: int | None, + rank_post: int | None, + score_pre: float | None, + score_post: float | None, + text_preview: str, +) -> dict[str, Any]: + weak = _weak_label(query, chunk_id, doc_id) + return { + "run_name": run_name, + "query_id": query.id, + "kind": query.kind, + "question": query.question, + "chunk_id": chunk_id, + "doc_id": doc_id, + "rank_pre": "" if rank_pre is None else rank_pre, + "rank_post": "" if rank_post is None else rank_post, + "in_pre": int(rank_pre is not None), + "in_post": int(rank_post is not None), + "score_pre": "" if score_pre is None else score_pre, + "score_post": "" if score_post is None else score_post, + "weak_relevance": "" if weak is None else weak, + "text_preview": text_preview, + "human_relevance": "", + "human_notes": "", + } + + +def _write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8") + return + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build pooled retrieval chunk label candidates from eval run artifacts.") + parser.add_argument("--run-dirs", nargs="+", required=True, type=Path) + parser.add_argument("--out-csv", required=True, type=Path) + parser.add_argument("--out-json", default=None, type=Path) + parser.add_argument("--pre-k", type=int, default=30) + parser.add_argument("--post-k", type=int, default=30) + parser.add_argument( + "--kinds", + nargs="*", + default=["factual", "open_ended", "comparison", "distractor"], + choices=["factual", "open_ended", "comparison", "distractor", "refusal"], + ) + args = parser.parse_args() + + wanted = set(args.kinds or []) + rows: list[dict[str, Any]] = [] + per_run_stats: dict[str, dict[str, Any]] = {} + + for run_dir in args.run_dirs: + run_path = run_dir.expanduser().resolve() + query_by_id, generation_by_id = _load(run_path) + run_rows_before = len(rows) + + for query_id, query in query_by_id.items(): + if query.kind not in wanted: + continue + generation = generation_by_id.get(query_id) + if generation is None or generation.error: + continue + + pre_chunks = list(generation.retrieved_chunks or [])[: max(0, int(args.pre_k))] + post_chunks = list(generation.top_chunks or [])[: max(0, int(args.post_k))] + if not pre_chunks and not post_chunks: + continue + + pre_index: dict[str, tuple[int, float | None, str]] = {} + for idx, chunk in enumerate(pre_chunks, start=1): + if chunk.chunk_id in pre_index: + continue + pre_index[chunk.chunk_id] = (idx, float(chunk.score), chunk.doc_id) + + post_index: dict[str, tuple[int, float | None, str]] = {} + for idx, chunk in enumerate(post_chunks, start=1): + if chunk.chunk_id in post_index: + continue + post_index[chunk.chunk_id] = (idx, float(chunk.score), chunk.doc_id) + + merged_ids = list(dict.fromkeys(list(pre_index.keys()) + list(post_index.keys()))) + chunk_text_by_id: dict[str, str] = {} + for chunk in pre_chunks + post_chunks: + if chunk.chunk_id not in chunk_text_by_id: + chunk_text_by_id[chunk.chunk_id] = chunk.text or chunk.preview or "" + + for chunk_id in merged_ids: + pre_meta = pre_index.get(chunk_id) + post_meta = post_index.get(chunk_id) + doc_id = (post_meta[2] if post_meta is not None else (pre_meta[2] if pre_meta is not None else "")) + rows.append( + _row( + run_name=run_path.name, + query=query, + chunk_id=chunk_id, + doc_id=doc_id, + rank_pre=(pre_meta[0] if pre_meta is not None else None), + rank_post=(post_meta[0] if post_meta is not None else None), + score_pre=(pre_meta[1] if pre_meta is not None else None), + score_post=(post_meta[1] if post_meta is not None else None), + text_preview=_preview(chunk_text_by_id.get(chunk_id, "")), + ) + ) + + per_run_stats[run_path.name] = { + "n_queries": len(query_by_id), + "pooled_rows": len(rows) - run_rows_before, + } + + _write_csv(args.out_csv, rows) + out_json = args.out_json or args.out_csv.with_suffix(".stats.json") + out_json.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text( + json.dumps( + { + "run_dirs": [str(item.expanduser().resolve()) for item in args.run_dirs], + "kinds": sorted(wanted), + "pre_k": int(args.pre_k), + "post_k": int(args.post_k), + "rows": len(rows), + "per_run": per_run_stats, + }, + indent=2, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + print(f"Wrote: {args.out_csv}") + print(f"Wrote: {out_json}") + print(f"Rows: {len(rows)}") + + +if __name__ == "__main__": + main() + diff --git a/scripts/calibrate_eval_metrics.py b/scripts/calibrate_eval_metrics.py new file mode 100644 index 0000000..bdbcff0 --- /dev/null +++ b/scripts/calibrate_eval_metrics.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import math +import random +from pathlib import Path +from typing import Any + +from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score + + +def _parse_binary(raw: str) -> int | None: + text = (raw or "").strip() + if text not in {"0", "1"}: + return None + return int(text) + + +def _parse_float(raw: str) -> float | None: + text = (raw or "").strip() + if not text: + return None + try: + return float(text) + except Exception: + return None + + +def _metric_payload(y_true: list[int], y_pred: list[int]) -> dict[str, float | int]: + tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel() + zero_division: Any = 0 + precision = float(precision_score(y_true, y_pred, pos_label=1, zero_division=zero_division)) + recall = float(recall_score(y_true, y_pred, pos_label=1, zero_division=zero_division)) + f1 = float(f1_score(y_true, y_pred, pos_label=1, zero_division=zero_division)) + accuracy = float((tp + tn) / len(y_true)) if y_true else 0.0 + specificity = float(tn / (tn + fp)) if (tn + fp) else 0.0 + balanced_accuracy = float((recall + specificity) / 2.0) + return { + "n": len(y_true), + "tp": int(tp), + "fp": int(fp), + "tn": int(tn), + "fn": int(fn), + "accuracy": accuracy, + "precision_1": precision, + "recall_1": recall, + "f1_1": f1, + "specificity_0": specificity, + "balanced_accuracy": balanced_accuracy, + } + + +def _bootstrap_ci( + y_true: list[int], + y_pred: list[int], + *, + metric: str, + n_bootstrap: int, + seed: int, +) -> dict[str, float]: + if not y_true or len(y_true) != len(y_pred): + return {"mean": math.nan, "ci95_lo": math.nan, "ci95_hi": math.nan} + rng = random.Random(seed) + n = len(y_true) + vals: list[float] = [] + for _ in range(max(1, int(n_bootstrap))): + idx = [rng.randrange(0, n) for _ in range(n)] + bt_true = [y_true[i] for i in idx] + bt_pred = [y_pred[i] for i in idx] + payload = _metric_payload(bt_true, bt_pred) + value = payload.get(metric) + if not isinstance(value, (int, float)): + raise ValueError(f"Metric is not numeric: {metric}") + vals.append(float(value)) + vals.sort() + lo = vals[int(0.025 * (len(vals) - 1))] + hi = vals[int(0.975 * (len(vals) - 1))] + return {"mean": float(sum(vals) / len(vals)), "ci95_lo": float(lo), "ci95_hi": float(hi)} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Calibrate weak auto labels against human labels with bootstrap CIs.") + parser.add_argument("--labels-csv", required=True, type=Path) + parser.add_argument("--human-col", default="human_relevance") + parser.add_argument("--weak-col", default="weak_relevance") + parser.add_argument("--weak-threshold", type=float, default=0.5) + parser.add_argument("--n-bootstrap", type=int, default=2000) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--out-json", default=None, type=Path) + args = parser.parse_args() + + labels_csv = args.labels_csv.expanduser().resolve() + if not labels_csv.exists(): + raise SystemExit(f"Missing labels CSV: {labels_csv}") + + y_true: list[int] = [] + y_pred: list[int] = [] + with labels_csv.open("r", encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + human = _parse_binary(row.get(args.human_col, "")) + weak = _parse_float(row.get(args.weak_col, "")) + if human is None or weak is None: + continue + y_true.append(human) + y_pred.append(1 if weak >= float(args.weak_threshold) else 0) + + if not y_true: + raise SystemExit("No usable labeled rows (need both human and weak labels).") + + report = { + "labels_csv": str(labels_csv), + "human_col": args.human_col, + "weak_col": args.weak_col, + "weak_threshold": float(args.weak_threshold), + "metrics": _metric_payload(y_true, y_pred), + "bootstrap_ci95": { + "accuracy": _bootstrap_ci(y_true, y_pred, metric="accuracy", n_bootstrap=args.n_bootstrap, seed=args.seed), + "precision_1": _bootstrap_ci( + y_true, y_pred, metric="precision_1", n_bootstrap=args.n_bootstrap, seed=args.seed + 1 + ), + "recall_1": _bootstrap_ci( + y_true, y_pred, metric="recall_1", n_bootstrap=args.n_bootstrap, seed=args.seed + 2 + ), + "f1_1": _bootstrap_ci(y_true, y_pred, metric="f1_1", n_bootstrap=args.n_bootstrap, seed=args.seed + 3), + "balanced_accuracy": _bootstrap_ci( + y_true, y_pred, metric="balanced_accuracy", n_bootstrap=args.n_bootstrap, seed=args.seed + 4 + ), + }, + } + + out_json = args.out_json or labels_csv.with_suffix(".calibration.json") + out_json.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Wrote: {out_json}") + print(json.dumps(report["metrics"], indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() + diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py index 948cad9..df768d6 100644 --- a/scripts/eval_retrieval.py +++ b/scripts/eval_retrieval.py @@ -36,6 +36,17 @@ def _safe_round(value: float, digits: int = 4) -> float: return round(value, digits) +def _dedupe_order(values: list[str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for value in values: + if value in seen: + continue + seen.add(value) + out.append(value) + return out + + def _evidence_blocks(gen: EvalGeneration, *, max_blocks: int = 10, max_chars: int = 900) -> list[str]: blocks: list[str] = [] for chunk in (gen.top_chunks or [])[:max_blocks]: @@ -108,6 +119,9 @@ def main() -> None: parser.add_argument("--nli-max-open-ended", type=int, default=120) parser.add_argument("--nli-support-threshold", type=float, default=0.50) parser.add_argument("--nli-contradiction-threshold", type=float, default=0.50) + parser.add_argument("--nli-batch-size", type=int, default=128) + parser.add_argument("--nli-device", default=None) + parser.add_argument("--nli-chunk-size", type=int, default=None) args = parser.parse_args() run_dir = Path(args.run_dir).expanduser().resolve() @@ -133,8 +147,8 @@ def main() -> None: post_chunk_ids = [item.chunk_id for item in post_chunks] pre_chunk_ids = [item.chunk_id for item in pre_chunks] - post_doc_ids = [item.doc_id for item in post_chunks] - pre_doc_ids = [item.doc_id for item in pre_chunks] + post_doc_ids = _dedupe_order([item.doc_id for item in post_chunks]) + pre_doc_ids = _dedupe_order([item.doc_id for item in pre_chunks]) pre_chunk_metrics = metrics_for_ranked_ids( ranked_ids=pre_chunk_ids, @@ -288,7 +302,12 @@ def main() -> None: } if args.enable_nli: - scorer = EntailmentScorer(model_name=args.nli_model) + scorer = EntailmentScorer( + model_name=args.nli_model, + batch_size=args.nli_batch_size, + device=args.nli_device, + predict_chunk_size=args.nli_chunk_size, + ) support_rows: list[dict[str, Any]] = [] open_ended_queries = [item for item in eval_queries if item.kind == "open_ended"][: max(0, args.nli_max_open_ended)] for query in open_ended_queries: diff --git a/scripts/judge_reliability.py b/scripts/judge_reliability.py index 0a8b141..24ae5c9 100755 --- a/scripts/judge_reliability.py +++ b/scripts/judge_reliability.py @@ -14,12 +14,21 @@ from sklearn.metrics import cohen_kappa_score, confusion_matrix, f1_score, precision_score, recall_score from sklearn.model_selection import train_test_split -DEFAULT_JUDGES = ["faithfulness_v1", "factual_correctness_v1", "helpfulness_v1", "focus_v1"] +DEFAULT_JUDGES = [ + "faithfulness_v1", + "factual_correctness_v1", + "helpfulness_v1", + "focus_v1", + "comparison_v1", + "refusal_v1", +] DEFAULT_KINDS_BY_JUDGE = { "faithfulness_v1": {"open_ended"}, "factual_correctness_v1": {"factual"}, - "helpfulness_v1": {"factual"}, + "helpfulness_v1": {"factual", "open_ended", "distractor", "comparison"}, "focus_v1": {"distractor"}, + "comparison_v1": {"comparison"}, + "refusal_v1": {"refusal"}, } @@ -225,7 +234,7 @@ def _build_audit(args: argparse.Namespace) -> None: for judge_id in target_judges: if judge_id not in pred_map: continue - if kind not in DEFAULT_KINDS_BY_JUDGE.get(judge_id, {kind}): + if not args.disable_kind_filter and kind not in DEFAULT_KINDS_BY_JUDGE.get(judge_id, {kind}): continue key = AuditKey(query_id=query_id, judge_id=judge_id) if args.dedupe and key.decision_id in seen: @@ -406,6 +415,12 @@ def _parse_args() -> argparse.Namespace: build.add_argument("--run-dirs", nargs="+", required=True, type=Path, help="Run directories containing review.csv") build.add_argument("--out-csv", type=Path, required=True, help="Output decision-level audit CSV") build.add_argument("--judges", nargs="*", default=None, help="Judge IDs to include") + build.add_argument( + "--disable-kind-filter", + action="store_true", + default=False, + help="Do not filter by DEFAULT_KINDS_BY_JUDGE; include any available judge prediction rows.", + ) build.add_argument( "--dedupe", action="store_true", diff --git a/src/andromeda/eval/evidence_support.py b/src/andromeda/eval/evidence_support.py index 4e0a7a5..0c8deff 100644 --- a/src/andromeda/eval/evidence_support.py +++ b/src/andromeda/eval/evidence_support.py @@ -88,9 +88,19 @@ class EntailmentScorer: Local cross-encoder entailment scorer for claim-evidence support checks. """ - def __init__(self, model_name: str = "cross-encoder/nli-deberta-v3-base", max_length: int = 512): + def __init__( + self, + model_name: str = "cross-encoder/nli-deberta-v3-base", + max_length: int = 512, + batch_size: int = 128, + device: str | None = None, + predict_chunk_size: int | None = None, + ): self.model_name = model_name self.model = CrossEncoder(model_name, max_length=max_length) + self.batch_size = max(1, int(batch_size)) + self.predict_chunk_size = predict_chunk_size + self.device = self._resolve_device(device) id2label = getattr(self.model.model.config, "id2label", {}) or {} entailment_id: int | None = None contradiction_id: int | None = None @@ -103,6 +113,23 @@ def __init__(self, model_name: str = "cross-encoder/nli-deberta-v3-base", max_le self.entailment_id = entailment_id if entailment_id is not None else 2 self.contradiction_id = contradiction_id if contradiction_id is not None else 0 + @staticmethod + def _resolve_device(device: str | None) -> str | None: + """ + Resolve a concrete device for CrossEncoder.predict. + """ + + if device and device.strip(): + return device.strip() + try: + import torch + + if torch.cuda.is_available(): + return "cuda" + except Exception: + return None + return None + def score_claims_against_evidence( self, *, @@ -142,7 +169,14 @@ def score_claims_against_evidence( for claim in valid_claims: for evidence in valid_evidence: pairs.append((claim, evidence)) - outputs = self.model.predict(pairs, apply_softmax=True) + outputs = self.model.predict( + pairs, + apply_softmax=True, + batch_size=self.batch_size, + show_progress_bar=False, + device=self.device, + chunk_size=self.predict_chunk_size, + ) supported = 0 contradicted = 0 diff --git a/src/andromeda/eval/retrieval_metrics.py b/src/andromeda/eval/retrieval_metrics.py index 65fdb53..c2d7d29 100644 --- a/src/andromeda/eval/retrieval_metrics.py +++ b/src/andromeda/eval/retrieval_metrics.py @@ -40,6 +40,23 @@ def rank_of_id(ranked_ids: list[str], target_id: str) -> int | None: return None +def _unique_prefix(items: list[str], limit: int) -> list[str]: + """ + Return an order-preserving unique top-k prefix. + """ + + if limit <= 0: + return [] + out: list[str] = [] + seen: set[str] = set() + for item in items[:limit]: + if item in seen: + continue + seen.add(item) + out.append(item) + return out + + def reciprocal_rank(rank: int | None) -> float: """ Compute reciprocal rank from a 1-based rank value. @@ -57,7 +74,7 @@ def hit_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: if k <= 0: return 0.0 - return 1.0 if any(item in relevant_ids for item in ranked_ids[:k]) else 0.0 + return 1.0 if any(item in relevant_ids for item in _unique_prefix(ranked_ids, k)) else 0.0 def precision_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: @@ -67,7 +84,7 @@ def precision_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> flo if k <= 0 or not ranked_ids: return 0.0 - topk = ranked_ids[:k] + topk = _unique_prefix(ranked_ids, k) if not topk: return 0.0 hits = sum(1 for item in topk if item in relevant_ids) @@ -81,7 +98,7 @@ def recall_at_k(ranked_ids: list[str], relevant_ids: set[str], k: int) -> float: if k <= 0 or not relevant_ids: return 0.0 - topk = ranked_ids[:k] + topk = _unique_prefix(ranked_ids, k) hits = sum(1 for item in topk if item in relevant_ids) return float(hits) / float(len(relevant_ids)) diff --git a/src/andromeda/eval/scoring.py b/src/andromeda/eval/scoring.py index dbcf216..5197856 100644 --- a/src/andromeda/eval/scoring.py +++ b/src/andromeda/eval/scoring.py @@ -123,6 +123,21 @@ def _chunk_tickers(chunks: list[RetrievedChunk]) -> list[str]: return out +def _dedupe_order(values: list[str]) -> list[str]: + """ + Return an order-preserving unique list. + """ + + out: list[str] = [] + seen: set[str] = set() + for value in values: + if value in seen: + continue + seen.add(value) + out.append(value) + return out + + def _mentions_token(text: str, token: str) -> bool: if not token or not token.strip(): return False @@ -202,9 +217,9 @@ def score_one( pre_chunks = list(top_chunks) cited_chunk_ids = _cited_chunk_ids(final) retrieved_chunk_ids = [c.chunk_id for c in top_chunks] - retrieved_doc_ids = [c.doc_id for c in top_chunks] + retrieved_doc_ids = _dedupe_order([c.doc_id for c in top_chunks]) pre_chunk_ids = [c.chunk_id for c in pre_chunks] - pre_doc_ids = [c.doc_id for c in pre_chunks] + pre_doc_ids = _dedupe_order([c.doc_id for c in pre_chunks]) retrieved_tickers = _chunk_tickers(top_chunks) score.retrieval["retrieved_chunks"] = len(retrieved_chunk_ids) From b5388e2f904e1a092986d422c432a1153d0b75aa Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Thu, 19 Feb 2026 11:15:20 +0800 Subject: [PATCH 08/10] chore: apply formatting in query planner/runtime modules --- src/andromeda/query/planner_heuristics.py | 4 +++- src/andromeda/query/runtime.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/andromeda/query/planner_heuristics.py b/src/andromeda/query/planner_heuristics.py index 90bf3a9..c48a4e4 100644 --- a/src/andromeda/query/planner_heuristics.py +++ b/src/andromeda/query/planner_heuristics.py @@ -151,7 +151,9 @@ def question_is_simple_numeric_metric(cls, question: str) -> bool: Return whether the question is a direct numeric metric lookup. """ - mentions_metrics = cls.question_mentions_financial_metrics(question) or cls.question_mentions_market_data(question) + mentions_metrics = cls.question_mentions_financial_metrics(question) or cls.question_mentions_market_data( + question + ) mentions_narrative = cls.question_mentions_filing_narrative(question) mentions_comparison = cls.question_mentions_comparison(question) has_period_scope = cls.question_has_explicit_period_scope(question) diff --git a/src/andromeda/query/runtime.py b/src/andromeda/query/runtime.py index 307353b..ce75272 100644 --- a/src/andromeda/query/runtime.py +++ b/src/andromeda/query/runtime.py @@ -519,9 +519,7 @@ def _planner_prompt( }, { "role": "system", - "content": ( - f"Indexed ticker catalog (first {len(preview_rows)} of {len(companies)}):\n{catalog}\n\n" - ) + "content": (f"Indexed ticker catalog (first {len(preview_rows)} of {len(companies)}):\n{catalog}\n\n"), }, { "role": "user", From 424a884d4b5cc8b3497f600dc434402f818ee7dd Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Thu, 19 Feb 2026 11:15:25 +0800 Subject: [PATCH 09/10] docs: add reduced-heuristics benchmark reports and experiment artifacts --- BENCHMARK_REDUCED_HEURISTICS.md | 142 +++++++++ BENCHMARK_RETRIEVAL.md | 178 ++++++++++++ agent_logs/LOGBOOK.md | 163 +++++++++++ agent_logs/plans/18Feb2026_local-models.md | 26 ++ .../plans/18Feb2026_retrieval_metrics_plan.md | 48 ++++ .../plans/20260218_task3_retrieval_finish.md | 48 ++++ .../eval_queries.jsonl | 1 + .../generation_summary.json | 25 ++ .../generations.jsonl | 1 + .../run_config.json | 18 ++ .../lite_single_query.jsonl | 1 + .../manual_sample300_summary.json | 170 +++++++++++ .../manual_sample300_summary.md | 57 ++++ ...91900_run_reduced_heuristics_full_suite.sh | 14 + ...un_reduced_heuristics_full_suite_retry1.sh | 14 + ...un_reduced_heuristics_full_suite_retry2.sh | 14 + ...ristics_full_suite_retry3_pinned_schema.sh | 16 ++ ...00300_reduced_heuristics_eval_override.env | 4 + ...uristics_full_suite_retry4_env_override.sh | 15 + ...00_build_reduced_heuristics_judge_audit.sh | 31 ++ ...ced_heuristics_judge_audit_fullcoverage.sh | 33 +++ ...8_211000_run_retrieval_pool_and_metrics.sh | 39 +++ .../20260218_215100_eval_retrieval_multi60.sh | 15 + ...15700_summarize_retrieval_manual_sample.sh | 272 ++++++++++++++++++ ...0218_220200_probe_lite_isolated_latency.sh | 62 ++++ ...21400_probe_lite_single_eval_timeout350.sh | 37 +++ 26 files changed, 1444 insertions(+) create mode 100644 BENCHMARK_REDUCED_HEURISTICS.md create mode 100644 BENCHMARK_RETRIEVAL.md create mode 100644 agent_logs/plans/18Feb2026_local-models.md create mode 100644 agent_logs/plans/18Feb2026_retrieval_metrics_plan.md create mode 100644 agent_logs/plans/20260218_task3_retrieval_finish.md create mode 100644 agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/eval_queries.jsonl create mode 100644 agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generation_summary.json create mode 100644 agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generations.jsonl create mode 100644 agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/run_config.json create mode 100644 agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl create mode 100644 agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json create mode 100644 agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md create mode 100755 agent_logs/scripts/eval/20260218_191900_run_reduced_heuristics_full_suite.sh create mode 100755 agent_logs/scripts/eval/20260218_192100_run_reduced_heuristics_full_suite_retry1.sh create mode 100755 agent_logs/scripts/eval/20260218_193600_run_reduced_heuristics_full_suite_retry2.sh create mode 100755 agent_logs/scripts/eval/20260218_195700_run_reduced_heuristics_full_suite_retry3_pinned_schema.sh create mode 100644 agent_logs/scripts/eval/20260218_200300_reduced_heuristics_eval_override.env create mode 100755 agent_logs/scripts/eval/20260218_200300_run_reduced_heuristics_full_suite_retry4_env_override.sh create mode 100755 agent_logs/scripts/eval/20260218_205200_build_reduced_heuristics_judge_audit.sh create mode 100755 agent_logs/scripts/eval/20260218_210000_build_reduced_heuristics_judge_audit_fullcoverage.sh create mode 100755 agent_logs/scripts/eval/20260218_211000_run_retrieval_pool_and_metrics.sh create mode 100755 agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh create mode 100755 agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh create mode 100755 agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh create mode 100755 agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh diff --git a/BENCHMARK_REDUCED_HEURISTICS.md b/BENCHMARK_REDUCED_HEURISTICS.md new file mode 100644 index 0000000..008d75a --- /dev/null +++ b/BENCHMARK_REDUCED_HEURISTICS.md @@ -0,0 +1,142 @@ +# Benchmark: Reduced-Heuristics Branch + +_Last updated: 2026-02-18_ + +## 1) Scope +This report covers Task 2 for branch `mlin/reduce-hardcoded-heuristics`: +- re-run the eval pipeline with current deploy-matched settings, +- analyze failures, +- perform a manual judge audit (Codex reasoning, no judge-LLM self-audit), +- compare against previously recorded benchmark baselines in `BENCHMARK.md`. + +## 2) Run Configuration +Core settings used: +- mode: `normal` +- tools: enabled +- refine: disabled +- generation workers: `12` (thread backend) +- query timeout/retries: `350s`, `1` +- judge workers: `12` +- judge context: `80000` +- judge timeout/retries: `350s`, `1` +- schema: `eval_revamp_combined_512_20260217` + +Run group manifest: +- `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.manifest.json` + +Run dirs: +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034` +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838` +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301` + +## 3) Topline Metrics (This Run) + +### 3.1 Generation throughput/latency +| suite | n | n_err | avg_total_ms | wall_total_ms | qps | +|---|---:|---:|---:|---:|---:| +| single100 | 100 | 1 | 59,540.45 | 846,854.45 | 0.1181 | +| multi60 | 60 | 0 | 134,343.77 | 708,687.42 | 0.0847 | +| open200 | 200 | 0 | 63,161.56 | 1,074,835.48 | 0.1861 | + +### 3.2 Judge-facing fail rates +- single100 (`score_summary.json`): + - factual fail: `0.0882` + - open faithfulness fail: `0.1333` + - refusal fail: `0.0000` + - distractor focus fail: `0.0667` +- multi60: + - comparison fail: `0.1167` +- open200: + - open faithfulness fail: `0.1350` + - open helpfulness fail: `0.0050` + +## 4) Comparison vs Previous Baseline (`BENCHMARK.md`) +Reference row in `BENCHMARK.md`: +- `baseline_normal`: factual fail `0.0857`, open faith fail `0.1667`, comparison fail `0.0167`. + +Comparison (closest axes): +- factual fail: `0.0882` (near parity; slightly worse by +0.0025) +- open faithfulness fail: `0.1333` (improved by -0.0334) +- comparison fail: `0.1167` (material regression, +0.1000) + +Interpretation: +- removing brittle heuristics did **not** materially hurt factual fail rate, +- faithfulness on open-ended remained improved versus historical baseline, +- comparison handling regressed strongly and is now the dominant quality gap on multi-ticker prompts. + +## 5) Timeout Incident Log (Generation) +Observed and captured in `agent_logs/LOGBOOK.md`: +- hard failure query: + - `query_id=1dd6251b-e62b-4e58-ae52-35a1253e14c3` + - question: "What was LITE's net income in its 10-Q filed 2026-02-04?" + - failure: timed out after 2 attempts (`350s` + retry), `n_err=1` + - scavenged output: no draft/final answer, no tool trace, generation error record persisted. +- recovered long-tail examples: + - `aada22de-6020-41aa-be15-5516f64b0aca` (MSFT total revenue) succeeded on retry. + - `598beb04-ec0c-4314-893e-2deb8f167179` (INTC vs NVDA comparison) succeeded on retry. + +### 5.1 Isolated replay of the LITE timeout query +To check whether this was purely batch-queue starvation, I ran the same query in isolation. + +- Query: `What was LITE's net income in its 10-Q filed 2026-02-04?` +- Probe A (`agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh`): + - direct single-call runtime probe with outer `timeout 500s` + - outcome: process timed out (`exit 124`) before returning +- Probe B (`agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh`): + - single-query `run_eval` with `concurrency=1`, `query_timeout_s=350`, `query_max_retries=0` + - run dir: `agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015` + - outcome: success in `20601 ms` (`n_ok=1`, `n_err=0`), with tool trace and retrieved/reranked chunks present. + +Inference: +- The long-tail timeout is not only a batching artifact; isolated calls can still hit pathological slow behavior. +- However, the same query can also complete quickly in isolated eval mode, which is consistent with intermittent decode/runtime stalls rather than deterministic query complexity. + +## 6) Manual Judge Audit (Codex, Non-Circular) + +### 6.1 Method +To avoid circularity, the audit did **not** call the judge LLM. +- Built decision table: + - `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.judge_audit_manual/decision_audit.raw.csv` +- Full 698-row decision set was split into six shards and manually labeled by Codex workers using rubric-by-`judge_id` reasoning. +- Merged labeled output: + - `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.judge_audit_manual/decision_audit.codex_manual.csv` +- Reliability report: + - `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.judge_audit_manual/judge_reliability_report.codex_manual.json` + +### 6.2 Alignment summary (test split) +| judge | n_test | accuracy | precision_fail | recall_fail | notes | +|---|---:|---:|---:|---:|---| +| faithfulness_v1 | 58 | 0.9828 | 0.8750 | 1.0000 | strong alignment | +| factual_correctness_v1 | 9 | 1.0000 | 1.0000 | 1.0000 | tiny sample | +| helpfulness_v1 | 85 | 0.9882 | 1.0000 | 0.5000 | under-calls fail cases | +| comparison_v1 | 15 | 1.0000 | 1.0000 | 1.0000 | aligned on this set | +| focus_v1 | 4 | 1.0000 | 0.0000 | 0.0000 | no fail cases in test split | +| refusal_v1 | 5 | 0.8000 | 0.0000 | 0.0000 | misses refusal-needed cases | + +### 6.3 Key disagreement patterns +Confusion from full 698 labeled decisions: +- false positives: `4` total + - mostly faithfulness over-flags (`3`) and one factual false positive. +- false negatives: `10` total + - helpfulness under-flags (`6`) for non-responsive comparison/analysis answers, + - refusal under-flags (`3`) where out-of-scope prompts were met with clarification instead of refusal, + - faithfulness under-flag (`1`) on unsupported filing-availability claim. + +## 7) Genuine Pipeline Failures (from manual audit) +Main categories of true failures (`human_label=1`): +- comparison completeness failures (`comparison_v1`, `7`): model defers/clarifies instead of producing requested side-by-side analysis. +- open-ended faithfulness failures (`faithfulness_v1`, `29`): period mismatch and unsupported specific claims remain the largest category. +- refusal behavior gaps (`refusal_v1`, `3`): out-of-scope ticker prompts not refused strongly enough. +- helpfulness failures (`helpfulness_v1`, `8`): mostly non-answers/deferrals for requested comparative analysis. + +## 8) Surprising Findings and Hypotheses +1. Removing brittle heuristics improved maintainability without collapsing factual/open-ended quality. +2. Multi-ticker comparison degraded sharply; likely because previous heuristic scaffolding implicitly forced comparative structure. +3. Faithfulness judge quality is now relatively strong under Codex-manual audit; biggest remaining reliability issue is helpfulness/refusal under-calling. +4. Timeout outliers still materially affect wall-clock and can dominate throughput for small suites. + +## 9) Immediate Follow-ups +1. Improve comparison answer planning (explicit required-output structure for multi-ticker comparison prompts). +2. Tighten refusal policy for out-of-scope tickers (prefer explicit refusal over vague clarifying loops). +3. Add retry+continue safeguards for long-tail timed-out generations (already partially in place). +4. Keep judge audits separated from judge-model outputs (Codex-manual process retained as the non-circular check). diff --git a/BENCHMARK_RETRIEVAL.md b/BENCHMARK_RETRIEVAL.md new file mode 100644 index 0000000..bc4a1aa --- /dev/null +++ b/BENCHMARK_RETRIEVAL.md @@ -0,0 +1,178 @@ +# Benchmark: Retrieval + Rerank Quality (Reduced-Heuristics Branch) + +## 1) Scope +This report continues from the completed 300-sample manual retrieval audit and closes the retrieval-focused evaluation workstream. + +Primary goals: +- quantify retriever vs reranker behavior, +- quantify evidence-support quality on open-ended answers, +- audit retrieval relevance with non-circular manual labels (Codex reasoning, not judge-LLM), +- calibrate weak labels against manual labels. + +All runs below use the same reduced-heuristics full-suite manifest: +- `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.manifest.json` + +## 2) Experiments Run + +| ID | Experiment | Input artifacts | Output artifacts | What it measures | +|---|---|---|---|---| +| E1 | Factual retrieval/rerank IR metrics (`single100`) | `eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034` | `retrieval_rerank_metrics.json`, `retrieval_rerank_metrics.csv`, `retrieval_nli_claim_support.csv` | Pre vs post rerank MRR/hit/precision/recall using factual gold evidence anchors | +| E2 | Open-ended NLI evidence support (`open200`) | `eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301` | `retrieval_rerank_metrics.json`, `retrieval_nli_claim_support.csv` | Claim support / contradiction / unsupported rates | +| E3 | Multi slice retrieval pass (`multi60`) | `eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838` | `retrieval_rerank_metrics.json` | Completeness check for full-suite parity (no factual/open-ended rows in this slice) | +| E4 | Retrieval candidate pool build | all three run dirs above | `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.retrieval_pool.csv`, `.stats.json` | pooled chunk candidates for manual relevance auditing | +| E5 | Manual relevance audit (300 rows) | `...retrieval_pool.sample300.csv` | `...retrieval_pool.sample300.codex_manual.csv` | Human-proxy relevance labels via Codex reasoning (non-circular) | +| E6 | Weak-label calibration | `...sample300.codex_manual.csv` | `...sample300.calibration.json` | Weak-label precision/recall/alignment vs manual labels | +| E7 | Manual audit summary rollup | `...sample300.codex_manual.csv` | `agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json`, `.md` | relevance prevalence, pre/post membership, rank movement, top-k relevance slices | + +## 3) Core Results + +### 3.1 Retriever vs reranker on factual anchors (E1) + +| Metric | Pre-rerank | Post-rerank | Delta | +|---|---:|---:|---:| +| factual_n | 34 | 34 | - | +| chunk MRR | 0.3092 | 0.1743 | -0.1349 | +| chunk win rate | - | - | 0.1765 | +| chunk precision@5 | 0.1118 | 0.0647 | -0.0471 | +| chunk precision@10 | 0.0647 | 0.0471 | -0.0176 | +| chunk precision@25 | 0.0294 | 0.0282 | -0.0012 | +| chunk recall@25 | 0.7353 | 0.7059 | -0.0294 | +| doc MRR | 1.0000 | 1.0000 | 0.0000 | + +Interpretation: +- On factual gold-anchor queries, current reranking is net negative on chunk-level relevance concentration. +- Doc-level MRR is saturated at 1.0 and is not discriminative for this run. + +### 3.2 NLI claim support on open-ended generations (E1/E2) + +| Slice | n_open_ended_scored | support_rate | contradiction_rate | unsupported_rate | +|---|---:|---:|---:|---:| +| `single100` subset | 30 | 0.1000 | 0.3958 | 0.5042 | +| `open200` | 120 | 0.1292 | 0.4115 | 0.4594 | + +Interpretation: +- Support is low and contradiction/unsupported are high. +- This aligns directionally with remaining faithfulness pressure points in answer-level evals. + +### 3.3 Multi slice parity check (E3) + +`multi60` has no factual or open-ended rows, so retrieval IR/NLI outputs are expectedly `NaN`/empty for these specific metric families. + +## 4) Manual 300-Sample Relevance Audit (E5/E7) + +Manual labels are from Codex reasoning on each row, not from the judge LLM. + +Source: +- `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.codex_manual.csv` + +Summary: +- `n_labeled = 300` +- overall positive relevance rate = `0.4167` + +By query kind: + +| Kind | n | n_positive | positive_rate | +|---|---:|---:|---:| +| factual | 140 | 19 | 0.1357 | +| open_ended | 90 | 53 | 0.5889 | +| comparison | 55 | 44 | 0.8000 | +| distractor | 15 | 9 | 0.6000 | + +Pre/post membership buckets: + +| Bucket | n | n_positive | positive_rate | +|---|---:|---:|---:| +| both pre+post | 195 | 74 | 0.3795 | +| pre-only | 53 | 26 | 0.4906 | +| post-only | 52 | 25 | 0.4808 | + +Relevant-rank movement (rows present in both pre and post, relevance=1): +- n=74, promoted=32, demoted=37, unchanged=5, avg delta(post-pre)=+0.0676 + +By kind (same movement view): + +| Kind | n | promoted | demoted | unchanged | avg_delta(post-pre) | +|---|---:|---:|---:|---:|---:| +| factual | 15 | 4 | 9 | 2 | +2.8000 | +| open_ended | 38 | 19 | 17 | 2 | -1.8421 | +| comparison | 17 | 6 | 10 | 1 | +3.4118 | +| distractor | 4 | 3 | 1 | 0 | -6.2500 | + +Sample top-k relevance slices (not an unbiased absolute P@k estimator; useful as directional diagnostics): + +| Phase | k | n_rows_in_slice | n_positive | positive_rate | +|---|---:|---:|---:|---:| +| pre | 5 | 63 | 40 | 0.6349 | +| post | 5 | 64 | 37 | 0.5781 | +| pre | 10 | 128 | 60 | 0.4688 | +| post | 10 | 134 | 71 | 0.5299 | + +Interpretation: +- At very early ranks (top-5), audited relevance is lower post-rerank than pre-rerank. +- At top-10, post-rerank recovers and slightly exceeds pre-rerank in this sample. +- Factual and comparison rows show more demotions than promotions, consistent with E1 factual-anchor degradation. + +## 5) Weak-Label Calibration (E6) + +Source: +- `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.calibration.json` + +Threshold: weak relevance >= 0.5 => relevant. + +| Metric | Value | +|---|---:| +| n | 140 | +| accuracy | 0.1357 | +| precision_1 | 0.1357 | +| recall_1 | 1.0000 | +| f1_1 | 0.2390 | +| balanced_accuracy | 0.5000 | +| tp / fp / tn / fn | 19 / 121 / 0 / 0 | + +Interpretation: +- Current weak labels are recall-maximal but extremely low precision (many false positives). +- They are suitable as high-recall candidate generation tags, not as ground-truth proxies for precision-sensitive decisions. + +## 6) Surprising Findings and Hypotheses + +1. Reranking currently hurts factual chunk concentration. +- Evidence: negative chunk MRR and precision deltas on factual anchors. +- Hypothesis: reranker objective overweights semantic fluency/contextual breadth vs exact numeric-evidence grounding. + +2. Doc-level metrics saturate and hide problems. +- Evidence: doc MRR fixed at 1.0 while chunk metrics degrade. +- Hypothesis: relevant document is often retrieved, but best evidence chunk inside that document is not prioritized. + +3. Weak labels are not precision-usable. +- Evidence: 121 FP out of 140 weak-positive rows in calibrated subset. +- Hypothesis: doc-match score 0.7 is too permissive for relevance labeling in factual settings. + +4. NLI flags substantial unsupported/contradicted claim mass. +- Evidence: contradiction ~0.40 and unsupported ~0.46-0.50. +- Hypothesis: long answers contain extrapolative claims beyond retrieved evidence granularity. + +## 7) Actionable Next Steps + +1. Rerank objective/feature tuning with factual-priority constraints. +- Add hard/soft boosts for period-aligned numeric/table chunks in rerank scoring. +- Re-run E1 and require non-negative delta on chunk MRR and P@5 before promotion. + +2. Improve weak-label scheme. +- Replace binary doc-match surrogate with graded weak labels including period/type alignment. +- Keep manual 300+ audits for calibration and CIs. + +3. Expand manual audit slices where signal is weakest. +- Increase factual sample beyond 140 rows and stratify by rerank disagreement bands. + +4. Keep this retrieval benchmark as a standing gate. +- Run E1+E5+E6 for major retrieval/prompt changes and block merges on consistent factual rerank regressions. + +## 8) Repro Commands + +Scripts executed: +- `agent_logs/scripts/eval/20260218_211000_run_retrieval_pool_and_metrics.sh` +- `agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh` +- `agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh` + +Calibration command: +- `source .venv/bin/activate && python scripts/calibrate_eval_metrics.py --labels-csv eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.codex_manual.csv --human-col human_relevance --weak-col weak_relevance --weak-threshold 0.5 --n-bootstrap 2000 --out-json eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.calibration.json` diff --git a/agent_logs/LOGBOOK.md b/agent_logs/LOGBOOK.md index 4d43983..54c2e96 100644 --- a/agent_logs/LOGBOOK.md +++ b/agent_logs/LOGBOOK.md @@ -2753,3 +2753,166 @@ ### Notes - Branch is clean after checkpoint; proceeding to eval rerun and benchmark analysis. +## 2026-02-18 - Local open-source model survey for NLI & finance-grade reranking + +### Scope completed +- Audited `pyproject.toml` to confirm the stack already brings `sentence-transformers`, `huggingface-hub`, and other HF/torch infrastructure needed for local models. +- Reviewed `CrossEncoderReranker`, `build_reranker()`, and the `LLMClient`/`llm_for_embeddings` plumbing so the existing retriever/reranker pair can accept new Hugging Face models with minimal code. +- Cataloged Apache-2.0 Hugging Face checkpoints for entailment (e.g., `cross-encoder/nli-roberta-base`, `cross-encoder/nli-distilroberta-base`) and finance-honed dense/cross encoders (e.g., `shail-2512/nomic-embed-financial-matryoshka`, `hutuhehe/finretriever-cross-reranker`) that can be dropped into the stack. + +### Key observations +- The reranker already instantiates a `SentenceTransformers` `CrossEncoder` via the `RERANKER_MODEL` env var (`src/andromeda/runtime/builders.py:424-427`), so swapping in a finance-tuned checkpoint only requires updating that variable and refreshing caches. +- Dense retrieval embeddings flow through `PostgresHybridRetriever` which leans on the `LLMClient` `embed_texts` hook (`src/andromeda/retrieval/retriever.py:366-383`); adding a lightweight `SentenceTransformer`‑backed `LLMClient` variant can plug into `llm_for_embeddings()` for on-prem embeddings. + +### Why this matters +- Capturing the above ensures future work can tie the finance-grade checkpoints and NLI cross-encoders into the QA + evidence pipeline without guessing at compatibility, saving a follow-on research step. + +## 2026-02-18 - Reduced-heuristics full-suite retry4: long-tail timeout incident handling + +### Context +- Active run: `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034` +- Settings: `mode=normal`, `gen_workers=12`, `parallel_backend=thread`, `query_timeout_s=350`, `query_max_retries=1`, `judge_context_chars=80000`, `judge_workers=12`. + +### What happened +- Generation progressed to `99/100` then long-tailed on one query. +- Runtime logs showed retry warnings for two queries: + - `1dd6251b-e62b-4e58-ae52-35a1253e14c3` (LITE net income factual query) + - `aada22de-6020-41aa-be15-5516f64b0aca` (MSFT total revenue factual query) +- Final outcome: + - `1dd6251b-e62b-4e58-ae52-35a1253e14c3` failed after retry budget exhausted with `Timed out after 350.0s`. + - `aada22de-6020-41aa-be15-5516f64b0aca` succeeded on retry attempt 2. +- Single100 generation summary: `n=100, n_ok=99, n_err=1`, `wall_total_ms=846854.446`. + +### Failed query summary (required incident detail) +- Query ID: `1dd6251b-e62b-4e58-ae52-35a1253e14c3` +- Query text: `What was LITE's net income in its 10-Q filed 2026-02-04?` +- Failure mode: timeout after retry (`query_attempts` exhausted). +- Scavenged artifacts: + - Runtime warning: `Retrying ... failed: Timed out after 350.0s` + - Runtime error: `Error during eval generation ... Timed out after 350.0s` + - Generation record: `error="Timed out after 350.0s"`, `timing_ms.total_ms=700505.2527501248` + - No draft/final answer captured, `tool_trace_len=0`, `tool_results_len=0` + +### Slow-but-recovered query summary +- Query ID: `aada22de-6020-41aa-be15-5516f64b0aca` +- Query text: `What was MSFT's total revenue in its 10-K filed 2025-07-30?` +- Behavior: timed out once, succeeded on retry (`query_attempts=2`) +- Scavenged output preview: + - Final answer began with `MSFT’s Total Revenue ... $281,724 million`. + - `timing_ms.total_ms=382516.4867863059` + - `tool_trace_len=8`, `tool_results_len=3` + +### Immediate action +- Proceeding without panic per instruction: run continues into scoring. +- This incident will be included in the reduced-heuristics benchmark report under long-tail decoding/timeout behavior. + +## 2026-02-18 - Retrieval eval instrumentation update (precision/recall + NLI support) + +### Commit +- `d9220cf` + +### What changed +- Added retrieval/rerank subsystem metric modules and integrated them into scoring/report surfaces: + - `src/andromeda/eval/retrieval_metrics.py` + - `src/andromeda/eval/rerank_metrics.py` + - `src/andromeda/eval/evidence_support.py` + - `src/andromeda/eval/scoring.py` + - `scripts/score_eval.py` + - `scripts/eval_retrieval.py` +- Added explicit precision/recall tracking to address retrieval-plan feedback: + - chunk/doc: `precision_at_{5,10,25}` + - chunk/doc: `recall_at_{5,10,25}` + - rerank deltas for precision/recall +- Added retrieval/NLI tests: + - `tests/test_eval_retrieval_metrics.py` + - updated `tests/test_eval_schema_scoring.py` + +### Validation +- `source .venv/bin/activate && pytest -vvv tests/test_eval_retrieval_metrics.py tests/test_eval_schema_scoring.py` +- Result: `10 passed`. + +### Why this matters +- Precision is now a first-class retrieval KPI in both per-query artifacts and topline summaries. +- Reranker evaluation now measures not only MRR/hit uplift, but precision/recall movement as well. + +## 2026-02-18 - Retrieval benchmark continuation from 300-sample manual audit (Task 3) + +### Context +- Resumed from completed Codex-manual retrieval annotation set: + - `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.codex_manual.csv` +- Goal: close the retrieval/rerank evaluation loop with explicit metrics tables, calibration interpretation, and final report. + +### Scripts executed +- `agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh` + - Added missing retrieval metric artifacts for `multi60` slice. + - Uses local HF cache under `/tmp/hf_home` to avoid permission errors on `~/.cache/huggingface`. +- `agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh` + - Produces manual-audit aggregate summaries and rank-movement diagnostics. + +### Artifacts produced/updated +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838/retrieval_rerank_metrics.json` +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838/retrieval_rerank_metrics.csv` +- `eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838/retrieval_rerank_metrics.md` +- `agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json` +- `agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md` +- `BENCHMARK_RETRIEVAL.md` + +### Key metrics and observations +- Factual-anchor retrieval/rerank (`single100`, `factual_n=34`): + - chunk MRR: `0.3092 -> 0.1743` (`delta=-0.1349`) + - chunk P@5: `0.1118 -> 0.0647` + - chunk P@10: `0.0647 -> 0.0471` + - chunk R@25: `0.7353 -> 0.7059` + - rerank chunk win-rate: `0.1765` +- NLI evidence support: + - `single100` open-ended subset (`n=30`): support `0.1000`, contradiction `0.3958`, unsupported `0.5042` + - `open200` (`n=120`): support `0.1292`, contradiction `0.4115`, unsupported `0.4594` +- Manual relevance audit (`n=300`, Codex-manual labels): + - overall relevance positive rate: `0.4167` + - factual relevance positive rate: `0.1357` (`19/140`) + - relevant rows in both pre/post: promoted `32`, demoted `37`, unchanged `5` (avg delta `+0.0676` where positive means post worse rank) + - by-kind movement indicates more factual/comparison demotions than promotions. +- Weak-label calibration vs manual labels (`n=140`, threshold `0.5`): + - `tp=19`, `fp=121`, `tn=0`, `fn=0` + - precision_1 `0.1357`, recall_1 `1.0000`, balanced accuracy `0.5000` + - weak labels are high-recall but too noisy for precision-sensitive ranking conclusions. + +### Surprising findings +- Reranking under current settings degrades chunk-level factual relevance concentration despite correct-doc saturation. +- Early-rank sample relevance (top-5) declines post-rerank in audited rows, while top-10 partly recovers; this suggests reordering behavior that does not consistently prioritize exact evidence. +- Weak-label doc-match heuristic overstates relevance for factual tasks and must not be treated as proxy precision ground truth. + +### Immediate actions taken +- Finalized a dedicated retrieval benchmark report with explicit experiment definitions and tables: + - `BENCHMARK_RETRIEVAL.md` +- Structured findings around actionable follow-ups: + - reranker tuning for factual numeric/period-aware evidence, + - improved weak-label design, + - continued manual calibration slices as release gate. + +## 2026-02-18 - Isolated latency probe for LITE timeout query + +### Why +- Follow-up to the earlier timeout incident for: + - `query_id=1dd6251b-e62b-4e58-ae52-35a1253e14c3` + - question: `What was LITE's net income in its 10-Q filed 2026-02-04?` +- Goal: test isolated behavior (no batch competition) and validate whether >350s was purely queueing. + +### Scripts executed +- `agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh` + - direct runtime call (`answer_question`) with outer shell timeout `500s` + - outcome: timed out (`exit=124`) before returning payload. +- `agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh` + - single-query eval (`run_eval`) with `concurrency=1`, `query_timeout_s=350`, `query_max_retries=0` + - run dir: `agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015` + +### Results +- Probe A: no completed response within `500s` (pathological slow/stuck behavior still reproducible in isolation). +- Probe B: same query completed successfully in `20601 ms`. + - generation summary: `n=1`, `n_ok=1`, `n_err=0`, `avg_total_ms=20601.03` + - response had `tool_trace_len=8`, `tool_results_len=3`, `retrieved_chunks=40`, `top_chunks=25`. + +### Interpretation +- Not purely a batching starvation issue: isolated runtime can still stall in rare cases. +- Also not deterministically expensive: isolated runs can finish quickly (~20.6s). +- Most plausible explanation remains intermittent model/runtime stall behavior (decoding or backend-level transient), reinforcing timeout+retry as the right control mechanism. diff --git a/agent_logs/plans/18Feb2026_local-models.md b/agent_logs/plans/18Feb2026_local-models.md new file mode 100644 index 0000000..8280351 --- /dev/null +++ b/agent_logs/plans/18Feb2026_local-models.md @@ -0,0 +1,26 @@ +# 18Feb2026 local-model integration research + +## Context +Need to survey the repo's current dependency stack and model-loading patterns to recommend low-friction open-source local models (HF transformers / sentence-transformers / cross-encoders) for NLI/evidence support and finance-oriented retrieval/reranking tasks. The report should map candidates to existing dependencies/code paths and clearly state the integration path. + +## Files to change / new files +- `agent_logs/plans/18Feb2026_local-models.md` (this planning document) +- `agent_logs/LOGBOOK.md` (append an entry summarizing the research findings and next steps) +- None planned beyond documentation updates. + +## Phases +1. **Dependency & model-loading inventory** + *Scope:* Inspect `pyproject.toml`, `package.json`, `src/` modules, and any scripts that instantiate HF sentence/transformer models to understand supported frameworks and wrappers. + *Acceptance criteria:* Document key libraries (e.g., `transformers`, `sentence-transformers`, `faiss`, etc.) and point to the code locations where embeddings/rerankers are instantiated. + +2. **Candidate model matching** + *Scope:* Identify open-source local models (HF checkpoints) that align with NLI/evidence support and finance retrieval/reranking, ensuring they are usable with the repo's stack (Python version, dependencies, libs). + *Acceptance criteria:* Produce a shortlist of ≥3 models per task category, highlighting license, tokenizer compatibility, required tooling (e.g., CUDA, quantization) and why they fit the repo's stack. + +3. **Integration recommendation** + *Scope:* Tie each model back to the existing modules/dependency footprint, outlining minimal code to extend a current loader (embedding retriever, reranker) and noting any dependency gaps. + *Acceptance criteria:* Provide a brief roadmap that references the specific files/classes/functions to update and suggests whether HF `AutoModel...`, `sentence-transformers`, or cross-encoder wrappers are the lowest-friction option. + +## Potential Add-ons (not in current scope) +- Benchmark a selected model locally using existing eval harness to confirm quality/latency. +- Implement adapters or wrappers to load quantized variants via `bitsandbytes` or `transformers` quantization utilities. diff --git a/agent_logs/plans/18Feb2026_retrieval_metrics_plan.md b/agent_logs/plans/18Feb2026_retrieval_metrics_plan.md new file mode 100644 index 0000000..594ae96 --- /dev/null +++ b/agent_logs/plans/18Feb2026_retrieval_metrics_plan.md @@ -0,0 +1,48 @@ +# 18Feb2026 Retrieval Metrics Plan + +## Objective +Document the current eval retrieval/rerank observability and plan the implementation of +metrics for (a) pre-rerank retrieval coverage, (b) reranker uplift (gold evidence rank +differentials), and (c) claim–evidence support. The implementation must reuse the +existing metrics helpers, keep evaluations readable, and keep CLI outputs in sync. + +## Phases +1. **Inventory & touchpoints** + - Acceptance: Identify which files/functions already emit retrieval information (`EvalGeneration`, + `score_one`, `summarize`, `scripts/score_eval.py`, `EvalScore.retrieval` dict, `EvalSummary`). + - Document where `retrieved_chunks` vs `top_chunks` are populated so downstream scoring can observe + pre-rerank data and citations. +2. **Extend scoring helpers** + - Acceptance: `score_one` computes per-query metrics for retrieval recall/MRR, reranker uplift (rank delta for + gold chunk/doc, recall jump), and claim-evidence coverage (citation counts and chunk coverage per claim) using + helpers in `andromeda.eval.metrics` and `_cited_chunk_ids`. + - Update `summarize` to surface aggregated uplift/coverage (e.g., mean rerank gain, claim citation rate). Add + targeted unit tests for the new helpers (`tests/test_eval_schema_scoring.py` and/or a new metrics test). +3. **Surface & document results** + - Acceptance: `scripts/score_eval.py` review CSV + cases now include the new metrics; generated `score_summary.json` and + HTML report display pre-rerank vs rerank recall plus evidence support stats. + - Note: Update `agent_logs/LOGBOOK.md` with a short entry describing the new metric coverage. Plan for `pre-commit run --all` + and `pytest -vvv tests/` after the implementation. + +## File-level edits +- `src/andromeda/eval/scoring.py` (score_one, helper functions, summary aggregation) +- `src/andromeda/eval/metrics.py` (add helpers for citation counts/rerank deltas if needed) +- `src/andromeda/eval/report.py` (display new metric cards/details) +- `scripts/score_eval.py` (add repo rows/columns for new metrics, keep cases consistent) +- `tests/test_eval_schema_scoring.py` (extend to cover the new metrics) +- Possibly `tests/test_eval_metrics.py` if new helpers live there. + +## New files +- None planned yet. If new helper module is required for claim-evidence parsing, create e.g. + `src/andromeda/eval/claim_support.py` and add to `files_to_change` above. + +## Existing utilities to reuse +- `andromeda.eval.metrics.recall_at_k`, `mrr`, `coverage_at_k` for pre-rerank metric calculations. +- `_cited_chunk_ids` / `cited_doc_ids` (in `scoring.py` / `metrics.py`) for claim–evidence support. +- `EvalGeneration.retrieved_chunks` vs `top_chunks` as the data sources. `summarize()` already folds `score.retrieval` into + `score_summary.json`. + +## Potential add-ons / future work +- Add a CLI flag to `scripts/run_eval.py` / `score_eval.py` to emit per-stage recall CSVs. +- Hook new metrics into the HTML report generator for interactive exploration. +- Record a runnable script under `agent_logs/scripts/` if any bespoke data extraction is needed later. diff --git a/agent_logs/plans/20260218_task3_retrieval_finish.md b/agent_logs/plans/20260218_task3_retrieval_finish.md new file mode 100644 index 0000000..e3be310 --- /dev/null +++ b/agent_logs/plans/20260218_task3_retrieval_finish.md @@ -0,0 +1,48 @@ +# 20260218 Task3 Retrieval Finish Plan + +## Goal +Complete Task 3 from the reduced-heuristics branch work: finalize retrieval/rerank evaluation with Codex-manual audit outputs and publish a readable benchmark report. + +## Technical Approach +1. Collect and verify existing artifacts from the completed full-suite run and manual 300-sample retrieval audit. +2. Compute additional summary metrics needed for interpretation: + - label coverage and relevance prevalence + - pre/post membership and rank shifts for relevant chunks + - precision-style summaries from manually audited rows + - weak-label calibration interpretation with bootstrap CI +3. Write `BENCHMARK_RETRIEVAL.md` with: + - exact experiments and artifact paths + - metrics tables + - key findings and hypotheses + - concrete follow-up actions +4. Append a detailed `agent_logs/LOGBOOK.md` entry with scripts, artifact paths, and conclusions. + +## Phases + Acceptance Criteria + +### Phase 1: Artifact Verification +Acceptance criteria: +- All required artifacts exist and are readable. +- Missing pieces are identified and backfilled if needed. + +### Phase 2: Retrieval Analysis +Acceptance criteria: +- Analysis outputs (JSON/CSV) include at least: + - manual-label prevalence + - pre/post relevant rank statistics + - calibration metrics with CI +- Outputs are written under `agent_logs/reports/`. + +### Phase 3: Reporting +Acceptance criteria: +- `BENCHMARK_RETRIEVAL.md` is created with reproducible commands and results tables. +- `agent_logs/LOGBOOK.md` has a new entry with a concise but complete lineage. + +## files_to_change +- `BENCHMARK_RETRIEVAL.md` +- `agent_logs/LOGBOOK.md` +- `agent_logs/scripts/eval/20260218_*.sh` + +## new_files +- `agent_logs/plans/20260218_task3_retrieval_finish.md` +- `agent_logs/reports/retrieval_eval_20260218/*.json` +- `agent_logs/reports/retrieval_eval_20260218/*.md` diff --git a/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/eval_queries.jsonl b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/eval_queries.jsonl new file mode 100644 index 0000000..f3c0c5e --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/eval_queries.jsonl @@ -0,0 +1 @@ +{"id": "1dd6251b-e62b-4e58-ae52-35a1253e14c3", "question": "What was LITE's net income in its 10-Q filed 2026-02-04?", "kind": "factual", "tags": ["factual", "sec", "LITE", "10-Q", "net income"], "created_at": "2026-02-16T20:30:23.567307Z", "factual": {"metric": "net income", "expected_numeric": {"value": 78.2, "unit": "USD", "scale": null, "raw": " 78.2 "}, "golden_evidence": {"doc_id": "LITE_000162828026005129_10-Q_2026-02-04", "chunk_id": "LITE_000162828026005129_10-Q_2026-02-04_1", "source": "/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/eval_revamp_combined_512_20260217/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md", "headings": ["PART I - FINANCIAL INFORMATION", "ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)", "CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"], "page_no": null, "section_path": "PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS", "snippet": "Ticker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 1199.3 | | $ | 739.1 | |\n| Cost of sales | | 405.8 | | | 281.2 | | | 738.6 | | | 517.7 | |\n| Amortization of acquired developed intangibles | | 19.6 | | | 21.4 | | | 39.1 | | | 43.9 | |\n| Gross profit | | 240.1 | | | 99.6 | | | 421.6 | | | 177.5 | |\n| Operating expenses: | | | | | | | | | | | | |\n| Research and development | | 80.1 | | | 74.2 | | | 161.5 | | | 148.5 | |\n| Selling, general and administrative | | 96.1 | | | 76.3 | | | 181.2 | | | 152.6 | |\n| Restructuring and related charges (reversals) | | (0.4) | | | 0.7 | | | 7.9 | | | 10.4 | |\n| Total operating expenses | | 175.8 | | | 151.2 | | | 350.6 | | | 311.5 | |\n| Income (loss) from operations | | 64.3 | | | (51.6) | | | 71.0 | | | (134.0) | |\n| Other income (expense), net: | | | | | | | | | | | | |\n| Escrow settlement | | 27.5 | | | — | | | 27.5 | | | — | |\n| Interest expense | | (6.3) | | | (5.6) | | | (12.0) | | | (11.1) | |\n| Other income, net | | 11.0 | | | 14.9 | | | 15.2 | | | 23.6 | |\n| Total other income, net | | 32.2 | | | 9.3 | | | 30.7 | | | 12.5 | |\n| Income (loss) before income taxes | | 96.5 | | | (42.3) | | | 101.7 | | | (121.5) | |\n| Income tax provision | | 18.3 | | | 18.6 | | | 19.3 | | | 21.8 | |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Net income (loss) per share: | | | | | | | | | | | | |\n| Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |\n| Diluted | $ | 0.89 | | $ | (0.88) | | $ | 0.99 | | $ | (2.09) | |\n| Shares used to compute net income (loss) per share: | | | | | | | | | | | | |\n| Basic | | 71.1 | | | 68.9 | | | 70.7 | | | 68.6 | |\n| Diluted | | 87.8 | | | 68.9 | | | 83.1 | | | 68.6 | |", "metadata": {}}}, "open_ended": null, "refusal": null, "distractor": null, "comparison": null, "generator": {"source": "chunk_exports", "seed": 20260217, "edgar_validation": {"status": "matched", "metric": "net income", "ticker": "LITE", "candidate_keys": ["net_income"], "rel_tol": 0.5, "best_rel_error": 0.0, "best_expected_scale": "millions", "best_expected_value": 78200000.0}}} diff --git a/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generation_summary.json b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generation_summary.json new file mode 100644 index 0000000..4288cb9 --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generation_summary.json @@ -0,0 +1,25 @@ +{ + "n": 1, + "n_ok": 1, + "n_err": 0, + "avg_total_ms": 20601.025926414877, + "wall_total_ms": 30414.335750974715, + "settings": { + "mode": "normal", + "top_k_retrieve": null, + "top_k_rerank": null, + "draft_max_tokens": null, + "final_max_tokens": null, + "brief_max_tokens": null, + "enable_rerank": null, + "enable_refine": null, + "answer_style": null, + "answering_effort": null, + "draft_temperature": null, + "concurrency": 1, + "parallel_backend": "thread", + "max_chunks": 50, + "query_timeout_s": 350.0, + "query_max_retries": 0 + } +} \ No newline at end of file diff --git a/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generations.jsonl b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generations.jsonl new file mode 100644 index 0000000..ebd4797 --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/generations.jsonl @@ -0,0 +1 @@ +{"query_id":"1dd6251b-e62b-4e58-ae52-35a1253e14c3","kind":"factual","question":"What was LITE's net income in its 10-Q filed 2026-02-04?","created_at":"2026-02-18T14:00:25.625443Z","settings":{"mode":"normal","top_k_retrieve":40,"top_k_rerank":25,"draft_max_tokens":65536,"final_max_tokens":32768,"enable_rerank":true,"enable_refine":false,"answer_style":"normal","draft_temperature":0.1,"concurrency":1,"query_attempts":1},"draft_answer":"**Analysis: LITE’s Net Income in 10-Q Filed 2026-02-04**\n\nThe 10-Q filing for Lumentum Holdings Inc. (Ticker: LITE), filed on 2026-02-04, covers the first quarter of fiscal year 2026 (ended December 27, 2025) and the six months ended December 27, 2025. The filing includes condensed consolidated statements of operations and comprehensive income, which report net income for both periods.\n\n---\n\n**1. Net Income for the Three Months Ended December 27, 2025** \n**Value:** $78.2 million \n**Why it matters:** This is the net income for the most recent quarter reported in the 10-Q, providing a key performance metric for Q1 2026. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**2. Net Income for the Six Months Ended December 27, 2025** \n**Value:** $82.4 million \n**Why it matters:** This reflects the company’s net income for the first half of fiscal year 2026, showing year-to-date performance. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**3. Net Income for the Three Months Ended December 28, 2024 (Prior Year Quarter)** \n**Value:** $(60.9) million (net loss) \n**Why it matters:** Provides a year-over-year comparison for Q1 2025, highlighting improvement in profitability. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**4. Net Income for the Six Months Ended December 28, 2024 (Prior Year Half-Year)** \n**Value:** $(143.3) million (net loss) \n**Why it matters:** Shows a significant improvement in year-to-date performance compared to the same period in the prior year. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**5. Net Income Per Share (Basic) for the Three Months Ended December 27, 2025** \n**Value:** $1.10 \n**Why it matters:** Indicates earnings per share for the most recent quarter, useful for investors. \n**Quote:** “Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**6. Net Income Per Share (Basic) for the Six Months Ended December 27, 2025** \n**Value:** $1.17 \n**Why it matters:** Reflects year-to-date earnings per share, showing strong improvement from prior year’s loss. \n**Quote:** “Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**Cited Sources:**\n\n- [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n- [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_3 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)]","final_answer":"**Analysis: LITE’s Net Income in 10-Q Filed 2026-02-04**\n\nThe 10-Q filing for Lumentum Holdings Inc. (Ticker: LITE), filed on 2026-02-04, covers the first quarter of fiscal year 2026 (ended December 27, 2025) and the six months ended December 27, 2025. The filing includes condensed consolidated statements of operations and comprehensive income, which report net income for both periods.\n\n---\n\n**1. Net Income for the Three Months Ended December 27, 2025** \n**Value:** $78.2 million \n**Why it matters:** This is the net income for the most recent quarter reported in the 10-Q, providing a key performance metric for Q1 2026. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**2. Net Income for the Six Months Ended December 27, 2025** \n**Value:** $82.4 million \n**Why it matters:** This reflects the company’s net income for the first half of fiscal year 2026, showing year-to-date performance. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**3. Net Income for the Three Months Ended December 28, 2024 (Prior Year Quarter)** \n**Value:** $(60.9) million (net loss) \n**Why it matters:** Provides a year-over-year comparison for Q1 2025, highlighting improvement in profitability. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**4. Net Income for the Six Months Ended December 28, 2024 (Prior Year Half-Year)** \n**Value:** $(143.3) million (net loss) \n**Why it matters:** Shows a significant improvement in year-to-date performance compared to the same period in the prior year. \n**Quote:** “Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**5. Net Income Per Share (Basic) for the Three Months Ended December 27, 2025** \n**Value:** $1.10 \n**Why it matters:** Indicates earnings per share for the most recent quarter, useful for investors. \n**Quote:** “Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**6. Net Income Per Share (Basic) for the Six Months Ended December 27, 2025** \n**Value:** $1.17 \n**Why it matters:** Reflects year-to-date earnings per share, showing strong improvement from prior year’s loss. \n**Quote:** “Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |” [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n\n---\n\n**Cited Sources:**\n\n- [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_1 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS]\n- [doc=e5a1040e-1347-4336-85bb-e927d853e5bd chunk=e5a1040e-1347-4336-85bb-e927d853e5bd_3 section=PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)]","tool_trace":[{"tool":"list_ingested_companies","args":{"count":15},"result":"Loaded 15 indexed tickers from PostgreSQL."},{"tool":"planner_llm","args":{"raw_action":"answer","tickers":["LITE"],"characteristics":["financial_metrics","period_scoped"],"use_rag":false,"use_yfinance":false,"use_edgar_financials":true,"use_multi_ticker_briefs":false},"result":"Planner produced structured query decision."},{"tool":"plan_tool_usage","args":{"use_rag":false,"use_yfinance":false,"use_edgar_financials":true},"result":"Resolved planner tool usage flags."},{"tool":"prepare_rag_function","args":{"tickers":["LITE"],"filing_date_from":"2026-02-04","filing_date_to":"2026-02-04","use_per_ticker_retrieval":false,"use_multi_ticker_briefs":false},"result":"Prepared RAG function call arguments from planner decision."},{"tool":"finance_tools_execute","args":{"tickers":["LITE"],"use_yfinance":false,"use_edgar_financials":true,"result_count":3},"result":"Executed finance tools and produced 3 result objects."},{"tool":"rag_function_fallback","args":{"reason":"no_actionable_tool_results"},"result":"Planner disabled RAG, but finance tools returned no usable data; falling back to retrieval."},{"tool":"retrieve_chunks","args":{"tickers":["LITE"],"top_k_retrieve":40,"retrieval_queries":["What was LITE's net income in its 10-Q filed 2026-02-04?"]},"result":"Retrieved 40 chunks."},{"tool":"rerank_chunks","args":{"enable_rerank":true,"top_k_rerank":25},"result":"Produced 25 reranked chunks."}],"tool_results":[{"tool":"edgar_get_financial_metrics","ticker":"LITE","status":"error","summary":"Failed to fetch annual financials: [Errno 13] Permission denied: '/home/mlin/.edgar/_tcache/data.sec.gov/submissions-CIK0001633978.json.meta'","payload":null},{"tool":"edgar_get_quarterly_financial_metrics","ticker":"LITE","status":"error","summary":"Failed to fetch quarterly financials: [Errno 13] Permission denied: '/home/mlin/.edgar/_tcache/data.sec.gov/submissions-CIK0001633978.json.meta'","payload":null},{"tool":"edgar_get_financial_statements","ticker":"LITE","status":"error","summary":"Failed to fetch annual statements: [Errno 13] Permission denied: '/home/mlin/.edgar/_tcache/data.sec.gov/submissions-CIK0001633978.json.meta'","payload":null}],"top_chunks":[{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_183","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":7.038593769073486,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\n\nOther income, net for the three months ended December 27, 2025 decreased by $3.9 million compared to the three months ended December 28, 2024 primarily driven by a decrease in net foreign exchange gains of $6.5 million as the U.S. dollar strengthened against the Japanese Yen, which is the underlying currency for our term loans. This was offset by an increase of $2.6 million in interest and investment income mainly due to the $2.0 million interest income from the Cloud Light escrow settlement.Other income, net for the six months ended December 27, 2025 decreased by $8.4 million from the six months ended December 28, 2024 primarily due to a $5.9 million inducement expense related to the partial repurchase of 2026 Notes and a decrease in net foreign exchange gains of $4.3 million mainly driven by the Japan term loans denominated in Japanese Yen offset by an increase of $2.0 million in interest and investment income related to the Cloud Light escrow settlement.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1179,"line_end":1179}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_181","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":6.787876605987549,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\n\n5. 6 million, respectively. the increase in interest expense for the three months ended december 27, 2025 is mainly due to the issuance of the 2032 notes in september 2025. for the six months ended december 27, 2025 and december 28, 2024, we recorded interest expense of $ 12. 0 million and $ 11. 1 million, respectively. the increase in interest expense for the six months ended december 27, 2025 is mainly due to the issuance of the 2032 notes in september 2025. interest expense is primarily driven by the amortization of the debt issuance costs of our convertible notes.\n\nThe components of other income, net are as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1169,"line_end":1169}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_182","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":5.690978527069092,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\nSummary: Other Income, Net (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Foreign exchange and other gains (losses), net | $ | (0.6) | | $ | 5.9 | | $ | 0.9 | | $ | 5.2 | |\n| Interest and investment income, net | | 11.6 | | | 9.0 | | | 20.2 | | | 18.4 | |\n| Inducement expense | | — | | | — | | | (5.9) | | | — | |\n| Total other income, net | $ | 11.0 | | $ | 14.9 | | $ | 15.2 | | $ | 23.6 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Other Income, Net (table). Columns: Three Months Ended, Six Months Ended.","line_start":1171,"line_end":1177}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_198","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Investing Cash Flow"],"score":5.672499179840088,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > I","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Investing Cash Flow\n\n$ 38. 7 million primarily due to higher inventory purchases and capital expenditures and an increase in income tax liabilities of $ 28. 5 million primarily due to income tax provision for the six months ended december 28, 2024, offset by an increase of $ 15. 0 million in prepayments and other current and non - current assets related mainly to value - added - tax receivables driven by higher recent capital expenditures and inventory purchases, and a decrease of $ 19. 9 million in accrued expenses and other current and non - current liabilities primarily due to payment of the net settlement amount of the oclaro merger litigation.\n\nCash used in investing activities of $299.2 million during the six months ended December 27, 2025 was attributable to capital expenditures of $159.8 million and net payments from sales or maturities of short-term investments of $139.5 million, offset by $0.1 million proceeds from sale of assets.Cash used in investing activities of $77.8 million during the six months ended December 28, 2024 was attributable to capital expenditures of $114.3 million, offset by net proceeds from sales or maturities of short-term investments of $36.3 million and proceeds from sales of property and equipment of $0.2 million.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1268,"line_end":1268}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_174","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":5.383613109588623,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\n\nFor the three and six months ended December 27, 2025, net revenue from customers outside the United States, based on customer shipping locations, represented 78.3% and 80.1% of net revenue, respectively. Our net revenue is primarily denominated in U.S. dollars, including our net revenue from customers outside the United States as presented above. We expect revenue from customers outside of the United States to continue to be an important part of our overall net revenue and an increasing focus for net revenue growth opportunities. However, regulatory and enforcement actions by the United States and other governmental agencies, as well as changes in tax and trade policies and tariffs, have impacted and may continue to negatively impact net revenue from customers outside the United States.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1141,"line_end":1141}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_197","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Operating Cash Flow"],"score":5.214409828186035,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > O","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Operating Cash Flow\n\nand $ 55. 6 million, respectively. we are unable to reliably estimate the timing of future payments related to uncertain tax positions. our balance of cash and cash equivalents increased by $ 137. 0 million from $ 520. 7 million as of june 28, 2025 to $ 657. 7 million as of december 27, 2025. the increase in cash and cash equivalents during the six months ended december 27, 2025 was due to cash from operating activities of $ 184. 6 million and cash from financing activities of $ 251. 6 million, offset by cash used in investing activities of $ 299. 2 million.\n\nCash from operating activities was $184.6 million during the six months ended December 27, 2025, which reflects a net income of $82.4 million and non-cash items of $229.5 million, offset by changes in operating assets and liabilities of $127.3 million. Changes in operating assets and liabilities were primarily driven by an increase in accounts payable of $79.9 million primarily due to higher inventory purchases and capital expenditures, an increase of $27.4 million in accrued payroll and related expenses mainly driven by our accrual on employee cash bonuses and outstanding payroll taxes mainly related to stock-based compensation, and an increase of $21.8 million in accrued expenses and other current and non-current liabilities driven by contractual liabilities and increase in provision for warranty reserves, offset by an increase in accounts receivable of $126.7 million mainly driven by higher revenue, an increase of $102.5 million in inventories driven by inventory builds to support market demand and an increase of $27.4 million in prepayments and other current and non-current assets primarily driven by increase in value-added-tax receivables due to higher capital expenditures and inventory purchases and deferred financing costs related to our revolving credit facility.Cash from operating activities was $63.9 million during the six months ended December 28, 2024, which reflects a net loss of $143.3 million, offset by non-cash items of $209.1 million and changes in operating assets and liabilities of $1.9 million. Changes in operating assets and liabilities were primarily driven by an increase in accounts payable of $38.7 million primarily due to higher inventory purchases and capital expenditures and an increase in income tax liabilities of $28.5 million primarily due to income tax provision for the six months ended December 28, 2024, offset by an increase of $15.0 million in prepayments and other current and non-current assets related mainly to value-added-tax receivables driven by higher recent capital expenditures and inventory purchases, and a decrease of $19.9 million in accrued expenses and other current and non-current liabilities primarily due to payment of the net settlement amount of the Oclaro merger litigation.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1264,"line_end":1264}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_108","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 10. Accumulated Other Comprehensive Income (Loss)"],"score":4.837074279785156,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 10. Accumulated Other Comprehensive Income (Loss)\nSummary: Note 10. Accumulated Other Comprehensive Income (Loss) (table). Columns: Foreign Currency Translation Adjustments, Net of Tax (1), Defined Benefit Obligations, Net of Tax (2), Unrealized Gain on Available-for-Sale Securities, Net of Tax (3), Total.\n\n| | | | Foreign Currency Translation Adjustments, Net of Tax (1) | | | Defined Benefit Obligations, Net of Tax (2) | | | Unrealized Gain on Available-for-Sale Securities, Net of Tax (3) | | | Total |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| Beginning balance as of June 28, 2025 | $ | 9.9 | | $ | (1.6) | | $ | 0.7 | | $ | 9.0 | |\n| Other comprehensive gain (loss), net | | (0.3) | | | — | | | 0.4 | | | 0.1 | |\n| Ending balance as of September 27, 2025 | $ | 9.6 | | $ | (1.6) | | $ | 1.1 | | $ | 9.1 | |\n| Other comprehensive gain, net | | — | | | — | | | 0.3 | | | 0.3 | |\n| Ending balance as of December 27, 2025 | $ | 9.6 | | $ | (1.6) | | $ | 1.4 | | $ | 9.4 | |\n| | | | Foreign Currency Translation Adjustments, Net of Tax (1) | | | Defined Benefit Obligations, Net of Tax (2) | | | Unrealized Gain (Loss) on Available-for-Sale Securities, Net of Tax (3) | | | Total |\n| Beginning balance as of June 29, 2024 | $ | 9.8 | | $ | 0.7 | | $ | (1.2) | | $ | 9.3 | |\n| Other comprehensive gain, net | | — | | | — | | | 2.3 | | | 2.3 | |\n| Ending balance as of September 28, 2024 | $ | 9.8 | | $ | 0.7 | | $ | 1.1 | | $ | 11.6 | |\n| Other comprehensive loss, net | | (0.3) | | | — | | | (1.1) | | | (1.4) | |\n| Ending balance as of December 28, 2024 | $ | 9.5 | | $ | 0.7 | | $ | — | | $ | 10.2 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Note 10. Accumulated Other Comprehensive Income (Loss) (table). Columns: Foreign Currency Translation Adjustments, Net of Tax (1), Defined Benefit Obligations, Net of Tax (2), Unrealized Gain on Available-for-Sale Securities, Net of Tax (3), Total.","line_start":744,"line_end":756}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_3","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)"],"score":4.816524028778076,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STAT","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited) (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Other comprehensive income (loss), net of tax: | | | | | | | | | | | | |\n| Foreign currency translation adjustments | | — | | | (0.3) | | | (0.3) | | | (0.3) | |\n| Net change in unrealized gain on available-for-sale securities | | 0.3 | | | (1.1) | | | 0.7 | | | 1.2 | |\n| Other comprehensive income (loss), net of tax | | 0.3 | | | (1.4) | | | 0.4 | | | 0.9 | |\n| Comprehensive income (loss), net of tax | $ | 78.5 | | $ | (62.3) | | $ | 82.8 | | $ | (142.4) | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited) (table). Columns: Three Months Ended, Six Months Ended.","line_start":45,"line_end":53}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_172","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":4.69216251373291,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\n\n##iver lines. we also continued the initial phase of optical circuit switch shipments, which contributed more than $ 10. 0 million of revenue during the six months ended december 27, 2025, and we remain on track for manufacturing expansion over the coming quarters to support future growth. during the three months ended december 27, 2025, two customers individually accounted for 24 % and 17 % of our total revenue, respectively. during the six months ended december 27, 2025, two customers individually accounted for 23 % and 19 % of our total net revenue, respectively. we had no other customers that represented 10 % or greater of our total net revenue.\n\nWe operate in three geographic regions: Americas, Asia-Pacific, and EMEA (Europe, Middle East, and Africa). Net revenue is assigned to the geographic region and country where our product is initially shipped. For example, certain customers may request shipment of our product to a contract manufacturer in one country, which may differ from the location of their end customers. The following table presents net revenue by the three geographic regions we operate in and net revenue from countries that generally represented 10% or more of our total net revenue based on customer shipping locations (in millions, except percentage data):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1119,"line_end":1119}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_168","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations"],"score":4.552690029144287,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations\nS","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations\nSummary: Results of Operations (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | Three Months Ended | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | December 27, 2025 | | December 28, 2024 | | December 27, 2025 | | December 28, 2024 |\n| Net revenue by type of products: | | | | | | | | |\n| Components | 66.7 | % | 65.6 | % | 68.6 | % | 67.0 | % |\n| Systems | 33.3 | | 34.4 | | 31.4 | | 33.0 | |\n| Net revenue | 100.0 | | 100.0 | | 100.0 | | 100.0 | |\n| Cost of sales | 61.0 | | 69.9 | | 61.5 | | 70.0 | |\n| Amortization of acquired developed intangibles | 2.9 | | 5.3 | | 3.3 | | 6.0 | |\n| Gross profit | 36.1 | | 24.8 | | 35.2 | | 24.0 | |\n| Operating expenses: | | | | | | | | |\n| Research and development | 12.0 | | 18.4 | | 13.5 | | 20.1 | |\n| Selling, general and administrative | 14.4 | | 19.0 | | 15.1 | | 20.6 | |\n| Restructuring and related charges | — | | 0.2 | | 0.7 | | 1.4 | |\n| Total operating expenses | 26.4 | | 37.6 | | 29.3 | | 42.1 | |\n| Income (loss) from operations | 9.7 | | (12.8) | | 5.9 | | (18.1) | |\n| Escrow settlement | 4.1 | | — | | 2.3 | | — | |\n| Interest expense | (0.9) | | (1.4) | | (1.0) | | (1.5) | |\n| Other income, net | 1.6 | | 3.7 | | 1.3 | | 3.2 | |\n| Income (loss) before income taxes | 14.5 | | (10.5) | | 8.5 | | (16.4) | |\n| Income tax provision | 2.7 | | 4.6 | | 1.6 | | 3.0 | |\n| Net income (loss) | 11.8 | % | (15.1) | % | 6.9 | % | (19.4) | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Results of Operations (table). Columns: Three Months Ended, Six Months Ended.","line_start":1070,"line_end":1091}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_60","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets"],"score":4.447673797607422,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets\n\nbuilding. the total carrying value of assets purchased was $ 58. 5 million at the purchase date, of which $ 33. 4 million was allocated to the land and $ 25. 1 million to the building. in addition, in connection with the sale of our brazilian entities, we recorded a gain on sale of approximately $ 1. 6 million recorded in selling, general and administrative expenses in our condensed consolidated statements of operations during the six months ended december 27, 2025. during the three and six months ended december 27, 2025, we recorded depreciation expense of $ 30. 6 million and $ 58. 4 million, respectively.\n\nOperating lease right-of-use assets, net were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":499,"line_end":499}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_1","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"],"score":4.312051773071289,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Co","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 1199.3 | | $ | 739.1 | |\n| Cost of sales | | 405.8 | | | 281.2 | | | 738.6 | | | 517.7 | |\n| Amortization of acquired developed intangibles | | 19.6 | | | 21.4 | | | 39.1 | | | 43.9 | |\n| Gross profit | | 240.1 | | | 99.6 | | | 421.6 | | | 177.5 | |\n| Operating expenses: | | | | | | | | | | | | |\n| Research and development | | 80.1 | | | 74.2 | | | 161.5 | | | 148.5 | |\n| Selling, general and administrative | | 96.1 | | | 76.3 | | | 181.2 | | | 152.6 | |\n| Restructuring and related charges (reversals) | | (0.4) | | | 0.7 | | | 7.9 | | | 10.4 | |\n| Total operating expenses | | 175.8 | | | 151.2 | | | 350.6 | | | 311.5 | |\n| Income (loss) from operations | | 64.3 | | | (51.6) | | | 71.0 | | | (134.0) | |\n| Other income (expense), net: | | | | | | | | | | | | |\n| Escrow settlement | | 27.5 | | | — | | | 27.5 | | | — | |\n| Interest expense | | (6.3) | | | (5.6) | | | (12.0) | | | (11.1) | |\n| Other income, net | | 11.0 | | | 14.9 | | | 15.2 | | | 23.6 | |\n| Total other income, net | | 32.2 | | | 9.3 | | | 30.7 | | | 12.5 | |\n| Income (loss) before income taxes | | 96.5 | | | (42.3) | | | 101.7 | | | (121.5) | |\n| Income tax provision | | 18.3 | | | 18.6 | | | 19.3 | | | 21.8 | |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Net income (loss) per share: | | | | | | | | | | | | |\n| Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |\n| Diluted | $ | 0.89 | | $ | (0.88) | | $ | 0.99 | | $ | (2.09) | |\n| Shares used to compute net income (loss) per share: | | | | | | | | | | | | |\n| Basic | | 71.1 | | | 68.9 | | | 70.7 | | | 68.6 | |\n| Diluted | | 87.8 | | | 68.9 | | | 83.1 | | | 68.6 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.","line_start":13,"line_end":39}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_196","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Cash Flows"],"score":4.302709102630615,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > C","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Cash Flows\n\nunder the revolving credit facility. for additional information regarding the credit agreement, refer to “ note 9. debt ”, in the condensed consolidated financial statements included in part 1, item 1 of this quarterly report on form 10 - q. for additional information, refer to part ii item 1a “ risk factors ”. as of december 27, 2025 and june 28, 2025, our other non - current liabilities include unrecognized tax benefit for uncertain tax positions of $ 60. 4 million and $ 55. 6 million, respectively. we are unable to reliably estimate the timing of future payments related to uncertain tax positions.\n\nOur balance of cash and cash equivalents increased by $137.0 million from $520.7 million as of June 28, 2025 to $657.7 million as of December 27, 2025. The increase in cash and cash equivalents during the six months ended December 27, 2025 was due to cash from operating activities of $184.6 million and cash from financing activities of $251.6 million, offset by cash used in investing activities of $299.2 million.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1260,"line_end":1260}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_173","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":4.229966163635254,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\nSummary: Revenue by Region (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Net revenue: | | | | | | | | | | | | | | | | | | | | |\n| Americas: | | | | | | | | | | | | | | | | | | | | |\n| United States | $ | 144.7 | | 21.7 | % | $ | 77.6 | | 19.3 | % | $ | 238.4 | | 19.9 | % | $ | 143.0 | | 19.3 | % |\n| Mexico | | 102.6 | | 15.4 | | | 37.4 | | 9.3 | | | 176.2 | | 14.7 | | | 71.3 | | 9.6 | |\n| Other Americas | | 2.0 | | 0.3 | | | 4.2 | | 1.0 | | | 10.6 | | 0.9 | | | 7.1 | | 1.0 | |\n| Total Americas | $ | 249.3 | | 37.4 | % | $ | 119.2 | | 29.6 | % | $ | 425.2 | | 35.5 | % | $ | 221.4 | | 29.9 | % |\n| Asia-Pacific: | | | | | | | | | | | | | | | | | | | | |\n| Hong Kong | $ | 118.9 | | 17.9 | % | $ | 100.5 | | 25.0 | % | $ | 211.8 | | 17.7 | % | $ | 189.2 | | 25.6 | % |\n| Thailand | | 123.0 | | 18.5 | | | 74.7 | | 18.6 | | | 232.1 | | 19.3 | | | 127.2 | | 17.2 | |\n| China | | 54.6 | | 8.2 | | | 18.1 | | 4.5 | | | 103.9 | | 8.7 | | | 32.7 | | 4.4 | |\n| Japan | | 23.8 | | 3.6 | | | 18.4 | | 4.5 | | | 44.8 | | 3.7 | | | 35.3 | | 4.8 | |\n| Other Asia-Pacific | | 55.8 | | 8.4 | | | 30.5 | | 7.6 | | | 105.2 | | 8.7 | | | 61.9 | | 8.4 | |\n| Total Asia-Pacific | $ | 376.1 | | 56.6 | % | $ | 242.2 | | 60.2 | % | $ | 697.8 | | 58.1 | % | $ | 446.3 | | 60.4 | % |\n| EMEA | $ | 40.1 | | 6.0 | % | $ | 40.8 | | 10.2 | % | $ | 76.3 | | 6.4 | % | $ | 71.4 | | 9.7 | % |\n| Total net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Revenue by Region (table). Columns: Three Months Ended, Six Months Ended.","line_start":1121,"line_end":1139}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_195","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Unrecognized Tax Benefits"],"score":4.1687397956848145,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > U","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Unrecognized Tax Benefits\n\non december 19, 2025, the company entered into a credit agreement providing for a senior secured revolving credit facility in an aggregate principal amount of $ 400. 0 million, including a $ 23. 0 million sublimit for the issuance of letters of credit. as of december 27, 2025, there were no borrowings outstanding under the revolving credit facility. for additional information regarding the credit agreement, refer to “ note 9. debt ”, in the condensed consolidated financial statements included in part 1, item 1 of this quarterly report on form 10 - q. for additional information, refer to part ii item 1a “ risk factors ”.\n\nAs of December 27, 2025 and June 28, 2025, our other non-current liabilities include unrecognized tax benefit for uncertain tax positions of $60.4 million and $55.6 million, respectively. We are unable to reliably estimate the timing of future payments related to uncertain tax positions.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1256,"line_end":1256}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_57","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Property, Plant and Equipment, Net"],"score":4.131467342376709,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Property, Plant and Equipment, Net\nSummary: Property, Plant and Equipment, Net (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | December 27, 2025 | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- |\n| Land | $ | 90.1 | | $ | 108.6 | |\n| Buildings and improvements | | 267.6 | | | 270.4 | |\n| Machinery and equipment | | 975.4 | | | 848.8 | |\n| Computer equipment and software | | 42.2 | | | 39.1 | |\n| Furniture and fixtures | | 12.8 | | | 14.7 | |\n| Leasehold improvements | | 46.6 | | | 45.9 | |\n| Construction in progress | | 180.8 | | | 152.3 | |\n| | | 1615.5 | | | 1479.8 | |\n| Less: Accumulated depreciation | | (802.0) | | | (753.4) | |\n| Property, plant and equipment, net | $ | 813.5 | | $ | 726.4 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Property, Plant and Equipment, Net (table). Columns: December 27, 2025, June 28, 2025.","line_start":478,"line_end":489}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_170","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Financial data for the three months ended December 27, 2025"],"score":4.037820339202881,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Financial data for the three months ended December 27, 2025\nSummary: Financial data for the three months ended December 27, 2025 (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | | Three Months Ended | | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | Change | | Percentage Change | | | December 27, 2025 | | | December 28, 2024 | | | Change | | Percentage Change |\n| Net revenue by type of products: | | | | | | | | | | | | | | | | | | | | | | |\n| Components | $ | 443.7 | | $ | 263.7 | | $ | 180.0 | | 68.3 | % | $ | 822.9 | | $ | 495.1 | | $ | 327.8 | | 66.2 | % |\n| System | | 221.8 | | | 138.5 | | | 83.3 | | 60.1 | % | | 376.4 | | | 244.0 | | | 132.4 | | 54.3 | % |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 263.3 | | 65.5 | % | $ | 1199.3 | | $ | 739.1 | | $ | 460.2 | | 62.3 | % |\n| Gross profit | $ | 240.1 | | $ | 99.6 | | $ | 140.5 | | 141.1 | % | $ | 421.6 | | $ | 177.5 | | $ | 244.1 | | 137.5 | % |\n| Gross margin | | 36.1 | % | | 24.8 | % | | | | | | | 35.2 | % | | 24.0 | % | | | | | |\n| Research and development | $ | 80.1 | | $ | 74.2 | | $ | 5.9 | | 8.0 | % | $ | 161.5 | | $ | 148.5 | | $ | 13.0 | | 8.8 | % |\n| Percentage of net revenue | | 12.0 | % | | 18.4 | % | | | | | | | 13.5 | % | | 20.1 | % | | | | | |\n| Selling, general and administrative | $ | 96.1 | | $ | 76.3 | | $ | 19.8 | | 26.0 | % | $ | 181.2 | | $ | 152.6 | | $ | 28.6 | | 18.7 | % |\n| Percentage of net revenue | | 14.4 | % | | 19.0 | % | | | | | | | 15.1 | % | | 20.6 | % | | | | | |\n| Restructuring and related charges (reversals) | $ | (0.4) | | $ | 0.7 | | $ | (1.1) | | (157.1) | % | $ | 7.9 | | $ | 10.4 | | $ | (2.5) | | (24.0) | % |\n| Percentage of net revenue | | — | % | | 0.2 | % | | | | | | | 0.7 | % | | 1.4 | % | | | | | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Financial data for the three months ended December 27, 2025 (table). Columns: Three Months Ended, Six Months Ended.","line_start":1097,"line_end":1111}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_164","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Critical Accounting Policies and Estimates","Income Taxes"],"score":4.0001606941223145,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Critical Accounting Pol","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Critical Accounting Policies and Estimates > Income Taxes\n\nestimates, judgments and assumptions and which we believe are the most critical to aid in fully understanding and evaluating our reported financial results include the following : - inventory valuation - revenue recognition - income taxes - business combinations - goodwill and intangible assets - impairment assessmentmanagement ’ s discussion and analysis of financial condition and results of operations contained in part ii, item 7 of our annual report on form 10 - k for our fiscal year ended june 28, 2025 provides a complete discussion of our critical accounting policies and estimates. there have been no changes to these policies during the three and six months ended december 27, 2025, except as noted below :","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"}}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_63","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets","Other Current Liabilities"],"score":3.9183943271636963,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets > Other Current Liabilities\n\nin connection with the purchase of land and building in sagamihara, japan in july 2024, we terminated our leases for the related facilities and recorded a $ 16. 3 million increase in the carrying value of building purchased, as a result of derecognizing $ 32. 0 million of net operating lease right - of - use asset, $ 1. 6 million of operating lease liabilities, current, and $ 14. 1 million of operating lease liabilities, non - current.\n\nThe components of other current liabilities were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":511,"line_end":511}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_191","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Indebtedness"],"score":3.7765538692474365,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > I","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Indebtedness\nSummary: Indebtedness (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | | | | December 27, 2025 | | | | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | Carrying Amount | | | Estimated Fair Value | | | Carrying Amount | | | Estimated Fair Value |\n| 2032 Notes | $ | 1255.3 | | $ | 2840.8 | | $ | — | | $ | — | |\n| 2029 Notes | | 600.6 | | | 3400.1 | | | 600.2 | | | 925.5 | |\n| 2028 Notes | | 858.3 | | | 2584.6 | | | 857.7 | | | 890.2 | |\n| 2026 Notes | | 468.3 | | | 1844.6 | | | 1048.3 | | | 1233.3 | |\n| | $ | 3182.5 | | $ | 10670.1 | | $ | 2506.2 | | $ | 3049.0 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Indebtedness (table). Columns: December 27, 2025, June 28, 2025.","line_start":1234,"line_end":1241}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_146","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Concentrations"],"score":3.6105124950408936,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Concentrations\nSummary: Concentrations (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Net revenue: | | | | | | | | | | | | | | | | | | | | |\n| Americas: | | | | | | | | | | | | | | | | | | | | |\n| United States | $ | 144.7 | | 21.7 | % | $ | 77.6 | | 19.3 | % | $ | 238.4 | | 19.9 | % | $ | 143.0 | | 19.3 | % |\n| Mexico | | 102.6 | | 15.4 | | | 37.4 | | 9.3 | | | 176.2 | | 14.7 | | | 71.3 | | 9.6 | |\n| Other Americas | | 2.0 | | 0.3 | | | 4.2 | | 1.0 | | | 10.6 | | 0.9 | | | 7.1 | | 1.0 | |\n| Total Americas | $ | 249.3 | | 37.4 | % | $ | 119.2 | | 29.6 | % | $ | 425.2 | | 35.5 | % | $ | 221.4 | | 29.9 | % |\n| Asia-Pacific: | | | | | | | | | | | | | | | | | | | | |\n| Hong Kong | $ | 118.9 | | 17.9 | % | $ | 100.5 | | 25.0 | % | $ | 211.8 | | 17.7 | % | $ | 189.2 | | 25.6 | % |\n| Thailand | | 123.0 | | 18.5 | | | 74.7 | | 18.6 | | | 232.1 | | 19.3 | | | 127.2 | | 17.2 | |\n| China | | 54.6 | | 8.2 | | | 18.1 | | 4.5 | | | 103.9 | | 8.7 | | | 32.7 | | 4.4 | |\n| Japan | | 23.8 | | 3.6 | | | 18.4 | | 4.5 | | | 44.8 | | 3.7 | | | 35.3 | | 4.8 | |\n| Other Asia-Pacific | | 55.8 | | 8.4 | | | 30.5 | | 7.6 | | | 105.2 | | 8.7 | | | 61.9 | | 8.4 | |\n| Total Asia-Pacific | $ | 376.1 | | 56.6 | % | $ | 242.2 | | 60.2 | % | $ | 697.8 | | 58.1 | % | $ | 446.3 | | 60.4 | % |\n| EMEA | $ | 40.1 | | 6.0 | % | $ | 40.8 | | 10.2 | % | $ | 76.3 | | 6.4 | % | $ | 71.4 | | 9.7 | % |\n| Total net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Concentrations (table). Columns: Three Months Ended, Six Months Ended.","line_start":958,"line_end":976}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_152","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 16. Revenue Recognition","Disaggregation of Revenue"],"score":3.489960193634033,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 16. Revenue Recognition > Disaggregation of Revenue\nSummary: Disaggregation of Revenue (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Components | | 443.7 | | 66.7 | % | | 263.7 | | 65.6 | % | | 822.9 | | 68.6 | % | $ | 495.1 | | 67.0 | % |\n| Systems | | 221.8 | | 33.3 | % | | 138.5 | | 34.4 | % | | 376.4 | | 31.4 | % | | 244.0 | | 33.0 | % |\n| Net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Disaggregation of Revenue (table). Columns: Three Months Ended, Six Months Ended.","line_start":1005,"line_end":1011}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_4","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)"],"score":3.4391701221466064,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STAT","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)\n\nSee accompanying Notes to Condensed Consolidated Financial Statements.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":55,"line_end":55}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_187","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Contractual Obligations"],"score":3.2510499954223633,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > C","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Contractual Obligations\n\nour suppliers ; - volatility in fixed income and credit, which impact the liquidity and valuation of our investment portfolios ; - cost and availability of credit, which may impact available financing for us, our customers or others with whom we do business ; - volatility in foreign exchange markets, which impacts our financial results ; - possible investments or acquisitions of complementary businesses, products or technologies, or other strategic transactions or partnerships ; - issuance of debt or equity securities, or other financing transactions, including bank debt ; - potential funding of pension liabilities either voluntarily or as required by law or regulation ; and - acquisitions or strategic transactions.\n\nThe following table summarizes our contractual obligations as of December 27, 2025, and the effect such obligations are expected to have on our liquidity and cash flow (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1211,"line_end":1211}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_56","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Property, Plant and Equipment, Net"],"score":3.19584321975708,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Property, Plant and Equipment, Net\n\nThe components of property, plant and equipment, net were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":476,"line_end":476}}],"retrieved_chunks":[{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_182","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":0.01639344262295082,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\nSummary: Other Income, Net (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Foreign exchange and other gains (losses), net | $ | (0.6) | | $ | 5.9 | | $ | 0.9 | | $ | 5.2 | |\n| Interest and investment income, net | | 11.6 | | | 9.0 | | | 20.2 | | | 18.4 | |\n| Inducement expense | | — | | | — | | | (5.9) | | | — | |\n| Total other income, net | $ | 11.0 | | $ | 14.9 | | $ | 15.2 | | $ | 23.6 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Other Income, Net (table). Columns: Three Months Ended, Six Months Ended.","line_start":1171,"line_end":1177}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_168","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations"],"score":0.015391705069124424,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations\nS","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations\nSummary: Results of Operations (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | Three Months Ended | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | December 27, 2025 | | December 28, 2024 | | December 27, 2025 | | December 28, 2024 |\n| Net revenue by type of products: | | | | | | | | |\n| Components | 66.7 | % | 65.6 | % | 68.6 | % | 67.0 | % |\n| Systems | 33.3 | | 34.4 | | 31.4 | | 33.0 | |\n| Net revenue | 100.0 | | 100.0 | | 100.0 | | 100.0 | |\n| Cost of sales | 61.0 | | 69.9 | | 61.5 | | 70.0 | |\n| Amortization of acquired developed intangibles | 2.9 | | 5.3 | | 3.3 | | 6.0 | |\n| Gross profit | 36.1 | | 24.8 | | 35.2 | | 24.0 | |\n| Operating expenses: | | | | | | | | |\n| Research and development | 12.0 | | 18.4 | | 13.5 | | 20.1 | |\n| Selling, general and administrative | 14.4 | | 19.0 | | 15.1 | | 20.6 | |\n| Restructuring and related charges | — | | 0.2 | | 0.7 | | 1.4 | |\n| Total operating expenses | 26.4 | | 37.6 | | 29.3 | | 42.1 | |\n| Income (loss) from operations | 9.7 | | (12.8) | | 5.9 | | (18.1) | |\n| Escrow settlement | 4.1 | | — | | 2.3 | | — | |\n| Interest expense | (0.9) | | (1.4) | | (1.0) | | (1.5) | |\n| Other income, net | 1.6 | | 3.7 | | 1.3 | | 3.2 | |\n| Income (loss) before income taxes | 14.5 | | (10.5) | | 8.5 | | (16.4) | |\n| Income tax provision | 2.7 | | 4.6 | | 1.6 | | 3.0 | |\n| Net income (loss) | 11.8 | % | (15.1) | % | 6.9 | % | (19.4) | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Results of Operations (table). Columns: Three Months Ended, Six Months Ended.","line_start":1070,"line_end":1091}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_1","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"],"score":0.014434675935391534,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Co","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 1199.3 | | $ | 739.1 | |\n| Cost of sales | | 405.8 | | | 281.2 | | | 738.6 | | | 517.7 | |\n| Amortization of acquired developed intangibles | | 19.6 | | | 21.4 | | | 39.1 | | | 43.9 | |\n| Gross profit | | 240.1 | | | 99.6 | | | 421.6 | | | 177.5 | |\n| Operating expenses: | | | | | | | | | | | | |\n| Research and development | | 80.1 | | | 74.2 | | | 161.5 | | | 148.5 | |\n| Selling, general and administrative | | 96.1 | | | 76.3 | | | 181.2 | | | 152.6 | |\n| Restructuring and related charges (reversals) | | (0.4) | | | 0.7 | | | 7.9 | | | 10.4 | |\n| Total operating expenses | | 175.8 | | | 151.2 | | | 350.6 | | | 311.5 | |\n| Income (loss) from operations | | 64.3 | | | (51.6) | | | 71.0 | | | (134.0) | |\n| Other income (expense), net: | | | | | | | | | | | | |\n| Escrow settlement | | 27.5 | | | — | | | 27.5 | | | — | |\n| Interest expense | | (6.3) | | | (5.6) | | | (12.0) | | | (11.1) | |\n| Other income, net | | 11.0 | | | 14.9 | | | 15.2 | | | 23.6 | |\n| Total other income, net | | 32.2 | | | 9.3 | | | 30.7 | | | 12.5 | |\n| Income (loss) before income taxes | | 96.5 | | | (42.3) | | | 101.7 | | | (121.5) | |\n| Income tax provision | | 18.3 | | | 18.6 | | | 19.3 | | | 21.8 | |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Net income (loss) per share: | | | | | | | | | | | | |\n| Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |\n| Diluted | $ | 0.89 | | $ | (0.88) | | $ | 0.99 | | $ | (2.09) | |\n| Shares used to compute net income (loss) per share: | | | | | | | | | | | | |\n| Basic | | 71.1 | | | 68.9 | | | 70.7 | | | 68.6 | |\n| Diluted | | 87.8 | | | 68.9 | | | 83.1 | | | 68.6 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.","line_start":13,"line_end":39}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_56","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Property, Plant and Equipment, Net"],"score":0.014373024236037934,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Property, Plant and Equipment, Net\n\nThe components of property, plant and equipment, net were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":476,"line_end":476}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_181","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":0.01384493670886076,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\n\n5. 6 million, respectively. the increase in interest expense for the three months ended december 27, 2025 is mainly due to the issuance of the 2032 notes in september 2025. for the six months ended december 27, 2025 and december 28, 2024, we recorded interest expense of $ 12. 0 million and $ 11. 1 million, respectively. the increase in interest expense for the six months ended december 27, 2025 is mainly due to the issuance of the 2032 notes in september 2025. interest expense is primarily driven by the amortization of the debt issuance costs of our convertible notes.\n\nThe components of other income, net are as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1169,"line_end":1169}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_0","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"],"score":0.013626373626373627,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\n\n(in millions, except per share data)\n\n(Unaudited)","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\n\n(in millions, except per share data)\n\n(Unaudited)","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":9,"line_end":11}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_3","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)"],"score":0.013348164627363737,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STAT","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited) (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Other comprehensive income (loss), net of tax: | | | | | | | | | | | | |\n| Foreign currency translation adjustments | | — | | | (0.3) | | | (0.3) | | | (0.3) | |\n| Net change in unrealized gain on available-for-sale securities | | 0.3 | | | (1.1) | | | 0.7 | | | 1.2 | |\n| Other comprehensive income (loss), net of tax | | 0.3 | | | (1.4) | | | 0.4 | | | 0.9 | |\n| Comprehensive income (loss), net of tax | $ | 78.5 | | $ | (62.3) | | $ | 82.8 | | $ | (142.4) | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited) (table). Columns: Three Months Ended, Six Months Ended.","line_start":45,"line_end":53}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_183","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Other Income, Net"],"score":0.012942612942612942,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Other Income, Net\n\nOther income, net for the three months ended December 27, 2025 decreased by $3.9 million compared to the three months ended December 28, 2024 primarily driven by a decrease in net foreign exchange gains of $6.5 million as the U.S. dollar strengthened against the Japanese Yen, which is the underlying currency for our term loans. This was offset by an increase of $2.6 million in interest and investment income mainly due to the $2.0 million interest income from the Cloud Light escrow settlement.Other income, net for the six months ended December 27, 2025 decreased by $8.4 million from the six months ended December 28, 2024 primarily due to a $5.9 million inducement expense related to the partial repurchase of 2026 Notes and a decrease in net foreign exchange gains of $4.3 million mainly driven by the Japan term loans denominated in Japanese Yen offset by an increase of $2.0 million in interest and investment income related to the Cloud Light escrow settlement.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1179,"line_end":1179}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_57","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Property, Plant and Equipment, Net"],"score":0.012205882352941178,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Property, Plant and Equipment, Net\nSummary: Property, Plant and Equipment, Net (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | December 27, 2025 | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- |\n| Land | $ | 90.1 | | $ | 108.6 | |\n| Buildings and improvements | | 267.6 | | | 270.4 | |\n| Machinery and equipment | | 975.4 | | | 848.8 | |\n| Computer equipment and software | | 42.2 | | | 39.1 | |\n| Furniture and fixtures | | 12.8 | | | 14.7 | |\n| Leasehold improvements | | 46.6 | | | 45.9 | |\n| Construction in progress | | 180.8 | | | 152.3 | |\n| | | 1615.5 | | | 1479.8 | |\n| Less: Accumulated depreciation | | (802.0) | | | (753.4) | |\n| Property, plant and equipment, net | $ | 813.5 | | $ | 726.4 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Property, Plant and Equipment, Net (table). Columns: December 27, 2025, June 28, 2025.","line_start":478,"line_end":489}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_97","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)"],"score":0.012187028657616892,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: June 28, 2025, 2026 Notes, 2028 Notes, 2029 Notes, Total.\n\n| June 28, 2025 | | | 2026 Notes | | | 2028 Notes | | | 2029 Notes | | | Total |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| Principal | $ | 1050.0 | | $ | 861.0 | | $ | 603.7 | | $ | 2514.7 | |\n| Unamortized debt issuance costs | | (1.7) | | | (3.3) | | | (3.5) | | | (8.5) | |\n| Net carrying amount of the liability component | $ | 1048.3 | | $ | 857.7 | | $ | 600.2 | | $ | 2506.2 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: June 28, 2025, 2026 Notes, 2028 Notes, 2029 Notes, Total.","line_start":687,"line_end":691}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_198","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Investing Cash Flow"],"score":0.011738648947951274,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > I","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Investing Cash Flow\n\n$ 38. 7 million primarily due to higher inventory purchases and capital expenditures and an increase in income tax liabilities of $ 28. 5 million primarily due to income tax provision for the six months ended december 28, 2024, offset by an increase of $ 15. 0 million in prepayments and other current and non - current assets related mainly to value - added - tax receivables driven by higher recent capital expenditures and inventory purchases, and a decrease of $ 19. 9 million in accrued expenses and other current and non - current liabilities primarily due to payment of the net settlement amount of the oclaro merger litigation.\n\nCash used in investing activities of $299.2 million during the six months ended December 27, 2025 was attributable to capital expenditures of $159.8 million and net payments from sales or maturities of short-term investments of $139.5 million, offset by $0.1 million proceeds from sale of assets.Cash used in investing activities of $77.8 million during the six months ended December 28, 2024 was attributable to capital expenditures of $114.3 million, offset by net proceeds from sales or maturities of short-term investments of $36.3 million and proceeds from sales of property and equipment of $0.2 million.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1268,"line_end":1268}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_164","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Critical Accounting Policies and Estimates","Income Taxes"],"score":0.011662726556343577,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Critical Accounting Pol","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Critical Accounting Policies and Estimates > Income Taxes\n\nestimates, judgments and assumptions and which we believe are the most critical to aid in fully understanding and evaluating our reported financial results include the following : - inventory valuation - revenue recognition - income taxes - business combinations - goodwill and intangible assets - impairment assessmentmanagement ’ s discussion and analysis of financial condition and results of operations contained in part ii, item 7 of our annual report on form 10 - k for our fiscal year ended june 28, 2025 provides a complete discussion of our critical accounting policies and estimates. there have been no changes to these policies during the three and six months ended december 27, 2025, except as noted below :","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"}}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_2","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"],"score":0.011443932411674348,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\n\nSee accompanying Notes to Condensed Consolidated Financial Statemen","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\n\nSee accompanying Notes to Condensed Consolidated Financial Statements.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":41,"line_end":41}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_108","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 10. Accumulated Other Comprehensive Income (Loss)"],"score":0.011415882967607104,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 10. Accumulated Other Comprehensive Income (Loss)\nSummary: Note 10. Accumulated Other Comprehensive Income (Loss) (table). Columns: Foreign Currency Translation Adjustments, Net of Tax (1), Defined Benefit Obligations, Net of Tax (2), Unrealized Gain on Available-for-Sale Securities, Net of Tax (3), Total.\n\n| | | | Foreign Currency Translation Adjustments, Net of Tax (1) | | | Defined Benefit Obligations, Net of Tax (2) | | | Unrealized Gain on Available-for-Sale Securities, Net of Tax (3) | | | Total |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| Beginning balance as of June 28, 2025 | $ | 9.9 | | $ | (1.6) | | $ | 0.7 | | $ | 9.0 | |\n| Other comprehensive gain (loss), net | | (0.3) | | | — | | | 0.4 | | | 0.1 | |\n| Ending balance as of September 27, 2025 | $ | 9.6 | | $ | (1.6) | | $ | 1.1 | | $ | 9.1 | |\n| Other comprehensive gain, net | | — | | | — | | | 0.3 | | | 0.3 | |\n| Ending balance as of December 27, 2025 | $ | 9.6 | | $ | (1.6) | | $ | 1.4 | | $ | 9.4 | |\n| | | | Foreign Currency Translation Adjustments, Net of Tax (1) | | | Defined Benefit Obligations, Net of Tax (2) | | | Unrealized Gain (Loss) on Available-for-Sale Securities, Net of Tax (3) | | | Total |\n| Beginning balance as of June 29, 2024 | $ | 9.8 | | $ | 0.7 | | $ | (1.2) | | $ | 9.3 | |\n| Other comprehensive gain, net | | — | | | — | | | 2.3 | | | 2.3 | |\n| Ending balance as of September 28, 2024 | $ | 9.8 | | $ | 0.7 | | $ | 1.1 | | $ | 11.6 | |\n| Other comprehensive loss, net | | (0.3) | | | — | | | (1.1) | | | (1.4) | |\n| Ending balance as of December 28, 2024 | $ | 9.5 | | $ | 0.7 | | $ | — | | $ | 10.2 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Note 10. Accumulated Other Comprehensive Income (Loss) (table). Columns: Foreign Currency Translation Adjustments, Net of Tax (1), Defined Benefit Obligations, Net of Tax (2), Unrealized Gain on Available-for-Sale Securities, Net of Tax (3), Total.","line_start":744,"line_end":756}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_149","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)"],"score":0.011341016238868518,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | December 27, 2025 | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- |\n| Property, plant and equipment, net | | | | | | |\n| United States | $ | 80.1 | | $ | 123.0 | |\n| Thailand | | 266.0 | | | 218.6 | |\n| Japan | | 182.9 | | | 144.3 | |\n| United Kingdom | | 121.9 | | | 109.4 | |\n| China | | 111.4 | | | 76.8 | |\n| Other countries | | 51.2 | | | 54.3 | |\n| Total property, plant and equipment, net | $ | 813.5 | | $ | 726.4 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: December 27, 2025, June 28, 2025.","line_start":984,"line_end":993}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_174","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":0.010945994925697717,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\n\nFor the three and six months ended December 27, 2025, net revenue from customers outside the United States, based on customer shipping locations, represented 78.3% and 80.1% of net revenue, respectively. Our net revenue is primarily denominated in U.S. dollars, including our net revenue from customers outside the United States as presented above. We expect revenue from customers outside of the United States to continue to be an important part of our overall net revenue and an increasing focus for net revenue growth opportunities. However, regulatory and enforcement actions by the United States and other governmental agencies, as well as changes in tax and trade policies and tariffs, have impacted and may continue to negatively impact net revenue from customers outside the United States.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1141,"line_end":1141}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_61","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets"],"score":0.010210526315789474,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets\nSummary: Operating Lease Right-of-Use Assets (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | December 27, 2025 | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- |\n| Operating lease right-of-use assets | $ | 58.7 | | $ | 54.4 | |\n| Less: accumulated amortization | | (29.1) | | | (26.5) | |\n| Operating lease right-of-use assets, net | $ | 29.6 | | $ | 27.9 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Operating Lease Right-of-Use Assets (table). Columns: December 27, 2025, June 28, 2025.","line_start":501,"line_end":505}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_173","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":0.009523809523809523,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\nSummary: Revenue by Region (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Net revenue: | | | | | | | | | | | | | | | | | | | | |\n| Americas: | | | | | | | | | | | | | | | | | | | | |\n| United States | $ | 144.7 | | 21.7 | % | $ | 77.6 | | 19.3 | % | $ | 238.4 | | 19.9 | % | $ | 143.0 | | 19.3 | % |\n| Mexico | | 102.6 | | 15.4 | | | 37.4 | | 9.3 | | | 176.2 | | 14.7 | | | 71.3 | | 9.6 | |\n| Other Americas | | 2.0 | | 0.3 | | | 4.2 | | 1.0 | | | 10.6 | | 0.9 | | | 7.1 | | 1.0 | |\n| Total Americas | $ | 249.3 | | 37.4 | % | $ | 119.2 | | 29.6 | % | $ | 425.2 | | 35.5 | % | $ | 221.4 | | 29.9 | % |\n| Asia-Pacific: | | | | | | | | | | | | | | | | | | | | |\n| Hong Kong | $ | 118.9 | | 17.9 | % | $ | 100.5 | | 25.0 | % | $ | 211.8 | | 17.7 | % | $ | 189.2 | | 25.6 | % |\n| Thailand | | 123.0 | | 18.5 | | | 74.7 | | 18.6 | | | 232.1 | | 19.3 | | | 127.2 | | 17.2 | |\n| China | | 54.6 | | 8.2 | | | 18.1 | | 4.5 | | | 103.9 | | 8.7 | | | 32.7 | | 4.4 | |\n| Japan | | 23.8 | | 3.6 | | | 18.4 | | 4.5 | | | 44.8 | | 3.7 | | | 35.3 | | 4.8 | |\n| Other Asia-Pacific | | 55.8 | | 8.4 | | | 30.5 | | 7.6 | | | 105.2 | | 8.7 | | | 61.9 | | 8.4 | |\n| Total Asia-Pacific | $ | 376.1 | | 56.6 | % | $ | 242.2 | | 60.2 | % | $ | 697.8 | | 58.1 | % | $ | 446.3 | | 60.4 | % |\n| EMEA | $ | 40.1 | | 6.0 | % | $ | 40.8 | | 10.2 | % | $ | 76.3 | | 6.4 | % | $ | 71.4 | | 9.7 | % |\n| Total net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Revenue by Region (table). Columns: Three Months Ended, Six Months Ended.","line_start":1121,"line_end":1139}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_170","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Financial data for the three months ended December 27, 2025"],"score":0.009375,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Financial data for the three months ended December 27, 2025\nSummary: Financial data for the three months ended December 27, 2025 (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | | Three Months Ended | | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | Change | | Percentage Change | | | December 27, 2025 | | | December 28, 2024 | | | Change | | Percentage Change |\n| Net revenue by type of products: | | | | | | | | | | | | | | | | | | | | | | |\n| Components | $ | 443.7 | | $ | 263.7 | | $ | 180.0 | | 68.3 | % | $ | 822.9 | | $ | 495.1 | | $ | 327.8 | | 66.2 | % |\n| System | | 221.8 | | | 138.5 | | | 83.3 | | 60.1 | % | | 376.4 | | | 244.0 | | | 132.4 | | 54.3 | % |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 263.3 | | 65.5 | % | $ | 1199.3 | | $ | 739.1 | | $ | 460.2 | | 62.3 | % |\n| Gross profit | $ | 240.1 | | $ | 99.6 | | $ | 140.5 | | 141.1 | % | $ | 421.6 | | $ | 177.5 | | $ | 244.1 | | 137.5 | % |\n| Gross margin | | 36.1 | % | | 24.8 | % | | | | | | | 35.2 | % | | 24.0 | % | | | | | |\n| Research and development | $ | 80.1 | | $ | 74.2 | | $ | 5.9 | | 8.0 | % | $ | 161.5 | | $ | 148.5 | | $ | 13.0 | | 8.8 | % |\n| Percentage of net revenue | | 12.0 | % | | 18.4 | % | | | | | | | 13.5 | % | | 20.1 | % | | | | | |\n| Selling, general and administrative | $ | 96.1 | | $ | 76.3 | | $ | 19.8 | | 26.0 | % | $ | 181.2 | | $ | 152.6 | | $ | 28.6 | | 18.7 | % |\n| Percentage of net revenue | | 14.4 | % | | 19.0 | % | | | | | | | 15.1 | % | | 20.6 | % | | | | | |\n| Restructuring and related charges (reversals) | $ | (0.4) | | $ | 0.7 | | $ | (1.1) | | (157.1) | % | $ | 7.9 | | $ | 10.4 | | $ | (2.5) | | (24.0) | % |\n| Percentage of net revenue | | — | % | | 0.2 | % | | | | | | | 0.7 | % | | 1.4 | % | | | | | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Financial data for the three months ended December 27, 2025 (table). Columns: Three Months Ended, Six Months Ended.","line_start":1097,"line_end":1111}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_191","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Indebtedness"],"score":0.00909090909090909,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > I","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Indebtedness\nSummary: Indebtedness (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | | | | December 27, 2025 | | | | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | Carrying Amount | | | Estimated Fair Value | | | Carrying Amount | | | Estimated Fair Value |\n| 2032 Notes | $ | 1255.3 | | $ | 2840.8 | | $ | — | | $ | — | |\n| 2029 Notes | | 600.6 | | | 3400.1 | | | 600.2 | | | 925.5 | |\n| 2028 Notes | | 858.3 | | | 2584.6 | | | 857.7 | | | 890.2 | |\n| 2026 Notes | | 468.3 | | | 1844.6 | | | 1048.3 | | | 1233.3 | |\n| | $ | 3182.5 | | $ | 10670.1 | | $ | 2506.2 | | $ | 3049.0 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Indebtedness (table). Columns: December 27, 2025, June 28, 2025.","line_start":1234,"line_end":1241}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_193","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Indebtedness"],"score":0.008823529411764706,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > I","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Indebtedness\nSummary: Indebtedness (table). Columns: Conversion Price (1), 130% of Conversion Price (1).\n\n| | | | Conversion Price (1) | | | 130% of Conversion Price (1) |\n| --- | --- | --- | --- | --- | --- | --- |\n| 2032 Notes | $ | 187.77 | | $ | 244.1 | |\n| 2029 Notes | | 69.54 | | | 90.4 | |\n| 2028 Notes | | 131.03 | | | 170.34 | |\n| 2026 Notes | | 99.29 | | | 129.08 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Indebtedness (table). Columns: Conversion Price (1), 130% of Conversion Price (1).","line_start":1245,"line_end":1250}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_187","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Contractual Obligations"],"score":0.008695652173913044,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > C","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Contractual Obligations\n\nour suppliers ; - volatility in fixed income and credit, which impact the liquidity and valuation of our investment portfolios ; - cost and availability of credit, which may impact available financing for us, our customers or others with whom we do business ; - volatility in foreign exchange markets, which impacts our financial results ; - possible investments or acquisitions of complementary businesses, products or technologies, or other strategic transactions or partnerships ; - issuance of debt or equity securities, or other financing transactions, including bank debt ; - potential funding of pension liabilities either voluntarily or as required by law or regulation ; and - acquisitions or strategic transactions.\n\nThe following table summarizes our contractual obligations as of December 27, 2025, and the effect such obligations are expected to have on our liquidity and cash flow (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1211,"line_end":1211}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_188","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Contractual Obligations"],"score":0.00857142857142857,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > C","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Contractual Obligations\nSummary: Contractual Obligations (table). Columns: Payments Due.\n\n| | | | | | | | | | Payments Due |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | Total | | | Less Than 1 Year | | | More Than 1 Year |\n| Contractual Obligations | | | | | | | | | |\n| Asset retirement obligations | $ | 7.1 | | $ | — | | $ | 7.1 | |\n| Operating lease liabilities, including imputed interest (1) | | 37.8 | | | 14.0 | | | 23.8 | |\n| Pension plan contributions (2) | | 2.0 | | | 2.0 | | | — | |\n| Purchase obligations (3) | | 1086.0 | | | 1022.8 | | | 63.2 | |\n| Term loans - principal (5) | | 104.8 | | | 57.7 | | | 47.1 | |\n| Term loans - interest (5) | | 1.8 | | | 1.0 | | | 0.8 | |\n| Convertible notes - principal (4) | | 3198.5 | | | 468.8 | | | 2729.7 | |\n| Convertible notes - interest (4) | | 80.1 | | | 20.5 | | | 59.6 | |\n| Others | | 15.0 | | | 1.4 | | | 13.6 | |\n| Total | $ | 4533.1 | | $ | 1588.2 | | $ | 2944.9 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Contractual Obligations (table). Columns: Payments Due.","line_start":1213,"line_end":1226}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_152","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 16. Revenue Recognition","Disaggregation of Revenue"],"score":0.008450704225352112,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 16. Revenue Recognition > Disaggregation of Revenue\nSummary: Disaggregation of Revenue (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Components | | 443.7 | | 66.7 | % | | 263.7 | | 65.6 | % | | 822.9 | | 68.6 | % | $ | 495.1 | | 67.0 | % |\n| Systems | | 221.8 | | 33.3 | % | | 138.5 | | 34.4 | % | | 376.4 | | 31.4 | % | | 244.0 | | 33.0 | % |\n| Net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Disaggregation of Revenue (table). Columns: Three Months Ended, Six Months Ended.","line_start":1005,"line_end":1011}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_197","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Operating Cash Flow"],"score":0.008333333333333333,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > O","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Operating Cash Flow\n\nand $ 55. 6 million, respectively. we are unable to reliably estimate the timing of future payments related to uncertain tax positions. our balance of cash and cash equivalents increased by $ 137. 0 million from $ 520. 7 million as of june 28, 2025 to $ 657. 7 million as of december 27, 2025. the increase in cash and cash equivalents during the six months ended december 27, 2025 was due to cash from operating activities of $ 184. 6 million and cash from financing activities of $ 251. 6 million, offset by cash used in investing activities of $ 299. 2 million.\n\nCash from operating activities was $184.6 million during the six months ended December 27, 2025, which reflects a net income of $82.4 million and non-cash items of $229.5 million, offset by changes in operating assets and liabilities of $127.3 million. Changes in operating assets and liabilities were primarily driven by an increase in accounts payable of $79.9 million primarily due to higher inventory purchases and capital expenditures, an increase of $27.4 million in accrued payroll and related expenses mainly driven by our accrual on employee cash bonuses and outstanding payroll taxes mainly related to stock-based compensation, and an increase of $21.8 million in accrued expenses and other current and non-current liabilities driven by contractual liabilities and increase in provision for warranty reserves, offset by an increase in accounts receivable of $126.7 million mainly driven by higher revenue, an increase of $102.5 million in inventories driven by inventory builds to support market demand and an increase of $27.4 million in prepayments and other current and non-current assets primarily driven by increase in value-added-tax receivables due to higher capital expenditures and inventory purchases and deferred financing costs related to our revolving credit facility.Cash from operating activities was $63.9 million during the six months ended December 28, 2024, which reflects a net loss of $143.3 million, offset by non-cash items of $209.1 million and changes in operating assets and liabilities of $1.9 million. Changes in operating assets and liabilities were primarily driven by an increase in accounts payable of $38.7 million primarily due to higher inventory purchases and capital expenditures and an increase in income tax liabilities of $28.5 million primarily due to income tax provision for the six months ended December 28, 2024, offset by an increase of $15.0 million in prepayments and other current and non-current assets related mainly to value-added-tax receivables driven by higher recent capital expenditures and inventory purchases, and a decrease of $19.9 million in accrued expenses and other current and non-current liabilities primarily due to payment of the net settlement amount of the Oclaro merger litigation.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1264,"line_end":1264}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_81","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 9. Debt"],"score":0.008108108108108109,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 9. Debt\nSummary: Note 9. Debt (table). Columns: Conversion Price, 130% of Conversion Price.\n\n| | | | Conversion Price | | | 130% of Conversion Price |\n| --- | --- | --- | --- | --- | --- | --- |\n| 2032 Notes | $ | 187.77 | | $ | 244.1 | |\n| 2029 Notes | | 69.54 | | | 90.4 | |\n| 2028 Notes | | 131.03 | | | 170.34 | |\n| 2026 Notes | | 99.29 | | | 129.08 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Note 9. Debt (table). Columns: Conversion Price, 130% of Conversion Price.","line_start":615,"line_end":620}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_256","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART II - OTHER INFORMATION","ITEM 6. EXHIBITS"],"score":0.008,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART II - OTHER INFORMATION > ITEM 6. EXHIBITS\n\n† The certifications furnished in Exhibits 32.1 and 32.2 that accompany this report are not deemed filed with the Securities and Exchange Commis","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART II - OTHER INFORMATION > ITEM 6. EXHIBITS\n\n† The certifications furnished in Exhibits 32.1 and 32.2 that accompany this report are not deemed filed with the Securities and Exchange Commission and are not to be incorporated by reference into any filing of the Registrant under the Securities Act of 1933, as amended, or the Securities Exchange Act of 1934, as amended, whether made before or after the date of this report, irrespective of any general incorporation language contained in such filing.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1750,"line_end":1750}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_146","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Concentrations"],"score":0.007894736842105262,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Concentrations\nSummary: Concentrations (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | | | | | Three Months Ended | | | | | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | | | December 27, 2025 | | | | | December 28, 2024 | | | | | December 27, 2025 | | | | | December 28, 2024 |\n| | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total | | | Amount | | % of Total |\n| Net revenue: | | | | | | | | | | | | | | | | | | | | |\n| Americas: | | | | | | | | | | | | | | | | | | | | |\n| United States | $ | 144.7 | | 21.7 | % | $ | 77.6 | | 19.3 | % | $ | 238.4 | | 19.9 | % | $ | 143.0 | | 19.3 | % |\n| Mexico | | 102.6 | | 15.4 | | | 37.4 | | 9.3 | | | 176.2 | | 14.7 | | | 71.3 | | 9.6 | |\n| Other Americas | | 2.0 | | 0.3 | | | 4.2 | | 1.0 | | | 10.6 | | 0.9 | | | 7.1 | | 1.0 | |\n| Total Americas | $ | 249.3 | | 37.4 | % | $ | 119.2 | | 29.6 | % | $ | 425.2 | | 35.5 | % | $ | 221.4 | | 29.9 | % |\n| Asia-Pacific: | | | | | | | | | | | | | | | | | | | | |\n| Hong Kong | $ | 118.9 | | 17.9 | % | $ | 100.5 | | 25.0 | % | $ | 211.8 | | 17.7 | % | $ | 189.2 | | 25.6 | % |\n| Thailand | | 123.0 | | 18.5 | | | 74.7 | | 18.6 | | | 232.1 | | 19.3 | | | 127.2 | | 17.2 | |\n| China | | 54.6 | | 8.2 | | | 18.1 | | 4.5 | | | 103.9 | | 8.7 | | | 32.7 | | 4.4 | |\n| Japan | | 23.8 | | 3.6 | | | 18.4 | | 4.5 | | | 44.8 | | 3.7 | | | 35.3 | | 4.8 | |\n| Other Asia-Pacific | | 55.8 | | 8.4 | | | 30.5 | | 7.6 | | | 105.2 | | 8.7 | | | 61.9 | | 8.4 | |\n| Total Asia-Pacific | $ | 376.1 | | 56.6 | % | $ | 242.2 | | 60.2 | % | $ | 697.8 | | 58.1 | % | $ | 446.3 | | 60.4 | % |\n| EMEA | $ | 40.1 | | 6.0 | % | $ | 40.8 | | 10.2 | % | $ | 76.3 | | 6.4 | % | $ | 71.4 | | 9.7 | % |\n| Total net revenue | $ | 665.5 | | 100.0 | % | $ | 402.2 | | 100.0 | % | $ | 1199.3 | | 100.0 | % | $ | 739.1 | | 100.0 | % |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Concentrations (table). Columns: Three Months Ended, Six Months Ended.","line_start":958,"line_end":976}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_101","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)"],"score":0.007792207792207792,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: Fiscal Years, 2026 Notes, 2028 Notes, 2029 Notes, 2032 Notes, Total.\n\n| Fiscal Years | | | 2026 Notes | | | 2028 Notes | | | 2029 Notes | | | 2032 Notes | | | Total |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| 2026 | $ | 1.2 | | $ | 2.2 | | $ | 4.5 | | $ | 2.4 | | $ | 10.3 | |\n| 2027 | | 469.8 | | | 4.3 | | | 9.1 | | | 4.7 | | | 487.9 | |\n| 2028 | | — | | | 865.3 | | | 9.1 | | | 4.7 | | | 879.1 | |\n| 2029 | | — | | | — | | | 9.1 | | | 4.7 | | | 13.8 | |\n| 2030 | | — | | | — | | | 608.1 | | | 4.7 | | | 612.8 | |\n| Thereafter | | — | | | — | | | — | | | 1274.7 | | | 1274.7 | |\n| Total payments | $ | 471.0 | | $ | 871.8 | | $ | 639.9 | | $ | 1295.9 | | $ | 3278.6 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: Fiscal Years, 2026 Notes, 2028 Notes, 2029 Notes, 2032 Notes, Total.","line_start":706,"line_end":714}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_7","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS(in millions)(Unaudited)"],"score":0.007692307692307692,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STAT","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS(in millions)(Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS(in millions)(Unaudited) (table). Columns: Six Months Ended.\n\n| | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 |\n| OPERATING ACTIVITIES: | | | | | | |\n| Net income (loss) | $ | 82.4 | | $ | (143.3) | |\n| Adjustments to reconcile net income (loss) to net cash provided by operating activities: | | | | | | |\n| Depreciation expense | | 58.4 | | | 52.9 | |\n| Stock-based compensation | | 87.8 | | | 74.4 | |\n| Bad debt recovery | | (0.1) | | | — | |\n| Change in valuation allowance on deferred tax assets | | 1.4 | | | — | |\n| Amortization and write-off of acquired intangibles | | 68.4 | | | 82.6 | |\n| Write-down and loss on sales and dispositions of property, plant and equipment | | 12.0 | | | 0.8 | |\n| Amortization of debt discount and debt issuance costs | | 1.9 | | | 1.5 | |\n| Inducement expense on partial repurchase of 2026 Notes | | 5.9 | | | — | |\n| Write-off of right-of-use assets | | — | | | 5.5 | |\n| Other non-cash items | | (6.2) | | | (8.6) | |\n| Changes in operating assets and liabilities: | | | | | | |\n| Accounts receivable | | (126.7) | | | (32.2) | |\n| Inventories | | (102.5) | | | (5.0) | |\n| Operating lease right-of-use assets, net | | (1.7) | | | 2.4 | |\n| Prepayments and other current and non-currents assets | | (27.4) | | | (15.0) | |\n| Income taxes, net | | 1.5 | | | 28.5 | |\n| Accounts payable | | 79.9 | | | 38.7 | |\n| Accrued payroll and related expenses | | 27.4 | | | 4.0 | |\n| Operating lease liabilities | | 0.4 | | | (3.4) | |\n| Accrued expenses and other current and non-current liabilities | | 21.8 | | | (19.9) | |\n| Net cash provided by operating activities | | 184.6 | | | 63.9 | |\n| INVESTING ACTIVITIES: | | | | | | |\n| Payments for acquisition of property, plant and equipment | | (159.8) | | | (114.3) | |\n| Purchases of short-term investments | | (257.5) | | | (190.4) | |\n| Proceeds from maturities and sales of short-term investments | | 118.0 | | | 226.7 | |\n| Proceeds from the sales of property, plant and equipment | | 0.1 | | | 0.2 | |\n| Net cash used in investing activities | | (299.2) | | | (77.8) | |\n| FINANCING ACTIVITIES: | | | | | | |\n| Proceeds from the issuance of 2032 Notes, net of issuance costs | | 1254.7 | | | — | |\n| Proceeds from term loans | | 47.9 | | | 76.5 | |\n| Proceeds from employee stock plans | | 8.7 | | | 8.1 | |\n| Payment for partial repurchase of 2026 Notes | | (843.1) | | | — | |\n| Payment for 2032 capped call options | | (102.0) | | | — | |\n| Payment of withholding taxes related to net share settlement of restricted stock units | | (107.4) | | | (23.8) | |\n| Principal payments on term loans | | (5.1) | | | (2.9) | |\n| Payment for financing costs related to revolving credit facility | | (2.0) | | | — | |\n| Payment for conversions of convertible notes | | (0.1) | | | — | |\n| Payment of acquisition related holdback | | — | | | (1.0) | |\n| Net cash provided by financing activities | | 251.6 | | | 56.9 | |\n| Increase in cash and cash equivalents | | 137.0 | | | 43.0 | |\n| Cash and cash equivalents at beginning of period | | 520.7 | | | 436.7 | |\n| Cash and cash equivalents at end of period | $ | 657.7 | | $ | 479.7 | |\n| Supplemental disclosure of cash flow information: | | | | | | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS(in millions)(Unaudited) (table). Columns: Six Months Ended.","line_start":103,"line_end":151}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_77","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)"],"score":0.007317073170731707,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)\nSummary: Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: Fiscal Years.\n\n| Fiscal Years | | |\n| --- | --- | --- |\n| Remainder of 2026 | $ | 67.3 |\n| 2027 | | 123.6 |\n| 2028 | | 83.0 |\n| 2029 | | 52.6 |\n| 2030 | | 46.5 |\n| Thereafter | | 21.2 |\n| Total future amortization | $ | 394.2 |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) (table). Columns: Fiscal Years.","line_start":588,"line_end":596}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_195","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Unrecognized Tax Benefits"],"score":0.007228915662650603,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > U","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Unrecognized Tax Benefits\n\non december 19, 2025, the company entered into a credit agreement providing for a senior secured revolving credit facility in an aggregate principal amount of $ 400. 0 million, including a $ 23. 0 million sublimit for the issuance of letters of credit. as of december 27, 2025, there were no borrowings outstanding under the revolving credit facility. for additional information regarding the credit agreement, refer to “ note 9. debt ”, in the condensed consolidated financial statements included in part 1, item 1 of this quarterly report on form 10 - q. for additional information, refer to part ii item 1a “ risk factors ”.\n\nAs of December 27, 2025 and June 28, 2025, our other non-current liabilities include unrecognized tax benefit for uncertain tax positions of $60.4 million and $55.6 million, respectively. We are unable to reliably estimate the timing of future payments related to uncertain tax positions.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1256,"line_end":1256}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_63","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets","Other Current Liabilities"],"score":0.0067415730337078645,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets > Other Current Liabilities\n\nin connection with the purchase of land and building in sagamihara, japan in july 2024, we terminated our leases for the related facilities and recorded a $ 16. 3 million increase in the carrying value of building purchased, as a result of derecognizing $ 32. 0 million of net operating lease right - of - use asset, $ 1. 6 million of operating lease liabilities, current, and $ 14. 1 million of operating lease liabilities, non - current.\n\nThe components of other current liabilities were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":511,"line_end":511}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_60","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets"],"score":0.006666666666666667,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets\n\nbuilding. the total carrying value of assets purchased was $ 58. 5 million at the purchase date, of which $ 33. 4 million was allocated to the land and $ 25. 1 million to the building. in addition, in connection with the sale of our brazilian entities, we recorded a gain on sale of approximately $ 1. 6 million recorded in selling, general and administrative expenses in our condensed consolidated statements of operations during the six months ended december 27, 2025. during the three and six months ended december 27, 2025, we recorded depreciation expense of $ 30. 6 million and $ 58. 4 million, respectively.\n\nOperating lease right-of-use assets, net were as follows (in millions):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":499,"line_end":499}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_64","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Operating Lease Right-of-Use Assets","Other Current Liabilities"],"score":0.006382978723404255,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Operating Lease Right-of-Use Assets > Other Current Liabilities\nSummary: Other Current Liabilities (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | December 27, 2025 | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- |\n| Restructuring accrual and related charges (1) | $ | 2.2 | | $ | 2.5 | |\n| Warranty reserve (2) | | 22.7 | | | 14.4 | |\n| Deferred revenue and customer deposits | | 2.1 | | | 0.7 | |\n| Income tax payable (3) | | 12.2 | | | 29.1 | |\n| Other current liabilities | | 3.6 | | | 6.4 | |\n| Other current liabilities | $ | 42.8 | | $ | 53.1 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Other Current Liabilities (table). Columns: December 27, 2025, June 28, 2025.","line_start":513,"line_end":520}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_196","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Financial Condition","Cash Flows"],"score":0.00631578947368421,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > C","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Financial Condition > Cash Flows\n\nunder the revolving credit facility. for additional information regarding the credit agreement, refer to “ note 9. debt ”, in the condensed consolidated financial statements included in part 1, item 1 of this quarterly report on form 10 - q. for additional information, refer to part ii item 1a “ risk factors ”. as of december 27, 2025 and june 28, 2025, our other non - current liabilities include unrecognized tax benefit for uncertain tax positions of $ 60. 4 million and $ 55. 6 million, respectively. we are unable to reliably estimate the timing of future payments related to uncertain tax positions.\n\nOur balance of cash and cash equivalents increased by $137.0 million from $520.7 million as of June 28, 2025 to $657.7 million as of December 27, 2025. The increase in cash and cash equivalents during the six months ended December 27, 2025 was due to cash from operating activities of $184.6 million and cash from financing activities of $251.6 million, offset by cash used in investing activities of $299.2 million.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1260,"line_end":1260}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_172","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS","Forward-Looking Statements","Overview","Results of Operations","Revenue by Region"],"score":0.0062499999999999995,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations >","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 2. MANAGEMENT’S DISCUSSION AND ANALYSIS OF FINANCIAL CONDITION AND RESULTS OF OPERATIONS > Forward-Looking Statements > Overview > Results of Operations > Revenue by Region\n\n##iver lines. we also continued the initial phase of optical circuit switch shipments, which contributed more than $ 10. 0 million of revenue during the six months ended december 27, 2025, and we remain on track for manufacturing expansion over the coming quarters to support future growth. during the three months ended december 27, 2025, two customers individually accounted for 24 % and 17 % of our total revenue, respectively. during the six months ended december 27, 2025, two customers individually accounted for 23 % and 19 % of our total net revenue, respectively. we had no other customers that represented 10 % or greater of our total net revenue.\n\nWe operate in three geographic regions: Americas, Asia-Pacific, and EMEA (Europe, Middle East, and Africa). Net revenue is assigned to the geographic region and country where our product is initially shipped. For example, certain customers may request shipment of our product to a contract manufacturer in one country, which may differ from the location of their end customers. The following table presents net revenue by the three geographic regions we operate in and net revenue from countries that generally represented 10% or more of our total net revenue based on customer shipping locations (in millions, except percentage data):","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":1119,"line_end":1119}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_79","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 9. Debt"],"score":0.006185567010309278,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 9. Debt\nSummary: Note 9. Debt (table). Columns: December 27, 2025, June 28, 2025.\n\n| | | | | | | | | | December 27, 2025 | | | | | | | | | June 28, 2025 |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | Short-term | | | Long-term | | | Total | | | Short-term | | | Long-term | | | Total |\n| Convertible notes (1) | $ | 3182.5 | | $ | — | | $ | 3182.5 | | $ | — | | $ | 2506.2 | | $ | 2506.2 | |\n| Term loans | | 57.7 | | | 47.1 | | | 104.8 | | | 10.6 | | | 56.4 | | | 67.0 | |\n| Total | $ | 3240.2 | | $ | 47.1 | | $ | 3287.3 | | $ | 10.6 | | $ | 2562.6 | | $ | 2573.2 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Note 9. Debt (table). Columns: December 27, 2025, June 28, 2025.","line_start":604,"line_end":609}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_111","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited)","Note 11. Restructuring and Related Charges (Reversals)"],"score":0.006122448979591836,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLID","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.NOTES TO CONDENSED CONSOLIDATED FINANCIAL STATEMENTS (Continued) (Unaudited) > Note 11. Restructuring and Related Charges (Reversals)\nSummary: Note 11. Restructuring and Related Charges (Reversals) (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Balance as of beginning of period | $ | 5.9 | | $ | 6.3 | | $ | 2.5 | | $ | 11.1 | |\n| Charges (reversals) | | (0.4) | | | 0.7 | | | 7.9 | | | 10.4 | |\n| Payments and other adjustments | | (3.3) | | | (5.5) | | | (8.2) | | | (20.0) | |\n| Balance as of end of period | $ | 2.2 | | $ | 1.5 | | $ | 2.2 | | $ | 1.5 | |","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"summary":"Note 11. Restructuring and Related Charges (Reversals) (table). Columns: Three Months Ended, Six Months Ended.","line_start":770,"line_end":776}},{"chunk_id":"e5a1040e-1347-4336-85bb-e927d853e5bd_4","doc_id":"e5a1040e-1347-4336-85bb-e927d853e5bd","page_no":null,"headings":["PART I - FINANCIAL INFORMATION","ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)","CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS","Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)"],"score":0.0060606060606060615,"source":"/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/exp__chunk_1024_o128_tokenizer__ctx_none__index_m24_ef200/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md","preview":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STAT","text":"Company: Lumentum Holdings Inc.\nTicker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS > Table of ContentsLUMENTUM HOLDINGS INC.CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (LOSS)(in millions)(Unaudited)\n\nSee accompanying Notes to Condensed Consolidated Financial Statements.","context":null,"metadata":{"doc":{"company":"Lumentum Holdings Inc.","ticker":"LITE","cik":"0001628280","filing_type":"10-Q","filing_date":"2026-02-04","filing_quarter":"2026Q1","filing_quarter_basis":"filing_date"},"line_start":55,"line_end":55}}],"timing_ms":{"total_ms":20601.025926414877},"error":null} diff --git a/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/run_config.json b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/run_config.json new file mode 100644 index 0000000..9cb560f --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/eval_run.lite_isolated_timeout350.20260218_220015/run_config.json @@ -0,0 +1,18 @@ +{ + "mode": "normal", + "top_k_retrieve": null, + "top_k_rerank": null, + "draft_max_tokens": null, + "final_max_tokens": null, + "brief_max_tokens": null, + "enable_rerank": null, + "enable_refine": null, + "answer_style": null, + "answering_effort": null, + "draft_temperature": null, + "concurrency": 1, + "parallel_backend": "thread", + "max_chunks": 50, + "query_timeout_s": 350.0, + "query_max_retries": 0 +} \ No newline at end of file diff --git a/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl new file mode 100644 index 0000000..f3c0c5e --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl @@ -0,0 +1 @@ +{"id": "1dd6251b-e62b-4e58-ae52-35a1253e14c3", "question": "What was LITE's net income in its 10-Q filed 2026-02-04?", "kind": "factual", "tags": ["factual", "sec", "LITE", "10-Q", "net income"], "created_at": "2026-02-16T20:30:23.567307Z", "factual": {"metric": "net income", "expected_numeric": {"value": 78.2, "unit": "USD", "scale": null, "raw": " 78.2 "}, "golden_evidence": {"doc_id": "LITE_000162828026005129_10-Q_2026-02-04", "chunk_id": "LITE_000162828026005129_10-Q_2026-02-04_1", "source": "/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/eval_revamp_combined_512_20260217/sec_filings_md_secparser/processed_markdown/LITE_000162828026005129_10-Q_2026-02-04.md", "headings": ["PART I - FINANCIAL INFORMATION", "ITEM 1. FINANCIAL STATEMENTS (UNAUDITED)", "CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS"], "page_no": null, "section_path": "PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS", "snippet": "Ticker: LITE\nFiling: 10-Q, filed 2026-02-04\nFiling quarter: 2026Q1\nSection: PART I - FINANCIAL INFORMATION > ITEM 1. FINANCIAL STATEMENTS (UNAUDITED) > CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS\nSummary: CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (table). Columns: Three Months Ended, Six Months Ended.\n\n| | | | | | | Three Months Ended | | | | | | Six Months Ended |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| | | | December 27, 2025 | | | December 28, 2024 | | | December 27, 2025 | | | December 28, 2024 |\n| Net revenue | $ | 665.5 | | $ | 402.2 | | $ | 1199.3 | | $ | 739.1 | |\n| Cost of sales | | 405.8 | | | 281.2 | | | 738.6 | | | 517.7 | |\n| Amortization of acquired developed intangibles | | 19.6 | | | 21.4 | | | 39.1 | | | 43.9 | |\n| Gross profit | | 240.1 | | | 99.6 | | | 421.6 | | | 177.5 | |\n| Operating expenses: | | | | | | | | | | | | |\n| Research and development | | 80.1 | | | 74.2 | | | 161.5 | | | 148.5 | |\n| Selling, general and administrative | | 96.1 | | | 76.3 | | | 181.2 | | | 152.6 | |\n| Restructuring and related charges (reversals) | | (0.4) | | | 0.7 | | | 7.9 | | | 10.4 | |\n| Total operating expenses | | 175.8 | | | 151.2 | | | 350.6 | | | 311.5 | |\n| Income (loss) from operations | | 64.3 | | | (51.6) | | | 71.0 | | | (134.0) | |\n| Other income (expense), net: | | | | | | | | | | | | |\n| Escrow settlement | | 27.5 | | | — | | | 27.5 | | | — | |\n| Interest expense | | (6.3) | | | (5.6) | | | (12.0) | | | (11.1) | |\n| Other income, net | | 11.0 | | | 14.9 | | | 15.2 | | | 23.6 | |\n| Total other income, net | | 32.2 | | | 9.3 | | | 30.7 | | | 12.5 | |\n| Income (loss) before income taxes | | 96.5 | | | (42.3) | | | 101.7 | | | (121.5) | |\n| Income tax provision | | 18.3 | | | 18.6 | | | 19.3 | | | 21.8 | |\n| Net income (loss) | $ | 78.2 | | $ | (60.9) | | $ | 82.4 | | $ | (143.3) | |\n| Net income (loss) per share: | | | | | | | | | | | | |\n| Basic | $ | 1.10 | | $ | (0.88) | | $ | 1.17 | | $ | (2.09) | |\n| Diluted | $ | 0.89 | | $ | (0.88) | | $ | 0.99 | | $ | (2.09) | |\n| Shares used to compute net income (loss) per share: | | | | | | | | | | | | |\n| Basic | | 71.1 | | | 68.9 | | | 70.7 | | | 68.6 | |\n| Diluted | | 87.8 | | | 68.9 | | | 83.1 | | | 68.6 | |", "metadata": {}}}, "open_ended": null, "refusal": null, "distractor": null, "comparison": null, "generator": {"source": "chunk_exports", "seed": 20260217, "edgar_validation": {"status": "matched", "metric": "net income", "ticker": "LITE", "candidate_keys": ["net_income"], "rel_tol": 0.5, "best_rel_error": 0.0, "best_expected_scale": "millions", "best_expected_value": 78200000.0}}} diff --git a/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json b/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json new file mode 100644 index 0000000..9b9a408 --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json @@ -0,0 +1,170 @@ +{ + "n_rows_total": 300, + "n_labeled": 300, + "positive_count": 125, + "positive_rate": 0.4166666666666667, + "by_kind": { + "comparison": { + "n": 55, + "n_positive": 44, + "positive_rate": 0.8 + }, + "distractor": { + "n": 15, + "n_positive": 9, + "positive_rate": 0.6 + }, + "factual": { + "n": 140, + "n_positive": 19, + "positive_rate": 0.1357142857142857 + }, + "open_ended": { + "n": 90, + "n_positive": 53, + "positive_rate": 0.5888888888888889 + } + }, + "by_run": { + "eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838": { + "n": 55, + "n_positive": 44, + "positive_rate": 0.8 + }, + "eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301": { + "n": 76, + "n_positive": 45, + "positive_rate": 0.5921052631578947 + }, + "eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034": { + "n": 169, + "n_positive": 36, + "positive_rate": 0.21301775147928995 + } + }, + "by_membership": { + "both": { + "n": 195, + "n_positive": 74, + "positive_rate": 0.37948717948717947 + }, + "pre_only": { + "n": 53, + "n_positive": 26, + "positive_rate": 0.49056603773584906 + }, + "post_only": { + "n": 52, + "n_positive": 25, + "positive_rate": 0.4807692307692308 + }, + "neither": { + "n": 0, + "n_positive": 0, + "positive_rate": 0.0 + } + }, + "relevant_rank_movement_both": { + "n": 74, + "promoted": 32, + "demoted": 37, + "same_rank": 5, + "avg_delta_rank_post_minus_pre": 0.06756756756756757 + }, + "relevant_rank_movement_both_by_kind": { + "comparison": { + "n": 17, + "promoted": 6, + "demoted": 10, + "same_rank": 1, + "avg_delta_rank_post_minus_pre": 3.411764705882353 + }, + "distractor": { + "n": 4, + "promoted": 3, + "demoted": 1, + "same_rank": 0, + "avg_delta_rank_post_minus_pre": -6.25 + }, + "factual": { + "n": 15, + "promoted": 4, + "demoted": 9, + "same_rank": 2, + "avg_delta_rank_post_minus_pre": 2.8 + }, + "open_ended": { + "n": 38, + "promoted": 19, + "demoted": 17, + "same_rank": 2, + "avg_delta_rank_post_minus_pre": -1.8421052631578947 + } + }, + "topk_relevance_rate_sample": { + "pre": { + "k1": { + "n": 15, + "n_positive": 11, + "positive_rate": 0.7333333333333333 + }, + "k3": { + "n": 34, + "n_positive": 24, + "positive_rate": 0.7058823529411765 + }, + "k5": { + "n": 63, + "n_positive": 40, + "positive_rate": 0.6349206349206349 + }, + "k10": { + "n": 128, + "n_positive": 60, + "positive_rate": 0.46875 + }, + "k25": { + "n": 233, + "n_positive": 95, + "positive_rate": 0.40772532188841204 + } + }, + "post": { + "k1": { + "n": 18, + "n_positive": 11, + "positive_rate": 0.6111111111111112 + }, + "k3": { + "n": 36, + "n_positive": 23, + "positive_rate": 0.6388888888888888 + }, + "k5": { + "n": 64, + "n_positive": 37, + "positive_rate": 0.578125 + }, + "k10": { + "n": 134, + "n_positive": 71, + "positive_rate": 0.5298507462686567 + }, + "k25": { + "n": 247, + "n_positive": 99, + "positive_rate": 0.4008097165991903 + } + } + }, + "weak_label_alignment_subset": { + "n": 140, + "tp": 19, + "fp": 121, + "tn": 0, + "fn": 0, + "accuracy": 0.1357142857142857, + "precision_1": 0.1357142857142857, + "recall_1": 1.0 + } +} diff --git a/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md b/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md new file mode 100644 index 0000000..f42a36a --- /dev/null +++ b/agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md @@ -0,0 +1,57 @@ +# Retrieval Manual Sample (300) Summary + +- source: `eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.codex_manual.csv` +- n_rows_total: `300` +- n_labeled: `300` +- positive_rate: `0.4167` + +## By Kind + +| kind | n | n_positive | positive_rate | +|---|---:|---:|---:| +| comparison | 55 | 44 | 0.8000 | +| distractor | 15 | 9 | 0.6000 | +| factual | 140 | 19 | 0.1357 | +| open_ended | 90 | 53 | 0.5889 | + +## Membership + +| bucket | n | n_positive | positive_rate | +|---|---:|---:|---:| +| both | 195 | 74 | 0.3795 | +| pre_only | 53 | 26 | 0.4906 | +| post_only | 52 | 25 | 0.4808 | +| neither | 0 | 0 | 0.0000 | + +## Relevant Rank Movement (Rows Present in Pre and Post) + +- n: `74`, promoted: `32`, demoted: `37`, same: `5`, avg_delta(post-pre): `0.0676` + +### By Kind + +| kind | n | promoted | demoted | same | avg_delta(post-pre) | +|---|---:|---:|---:|---:|---:| +| comparison | 17 | 6 | 10 | 1 | 3.4118 | +| distractor | 4 | 3 | 1 | 0 | -6.2500 | +| factual | 15 | 4 | 9 | 2 | 2.8000 | +| open_ended | 38 | 19 | 17 | 2 | -1.8421 | + +## Top-k Relevance Rate (Sample-based) + +| phase | k | n | n_positive | positive_rate | +|---|---:|---:|---:|---:| +| pre | 1 | 15 | 11 | 0.7333 | +| pre | 3 | 34 | 24 | 0.7059 | +| pre | 5 | 63 | 40 | 0.6349 | +| pre | 10 | 128 | 60 | 0.4688 | +| pre | 25 | 233 | 95 | 0.4077 | +| post | 1 | 18 | 11 | 0.6111 | +| post | 3 | 36 | 23 | 0.6389 | +| post | 5 | 64 | 37 | 0.5781 | +| post | 10 | 134 | 71 | 0.5299 | +| post | 25 | 247 | 99 | 0.4008 | + +## Weak Label Alignment (subset with weak labels) + +- n: `140`, tp: `19`, fp: `121`, tn: `0`, fn: `0`, accuracy: `0.1357`, precision_1: `0.1357`, recall_1: `1.0000` + diff --git a/agent_logs/scripts/eval/20260218_191900_run_reduced_heuristics_full_suite.sh b/agent_logs/scripts/eval/20260218_191900_run_reduced_heuristics_full_suite.sh new file mode 100755 index 0000000..110da27 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_191900_run_reduced_heuristics_full_suite.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export MODE=normal +export RUN_PREFIX=reduced_heuristics_full +export GEN_WORKERS=12 +export JUDGE_WORKERS=12 +export QUERY_TIMEOUT_S=350 +export QUERY_MAX_RETRIES=1 +export JUDGE_CONTEXT_CHARS=80000 +export JUDGE_TIMEOUT_S=350 +export JUDGE_MAX_RETRIES=1 +export PARALLEL_BACKEND=thread +bash scripts/run_full_eval_suite.sh diff --git a/agent_logs/scripts/eval/20260218_192100_run_reduced_heuristics_full_suite_retry1.sh b/agent_logs/scripts/eval/20260218_192100_run_reduced_heuristics_full_suite_retry1.sh new file mode 100755 index 0000000..c7097bd --- /dev/null +++ b/agent_logs/scripts/eval/20260218_192100_run_reduced_heuristics_full_suite_retry1.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export MODE=normal +export RUN_PREFIX=reduced_heuristics_full_retry1 +export GEN_WORKERS=12 +export JUDGE_WORKERS=12 +export QUERY_TIMEOUT_S=350 +export QUERY_MAX_RETRIES=1 +export JUDGE_CONTEXT_CHARS=80000 +export JUDGE_TIMEOUT_S=350 +export JUDGE_MAX_RETRIES=1 +export PARALLEL_BACKEND=thread +bash scripts/run_full_eval_suite.sh diff --git a/agent_logs/scripts/eval/20260218_193600_run_reduced_heuristics_full_suite_retry2.sh b/agent_logs/scripts/eval/20260218_193600_run_reduced_heuristics_full_suite_retry2.sh new file mode 100755 index 0000000..4d31b96 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_193600_run_reduced_heuristics_full_suite_retry2.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export MODE=normal +export RUN_PREFIX=reduced_heuristics_full_retry2 +export GEN_WORKERS=12 +export JUDGE_WORKERS=12 +export QUERY_TIMEOUT_S=350 +export QUERY_MAX_RETRIES=1 +export JUDGE_CONTEXT_CHARS=80000 +export JUDGE_TIMEOUT_S=350 +export JUDGE_MAX_RETRIES=1 +export PARALLEL_BACKEND=thread +bash scripts/run_full_eval_suite.sh diff --git a/agent_logs/scripts/eval/20260218_195700_run_reduced_heuristics_full_suite_retry3_pinned_schema.sh b/agent_logs/scripts/eval/20260218_195700_run_reduced_heuristics_full_suite_retry3_pinned_schema.sh new file mode 100755 index 0000000..4bfb550 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_195700_run_reduced_heuristics_full_suite_retry3_pinned_schema.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export POSTGRES_SCHEMA=eval_revamp_combined_512_20260217 +export FINRAG_DOC_INDEX_PATH="$(pwd)/data/ingest_profiles/eval_revamp_combined_512_20260217/sec_filings_md_secparser/chunked_512_64/doc_index.jsonl" +export MODE=normal +export RUN_PREFIX=reduced_heuristics_full_retry3_pinned +export GEN_WORKERS=12 +export JUDGE_WORKERS=12 +export QUERY_TIMEOUT_S=350 +export QUERY_MAX_RETRIES=1 +export JUDGE_CONTEXT_CHARS=80000 +export JUDGE_TIMEOUT_S=350 +export JUDGE_MAX_RETRIES=1 +export PARALLEL_BACKEND=thread +bash scripts/run_full_eval_suite.sh diff --git a/agent_logs/scripts/eval/20260218_200300_reduced_heuristics_eval_override.env b/agent_logs/scripts/eval/20260218_200300_reduced_heuristics_eval_override.env new file mode 100644 index 0000000..1d2fa6b --- /dev/null +++ b/agent_logs/scripts/eval/20260218_200300_reduced_heuristics_eval_override.env @@ -0,0 +1,4 @@ +# shellcheck shell=bash +source /home/mlin/repos/z_scratch/financial-rag/.env +POSTGRES_SCHEMA=eval_revamp_combined_512_20260217 +FINRAG_DOC_INDEX_PATH=/home/mlin/repos/z_scratch/financial-rag/data/ingest_profiles/eval_revamp_combined_512_20260217/sec_filings_md_secparser/chunked_512_64/doc_index.jsonl diff --git a/agent_logs/scripts/eval/20260218_200300_run_reduced_heuristics_full_suite_retry4_env_override.sh b/agent_logs/scripts/eval/20260218_200300_run_reduced_heuristics_full_suite_retry4_env_override.sh new file mode 100755 index 0000000..754e0ff --- /dev/null +++ b/agent_logs/scripts/eval/20260218_200300_run_reduced_heuristics_full_suite_retry4_env_override.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export ENV_FILE=/home/mlin/repos/z_scratch/financial-rag/agent_logs/scripts/eval/20260218_200300_reduced_heuristics_eval_override.env +export MODE=normal +export RUN_PREFIX=reduced_heuristics_full_retry4_envoverride +export GEN_WORKERS=12 +export JUDGE_WORKERS=12 +export QUERY_TIMEOUT_S=350 +export QUERY_MAX_RETRIES=1 +export JUDGE_CONTEXT_CHARS=80000 +export JUDGE_TIMEOUT_S=350 +export JUDGE_MAX_RETRIES=1 +export PARALLEL_BACKEND=thread +bash scripts/run_full_eval_suite.sh diff --git a/agent_logs/scripts/eval/20260218_205200_build_reduced_heuristics_judge_audit.sh b/agent_logs/scripts/eval/20260218_205200_build_reduced_heuristics_judge_audit.sh new file mode 100755 index 0000000..3ece067 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_205200_build_reduced_heuristics_judge_audit.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate + +RUN_SINGLE="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034" +RUN_MULTI="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838" +RUN_OPEN="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301" + +AUDIT_DIR="eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.judge_audit" +mkdir -p "$AUDIT_DIR" + +python -m scripts.judge_reliability build-audit \ + --run-dirs "$RUN_SINGLE" "$RUN_MULTI" "$RUN_OPEN" \ + --out-csv "$AUDIT_DIR/decision_audit.raw.csv" + +python -m scripts.audit_judge_decisions \ + --audit-csv "$AUDIT_DIR/decision_audit.raw.csv" \ + --out-csv "$AUDIT_DIR/decision_audit.audited.csv" \ + --workers 12 \ + --context-chars 80000 \ + --timeout-s 350 \ + --max-retries 1 \ + --overwrite + +python -m scripts.judge_reliability evaluate \ + --audit-csv "$AUDIT_DIR/decision_audit.audited.csv" \ + --out-json "$AUDIT_DIR/judge_reliability_report.json" \ + --dev-fraction 0.75 \ + --seed 42 \ + --n-bootstrap 2000 \ + --write-split diff --git a/agent_logs/scripts/eval/20260218_210000_build_reduced_heuristics_judge_audit_fullcoverage.sh b/agent_logs/scripts/eval/20260218_210000_build_reduced_heuristics_judge_audit_fullcoverage.sh new file mode 100755 index 0000000..ce58e13 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_210000_build_reduced_heuristics_judge_audit_fullcoverage.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate + +RUN_SINGLE="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034" +RUN_MULTI="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838" +RUN_OPEN="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301" + +AUDIT_DIR="eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.judge_audit_full" +mkdir -p "$AUDIT_DIR" + +python -m scripts.judge_reliability build-audit \ + --run-dirs "$RUN_SINGLE" "$RUN_MULTI" "$RUN_OPEN" \ + --judges faithfulness_v1 factual_correctness_v1 helpfulness_v1 focus_v1 comparison_v1 refusal_v1 \ + --disable-kind-filter \ + --out-csv "$AUDIT_DIR/decision_audit.raw.csv" + +python -m scripts.audit_judge_decisions \ + --audit-csv "$AUDIT_DIR/decision_audit.raw.csv" \ + --out-csv "$AUDIT_DIR/decision_audit.audited.csv" \ + --workers 12 \ + --context-chars 80000 \ + --timeout-s 350 \ + --max-retries 1 \ + --overwrite + +python -m scripts.judge_reliability evaluate \ + --audit-csv "$AUDIT_DIR/decision_audit.audited.csv" \ + --out-json "$AUDIT_DIR/judge_reliability_report.json" \ + --dev-fraction 0.75 \ + --seed 42 \ + --n-bootstrap 2000 \ + --write-split diff --git a/agent_logs/scripts/eval/20260218_211000_run_retrieval_pool_and_metrics.sh b/agent_logs/scripts/eval/20260218_211000_run_retrieval_pool_and_metrics.sh new file mode 100755 index 0000000..d163f3a --- /dev/null +++ b/agent_logs/scripts/eval/20260218_211000_run_retrieval_pool_and_metrics.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate + +export HF_HOME="/tmp/hf_home" +export TRANSFORMERS_CACHE="/tmp/hf_home/transformers" +export HUGGINGFACE_HUB_CACHE="/tmp/hf_home/hub" +mkdir -p "$HF_HOME" "$TRANSFORMERS_CACHE" "$HUGGINGFACE_HUB_CACHE" + +RUN_SINGLE="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034" +RUN_MULTI="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838" +RUN_OPEN="eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.open200.normal.tools12.norefine.20260218_202301" + +NLI_MODEL="cross-encoder/nli-distilroberta-base" + +# Retrieval/rerank metrics + NLI support snapshot for single100. +python scripts/eval_retrieval.py \ + --run-dir "$RUN_SINGLE" \ + --enable-nli \ + --nli-model "$NLI_MODEL" \ + --nli-max-open-ended 30 \ + --nli-batch-size 128 + +# Open-ended NLI support snapshot at scale (bounded sample for runtime). +python scripts/eval_retrieval.py \ + --run-dir "$RUN_OPEN" \ + --enable-nli \ + --nli-model "$NLI_MODEL" \ + --nli-max-open-ended 120 \ + --nli-batch-size 128 + +# Build pooled chunk labels for manual retrieval/rerank relevance audit. +python scripts/build_retrieval_label_pool.py \ + --run-dirs "$RUN_SINGLE" "$RUN_MULTI" "$RUN_OPEN" \ + --out-csv eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.retrieval_pool.csv \ + --out-json eval/results_revamp/full_suite/reduced_heuristics_full_retry4_envoverride_20260218_195034.retrieval_pool.stats.json \ + --pre-k 30 \ + --post-k 25 \ + --kinds factual open_ended comparison distractor diff --git a/agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh b/agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh new file mode 100755 index 0000000..a824347 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_215100_eval_retrieval_multi60.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +export HF_HOME=/tmp/hf_home +export TRANSFORMERS_CACHE=/tmp/hf_home/transformers +export HUGGINGFACE_HUB_CACHE=/tmp/hf_home/hub +mkdir -p "$HF_HOME" "$TRANSFORMERS_CACHE" "$HUGGINGFACE_HUB_CACHE" +python scripts/eval_retrieval.py \ + --run-dir eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.multi60.normal.tools12.norefine.20260218_200838 \ + --enable-nli \ + --nli-model cross-encoder/nli-distilroberta-base \ + --nli-max-open-ended 120 \ + --nli-support-threshold 0.5 \ + --nli-contradiction-threshold 0.5 \ + --nli-batch-size 256 diff --git a/agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh b/agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh new file mode 100755 index 0000000..2d4e2cc --- /dev/null +++ b/agent_logs/scripts/eval/20260218_215700_summarize_retrieval_manual_sample.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +mkdir -p agent_logs/reports/retrieval_eval_20260218 +python - <<'PY' +from __future__ import annotations +import csv +import json +from collections import Counter, defaultdict +from pathlib import Path + +in_path = Path('eval/results_revamp/full_suite/reduced_heuristics_full_retry4_retrieval_pool.sample300.codex_manual.csv') +out_json = Path('agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.json') +out_md = Path('agent_logs/reports/retrieval_eval_20260218/manual_sample300_summary.md') + +rows = [] +with in_path.open('r', encoding='utf-8', newline='') as f: + for row in csv.DictReader(f): + rows.append(row) + +def to_i(v: str) -> int | None: + t = (v or '').strip() + if t in {'0','1'}: + return int(t) + return None + +def to_rank(v: str) -> int | None: + t = (v or '').strip() + if not t: + return None + try: + return int(float(t)) + except Exception: + return None + +labeled = [] +for r in rows: + hr = to_i(r.get('human_relevance', '')) + if hr is None: + continue + r2 = dict(r) + r2['human_relevance_i'] = hr + r2['in_pre_i'] = to_i(r.get('in_pre','')) or 0 + r2['in_post_i'] = to_i(r.get('in_post','')) or 0 + r2['rank_pre_i'] = to_rank(r.get('rank_pre','')) + r2['rank_post_i'] = to_rank(r.get('rank_post','')) + labeled.append(r2) + +summary: dict[str, object] = {} +summary['n_rows_total'] = len(rows) +summary['n_labeled'] = len(labeled) +summary['positive_count'] = sum(r['human_relevance_i'] for r in labeled) +summary['positive_rate'] = (summary['positive_count'] / len(labeled)) if labeled else 0.0 + +# By kind +kind_counts = Counter(r.get('kind','') for r in labeled) +kind_pos = Counter() +for r in labeled: + if r['human_relevance_i'] == 1: + kind_pos[r.get('kind','')] += 1 +summary['by_kind'] = { + k: { + 'n': kind_counts[k], + 'n_positive': kind_pos[k], + 'positive_rate': (kind_pos[k] / kind_counts[k]) if kind_counts[k] else 0.0, + } + for k in sorted(kind_counts) +} + +# By run +run_counts = Counter(r.get('run_name','') for r in labeled) +run_pos = Counter() +for r in labeled: + if r['human_relevance_i'] == 1: + run_pos[r.get('run_name','')] += 1 +summary['by_run'] = { + k: { + 'n': run_counts[k], + 'n_positive': run_pos[k], + 'positive_rate': (run_pos[k] / run_counts[k]) if run_counts[k] else 0.0, + } + for k in sorted(run_counts) +} + +# Membership (pre/post) +membership_map = { + (1,1): 'both', + (1,0): 'pre_only', + (0,1): 'post_only', + (0,0): 'neither', +} +member_counts = Counter() +member_pos = Counter() +for r in labeled: + key = membership_map[(r['in_pre_i'], r['in_post_i'])] + member_counts[key] += 1 + if r['human_relevance_i'] == 1: + member_pos[key] += 1 +summary['by_membership'] = { + k: { + 'n': member_counts[k], + 'n_positive': member_pos[k], + 'positive_rate': (member_pos[k] / member_counts[k]) if member_counts[k] else 0.0, + } + for k in ['both', 'pre_only', 'post_only', 'neither'] +} + +# Rank movement on relevant rows present in both +relevant_both = [ + r for r in labeled + if r['human_relevance_i'] == 1 and r['rank_pre_i'] is not None and r['rank_post_i'] is not None +] +promoted = 0 +same = 0 +demoted = 0 +deltas = [] +for r in relevant_both: + delta = r['rank_post_i'] - r['rank_pre_i'] + deltas.append(delta) + if delta < 0: + promoted += 1 + elif delta > 0: + demoted += 1 + else: + same += 1 +summary['relevant_rank_movement_both'] = { + 'n': len(relevant_both), + 'promoted': promoted, + 'demoted': demoted, + 'same_rank': same, + 'avg_delta_rank_post_minus_pre': (sum(deltas)/len(deltas)) if deltas else 0.0, +} + +# Rank movement by kind (relevant rows present in both) +movement_by_kind = {} +for kind in sorted({r.get('kind', '') for r in relevant_both}): + sub = [r for r in relevant_both if r.get('kind', '') == kind] + k_promoted = 0 + k_same = 0 + k_demoted = 0 + k_deltas = [] + for r in sub: + delta = r['rank_post_i'] - r['rank_pre_i'] + k_deltas.append(delta) + if delta < 0: + k_promoted += 1 + elif delta > 0: + k_demoted += 1 + else: + k_same += 1 + movement_by_kind[kind] = { + 'n': len(sub), + 'promoted': k_promoted, + 'demoted': k_demoted, + 'same_rank': k_same, + 'avg_delta_rank_post_minus_pre': (sum(k_deltas) / len(k_deltas)) if k_deltas else 0.0, + } +summary['relevant_rank_movement_both_by_kind'] = movement_by_kind + +# Top-k relevance slices (sample-estimate) + +def topk_slice(phase: str, k: int) -> tuple[int,int,float]: + assert phase in {'pre','post'} + key = 'rank_pre_i' if phase == 'pre' else 'rank_post_i' + sub = [r for r in labeled if r[key] is not None and r[key] <= k] + n = len(sub) + pos = sum(r['human_relevance_i'] for r in sub) + rate = (pos / n) if n else 0.0 + return n, pos, rate + +summary['topk_relevance_rate_sample'] = {'pre': {}, 'post': {}} +for phase in ['pre','post']: + for k in [1,3,5,10,25]: + n, pos, rate = topk_slice(phase, k) + summary['topk_relevance_rate_sample'][phase][f'k{k}'] = { + 'n': n, + 'n_positive': pos, + 'positive_rate': rate, + } + +# Weak-label coverage and confusion at threshold 0.5 +usable = [] +for r in labeled: + w = (r.get('weak_relevance','') or '').strip() + try: + wf = float(w) + except Exception: + continue + pred = 1 if wf >= 0.5 else 0 + usable.append((r['human_relevance_i'], pred)) + +tp = sum(1 for y,p in usable if y==1 and p==1) +fp = sum(1 for y,p in usable if y==0 and p==1) +tn = sum(1 for y,p in usable if y==0 and p==0) +fn = sum(1 for y,p in usable if y==1 and p==0) +precision = tp / (tp + fp) if (tp + fp) else 0.0 +recall = tp / (tp + fn) if (tp + fn) else 0.0 +acc = (tp + tn) / len(usable) if usable else 0.0 +summary['weak_label_alignment_subset'] = { + 'n': len(usable), + 'tp': tp, + 'fp': fp, + 'tn': tn, + 'fn': fn, + 'accuracy': acc, + 'precision_1': precision, + 'recall_1': recall, +} + +out_json.write_text(json.dumps(summary, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + +lines = [] +lines.append('# Retrieval Manual Sample (300) Summary') +lines.append('') +lines.append(f"- source: `{in_path}`") +lines.append(f"- n_rows_total: `{summary['n_rows_total']}`") +lines.append(f"- n_labeled: `{summary['n_labeled']}`") +lines.append(f"- positive_rate: `{summary['positive_rate']:.4f}`") +lines.append('') +lines.append('## By Kind') +lines.append('') +lines.append('| kind | n | n_positive | positive_rate |') +lines.append('|---|---:|---:|---:|') +for kind, payload in summary['by_kind'].items(): + lines.append(f"| {kind} | {payload['n']} | {payload['n_positive']} | {payload['positive_rate']:.4f} |") +lines.append('') +lines.append('## Membership') +lines.append('') +lines.append('| bucket | n | n_positive | positive_rate |') +lines.append('|---|---:|---:|---:|') +for bucket, payload in summary['by_membership'].items(): + lines.append(f"| {bucket} | {payload['n']} | {payload['n_positive']} | {payload['positive_rate']:.4f} |") +lines.append('') +rm = summary['relevant_rank_movement_both'] +lines.append('## Relevant Rank Movement (Rows Present in Pre and Post)') +lines.append('') +lines.append( + f"- n: `{rm['n']}`, promoted: `{rm['promoted']}`, demoted: `{rm['demoted']}`, same: `{rm['same_rank']}`, avg_delta(post-pre): `{rm['avg_delta_rank_post_minus_pre']:.4f}`" +) +lines.append('') +lines.append('### By Kind') +lines.append('') +lines.append('| kind | n | promoted | demoted | same | avg_delta(post-pre) |') +lines.append('|---|---:|---:|---:|---:|---:|') +for kind, payload in summary['relevant_rank_movement_both_by_kind'].items(): + lines.append( + f"| {kind} | {payload['n']} | {payload['promoted']} | {payload['demoted']} | " + f"{payload['same_rank']} | {payload['avg_delta_rank_post_minus_pre']:.4f} |" + ) +lines.append('') +lines.append('## Top-k Relevance Rate (Sample-based)') +lines.append('') +lines.append('| phase | k | n | n_positive | positive_rate |') +lines.append('|---|---:|---:|---:|---:|') +for phase in ['pre','post']: + for k in ['k1','k3','k5','k10','k25']: + p = summary['topk_relevance_rate_sample'][phase][k] + lines.append(f"| {phase} | {k[1:]} | {p['n']} | {p['n_positive']} | {p['positive_rate']:.4f} |") +lines.append('') +wa = summary['weak_label_alignment_subset'] +lines.append('## Weak Label Alignment (subset with weak labels)') +lines.append('') +lines.append( + f"- n: `{wa['n']}`, tp: `{wa['tp']}`, fp: `{wa['fp']}`, tn: `{wa['tn']}`, fn: `{wa['fn']}`, accuracy: `{wa['accuracy']:.4f}`, precision_1: `{wa['precision_1']:.4f}`, recall_1: `{wa['recall_1']:.4f}`" +) +lines.append('') +out_md.write_text('\n'.join(lines) + '\n', encoding='utf-8') + +print('Wrote', out_json) +print('Wrote', out_md) +print(json.dumps(summary, indent=2, ensure_ascii=False)) +PY diff --git a/agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh b/agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh new file mode 100755 index 0000000..34c1af8 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_220200_probe_lite_isolated_latency.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +mkdir -p agent_logs/reports/retrieval_eval_20260218 +export PYTHONPATH=src +# hard cap to avoid indefinite hangs +/usr/bin/timeout 500s python - <<'PY' +from __future__ import annotations +import json +import time +from pathlib import Path + +from dotenv import load_dotenv + +from andromeda.llm.generation_controls import resolve_generation_settings +from andromeda.main import get_rag_service + +load_dotenv(Path('.env')) +query = "What was LITE's net income in its 10-Q filed 2026-02-04?" +settings = resolve_generation_settings(mode='normal') +service = get_rag_service() + +attempts = [] +for i in range(2): + start = time.perf_counter() + payload = { + 'attempt': i + 1, + 'query': query, + 'mode': settings.mode, + 'top_k_retrieve': settings.top_k_retrieve, + 'top_k_rerank': settings.top_k_rerank, + 'draft_max_tokens': settings.draft_max_tokens, + 'final_max_tokens': settings.final_max_tokens, + 'answering_effort': settings.answering_effort.value, + } + try: + response = service.answer_question(query, settings) + elapsed = time.perf_counter() - start + payload.update( + { + 'ok': True, + 'latency_s': elapsed, + 'tool_trace_len': len(response.tool_trace), + 'tool_results_len': len(response.tool_results), + 'top_chunks_len': len(response.top_chunks), + 'answer_preview': (response.final_answer or '')[:220], + } + ) + except Exception as exc: # pragma: no cover - runtime probe path + elapsed = time.perf_counter() - start + payload.update({'ok': False, 'latency_s': elapsed, 'error': str(exc)}) + attempts.append(payload) + +out = { + 'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'), + 'attempts': attempts, +} +out_path = Path('agent_logs/reports/retrieval_eval_20260218/lite_query_isolated_latency.json') +out_path.write_text(json.dumps(out, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') +print(json.dumps(out, indent=2, ensure_ascii=False)) +print(f'Wrote: {out_path}') +PY diff --git a/agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh b/agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh new file mode 100755 index 0000000..706fb05 --- /dev/null +++ b/agent_logs/scripts/eval/20260218_221400_probe_lite_single_eval_timeout350.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +source .venv/bin/activate +mkdir -p agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe +python - <<'PY' +from __future__ import annotations +import json +from pathlib import Path + +query_id = '1dd6251b-e62b-4e58-ae52-35a1253e14c3' +src = Path('eval/results_revamp/full_suite/eval_run.reduced_heuristics_full_retry4_envoverride_20260218_195034.single100.normal.tools12.norefine.20260218_195034/eval_queries.jsonl') +out = Path('agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl') +line = None +with src.open('r', encoding='utf-8') as handle: + for raw in handle: + raw = raw.rstrip('\n') + if not raw.strip(): + continue + item = json.loads(raw) + if item.get('id') == query_id: + line = json.dumps(item, ensure_ascii=False) + break +if line is None: + raise SystemExit(f'Query id not found: {query_id}') +out.write_text(line + '\n', encoding='utf-8') +print(f'Wrote: {out}') +PY + +python scripts/run_eval.py \ + --eval-queries agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe/lite_single_query.jsonl \ + --out-dir agent_logs/reports/retrieval_eval_20260218/lite_single_eval_probe \ + --run-name lite_isolated_timeout350 \ + --mode normal \ + --concurrency 1 \ + --parallel-backend thread \ + --query-timeout-s 350 \ + --query-max-retries 0 From 9d3a8b7fc79a9a1b88769cef4d6b1bdae57adef7 Mon Sep 17 00:00:00 2001 From: Lin Min Htoo Date: Thu, 19 Feb 2026 11:32:40 +0800 Subject: [PATCH 10/10] doc(CHANGELOG): bump version to 1.10.0 --- CHANGELOG.md | 37 ++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3052ab3..d95184a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,38 @@ this file. This format is based on [Keep a Changelog](https://keepachangelog.com/). -## Unreleased +--- + +## Template (do not modify this) + +### Added + +### Changed + +### Fixed + +### Removed + +### Dev + +--- + +## Unreleased (modify this) + +### Added + +### Changed + +### Fixed + +### Removed + +### Dev + +--- + + +## v1.10.0 - 18 Feb 2026 ### Added - Planner fallback heuristics module at `src/andromeda/query/planner_heuristics.py` to isolate regex/keyword logic from normal runtime flow. @@ -18,8 +49,6 @@ This format is based on [Keep a Changelog](https://keepachangelog.com/). - non-narrative market/financial metric requests default to finance tools without mandatory RAG, - mixed narrative + market/financial requests can enable both RAG and tools. -### Fixed - ### Removed - Removed brittle runtime heuristic stages from active execution path: - narrative retrieval-query expansion @@ -28,8 +57,6 @@ This format is based on [Keep a Changelog](https://keepachangelog.com/). - adaptive retrieval-budget lowering - Removed corresponding heuristic helper implementations from `src/andromeda/query/runtime.py`; fallback heuristics now live in the dedicated planner fallback module. -### Dev - ## v1.9.0 - 18 Feb 2026 diff --git a/pyproject.toml b/pyproject.toml index eed279a..89f5b36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "andromeda" -version = "1.9.0" +version = "1.10.0" description = "A financial Question-Answering assistant grounded on SEC filings" requires-python = ">=3.12" # TODO: clean up this dependencies list