From 2f2849baa54acf1088f1dc560bdb17cea043639d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:11:43 +0200 Subject: [PATCH 01/38] feat(config): per-stage model settings (FLYDOCS__MODEL) --- src/flydocs/config.py | 20 ++++ tests/unit/test_per_stage_models.py | 178 ++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 tests/unit/test_per_stage_models.py diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 9af1dee..1799546 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -90,6 +90,26 @@ class IDPSettings(BaseSettings): # -- Extraction ----------------------------------------------------- model: str = "anthropic:claude-sonnet-4-6" fallback_model: str | None = "openai:gpt-4o" + + # -- Per-stage model routing ---------------------------------------- + # Pin an individual pipeline stage to its own model id. ``None`` + # (default) means the stage runs on the shared request model + # (``options.model`` or ``FLYDOCS_MODEL``). A pinned stage wins over + # ``options.model`` so operator stage-tuning survives per-request + # overrides. Typical policy: cheap models for splitter/classifier, + # the default for extract, a stronger model for judge. + splitter_model: str | None = None + classifier_model: str | None = None + extract_model: str | None = None + visual_authenticity_model: str | None = None + content_authenticity_model: str | None = None + judge_model: str | None = None + rule_engine_model: str | None = None + # ``transform_model`` and ``bbox_matcher_model`` are constructor-level: + # the transformation engine and the bbox LLM matcher receive their + # model at bean construction, not per pipeline call. + transform_model: str | None = None + bbox_matcher_model: str | None = None # Optional pre-extraction text rendering. ``"none"`` (default) sends # the binary only. ``"docling"`` runs Docling # over the document and splices the resulting Markdown into the diff --git a/tests/unit/test_per_stage_models.py b/tests/unit/test_per_stage_models.py new file mode 100644 index 0000000..561a7cd --- /dev/null +++ b/tests/unit/test_per_stage_models.py @@ -0,0 +1,178 @@ +# Copyright 2024-2026 Firefly Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for per-stage model routing. + +Each LLM pipeline stage can be pinned to its own model via a +``FLYDOCS__MODEL`` setting. Resolution precedence per stage: + + per-stage setting > ``options.model`` > ``FLYDOCS_MODEL`` +""" + +from __future__ import annotations + +import base64 +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from flydocs.config import IDPSettings +from flydocs.core.services.pipeline.orchestrator import ( + PipelineOrchestrator, + _stage_models, +) +from flydocs.interfaces.dtos.document_type import DocumentTypeSpec +from flydocs.interfaces.dtos.extract import ( + ExtractionOptions, + ExtractionRequest, + FileInput, + StageToggles, +) +from flydocs.interfaces.dtos.field import ExtractedFieldGroup, Field, FieldGroup +from flydocs.interfaces.enums.field_type import FieldType + +_DUMMY = base64.b64encode(b"%PDF-1.4").decode("ascii") + +_STAGES = ( + "splitter", + "classifier", + "extract", + "visual_authenticity", + "content_authenticity", + "judge", + "rules", +) + + +def test_settings_per_stage_models_default_none() -> None: + settings = IDPSettings() + assert settings.splitter_model is None + assert settings.classifier_model is None + assert settings.extract_model is None + assert settings.visual_authenticity_model is None + assert settings.content_authenticity_model is None + assert settings.judge_model is None + assert settings.rule_engine_model is None + assert settings.transform_model is None + assert settings.bbox_matcher_model is None + + +def test_settings_per_stage_models_env_binding(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FLYDOCS_JUDGE_MODEL", "anthropic:claude-opus-4-8") + monkeypatch.setenv("FLYDOCS_CLASSIFIER_MODEL", "anthropic:claude-haiku-4-5") + settings = IDPSettings() + assert settings.judge_model == "anthropic:claude-opus-4-8" + assert settings.classifier_model == "anthropic:claude-haiku-4-5" + + +def test_stage_models_all_fall_back_to_model_id() -> None: + resolved = _stage_models(IDPSettings(), "default-model") + assert set(resolved) == set(_STAGES) + assert all(resolved[stage] == "default-model" for stage in _STAGES) + + +def test_stage_models_per_stage_setting_wins_over_request_model() -> None: + settings = IDPSettings( + judge_model="judge-pin", + classifier_model="classifier-pin", + ) + # "request-override" plays the role of ``options.model or FLYDOCS_MODEL``. + resolved = _stage_models(settings, "request-override") + assert resolved["judge"] == "judge-pin" + assert resolved["classifier"] == "classifier-pin" + for stage in set(_STAGES) - {"judge", "classifier"}: + assert resolved[stage] == "request-override" + + +def _doc_spec() -> DocumentTypeSpec: + return DocumentTypeSpec( + id="passport", + description="x", + country="ES", + field_groups=[ + FieldGroup( + name="g", + fields=[Field(name="a", description="x", type=FieldType.STRING)], + ) + ], + ) + + +def _request() -> ExtractionRequest: + return ExtractionRequest( + intention="test", + files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], + document_types=[_doc_spec()], + options=ExtractionOptions(stages=StageToggles(judge=True)), + ) + + +def _fake_normalizer() -> Any: + row = SimpleNamespace( + bytes=b"%PDF-1.4", + media_type="application/pdf", + page_count=1, + filename="doc.pdf", + derived_from=[], + ) + normalizer = MagicMock() + normalizer.normalise = AsyncMock(return_value=[row]) + return normalizer + + +def _orchestrator(settings: IDPSettings, extractor: Any, judge: Any) -> PipelineOrchestrator: + field_validator = MagicMock() + field_validator.validate_groups = MagicMock(return_value=None) + bbox_validator = MagicMock() + bbox_validator.validate_groups = MagicMock(return_value=None) + return PipelineOrchestrator( + extractor=extractor, + splitter=MagicMock(), + classifier=MagicMock(), + field_validator=field_validator, + bbox_validator=bbox_validator, + bbox_refiner=MagicMock(), + binary_normalizer=_fake_normalizer(), + visual_checker=MagicMock(), + content_checker=MagicMock(), + judge=judge, + rule_engine=MagicMock(), + judge_escalator=MagicMock(), + transformation_engine=MagicMock(), + settings=settings, + default_model="default-model", + ) + + +@pytest.mark.asyncio +async def test_orchestrator_routes_stage_pinned_models() -> None: + """extract + judge receive their pinned models, not the shared default.""" + groups = [ExtractedFieldGroup(name="g", fields=[])] + extractor = MagicMock() + extractor.extract = AsyncMock(return_value=(groups, "extract-pin")) + judge = MagicMock() + judge.judge = AsyncMock(return_value=None) + + settings = IDPSettings(extract_model="extract-pin", judge_model="judge-pin") + orchestrator = _orchestrator(settings, extractor, judge) + + result = await orchestrator.execute(_request()) + + assert extractor.extract.await_count == 1 + assert extractor.extract.await_args.kwargs["model"] == "extract-pin" + assert judge.judge.await_count == 1 + assert judge.judge.await_args.kwargs["model"] == "judge-pin" + assert result.model == "extract-pin" From 221db5a0233ba58a13b1e548b9a8c6b7c85ed51b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:14:18 +0200 Subject: [PATCH 02/38] feat(pipeline): route each stage to its pinned model via stage_models map --- .../core/services/pipeline/orchestrator.py | 34 +++++++++++++++---- tests/unit/test_per_stage_models.py | 2 +- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/flydocs/core/services/pipeline/orchestrator.py b/src/flydocs/core/services/pipeline/orchestrator.py index f875fa0..0872a8e 100644 --- a/src/flydocs/core/services/pipeline/orchestrator.py +++ b/src/flydocs/core/services/pipeline/orchestrator.py @@ -400,6 +400,7 @@ async def _execute_inner( "request": request, "result_id": result_id, "model_id": model_id, + "stage_models": _stage_models(self._settings, model_id), "pipeline_errors": [], "unmatched_segments": [], # segments the classifier left without a docType }, @@ -509,7 +510,7 @@ async def _discover_one(slot: _FileSlot) -> None: page_count=slot.page_count, targets=request.document_types, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["splitter"], ) except Exception as exc: # noqa: BLE001 self._record_error(ctx, "discover", "SPLITTER_ERROR", exc, doc_type=slot.filename) @@ -571,7 +572,7 @@ async def _classify_one(slot: _FileSlot, seg: _Segment) -> None: filename=slot.filename, candidates=request.document_types, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["classifier"], ) seg.classification = result if result.matched and result.document_type in docs_by_type: @@ -647,7 +648,7 @@ async def _extract_one(task: _ExtractionTask) -> None: doc=task.doc_spec, intention=request.intention, language_hint=request.options.language_hint, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["extract"], ) task.extracted_groups = groups task.model_used = used @@ -717,7 +718,7 @@ async def _check_one(task: _ExtractionTask) -> None: media_type=task.segment.media_type, doc=task.doc_spec, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["visual_authenticity"], ) task.visual = outcomes except Exception as exc: # noqa: BLE001 @@ -741,7 +742,7 @@ async def _audit_one(task: _ExtractionTask) -> None: media_type=task.segment.media_type, doc=task.doc_spec, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["content_authenticity"], ) except Exception as exc: # noqa: BLE001 self._record_error( @@ -765,7 +766,7 @@ async def _judge_one(task: _ExtractionTask) -> None: doc=task.doc_spec, extracted_groups=task.extracted_groups, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["judge"], ) except Exception as exc: # noqa: BLE001 self._record_error(ctx, "judge", "JUDGE_ERROR", exc, doc_type=task.task_id) @@ -831,7 +832,7 @@ async def _step_rules(self, ctx: PipelineContext, _inputs: dict[str, Any]) -> An extracted_by_doc=extracted_by_doc, visual_by_doc=visual_by_doc, intention=request.intention, - model=ctx.metadata["model_id"], + model=ctx.metadata["stage_models"]["rules"], ) ctx.metadata["rule_results"] = rule_results return {"rules_evaluated": len(rule_results)} @@ -1041,6 +1042,25 @@ def _build_result( # --------------------------------------------------------------------------- +def _stage_models(settings: IDPSettings, model_id: str) -> dict[str, str]: + """Resolve the model id each pipeline stage runs on. + + Precedence per stage: the ``FLYDOCS__MODEL`` setting when set, + else ``model_id`` (which already resolved ``options.model`` against + ``FLYDOCS_MODEL``). A pinned stage wins over ``options.model`` so + operator stage-tuning survives per-request overrides. + """ + return { + "splitter": settings.splitter_model or model_id, + "classifier": settings.classifier_model or model_id, + "extract": settings.extract_model or model_id, + "visual_authenticity": settings.visual_authenticity_model or model_id, + "content_authenticity": settings.content_authenticity_model or model_id, + "judge": settings.judge_model or model_id, + "rules": settings.rule_engine_model or model_id, + } + + def _pages_range(start: int | None, end: int | None) -> list[int]: if start is None or end is None or end < start: return [] diff --git a/tests/unit/test_per_stage_models.py b/tests/unit/test_per_stage_models.py index 561a7cd..5d1017c 100644 --- a/tests/unit/test_per_stage_models.py +++ b/tests/unit/test_per_stage_models.py @@ -175,4 +175,4 @@ async def test_orchestrator_routes_stage_pinned_models() -> None: assert extractor.extract.await_args.kwargs["model"] == "extract-pin" assert judge.judge.await_count == 1 assert judge.judge.await_args.kwargs["model"] == "judge-pin" - assert result.model == "extract-pin" + assert result.pipeline.model == "extract-pin" From 247ce89f289dc7efcacafc06fbc60c0061151429 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:16:10 +0200 Subject: [PATCH 03/38] feat(config): per-stage models for bbox LLM matcher and transformer beans --- src/flydocs/core/configuration.py | 10 +++++++--- tests/unit/test_per_stage_models.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/flydocs/core/configuration.py b/src/flydocs/core/configuration.py index 65e9a53..0d8654b 100644 --- a/src/flydocs/core/configuration.py +++ b/src/flydocs/core/configuration.py @@ -230,19 +230,20 @@ def bbox_value_matcher(self, settings: IDPSettings, prompts: PromptCatalog) -> B single strategy with no cascade. """ kind = (settings.bbox_refine_matcher or "hybrid").lower() + matcher_model = settings.bbox_matcher_model or settings.model if kind == "hybrid": return HybridValueMatcher( fuzzy=ValueMatcher(settings=settings), llm=LlmValueMatcher( template=prompts.bbox_matcher, - model=settings.model, + model=matcher_model, threshold=settings.bbox_refine_threshold, ), ) if kind == "llm": return LlmValueMatcher( template=prompts.bbox_matcher, - model=settings.model, + model=matcher_model, threshold=settings.bbox_refine_threshold, ) if kind == "fuzzy": @@ -331,7 +332,10 @@ def llm_transformer(self, settings: IDPSettings, prompts: PromptCatalog) -> LlmT ``@service`` that autowires this bean alongside the ``EntityResolutionTransformer``. """ - return LlmTransformer(template=prompts.transform, model=settings.model) + return LlmTransformer( + template=prompts.transform, + model=settings.transform_model or settings.model, + ) @bean def rule_engine(self, settings: IDPSettings, prompts: PromptCatalog) -> RuleEngine: diff --git a/tests/unit/test_per_stage_models.py b/tests/unit/test_per_stage_models.py index 5d1017c..676fd0b 100644 --- a/tests/unit/test_per_stage_models.py +++ b/tests/unit/test_per_stage_models.py @@ -30,6 +30,8 @@ import pytest from flydocs.config import IDPSettings +from flydocs.core.configuration import IDPCoreConfiguration +from flydocs.core.services.extraction.prompts import PromptCatalog from flydocs.core.services.pipeline.orchestrator import ( PipelineOrchestrator, _stage_models, @@ -97,6 +99,26 @@ def test_stage_models_per_stage_setting_wins_over_request_model() -> None: assert resolved[stage] == "request-override" +def test_bbox_matcher_bean_uses_pinned_model_with_global_fallback() -> None: + config = IDPCoreConfiguration() + prompts = PromptCatalog.from_resources() + pinned = config.bbox_value_matcher( + IDPSettings(bbox_refine_matcher="llm", bbox_matcher_model="matcher-pin"), prompts + ) + assert pinned._model == "matcher-pin" + fallback = config.bbox_value_matcher(IDPSettings(bbox_refine_matcher="llm"), prompts) + assert fallback._model == IDPSettings().model + + +def test_llm_transformer_bean_uses_pinned_model_with_global_fallback() -> None: + config = IDPCoreConfiguration() + prompts = PromptCatalog.from_resources() + pinned = config.llm_transformer(IDPSettings(transform_model="transform-pin"), prompts) + assert pinned._model == "transform-pin" + fallback = config.llm_transformer(IDPSettings(), prompts) + assert fallback._model == IDPSettings().model + + def _doc_spec() -> DocumentTypeSpec: return DocumentTypeSpec( id="passport", From 4eb17224ea2359f239684e5ca39db3e902aa3f75 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:17:10 +0200 Subject: [PATCH 04/38] docs(env): document per-stage model vars with recommended tiering --- env_template | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/env_template b/env_template index 5830ba6..3386d80 100644 --- a/env_template +++ b/env_template @@ -54,6 +54,35 @@ FLYDOCS_JOBS_TOPIC=flydocs.extractions FLYDOCS_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_FALLBACK_MODEL=openai:gpt-4o +# Per-stage model routing (all optional). Pin an individual pipeline stage to +# its own model; any stage left unset falls back to the global FLYDOCS_MODEL +# (or the request's ``options.model`` when the caller sets one). A pinned +# stage wins over ``options.model``. +# +# Recommended policy (uncomment to adopt): cheap models where the task is +# mechanical, the default where quality matters, a stronger model where +# verification pays for itself. +# +# Structural, low-difficulty tasks -- page-range detection, picking one +# doctype from a declared list, residual bbox value matching: +# FLYDOCS_SPLITTER_MODEL=anthropic:claude-haiku-4-5 +# FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-haiku-4-5 +# FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-haiku-4-5 +# Binary presence checks (signature/stamp/photo) -- simple verdicts: +# FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-haiku-4-5 +# +# Core quality drivers -- keep on the global default (sonnet-tier); +# uncomment only to diverge from FLYDOCS_MODEL: +# FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_CONTENT_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_RULE_ENGINE_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_TRANSFORM_MODEL=anthropic:claude-sonnet-4-6 +# +# Verification -- a strong model *different* from the extractor catches +# errors the extractor is systematically blind to; pairs with +# FLYDOCS_ESCALATION_MODEL for the judge-triggered re-run: +# FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 + # Maximum pages per single extraction request. Documents above this go through # the async API only. The document bytes are sent directly to the multimodal # LLM -- no local rasterisation, so format is whatever the provider accepts From b371f99b6efe57398ef6b7a43d424fd0b9de0cd7 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:17:49 +0200 Subject: [PATCH 05/38] docs(env): sonnet as minimum tier for all stages, opus for judge --- env_template | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/env_template b/env_template index 3386d80..01fb41c 100644 --- a/env_template +++ b/env_template @@ -59,29 +59,20 @@ FLYDOCS_FALLBACK_MODEL=openai:gpt-4o # (or the request's ``options.model`` when the caller sets one). A pinned # stage wins over ``options.model``. # -# Recommended policy (uncomment to adopt): cheap models where the task is -# mechanical, the default where quality matters, a stronger model where -# verification pays for itself. -# -# Structural, low-difficulty tasks -- page-range detection, picking one -# doctype from a declared list, residual bbox value matching: -# FLYDOCS_SPLITTER_MODEL=anthropic:claude-haiku-4-5 -# FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-haiku-4-5 -# FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-haiku-4-5 -# Binary presence checks (signature/stamp/photo) -- simple verdicts: -# FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-haiku-4-5 -# -# Core quality drivers -- keep on the global default (sonnet-tier); -# uncomment only to diverge from FLYDOCS_MODEL: +# Recommended policy (uncomment to adopt): sonnet-tier is the floor for +# every stage; the judge gets a stronger model because verification by a +# model *different* from the extractor catches errors the extractor is +# systematically blind to (pairs with FLYDOCS_ESCALATION_MODEL for the +# judge-triggered re-run): +# FLYDOCS_SPLITTER_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-sonnet-4-6 # FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 # FLYDOCS_CONTENT_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 +# FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 # FLYDOCS_RULE_ENGINE_MODEL=anthropic:claude-sonnet-4-6 # FLYDOCS_TRANSFORM_MODEL=anthropic:claude-sonnet-4-6 -# -# Verification -- a strong model *different* from the extractor catches -# errors the extractor is systematically blind to; pairs with -# FLYDOCS_ESCALATION_MODEL for the judge-triggered re-run: -# FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 +# FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-sonnet-4-6 # Maximum pages per single extraction request. Documents above this go through # the async API only. The document bytes are sent directly to the multimodal From 4462e9be6e4d444b1cd165294086ddeecfde4e8c Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:18:43 +0200 Subject: [PATCH 06/38] docs(env): sonnet-or-higher policy with per-stage opus guidance --- env_template | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/env_template b/env_template index 01fb41c..16ab349 100644 --- a/env_template +++ b/env_template @@ -59,11 +59,24 @@ FLYDOCS_FALLBACK_MODEL=openai:gpt-4o # (or the request's ``options.model`` when the caller sets one). A pinned # stage wins over ``options.model``. # -# Recommended policy (uncomment to adopt): sonnet-tier is the floor for -# every stage; the judge gets a stronger model because verification by a -# model *different* from the extractor catches errors the extractor is -# systematically blind to (pairs with FLYDOCS_ESCALATION_MODEL for the -# judge-triggered re-run): +# Recommended policy (uncomment to adopt): sonnet or higher for every +# stage. Upgrade a stage to opus when its judgment is what the caller +# pays for: +# +# * judge -- the strongest opus case: verification by a model different +# from (and stronger than) the extractor catches errors the extractor +# is systematically blind to. Pairs with FLYDOCS_ESCALATION_MODEL. +# * extract -- opus when documents are hard (dense/handwritten scans, +# long tables, high-stakes fields) or when judge fail-rates trigger +# frequent escalation re-runs; pinning extract to opus up front is +# then cheaper than paying extract twice. +# * content_authenticity -- opus when tampering/fraud detection is a +# core product promise (cross-page coherence needs deep reasoning). +# * rules -- opus when rule DAGs encode multi-level compliance +# decisions rather than simple field predicates. +# * splitter / classifier / visual_authenticity / transform / +# bbox_matcher -- sonnet is enough; opus rarely changes the outcome. +# # FLYDOCS_SPLITTER_MODEL=anthropic:claude-sonnet-4-6 # FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-sonnet-4-6 # FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 From a9b60093af2f5ce97bd64ff1d72674e9345d8e7c Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:20:27 +0200 Subject: [PATCH 07/38] docs(env): uncomment per-stage default models, concise fallback note --- env_template | 43 +++++++++++-------------------------------- src/flydocs/config.py | 3 +-- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/env_template b/env_template index 16ab349..b598331 100644 --- a/env_template +++ b/env_template @@ -54,38 +54,17 @@ FLYDOCS_JOBS_TOPIC=flydocs.extractions FLYDOCS_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_FALLBACK_MODEL=openai:gpt-4o -# Per-stage model routing (all optional). Pin an individual pipeline stage to -# its own model; any stage left unset falls back to the global FLYDOCS_MODEL -# (or the request's ``options.model`` when the caller sets one). A pinned -# stage wins over ``options.model``. -# -# Recommended policy (uncomment to adopt): sonnet or higher for every -# stage. Upgrade a stage to opus when its judgment is what the caller -# pays for: -# -# * judge -- the strongest opus case: verification by a model different -# from (and stronger than) the extractor catches errors the extractor -# is systematically blind to. Pairs with FLYDOCS_ESCALATION_MODEL. -# * extract -- opus when documents are hard (dense/handwritten scans, -# long tables, high-stakes fields) or when judge fail-rates trigger -# frequent escalation re-runs; pinning extract to opus up front is -# then cheaper than paying extract twice. -# * content_authenticity -- opus when tampering/fraud detection is a -# core product promise (cross-page coherence needs deep reasoning). -# * rules -- opus when rule DAGs encode multi-level compliance -# decisions rather than simple field predicates. -# * splitter / classifier / visual_authenticity / transform / -# bbox_matcher -- sonnet is enough; opus rarely changes the outcome. -# -# FLYDOCS_SPLITTER_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_CONTENT_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 -# FLYDOCS_RULE_ENGINE_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_TRANSFORM_MODEL=anthropic:claude-sonnet-4-6 -# FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-sonnet-4-6 +# Default models for each pipeline stage. Any stage left unset falls back +# to the global FLYDOCS_MODEL (or the request's ``options.model``). +FLYDOCS_SPLITTER_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_CONTENT_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 +FLYDOCS_RULE_ENGINE_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_TRANSFORM_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-sonnet-4-6 # Maximum pages per single extraction request. Documents above this go through # the async API only. The document bytes are sent directly to the multimodal diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 1799546..5743f05 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -96,8 +96,7 @@ class IDPSettings(BaseSettings): # (default) means the stage runs on the shared request model # (``options.model`` or ``FLYDOCS_MODEL``). A pinned stage wins over # ``options.model`` so operator stage-tuning survives per-request - # overrides. Typical policy: cheap models for splitter/classifier, - # the default for extract, a stronger model for judge. + # overrides. See env_template for the default model of each stage. splitter_model: str | None = None classifier_model: str | None = None extract_model: str | None = None From 025ff7df88e46540db983776f3b870d0c593d166 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:35:24 +0200 Subject: [PATCH 08/38] feat(prompts): extract_repair template for the focused repair pass --- .../core/services/extraction/prompts.py | 5 + .../resources/prompts/extract_repair.yaml | 35 ++ tests/unit/test_field_repairer.py | 327 ++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 src/flydocs/resources/prompts/extract_repair.yaml create mode 100644 tests/unit/test_field_repairer.py diff --git a/src/flydocs/core/services/extraction/prompts.py b/src/flydocs/core/services/extraction/prompts.py index f9c83f9..eee88a2 100644 --- a/src/flydocs/core/services/extraction/prompts.py +++ b/src/flydocs/core/services/extraction/prompts.py @@ -48,6 +48,7 @@ _PROMPT_FILES: dict[str, str] = { "extract": "extract.yaml", "extract_retry_arrays": "extract_retry_arrays.yaml", + "extract_repair": "extract_repair.yaml", "splitter": "splitter.yaml", "classifier": "classifier.yaml", "content_authenticity": "content_authenticity.yaml", @@ -89,6 +90,10 @@ def extract(self) -> PromptTemplate: def extract_retry_arrays(self) -> PromptTemplate: return self._templates["extract_retry_arrays"] + @property + def extract_repair(self) -> PromptTemplate: + return self._templates["extract_repair"] + @property def splitter(self) -> PromptTemplate: return self._templates["splitter"] diff --git a/src/flydocs/resources/prompts/extract_repair.yaml b/src/flydocs/resources/prompts/extract_repair.yaml new file mode 100644 index 0000000..5648291 --- /dev/null +++ b/src/flydocs/resources/prompts/extract_repair.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Firefly Software Foundation +# +# Loaded by PromptCatalog at boot and registered with the +# fireflyframework-agentic PromptRegistry under (name, version). Used by +# MultimodalExtractor for the REPAIR pass: after judge + field +# validation, fields that failed verification are re-extracted in a +# focused second call that carries the failure evidence (the judge's +# reasoning and/or the deterministic validator errors), so the model +# knows exactly what was wrong with the previous value instead of +# repeating the same mistake blind. +name: flydocs/extract_repair +version: "1.0.0" +description: >- + Focused repair pass used by FieldRepairer when extracted fields + failed the judge re-check or a deterministic validator. Re-extracts + only the failing fields, with the previous value and the failure + reason quoted per field. +required_variables: + - failing_fields_text + - page_count +system_template: | + You are an expert document understanding system performing a focused + repair pass. A previous extraction produced values that failed + verification. Re-examine the document carefully and re-extract ONLY + the fields listed, correcting the reported problems. For every field + return value, confidence, page, bbox and notes. Do not guess: if the + document genuinely contains no value for a field, return null and + explain in notes. +user_template: | + The document has {{ page_count }} page(s). Each field below failed + verification -- the previously extracted value and the reason it was + rejected are given. Locate the correct value in the document and + re-extract these fields: + + {{ failing_fields_text }} diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py new file mode 100644 index 0000000..b8ef717 --- /dev/null +++ b/tests/unit/test_field_repairer.py @@ -0,0 +1,327 @@ +# Copyright 2024-2026 Firefly Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for :class:`FieldRepairer` -- closed-loop targeted repair. + +After judge + field_validation, fields that failed verification are +re-extracted in a focused second pass that carries the failure evidence +(judge reasoning + validator errors). Repaired values replace the +originals only when they pass re-verification. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from flydocs.core.services.extraction.prompts import PromptCatalog +from flydocs.core.services.repair import FieldRepairer, collect_failing_fields +from flydocs.core.services.validation.field_validator import FieldValidator +from flydocs.interfaces.dtos.document_type import DocumentTypeSpec +from flydocs.interfaces.dtos.extract import ( + ExtractionOptions, + ExtractionRequest, + FileInput, + StageToggles, +) +from flydocs.interfaces.dtos.field import ( + ExtractedField, + ExtractedFieldGroup, + Field, + FieldGroup, + FieldValidation, + FieldValidationError, + JudgeOutcome, +) +from flydocs.interfaces.enums.field_type import FieldType +from flydocs.interfaces.enums.status import JudgeStatus, ValidationRule + +import base64 + +_DUMMY = base64.b64encode(b"%PDF-1.4").decode("ascii") + + +def _doc_spec() -> DocumentTypeSpec: + return DocumentTypeSpec( + id="passport", + description="x", + country="ES", + field_groups=[ + FieldGroup( + name="identity", + fields=[ + Field(name="number", description="x", type=FieldType.STRING), + Field(name="name", description="x", type=FieldType.STRING), + ], + ) + ], + ) + + +def _clean_field(name: str, value: str) -> ExtractedField: + return ExtractedField( + name=name, + value=value, + judge=JudgeOutcome(status=JudgeStatus.PASS), + validation=FieldValidation(valid=True), + ) + + +def _judge_failed_field(name: str, value: str, evidence: str) -> ExtractedField: + return ExtractedField( + name=name, + value=value, + judge=JudgeOutcome(status=JudgeStatus.FAIL, evidence=evidence), + validation=FieldValidation(valid=True), + ) + + +def _validator_failed_field(name: str, value: str, message: str) -> ExtractedField: + return ExtractedField( + name=name, + value=value, + judge=JudgeOutcome(status=JudgeStatus.PASS), + validation=FieldValidation( + valid=False, + errors=[FieldValidationError(rule=ValidationRule.VALIDATOR, message=message)], + ), + ) + + +def _task(groups: list[ExtractedFieldGroup]) -> Any: + return SimpleNamespace( + task_id="file0/seg0/passport", + doc_spec=_doc_spec(), + slice_bytes=b"%PDF-1.4", + slice_pages=1, + segment=SimpleNamespace(media_type="application/pdf"), + extracted_groups=groups, + model_used=None, + ) + + +def _ctx(tasks: list[Any], model_id: str = "base-model") -> Any: + return SimpleNamespace(metadata={"tasks": tasks, "model_id": model_id}) + + +def _request(*, judge: bool = True) -> ExtractionRequest: + return ExtractionRequest( + intention="test", + files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], + document_types=[_doc_spec()], + options=ExtractionOptions(stages=StageToggles(judge=judge, repair=True)), + ) + + +# --------------------------------------------------------------------------- +# Failure collection +# --------------------------------------------------------------------------- + + +def test_collect_failing_fields_picks_judge_and_validator_failures() -> None: + groups = [ + ExtractedFieldGroup( + name="identity", + fields=[ + _clean_field("name", "JOHN"), + _judge_failed_field("number", "X123", "value not present on page 1"), + _validator_failed_field("iban", "ES00", "IBAN checksum failed"), + ], + ) + ] + failures = collect_failing_fields(groups) + names = [(f.group, f.field) for f in failures] + assert ("identity", "number") in names + assert ("identity", "iban") in names + assert ("identity", "name") not in names + evidence = " ".join(f.evidence for f in failures) + assert "value not present on page 1" in evidence + assert "IBAN checksum failed" in evidence + + +def test_collect_failing_fields_includes_flag_for_review() -> None: + flagged = ExtractedField( + name="number", + value="X123", + judge=JudgeOutcome(status=JudgeStatus.PASS, flag_for_review=True), + ) + groups = [ExtractedFieldGroup(name="identity", fields=[flagged])] + failures = collect_failing_fields(groups) + assert [(f.group, f.field) for f in failures] == [("identity", "number")] + + +def test_collect_failing_fields_empty_when_all_clean() -> None: + groups = [ExtractedFieldGroup(name="identity", fields=[_clean_field("name", "JOHN")])] + assert collect_failing_fields(groups) == [] + + +# --------------------------------------------------------------------------- +# Repair round-trip +# --------------------------------------------------------------------------- + + +def _repairer( + extractor: Any, + judge: Any, + *, + default_model: str | None = None, +) -> FieldRepairer: + return FieldRepairer( + extractor=extractor, + judge=judge, + field_validator=FieldValidator(), + default_model=default_model, + ) + + +@pytest.mark.asyncio +async def test_maybe_repair_returns_none_when_nothing_failed() -> None: + task = _task([ExtractedFieldGroup(name="identity", fields=[_clean_field("name", "JOHN")])]) + repairer = _repairer(MagicMock(), MagicMock()) + info = await repairer.maybe_repair(_ctx([task]), _request()) + assert info is None + + +@pytest.mark.asyncio +async def test_maybe_repair_accepts_field_that_passes_recheck() -> None: + task = _task( + [ + ExtractedFieldGroup( + name="identity", + fields=[ + _clean_field("name", "JOHN"), + _judge_failed_field("number", "X123", "misread"), + ], + ) + ] + ) + repaired_field = ExtractedField(name="number", value="Y456") + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ExtractedFieldGroup(name="identity", fields=[repaired_field])] + ) + + async def _stamp_pass(*, extracted_groups: list[ExtractedFieldGroup], **_: Any) -> Any: + for group in extracted_groups: + for f in group.fields: + f.judge = JudgeOutcome(status=JudgeStatus.PASS) + return extracted_groups + + judge = MagicMock() + judge.judge = AsyncMock(side_effect=_stamp_pass) + + repairer = _repairer(extractor, judge, default_model="repair-model") + info = await repairer.maybe_repair(_ctx([task]), _request()) + + assert info is not None and info.triggered + assert info.model == "repair-model" + assert info.fields_flagged == 1 + assert info.fields_repaired == 1 + assert info.repaired_fields == ["identity.number"] + # The repaired value replaced the original; the clean field is untouched. + by_name = {f.name: f for f in task.extracted_groups[0].fields} + assert by_name["number"].value == "Y456" + assert by_name["name"].value == "JOHN" + # The focused re-ask carried the failure evidence. + kwargs = extractor.extract_repair.await_args.kwargs + assert "misread" in kwargs["failing_fields_text"] + assert kwargs["model"] == "repair-model" + # The subset spec only contains the failing field. + subset: DocumentTypeSpec = kwargs["doc"] + assert [f.name for g in subset.field_groups for f in g.fields] == ["number"] + + +@pytest.mark.asyncio +async def test_maybe_repair_keeps_original_when_recheck_still_fails() -> None: + original = _judge_failed_field("number", "X123", "misread") + task = _task([ExtractedFieldGroup(name="identity", fields=[original])]) + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup(name="identity", fields=[ExtractedField(name="number", value="Z999")]) + ] + ) + + async def _stamp_fail(*, extracted_groups: list[ExtractedFieldGroup], **_: Any) -> Any: + for group in extracted_groups: + for f in group.fields: + f.judge = JudgeOutcome(status=JudgeStatus.FAIL, evidence="still wrong") + return extracted_groups + + judge = MagicMock() + judge.judge = AsyncMock(side_effect=_stamp_fail) + + repairer = _repairer(extractor, judge) + info = await repairer.maybe_repair(_ctx([task]), _request()) + + assert info is not None and info.triggered + assert info.fields_flagged == 1 + assert info.fields_repaired == 0 + assert task.extracted_groups[0].fields[0].value == "X123" + + +@pytest.mark.asyncio +async def test_maybe_repair_without_judge_stage_uses_validator_only() -> None: + """When the judge stage is off, acceptance rests on the validator re-check alone.""" + spec = _doc_spec() + spec.field_groups[0].fields[0].pattern = r"^[A-Z]\d{3}$" # number must match + task = _task( + [ + ExtractedFieldGroup( + name="identity", + fields=[_validator_failed_field("number", "bad!", "pattern mismatch")], + ) + ] + ) + task.doc_spec = spec + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup(name="identity", fields=[ExtractedField(name="number", value="X123")]) + ] + ) + judge = MagicMock() + judge.judge = AsyncMock() + + repairer = _repairer(extractor, judge) + info = await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + assert info is not None and info.fields_repaired == 1 + assert task.extracted_groups[0].fields[0].value == "X123" + judge.judge.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_maybe_repair_falls_back_to_request_model() -> None: + task = _task( + [ExtractedFieldGroup(name="identity", fields=[_judge_failed_field("number", "X", "bad")])] + ) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), default_model=None) + await repairer.maybe_repair(_ctx([task], model_id="request-model"), _request()) + assert extractor.extract_repair.await_args.kwargs["model"] == "request-model" + + +# --------------------------------------------------------------------------- +# Prompt catalog +# --------------------------------------------------------------------------- + + +def test_prompt_catalog_ships_extract_repair_template() -> None: + catalog = PromptCatalog.from_resources() + assert catalog.extract_repair is not None From ed2a618a27f36160b0683e76c6b6a8f4678be3ee Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:36:41 +0200 Subject: [PATCH 09/38] feat(dtos,config): repair stage toggle, RepairInfo audit block, repair settings --- src/flydocs/config.py | 11 +++++++++++ src/flydocs/interfaces/dtos/extract.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 9af1dee..0d3f52b 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -207,6 +207,17 @@ class IDPSettings(BaseSettings): escalation_threshold: float = 0.0 escalation_model: str | None = None + # -- Closed-loop targeted repair ------------------------------------ + # When ``stages.repair`` is on, fields that failed the judge re-check + # or a deterministic validator are re-extracted in ONE focused pass + # that quotes the failure evidence back to the model. Repaired values + # replace the originals only when they pass re-verification. + # ``repair_model`` pins the pass to its own model (``None`` = the + # request model). Runs BEFORE judge_escalation, so the full re-run + # only fires when targeted repair was not enough. + repair_model: str | None = None + repair_timeout_s: int = 300 + # -- Webhook -------------------------------------------------------- # The result webhook delivers the full extraction (split docs + fields + # rule evaluations) to the consumer, which persists it synchronously before diff --git a/src/flydocs/interfaces/dtos/extract.py b/src/flydocs/interfaces/dtos/extract.py index aa4f02b..22ae2a3 100644 --- a/src/flydocs/interfaces/dtos/extract.py +++ b/src/flydocs/interfaces/dtos/extract.py @@ -106,6 +106,7 @@ class StageToggles(BaseModel): content_authenticity: bool = False judge: bool = False judge_escalation: bool = False + repair: bool = False bbox_refine: bool = False transform: bool = False rule_engine: bool = False @@ -249,6 +250,18 @@ class EscalationInfo(BaseModel): accepted: bool = False +class RepairInfo(BaseModel): + """Audit block for the closed-loop targeted repair pass.""" + + model_config = ConfigDict(extra="forbid") + + triggered: bool = False + model: str | None = None + fields_flagged: int = Field(default=0, ge=0) + fields_repaired: int = Field(default=0, ge=0) + repaired_fields: list[str] = Field(default_factory=list) + + class UsageBreakdown(BaseModel): """Aggregated token usage and cost across every LLM call of one request.""" @@ -277,6 +290,7 @@ class PipelineMeta(BaseModel): trace: list[TraceEntry] = Field(default_factory=list) errors: list[PipelineError] = Field(default_factory=list) escalation: EscalationInfo | None = None + repair: RepairInfo | None = None usage: UsageBreakdown | None = None From e0f35e00a6c1f58b91441603b73170abee8ab207 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:37:56 +0200 Subject: [PATCH 10/38] feat(repair): FieldRepairer with evidence collection and monotonic accept guard --- src/flydocs/core/services/repair/__init__.py | 23 ++ .../core/services/repair/field_repairer.py | 226 ++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 src/flydocs/core/services/repair/__init__.py create mode 100644 src/flydocs/core/services/repair/field_repairer.py diff --git a/src/flydocs/core/services/repair/__init__.py b/src/flydocs/core/services/repair/__init__.py new file mode 100644 index 0000000..98deb4d --- /dev/null +++ b/src/flydocs/core/services/repair/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2024-2026 Firefly Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Closed-loop targeted repair -- focused re-extraction of failing fields.""" + +from flydocs.core.services.repair.field_repairer import ( + FailingField, + FieldRepairer, + collect_failing_fields, +) + +__all__ = ["FailingField", "FieldRepairer", "collect_failing_fields"] diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py new file mode 100644 index 0000000..3285c74 --- /dev/null +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -0,0 +1,226 @@ +# Copyright 2024-2026 Firefly Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``FieldRepairer`` -- closed-loop targeted repair of failing fields. + +The judge stamps every extracted field with PASS / FAIL / UNCERTAIN and +the deterministic validators record per-field errors. Escalation re-runs +the WHOLE extraction blind; this service instead re-asks ONLY the +failing fields, quoting the failure evidence (judge reasoning + +validator messages) back to the model so the second pass corrects the +specific mistake instead of re-rolling the dice on everything. + +Repaired values are re-verified (validators always; judge subset +re-check when the judge stage is enabled) and replace the originals +only when they come back clean -- the repair is monotonic: it can fix +fields, never degrade them. + +Opt-in via ``options.stages.repair``. Runs after judge and BEFORE +judge_escalation, so the expensive full re-run only fires when targeted +repair was not enough. Top-level fields only: rows nested inside array +fields are repaired by re-extracting the array field as a whole. +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import Any + +from flydocs.core.services.extraction.extractor import MultimodalExtractor +from flydocs.core.services.judge import Judge +from flydocs.core.services.validation.field_validator import FieldValidator +from flydocs.interfaces.dtos.document_type import DocumentTypeSpec +from flydocs.interfaces.dtos.extract import ExtractionRequest, RepairInfo +from flydocs.interfaces.dtos.field import ExtractedField, ExtractedFieldGroup +from flydocs.interfaces.enums.status import JudgeStatus + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FailingField: + """One field that failed verification, with the evidence to feed back.""" + + group: str + field: str + value: Any + evidence: str + + +def collect_failing_fields(groups: list[ExtractedFieldGroup]) -> list[FailingField]: + """Top-level fields whose judge verdict or validation failed. + + A field fails when ``judge.status == FAIL``, ``judge.flag_for_review`` + is set, or ``validation.valid`` is false. The evidence string joins + the judge's reasoning with every validator error message. + """ + failures: list[FailingField] = [] + for group in groups: + for field in group.fields: + reasons: list[str] = [] + if field.judge.status == JudgeStatus.FAIL or field.judge.flag_for_review: + reasons.append(field.judge.evidence or field.judge.notes or "judge rejected the value") + if not field.validation.valid: + reasons.extend(e.message for e in field.validation.errors) + if reasons: + failures.append( + FailingField( + group=group.name, + field=field.name, + value=field.value, + evidence="; ".join(reasons), + ) + ) + return failures + + +class FieldRepairer: + """Re-extract failing fields with their failure evidence, merge back what passes.""" + + def __init__( + self, + *, + extractor: MultimodalExtractor, + judge: Judge, + field_validator: FieldValidator, + default_model: str | None, + ) -> None: + self._extractor = extractor + self._judge = judge + self._field_validator = field_validator + self._default_model = default_model + + async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo | None: + """Return a :class:`RepairInfo` when at least one field was flagged, else ``None``. + + Mutates each task's ``extracted_groups`` in place: a repaired + field replaces the original only when it passes re-verification. + """ + tasks: list[Any] = ctx.metadata.get("tasks", []) + model = self._default_model or ctx.metadata.get("model_id") + flagged = 0 + repaired_paths: list[str] = [] + + async def _repair_task(task: Any) -> None: + nonlocal flagged + failures = collect_failing_fields(task.extracted_groups) + if not failures: + return + flagged += len(failures) + try: + accepted = await self._repair_one(task, failures, request, model) + except Exception as exc: # noqa: BLE001 -- repair must never break the pipeline + logger.warning("repair failed for %s: %s; keeping original fields", task.task_id, exc) + return + repaired_paths.extend(accepted) + + await asyncio.gather(*(_repair_task(t) for t in tasks)) + + if flagged == 0: + return None + info = RepairInfo( + triggered=True, + model=model, + fields_flagged=flagged, + fields_repaired=len(repaired_paths), + repaired_fields=sorted(repaired_paths), + ) + logger.info( + "repair flagged=%d repaired=%d model=%s", + info.fields_flagged, + info.fields_repaired, + model, + ) + return info + + async def _repair_one( + self, + task: Any, + failures: list[FailingField], + request: ExtractionRequest, + model: str | None, + ) -> list[str]: + """Repair one task; return the ``group.field`` paths that were accepted.""" + subset = _subset_spec(task.doc_spec, failures) + repaired_groups = await self._extractor.extract_repair( + document_bytes=task.slice_bytes, + media_type=task.segment.media_type, + page_count=task.slice_pages, + doc=subset, + failing_fields_text=_failures_text(failures), + language_hint=request.options.language_hint, + model=model, + ) + if not repaired_groups: + return [] + + # Re-verify: validators always (deterministic, free); judge subset + # re-check only when the caller enabled the judge stage. + self._field_validator.validate(subset.field_groups, repaired_groups) + if request.options.stages.judge: + await self._judge.judge( + document_bytes=task.slice_bytes, + media_type=task.segment.media_type, + doc=subset, + extracted_groups=repaired_groups, + intention=request.intention, + model=model, + ) + + failing_keys = {(f.group, f.field) for f in failures} + repaired_by_key: dict[tuple[str, str], ExtractedField] = { + (group.name, field.name): field + for group in repaired_groups + for field in group.fields + } + accepted: list[str] = [] + for group in task.extracted_groups: + for index, field in enumerate(group.fields): + key = (group.name, field.name) + if key not in failing_keys: + continue + candidate = repaired_by_key.get(key) + if candidate is None or not _is_clean(candidate): + continue + group.fields[index] = candidate + accepted.append(f"{group.name}.{field.name}") + return accepted + + +def _is_clean(field: ExtractedField) -> bool: + """A repaired field is accepted only when re-verification passed.""" + if not field.validation.valid: + return False + return field.judge.status != JudgeStatus.FAIL and not field.judge.flag_for_review + + +def _subset_spec(doc: DocumentTypeSpec, failures: list[FailingField]) -> DocumentTypeSpec: + """A copy of ``doc`` whose field_groups contain only the failing fields.""" + failing_keys = {(f.group, f.field) for f in failures} + groups = [] + for group in doc.field_groups: + fields = [f for f in group.fields if (group.name, f.name) in failing_keys] + if fields: + groups.append(group.model_copy(update={"fields": fields})) + return doc.model_copy(update={"field_groups": groups}) + + +def _failures_text(failures: list[FailingField]) -> str: + """Render the per-field failure evidence block for the repair prompt.""" + lines = [] + for f in failures: + lines.append(f"- ``{f.group}.{f.field}``: previous value {f.value!r} -- rejected because: {f.evidence}") + return "\n".join(lines) From d467f29ebe38056da9823886f6aed772ce49b151 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:39:28 +0200 Subject: [PATCH 11/38] feat(extractor): extract_repair focused pass with failure-evidence prompt --- .../core/services/extraction/extractor.py | 44 ++++++++ tests/unit/test_extract_repair.py | 101 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 tests/unit/test_extract_repair.py diff --git a/src/flydocs/core/services/extraction/extractor.py b/src/flydocs/core/services/extraction/extractor.py index a3c003f..07cad8f 100644 --- a/src/flydocs/core/services/extraction/extractor.py +++ b/src/flydocs/core/services/extraction/extractor.py @@ -55,6 +55,7 @@ def __init__( *, template: PromptTemplate, retry_arrays_template: PromptTemplate | None = None, + repair_template: PromptTemplate | None = None, model: str, fallback_model: str | None = None, agent_name: str = "flydocs-extractor", @@ -62,6 +63,7 @@ def __init__( ) -> None: self._template = template self._retry_arrays_template = retry_arrays_template + self._repair_template = repair_template self._model = model self._fallback_model = fallback_model self._agent_name = agent_name @@ -181,6 +183,48 @@ async def _extract_retry_arrays( result = await timed_agent_run(agent, content, op="extract.retry_arrays", model=model_id) return normalise_doc(result.output, doc), model_id + async def extract_repair( + self, + *, + document_bytes: bytes, + media_type: str, + page_count: int, + doc: DocumentTypeSpec, + failing_fields_text: str, + language_hint: str | None = None, + model: str | None = None, + ) -> list[ExtractedFieldGroup]: + """Focused repair pass: re-extract only the failing fields. + + ``doc`` is the SUBSET spec containing just the fields that failed + verification; ``failing_fields_text`` quotes each previous value + and the reason it was rejected. Uses the dedicated + :pyattr:`_repair_template` (``flydocs/extract_repair``). + Returns ``[]`` when no repair template is configured. + """ + if self._repair_template is None: + return [] + prompt = self._repair_template.render( + failing_fields_text=failing_fields_text, + page_count=page_count, + ) + model_id = model or self._model + output_model = build_extraction_output_model(doc) + agent = self._build_agent(model_id, output_model, instructions=prompt.system) + # Same shape as the retry-arrays pass: short action-oriented user + # text plus the (subset) schema so the LLM knows the contract. + schema_json = self._schema_payload(doc) + user_text = f"{prompt.user.strip()}\n\nSchema:\n```json\n{schema_json}\n```" + if language_hint: + user_text = f"{user_text}\n\nDocument language hint: {language_hint}" + content = self._build_user_content( + user_text=user_text, + document_bytes=document_bytes, + media_type=media_type, + ) + result = await timed_agent_run(agent, content, op="extract.repair", model=model_id) + return normalise_doc(result.output, doc) + async def _extract_once( self, *, diff --git a/tests/unit/test_extract_repair.py b/tests/unit/test_extract_repair.py new file mode 100644 index 0000000..c77b3f6 --- /dev/null +++ b/tests/unit/test_extract_repair.py @@ -0,0 +1,101 @@ +# Copyright 2024-2026 Firefly Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for :meth:`MultimodalExtractor.extract_repair`.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from pydantic import BaseModel + +from flydocs.core.services.extraction import extractor as extractor_module +from flydocs.core.services.extraction.extractor import MultimodalExtractor +from flydocs.core.services.extraction.prompts import PromptCatalog +from flydocs.interfaces.dtos.document_type import DocumentTypeSpec +from flydocs.interfaces.dtos.field import Field, FieldGroup +from flydocs.interfaces.enums.field_type import FieldType + + +def _doc_subset() -> DocumentTypeSpec: + return DocumentTypeSpec( + id="passport", + description="x", + country="ES", + field_groups=[ + FieldGroup( + name="identity", + fields=[Field(name="number", description="x", type=FieldType.STRING)], + ) + ], + ) + + +@pytest.mark.asyncio +async def test_extract_repair_renders_evidence_and_uses_repair_template( + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog = PromptCatalog.from_resources() + captured: dict[str, Any] = {} + + class _EmptyOutput(BaseModel): + pass + + async def _fake_run(agent: Any, content: list[Any], *, op: str, model: str) -> Any: + captured["op"] = op + captured["model"] = model + captured["user_text"] = content[0] + # Minimal structured output: normalise_doc fills the spec shape + # with null values when the payload is empty. + return SimpleNamespace(output=_EmptyOutput()) + + monkeypatch.setattr(extractor_module, "timed_agent_run", _fake_run) + + ext = MultimodalExtractor( + template=catalog.extract, + repair_template=catalog.extract_repair, + model="base-model", + ) + groups = await ext.extract_repair( + document_bytes=b"%PDF-1.4", + media_type="application/pdf", + page_count=2, + doc=_doc_subset(), + failing_fields_text="- ``identity.number``: previous value 'X1' -- rejected because: misread", + model="repair-model", + ) + + # normalise_doc shapes the output after the subset spec. + assert [g.name for g in groups] == ["identity"] + assert [f.name for f in groups[0].fields] == ["number"] + assert captured["op"] == "extract.repair" + assert captured["model"] == "repair-model" + assert "misread" in captured["user_text"] + assert "identity.number" in captured["user_text"] + + +@pytest.mark.asyncio +async def test_extract_repair_without_template_returns_empty() -> None: + catalog = PromptCatalog.from_resources() + ext = MultimodalExtractor(template=catalog.extract, model="base-model") + groups = await ext.extract_repair( + document_bytes=b"%PDF-1.4", + media_type="application/pdf", + page_count=1, + doc=_doc_subset(), + failing_fields_text="x", + ) + assert groups == [] From 055536001adbfcc12ac0918e2bca36b629714115 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:40:29 +0200 Subject: [PATCH 12/38] test(extractor): use pydantic-ai TestModel id in extract_repair test --- tests/unit/test_extract_repair.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_extract_repair.py b/tests/unit/test_extract_repair.py index c77b3f6..310e621 100644 --- a/tests/unit/test_extract_repair.py +++ b/tests/unit/test_extract_repair.py @@ -75,14 +75,14 @@ async def _fake_run(agent: Any, content: list[Any], *, op: str, model: str) -> A page_count=2, doc=_doc_subset(), failing_fields_text="- ``identity.number``: previous value 'X1' -- rejected because: misread", - model="repair-model", + model="test", ) # normalise_doc shapes the output after the subset spec. assert [g.name for g in groups] == ["identity"] assert [f.name for f in groups[0].fields] == ["number"] assert captured["op"] == "extract.repair" - assert captured["model"] == "repair-model" + assert captured["model"] == "test" assert "misread" in captured["user_text"] assert "identity.number" in captured["user_text"] From 7d63bb6835231c92e68815e107b5aca294a79b15 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:45:11 +0200 Subject: [PATCH 13/38] feat(pipeline): repair node between judge and escalation; DI wiring; audit block --- src/flydocs/core/configuration.py | 20 ++++ .../core/services/pipeline/orchestrator.py | 37 +++++- .../core/services/repair/field_repairer.py | 8 +- tests/unit/test_field_repairer.py | 113 +++++++++++++++++- 4 files changed, 168 insertions(+), 10 deletions(-) diff --git a/src/flydocs/core/configuration.py b/src/flydocs/core/configuration.py index 65e9a53..56f9262 100644 --- a/src/flydocs/core/configuration.py +++ b/src/flydocs/core/configuration.py @@ -78,6 +78,7 @@ ) from flydocs.core.services.judge import Judge from flydocs.core.services.pipeline import PipelineOrchestrator +from flydocs.core.services.repair import FieldRepairer from flydocs.core.services.rules import RuleEngine from flydocs.core.services.splitting import DocumentSplitter from flydocs.core.services.transformations import LlmTransformer, TransformationEngine @@ -167,6 +168,7 @@ def extractor( return MultimodalExtractor( template=prompts.extract, retry_arrays_template=prompts.extract_retry_arrays, + repair_template=prompts.extract_repair, model=settings.model, fallback_model=settings.fallback_model, text_anchor=text_anchor, @@ -352,6 +354,22 @@ def judge_escalator( default_model=settings.escalation_model, ) + @bean + def field_repairer( + self, + extractor: MultimodalExtractor, + judge: Judge, + field_validator: FieldValidator, + settings: IDPSettings, + ) -> FieldRepairer: + """Focused re-extraction of judge/validator-failing fields.""" + return FieldRepairer( + extractor=extractor, + judge=judge, + field_validator=field_validator, + default_model=settings.repair_model, + ) + # ------------------------------------------------------------------ # Orchestrator + async worker # ------------------------------------------------------------------ @@ -372,6 +390,7 @@ def orchestrator( rule_engine: RuleEngine, judge_escalator: JudgeEscalator, transformation_engine: TransformationEngine, + field_repairer: FieldRepairer, settings: IDPSettings, ) -> PipelineOrchestrator: return PipelineOrchestrator( @@ -388,6 +407,7 @@ def orchestrator( rule_engine=rule_engine, judge_escalator=judge_escalator, transformation_engine=transformation_engine, + field_repairer=field_repairer, settings=settings, default_model=settings.model, ) diff --git a/src/flydocs/core/services/pipeline/orchestrator.py b/src/flydocs/core/services/pipeline/orchestrator.py index f875fa0..7f6fe28 100644 --- a/src/flydocs/core/services/pipeline/orchestrator.py +++ b/src/flydocs/core/services/pipeline/orchestrator.py @@ -20,7 +20,7 @@ load -> discover? -> classify? -> plan_tasks -> extract -> bbox_validation -> field_validation? -> visual? -> content? -> - judge? -> judge_escalation? -> rules? -> assemble + judge? -> repair? -> judge_escalation? -> rules? -> assemble The discover stage (``stages.splitter``) enumerates every distinct sub-document inside a file, so a single uploaded PDF that happens to @@ -78,6 +78,7 @@ from flydocs.core.services.extraction.extractor import MultimodalExtractor from flydocs.core.services.extraction.pdf_slicer import PageRange, slice_pdf from flydocs.core.services.judge import Judge +from flydocs.core.services.repair import FieldRepairer from flydocs.core.services.rules import RuleEngine from flydocs.core.services.splitting import DiscoveredSegment, DocumentSplitter from flydocs.core.services.transformations import TransformationEngine @@ -97,6 +98,7 @@ FileSummary, PipelineError, PipelineMeta, + RepairInfo, TraceEntry, UsageBreakdown, ) @@ -231,6 +233,7 @@ def __init__( rule_engine: RuleEngine, judge_escalator: JudgeEscalator, transformation_engine: TransformationEngine, + field_repairer: FieldRepairer, settings: IDPSettings, default_model: str, ) -> None: @@ -246,6 +249,7 @@ def __init__( self._judge = judge self._rule_engine = rule_engine self._judge_escalator = judge_escalator + self._field_repairer = field_repairer self._transformation_engine = transformation_engine self._settings = settings self._default_model = default_model @@ -363,6 +367,18 @@ async def _execute_inner( ) chain.append("judge") + # Targeted repair. Runs AFTER judge (so it sees the verdicts) and + # BEFORE judge_escalation, so the expensive full re-run only fires + # when the focused per-field repair was not enough. Needs at least + # one verification stage to produce failure signals. + if stages.repair and (stages.judge or stages.field_validation): + builder.add_node( + "repair", + CallableStep(self._step_repair), + timeout_seconds=self._settings.repair_timeout_s, + ) + chain.append("repair") + if stages.judge and stages.judge_escalation: builder.add_node( "judge_escalation", @@ -773,6 +789,23 @@ async def _judge_one(task: _ExtractionTask) -> None: await asyncio.gather(*(_judge_one(t) for t in tasks)) return {"judged": True} + async def _step_repair(self, ctx: PipelineContext, _inputs: dict[str, Any]) -> Any: + request: ExtractionRequest = ctx.metadata["request"] + try: + info = await self._field_repairer.maybe_repair(ctx, request) + except Exception as exc: # noqa: BLE001 + self._record_error(ctx, "repair", "REPAIR_ERROR", exc) + return {"failed": True} + if info is None: + return {"repair_triggered": False} + ctx.metadata["repair"] = info + # Repaired values carry fresh judge/validation verdicts but their + # bboxes came from a new LLM pass -- re-grade the geometry. + for task in ctx.metadata["tasks"]: + if task.extracted_groups: + self._bbox_validator.validate_groups(task.extracted_groups) + return {"repair_triggered": True, "fields_repaired": info.fields_repaired} + async def _step_judge_escalation(self, ctx: PipelineContext, _inputs: dict[str, Any]) -> Any: request: ExtractionRequest = ctx.metadata["request"] tasks: list[_ExtractionTask] = ctx.metadata["tasks"] @@ -996,6 +1029,7 @@ def _build_result( ] escalation: EscalationInfo | None = ctx.metadata.get("escalation") + repair: RepairInfo | None = ctx.metadata.get("repair") usage_breakdown = _usage_breakdown( request_id=result_id, pipeline_result=pipeline_result, @@ -1011,6 +1045,7 @@ def _build_result( trace=trace, errors=pipeline_errors, escalation=escalation, + repair=repair, usage=usage_breakdown, ) # Determine overall status: ``partial`` when at least one task diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 3285c74..68792f4 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -182,9 +182,7 @@ async def _repair_one( failing_keys = {(f.group, f.field) for f in failures} repaired_by_key: dict[tuple[str, str], ExtractedField] = { - (group.name, field.name): field - for group in repaired_groups - for field in group.fields + (group.name, field.name): field for group in repaired_groups for field in group.fields } accepted: list[str] = [] for group in task.extracted_groups: @@ -222,5 +220,7 @@ def _failures_text(failures: list[FailingField]) -> str: """Render the per-field failure evidence block for the repair prompt.""" lines = [] for f in failures: - lines.append(f"- ``{f.group}.{f.field}``: previous value {f.value!r} -- rejected because: {f.evidence}") + lines.append( + f"- ``{f.group}.{f.field}``: previous value {f.value!r} -- rejected because: {f.evidence}" + ) return "\n".join(lines) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index b8ef717..8ee723c 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -22,6 +22,7 @@ from __future__ import annotations +import base64 from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -50,8 +51,6 @@ from flydocs.interfaces.enums.field_type import FieldType from flydocs.interfaces.enums.status import JudgeStatus, ValidationRule -import base64 - _DUMMY = base64.b64encode(b"%PDF-1.4").decode("ascii") @@ -307,9 +306,7 @@ async def test_maybe_repair_without_judge_stage_uses_validator_only() -> None: @pytest.mark.asyncio async def test_maybe_repair_falls_back_to_request_model() -> None: - task = _task( - [ExtractedFieldGroup(name="identity", fields=[_judge_failed_field("number", "X", "bad")])] - ) + task = _task([ExtractedFieldGroup(name="identity", fields=[_judge_failed_field("number", "X", "bad")])]) extractor = MagicMock() extractor.extract_repair = AsyncMock(return_value=[]) repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), default_model=None) @@ -325,3 +322,109 @@ async def test_maybe_repair_falls_back_to_request_model() -> None: def test_prompt_catalog_ships_extract_repair_template() -> None: catalog = PromptCatalog.from_resources() assert catalog.extract_repair is not None + + +# --------------------------------------------------------------------------- +# Orchestrator wiring +# --------------------------------------------------------------------------- + + +def _fake_normalizer() -> Any: + row = SimpleNamespace( + bytes=b"%PDF-1.4", + media_type="application/pdf", + page_count=1, + filename="doc.pdf", + derived_from=[], + ) + normalizer = MagicMock() + normalizer.normalise = AsyncMock(return_value=[row]) + return normalizer + + +def _orchestrator(repairer: Any) -> Any: + from flydocs.config import IDPSettings + from flydocs.core.services.pipeline.orchestrator import PipelineOrchestrator + + groups = [ExtractedFieldGroup(name="identity", fields=[])] + extractor = MagicMock() + extractor.extract = AsyncMock(return_value=(groups, "base-model")) + judge = MagicMock() + judge.judge = AsyncMock(return_value=None) + return PipelineOrchestrator( + extractor=extractor, + splitter=MagicMock(), + classifier=MagicMock(), + field_validator=MagicMock(validate=MagicMock(return_value=None)), + bbox_validator=MagicMock(validate_groups=MagicMock(return_value=None)), + bbox_refiner=MagicMock(), + binary_normalizer=_fake_normalizer(), + visual_checker=MagicMock(), + content_checker=MagicMock(), + judge=judge, + rule_engine=MagicMock(), + judge_escalator=MagicMock(), + transformation_engine=MagicMock(), + field_repairer=repairer, + settings=IDPSettings(), + default_model="base-model", + ) + + +def _wiring_request(*, repair: bool, judge: bool = True) -> ExtractionRequest: + return ExtractionRequest( + intention="test", + files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], + document_types=[_doc_spec()], + options=ExtractionOptions(stages=StageToggles(judge=judge, repair=repair)), + ) + + +@pytest.mark.asyncio +async def test_orchestrator_runs_repair_node_and_reports_audit_block() -> None: + from flydocs.interfaces.dtos.extract import RepairInfo + + repairer = MagicMock() + repairer.maybe_repair = AsyncMock( + return_value=RepairInfo( + triggered=True, + model="repair-model", + fields_flagged=2, + fields_repaired=1, + repaired_fields=["identity.number"], + ) + ) + orchestrator = _orchestrator(repairer) + result = await orchestrator.execute(_wiring_request(repair=True)) + repairer.maybe_repair.assert_awaited_once() + assert result.pipeline.repair is not None + assert result.pipeline.repair.fields_repaired == 1 + assert "repair" in [t.node for t in result.pipeline.trace] + + +@pytest.mark.asyncio +async def test_orchestrator_skips_repair_node_when_toggle_off() -> None: + repairer = MagicMock() + repairer.maybe_repair = AsyncMock() + orchestrator = _orchestrator(repairer) + result = await orchestrator.execute(_wiring_request(repair=False)) + repairer.maybe_repair.assert_not_awaited() + assert result.pipeline.repair is None + assert "repair" not in [t.node for t in result.pipeline.trace] + + +@pytest.mark.asyncio +async def test_orchestrator_skips_repair_without_any_signal_stage() -> None: + """repair needs judge or field_validation verdicts to act on.""" + repairer = MagicMock() + repairer.maybe_repair = AsyncMock() + orchestrator = _orchestrator(repairer) + request = ExtractionRequest( + intention="test", + files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], + document_types=[_doc_spec()], + options=ExtractionOptions(stages=StageToggles(judge=False, field_validation=False, repair=True)), + ) + result = await orchestrator.execute(request) + repairer.maybe_repair.assert_not_awaited() + assert "repair" not in [t.node for t in result.pipeline.trace] From 2ec569070217d0adb4c27f74f7a8bb0806a11ce9 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 13:45:51 +0200 Subject: [PATCH 14/38] docs(env): repair-pass model and timeout defaults --- env_template | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/env_template b/env_template index 5830ba6..2e2b82f 100644 --- a/env_template +++ b/env_template @@ -62,6 +62,15 @@ FLYDOCS_MAX_SYNC_PAGES=10 # Cap on total bytes accepted (after base64-decode) in a single submission. FLYDOCS_MAX_BYTES=33554432 +# Closed-loop targeted repair (stage is opt-in per request via +# ``options.stages.repair``). Fields that failed the judge or a validator +# are re-extracted once with the failure evidence quoted back to the +# model. Default model for the repair pass; unset falls back to the +# global FLYDOCS_MODEL. A stronger model than the extractor is +# recommended -- it is fixing what the extractor got wrong. +FLYDOCS_REPAIR_MODEL=anthropic:claude-opus-4-8 +FLYDOCS_REPAIR_TIMEOUT_S=300 + # Per-call timeouts (seconds). Sync requests get the shorter one; async jobs # retry up to ``JOB_MAX_ATTEMPTS`` times on timeout. FLYDOCS_SYNC_TIMEOUT_S=60 From cf507a4f0a9ffaac68a468f66da345fae79316e9 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 14:04:25 +0200 Subject: [PATCH 15/38] =?UTF-8?q?fix(repair):=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20array-failure=20collection,=20judge-PASS=20accept?= =?UTF-8?q?=20guard,=20never=20erase=20to=20null,=20compact=20array=20evid?= =?UTF-8?q?ence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/services/repair/field_repairer.py | 56 ++++++++++++-- tests/unit/test_field_repairer.py | 74 +++++++++++++++++++ 2 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 68792f4..091868f 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -23,8 +23,9 @@ Repaired values are re-verified (validators always; judge subset re-check when the judge stage is enabled) and replace the originals -only when they come back clean -- the repair is monotonic: it can fix -fields, never degrade them. +only when they come back clean -- with the judge on, that means an +explicit PASS, and a null candidate never replaces a value. The repair +is monotonic: it can fix fields, never degrade them. Opt-in via ``options.stages.repair``. Runs after judge and BEFORE judge_escalation, so the expensive full re-run only fires when targeted @@ -65,7 +66,10 @@ def collect_failing_fields(groups: list[ExtractedFieldGroup]) -> list[FailingFie A field fails when ``judge.status == FAIL``, ``judge.flag_for_review`` is set, or ``validation.valid`` is false. The evidence string joins - the judge's reasoning with every validator error message. + the judge's reasoning with every validator error message. Array + fields carry ``valid=False`` with an empty parent error list (the + messages live on the row sub-fields), so the evidence for arrays is + harvested from the rows. """ failures: list[FailingField] = [] for group in groups: @@ -74,7 +78,10 @@ def collect_failing_fields(groups: list[ExtractedFieldGroup]) -> list[FailingFie if field.judge.status == JudgeStatus.FAIL or field.judge.flag_for_review: reasons.append(field.judge.evidence or field.judge.notes or "judge rejected the value") if not field.validation.valid: - reasons.extend(e.message for e in field.validation.errors) + messages = [e.message for e in field.validation.errors] + if not messages and isinstance(field.value, list): + messages = _row_error_messages(field.value) + reasons.extend(messages or ["failed validation"]) if reasons: failures.append( FailingField( @@ -87,6 +94,19 @@ def collect_failing_fields(groups: list[ExtractedFieldGroup]) -> list[FailingFie return failures +def _row_error_messages(rows: list[Any]) -> list[str]: + """Validator messages stamped on the row sub-fields of an array field.""" + messages: list[str] = [] + for row in rows: + if not isinstance(row, ExtractedField) or not isinstance(row.value, list): + continue + for sub_field in row.value: + if not isinstance(sub_field, ExtractedField): + continue + messages.extend(f"{sub_field.name}: {e.message}" for e in sub_field.validation.errors) + return messages + + class FieldRepairer: """Re-extract failing fields with their failure evidence, merge back what passes.""" @@ -184,6 +204,7 @@ async def _repair_one( repaired_by_key: dict[tuple[str, str], ExtractedField] = { (group.name, field.name): field for group in repaired_groups for field in group.fields } + judged = request.options.stages.judge accepted: list[str] = [] for group in task.extracted_groups: for index, field in enumerate(group.fields): @@ -191,17 +212,28 @@ async def _repair_one( if key not in failing_keys: continue candidate = repaired_by_key.get(key) - if candidate is None or not _is_clean(candidate): + if candidate is None or not _is_clean(candidate, require_judge_pass=judged): continue group.fields[index] = candidate accepted.append(f"{group.name}.{field.name}") return accepted -def _is_clean(field: ExtractedField) -> bool: - """A repaired field is accepted only when re-verification passed.""" +def _is_clean(field: ExtractedField, *, require_judge_pass: bool) -> bool: + """A repaired field is accepted only when re-verification passed. + + A ``None`` value never replaces the original -- a repair that found + nothing must not erase data. When the judge stage is on, the + candidate needs an explicit PASS: a field the judge omitted keeps + the default UNCERTAIN outcome and would otherwise slip through + unverified. + """ + if field.value is None: + return False if not field.validation.valid: return False + if require_judge_pass: + return field.judge.status == JudgeStatus.PASS and not field.judge.flag_for_review return field.judge.status != JudgeStatus.FAIL and not field.judge.flag_for_review @@ -221,6 +253,14 @@ def _failures_text(failures: list[FailingField]) -> str: lines = [] for f in failures: lines.append( - f"- ``{f.group}.{f.field}``: previous value {f.value!r} -- rejected because: {f.evidence}" + f"- ``{f.group}.{f.field}``: previous value {_value_summary(f.value)} " + f"-- rejected because: {f.evidence}" ) return "\n".join(lines) + + +def _value_summary(value: Any) -> str: + """Human-readable value for the repair prompt; arrays stay compact.""" + if isinstance(value, list): + return f"" + return repr(value) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 8ee723c..21eab1f 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -168,6 +168,40 @@ def test_collect_failing_fields_empty_when_all_clean() -> None: assert collect_failing_fields(groups) == [] +def test_collect_failing_fields_includes_arrays_with_empty_error_list() -> None: + """_validate_array stamps the parent valid=False with errors=[]; the + evidence must be harvested from the row sub-fields instead.""" + bad_sub = ExtractedField( + name="quantity", + value="-3", + validation=FieldValidation( + valid=False, + errors=[FieldValidationError(rule=ValidationRule.MINIMUM, message="value below minimum 0")], + ), + ) + row = ExtractedField(name="row", value=[bad_sub]) + array_field = ExtractedField( + name="line_items", + value=[row], + validation=FieldValidation(valid=False, errors=[]), # exactly what _validate_array produces + ) + groups = [ExtractedFieldGroup(name="items", fields=[array_field])] + failures = collect_failing_fields(groups) + assert [(f.group, f.field) for f in failures] == [("items", "line_items")] + assert "value below minimum 0" in failures[0].evidence + + +def test_failures_text_renders_array_values_compactly() -> None: + from flydocs.core.services.repair.field_repairer import FailingField, _failures_text + + row = ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]) + text = _failures_text( + [FailingField(group="items", field="line_items", value=[row], evidence="bad rows")] + ) + assert "ExtractedField(" not in text + assert "1 row(s)" in text + + # --------------------------------------------------------------------------- # Repair round-trip # --------------------------------------------------------------------------- @@ -304,6 +338,46 @@ async def test_maybe_repair_without_judge_stage_uses_validator_only() -> None: judge.judge.assert_not_awaited() +@pytest.mark.asyncio +async def test_maybe_repair_rejects_candidate_the_judge_never_graded() -> None: + """With the judge stage ON, a candidate the judge omitted keeps the + default UNCERTAIN outcome and must NOT replace the original.""" + original = _judge_failed_field("number", "X123", "misread") + task = _task([ExtractedFieldGroup(name="identity", fields=[original])]) + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup(name="identity", fields=[ExtractedField(name="number", value="Z999")]) + ] + ) + judge = MagicMock() + judge.judge = AsyncMock(side_effect=lambda *, extracted_groups, **_: extracted_groups) # stamps nothing + + repairer = _repairer(extractor, judge) + info = await repairer.maybe_repair(_ctx([task]), _request()) + + assert info is not None and info.fields_repaired == 0 + assert task.extracted_groups[0].fields[0].value == "X123" + + +@pytest.mark.asyncio +async def test_maybe_repair_never_replaces_a_value_with_none() -> None: + """A null candidate (repair found nothing) must not erase the original.""" + original = _validator_failed_field("number", "bad!", "pattern mismatch") + task = _task([ExtractedFieldGroup(name="identity", fields=[original])]) + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup(name="identity", fields=[ExtractedField(name="number", value=None)]) + ] + ) + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + info = await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + assert info is not None and info.fields_repaired == 0 + assert task.extracted_groups[0].fields[0].value == "bad!" + + @pytest.mark.asyncio async def test_maybe_repair_falls_back_to_request_model() -> None: task = _task([ExtractedFieldGroup(name="identity", fields=[_judge_failed_field("number", "X", "bad")])]) From 65e3bb56846eb1191c9bf62f7b06c2caf4af6d10 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 14:05:34 +0200 Subject: [PATCH 16/38] feat(validation): warn when repair is enabled without any verification stage --- .../services/validation/request_validator.py | 16 +++++++++++++++ tests/unit/test_request_validator.py | 20 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/flydocs/core/services/validation/request_validator.py b/src/flydocs/core/services/validation/request_validator.py index acd3362..96fc60c 100644 --- a/src/flydocs/core/services/validation/request_validator.py +++ b/src/flydocs/core/services/validation/request_validator.py @@ -384,3 +384,19 @@ def _check_stage_consistency(self, request: ExtractionRequest, report: Validatio path="options.stages.splitter", ) ) + + # repair on but no stage produces failure signals => the node is + # never added to the DAG. Warn. + if stages.repair and not (stages.judge or stages.field_validation): + report.issues.append( + ValidationIssue( + severity="warning", + code="repair_no_verification_stage", + message=( + "stages.repair is enabled but both judge and " + "field_validation are off -- there are no failure " + "signals to repair from, so the stage is skipped." + ), + path="options.stages.repair", + ) + ) diff --git a/tests/unit/test_request_validator.py b/tests/unit/test_request_validator.py index bd95a53..f1c45c4 100644 --- a/tests/unit/test_request_validator.py +++ b/tests/unit/test_request_validator.py @@ -249,6 +249,26 @@ def test_splitter_single_doc_is_warning_only(validator: RequestValidator) -> Non assert "splitter_single_doc" in codes +# -- warning: repair on without any verification stage ------------------------ + + +def test_repair_without_verification_stage_is_warning_only(validator: RequestValidator) -> None: + options = ExtractionOptions( + stages=StageToggles(repair=True, judge=False, field_validation=False) + ) + report = validator.validate(_request(options=options)) + assert not report.has_errors + codes = [i.code for i in report.warnings] + assert "repair_no_verification_stage" in codes + + +def test_repair_with_judge_produces_no_warning(validator: RequestValidator) -> None: + options = ExtractionOptions(stages=StageToggles(repair=True, judge=True)) + report = validator.validate(_request(options=options)) + codes = [i.code for i in report.warnings] + assert "repair_no_verification_stage" not in codes + + # -- warning: visual_authenticity on but no visual checks -------------------- From 4ae86fd491f5d391cc3c2c6111e40afbe21f291d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 14:07:10 +0200 Subject: [PATCH 17/38] docs: repair stage in pipeline table, README diagram, timeout list, async-budget note --- README.md | 4 ++-- docs/pipeline.md | 15 ++++++++++----- env_template | 4 +++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4e54092..c5133f1 100644 --- a/README.md +++ b/README.md @@ -271,8 +271,8 @@ DAG for each call so the audit trail reflects exactly what executed. ┌──────────────────────────────────────────────────────────────────┐ POST ──────▶│ load → discover? → classify? → plan_tasks → extract → │──────▶ JSON (PDF/PNG/…) │ bbox_validation → bbox_refine? → field_validation? → │ (fields + bbox - │ visual_auth? → content_auth? → judge? → judge_escalation? → │ + verdicts) - │ transform? → rules? → assemble │ + │ visual_auth? → content_auth? → judge? → repair? → │ + verdicts) + │ judge_escalation? → transform? → rules? → assemble │ └──────────────────────────────────────────────────────────────────┘ │ │ per-segment concurrency (asyncio.gather) diff --git a/docs/pipeline.md b/docs/pipeline.md index aa2003c..b8ad801 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -55,10 +55,11 @@ reflect exactly what executed. | 9 | `visual_authenticity` | no | LLM evaluates caller-defined visual validators (signature present, stamp present, …). | 180 s | | 10 | `content_authenticity` | no | LLM audit: dates consistent, totals add up, expected boilerplate, tampering signals. | 180 s | | 11 | `judge` | no | Second LLM pass re-grades every extracted value against the source. | 300 s | -| 12 | `judge_escalation` | no | When the judge's failure rate exceeds `escalation_threshold`, re-run extract + judge with `escalation_model` and keep the better result. | 600 s | -| 13 | `transform` | no | Caller-declared post-extraction transformations: declarative entity resolution + free-form LLM transformations. See [transformations.md](transformations.md). | 600 s | -| 14 | `rules` | no | LLM evaluates the business-rule DAG, level by level. | 180 s | -| 15 | `assemble` | yes | Pure Python: compose the `ExtractionResult`. | 5 s | +| 12 | `repair` | no | Targeted repair: re-extract ONLY the fields that failed the judge or a validator, quoting the failure evidence; accept per field only when the re-check passes. | 300 s | +| 13 | `judge_escalation` | no | When the judge's failure rate exceeds `escalation_threshold`, re-run extract + judge with `escalation_model` and keep the better result. | 600 s | +| 14 | `transform` | no | Caller-declared post-extraction transformations: declarative entity resolution + free-form LLM transformations. See [transformations.md](transformations.md). | 600 s | +| 15 | `rules` | no | LLM evaluates the business-rule DAG, level by level. | 180 s | +| 16 | `assemble` | yes | Pure Python: compose the `ExtractionResult`. | 5 s | Optional stages are caller-toggled through `ExtractionOptions.stages`. Other short-circuits: @@ -75,6 +76,9 @@ Other short-circuits: - `bbox_refine` runs **inline only for sync** requests. On the async path `ExtractionWorker` skips it and delegates refinement to the dedicated `BboxRefineWorker` (out-of-band, idempotent). +- `repair` is silently skipped when both `judge` and + `field_validation` are off (no failure signals to act on); the + request validator emits a `repair_no_verification_stage` warning. - `judge_escalation` is silently skipped when `judge` is off. - `transform` is silently a no-op when `options.transformations` is empty even with the toggle on. @@ -91,6 +95,7 @@ FLYDOCS_BBOX_REFINE_INLINE_TIMEOUT_S=900 FLYDOCS_CLASSIFIER_TIMEOUT_S=180 FLYDOCS_SPLITTER_TIMEOUT_S=180 FLYDOCS_JUDGE_ESCALATION_TIMEOUT_S=600 +FLYDOCS_REPAIR_TIMEOUT_S=300 FLYDOCS_TRANSFORM_TIMEOUT_S=600 ``` @@ -140,7 +145,7 @@ builder.add_node("bbox_validation", CallableStep(self._step_bbox_validation), ti if stages.field_validation: builder.add_node("field_validation", ...) -# ...visual_authenticity, content_authenticity, judge, judge_escalation, rules... +# ...visual_authenticity, content_authenticity, judge, repair, judge_escalation, rules... builder.add_node("assemble", CallableStep(self._step_assemble), timeout_seconds=5) builder.chain(*chain) # linear order diff --git a/env_template b/env_template index 2e2b82f..ceafa1b 100644 --- a/env_template +++ b/env_template @@ -67,7 +67,9 @@ FLYDOCS_MAX_BYTES=33554432 # are re-extracted once with the failure evidence quoted back to the # model. Default model for the repair pass; unset falls back to the # global FLYDOCS_MODEL. A stronger model than the extractor is -# recommended -- it is fixing what the extractor got wrong. +# recommended -- it is fixing what the extractor got wrong. Note the +# repair pass consumes part of the FLYDOCS_ASYNC_TIMEOUT_S budget on +# async jobs -- raise that budget if you also raise this timeout. FLYDOCS_REPAIR_MODEL=anthropic:claude-opus-4-8 FLYDOCS_REPAIR_TIMEOUT_S=300 From 7b6eac3eb7af8d672e9b923712e3f7c533b1dff1 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 14:07:47 +0200 Subject: [PATCH 18/38] style: ruff format --- tests/unit/test_field_repairer.py | 4 +--- tests/unit/test_request_validator.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 21eab1f..d34c248 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -195,9 +195,7 @@ def test_failures_text_renders_array_values_compactly() -> None: from flydocs.core.services.repair.field_repairer import FailingField, _failures_text row = ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]) - text = _failures_text( - [FailingField(group="items", field="line_items", value=[row], evidence="bad rows")] - ) + text = _failures_text([FailingField(group="items", field="line_items", value=[row], evidence="bad rows")]) assert "ExtractedField(" not in text assert "1 row(s)" in text diff --git a/tests/unit/test_request_validator.py b/tests/unit/test_request_validator.py index f1c45c4..50846e3 100644 --- a/tests/unit/test_request_validator.py +++ b/tests/unit/test_request_validator.py @@ -253,9 +253,7 @@ def test_splitter_single_doc_is_warning_only(validator: RequestValidator) -> Non def test_repair_without_verification_stage_is_warning_only(validator: RequestValidator) -> None: - options = ExtractionOptions( - stages=StageToggles(repair=True, judge=False, field_validation=False) - ) + options = ExtractionOptions(stages=StageToggles(repair=True, judge=False, field_validation=False)) report = validator.validate(_request(options=options)) assert not report.has_errors codes = [i.code for i in report.warnings] From 044548e668d2d1f757bc84edbb5961db50661706 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 14:08:18 +0200 Subject: [PATCH 19/38] style(tests): move repair test imports to top of file --- tests/unit/test_field_repairer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index d34c248..bcb4be0 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -30,7 +30,8 @@ import pytest from flydocs.core.services.extraction.prompts import PromptCatalog -from flydocs.core.services.repair import FieldRepairer, collect_failing_fields +from flydocs.core.services.repair import FailingField, FieldRepairer, collect_failing_fields +from flydocs.core.services.repair.field_repairer import _failures_text from flydocs.core.services.validation.field_validator import FieldValidator from flydocs.interfaces.dtos.document_type import DocumentTypeSpec from flydocs.interfaces.dtos.extract import ( @@ -192,8 +193,6 @@ def test_collect_failing_fields_includes_arrays_with_empty_error_list() -> None: def test_failures_text_renders_array_values_compactly() -> None: - from flydocs.core.services.repair.field_repairer import FailingField, _failures_text - row = ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]) text = _failures_text([FailingField(group="items", field="line_items", value=[row], evidence="bad rows")]) assert "ExtractedField(" not in text From 5b24307b0e100e83724cde16aaccc4e4cb8d7d70 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:31:36 +0200 Subject: [PATCH 20/38] style(tests): move remaining inline imports to top of file --- tests/unit/test_field_repairer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index bcb4be0..1231913 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -29,7 +29,9 @@ import pytest +from flydocs.config import IDPSettings from flydocs.core.services.extraction.prompts import PromptCatalog +from flydocs.core.services.pipeline.orchestrator import PipelineOrchestrator from flydocs.core.services.repair import FailingField, FieldRepairer, collect_failing_fields from flydocs.core.services.repair.field_repairer import _failures_text from flydocs.core.services.validation.field_validator import FieldValidator @@ -38,6 +40,7 @@ ExtractionOptions, ExtractionRequest, FileInput, + RepairInfo, StageToggles, ) from flydocs.interfaces.dtos.field import ( @@ -414,9 +417,6 @@ def _fake_normalizer() -> Any: def _orchestrator(repairer: Any) -> Any: - from flydocs.config import IDPSettings - from flydocs.core.services.pipeline.orchestrator import PipelineOrchestrator - groups = [ExtractedFieldGroup(name="identity", fields=[])] extractor = MagicMock() extractor.extract = AsyncMock(return_value=(groups, "base-model")) @@ -453,8 +453,6 @@ def _wiring_request(*, repair: bool, judge: bool = True) -> ExtractionRequest: @pytest.mark.asyncio async def test_orchestrator_runs_repair_node_and_reports_audit_block() -> None: - from flydocs.interfaces.dtos.extract import RepairInfo - repairer = MagicMock() repairer.maybe_repair = AsyncMock( return_value=RepairInfo( From 2ed071c25ed73e250f9fe179392a1834f9c7d450 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:32:09 +0200 Subject: [PATCH 21/38] refactor(tests): one parameterized request builder instead of three --- tests/unit/test_field_repairer.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 1231913..47afe51 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -121,12 +121,16 @@ def _ctx(tasks: list[Any], model_id: str = "base-model") -> Any: return SimpleNamespace(metadata={"tasks": tasks, "model_id": model_id}) -def _request(*, judge: bool = True) -> ExtractionRequest: +def _request( + *, repair: bool = True, judge: bool = True, field_validation: bool = True +) -> ExtractionRequest: return ExtractionRequest( intention="test", files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], document_types=[_doc_spec()], - options=ExtractionOptions(stages=StageToggles(judge=judge, repair=True)), + options=ExtractionOptions( + stages=StageToggles(judge=judge, repair=repair, field_validation=field_validation) + ), ) @@ -442,15 +446,6 @@ def _orchestrator(repairer: Any) -> Any: ) -def _wiring_request(*, repair: bool, judge: bool = True) -> ExtractionRequest: - return ExtractionRequest( - intention="test", - files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], - document_types=[_doc_spec()], - options=ExtractionOptions(stages=StageToggles(judge=judge, repair=repair)), - ) - - @pytest.mark.asyncio async def test_orchestrator_runs_repair_node_and_reports_audit_block() -> None: repairer = MagicMock() @@ -464,7 +459,7 @@ async def test_orchestrator_runs_repair_node_and_reports_audit_block() -> None: ) ) orchestrator = _orchestrator(repairer) - result = await orchestrator.execute(_wiring_request(repair=True)) + result = await orchestrator.execute(_request()) repairer.maybe_repair.assert_awaited_once() assert result.pipeline.repair is not None assert result.pipeline.repair.fields_repaired == 1 @@ -476,7 +471,7 @@ async def test_orchestrator_skips_repair_node_when_toggle_off() -> None: repairer = MagicMock() repairer.maybe_repair = AsyncMock() orchestrator = _orchestrator(repairer) - result = await orchestrator.execute(_wiring_request(repair=False)) + result = await orchestrator.execute(_request(repair=False)) repairer.maybe_repair.assert_not_awaited() assert result.pipeline.repair is None assert "repair" not in [t.node for t in result.pipeline.trace] @@ -488,12 +483,6 @@ async def test_orchestrator_skips_repair_without_any_signal_stage() -> None: repairer = MagicMock() repairer.maybe_repair = AsyncMock() orchestrator = _orchestrator(repairer) - request = ExtractionRequest( - intention="test", - files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], - document_types=[_doc_spec()], - options=ExtractionOptions(stages=StageToggles(judge=False, field_validation=False, repair=True)), - ) - result = await orchestrator.execute(request) + result = await orchestrator.execute(_request(judge=False, field_validation=False)) repairer.maybe_repair.assert_not_awaited() assert "repair" not in [t.node for t in result.pipeline.trace] From 86ecb5f6105e1e7c93d09dcde879ec61e838f002 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:32:32 +0200 Subject: [PATCH 22/38] refactor(repair): export only FieldRepairer, matching the escalation sibling --- src/flydocs/core/services/repair/__init__.py | 8 ++------ tests/unit/test_field_repairer.py | 8 ++++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/flydocs/core/services/repair/__init__.py b/src/flydocs/core/services/repair/__init__.py index 98deb4d..ec24afb 100644 --- a/src/flydocs/core/services/repair/__init__.py +++ b/src/flydocs/core/services/repair/__init__.py @@ -14,10 +14,6 @@ """Closed-loop targeted repair -- focused re-extraction of failing fields.""" -from flydocs.core.services.repair.field_repairer import ( - FailingField, - FieldRepairer, - collect_failing_fields, -) +from flydocs.core.services.repair.field_repairer import FieldRepairer -__all__ = ["FailingField", "FieldRepairer", "collect_failing_fields"] +__all__ = ["FieldRepairer"] diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 47afe51..3f817b4 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -32,8 +32,12 @@ from flydocs.config import IDPSettings from flydocs.core.services.extraction.prompts import PromptCatalog from flydocs.core.services.pipeline.orchestrator import PipelineOrchestrator -from flydocs.core.services.repair import FailingField, FieldRepairer, collect_failing_fields -from flydocs.core.services.repair.field_repairer import _failures_text +from flydocs.core.services.repair import FieldRepairer +from flydocs.core.services.repair.field_repairer import ( + FailingField, + _failures_text, + collect_failing_fields, +) from flydocs.core.services.validation.field_validator import FieldValidator from flydocs.interfaces.dtos.document_type import DocumentTypeSpec from flydocs.interfaces.dtos.extract import ( From 7dd6ccbd3d37fb4a2a4694a81b9b76670dc7569b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:33:13 +0200 Subject: [PATCH 23/38] refactor(repair): inline single-caller _value_summary helper --- src/flydocs/core/services/repair/field_repairer.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 091868f..76faea0 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -252,15 +252,7 @@ def _failures_text(failures: list[FailingField]) -> str: """Render the per-field failure evidence block for the repair prompt.""" lines = [] for f in failures: - lines.append( - f"- ``{f.group}.{f.field}``: previous value {_value_summary(f.value)} " - f"-- rejected because: {f.evidence}" - ) + # Arrays stay compact -- a full repr of the nested rows would flood the prompt. + value = f"" if isinstance(f.value, list) else repr(f.value) + lines.append(f"- ``{f.group}.{f.field}``: previous value {value} -- rejected because: {f.evidence}") return "\n".join(lines) - - -def _value_summary(value: Any) -> str: - """Human-readable value for the repair prompt; arrays stay compact.""" - if isinstance(value, list): - return f"" - return repr(value) From ab06e4811e53dbd45bebe3e9f460636f7ad73568 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:33:46 +0200 Subject: [PATCH 24/38] refactor(tests): assert array-evidence rendering via the public repair path --- tests/unit/test_field_repairer.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 3f817b4..72d335b 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -33,11 +33,7 @@ from flydocs.core.services.extraction.prompts import PromptCatalog from flydocs.core.services.pipeline.orchestrator import PipelineOrchestrator from flydocs.core.services.repair import FieldRepairer -from flydocs.core.services.repair.field_repairer import ( - FailingField, - _failures_text, - collect_failing_fields, -) +from flydocs.core.services.repair.field_repairer import collect_failing_fields from flydocs.core.services.validation.field_validator import FieldValidator from flydocs.interfaces.dtos.document_type import DocumentTypeSpec from flydocs.interfaces.dtos.extract import ( @@ -203,9 +199,22 @@ def test_collect_failing_fields_includes_arrays_with_empty_error_list() -> None: assert "value below minimum 0" in failures[0].evidence -def test_failures_text_renders_array_values_compactly() -> None: - row = ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]) - text = _failures_text([FailingField(group="items", field="line_items", value=[row], evidence="bad rows")]) +@pytest.mark.asyncio +async def test_maybe_repair_renders_array_evidence_compactly() -> None: + """The repair prompt summarizes array values instead of dumping DTO reprs.""" + array_field = ExtractedField( + name="line_items", + value=[ExtractedField(name="row", value=[ExtractedField(name="a", value="1")])], + validation=FieldValidation(valid=False, errors=[]), + ) + task = _task([ExtractedFieldGroup(name="items", fields=[array_field])]) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + text = extractor.extract_repair.await_args.kwargs["failing_fields_text"] assert "ExtractedField(" not in text assert "1 row(s)" in text From ead10e24d9832a7fab8d0904e05bbc87a3a8a8ee Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:34:24 +0200 Subject: [PATCH 25/38] refactor(repair): build failing_keys once, pass to _subset_spec --- src/flydocs/core/services/repair/field_repairer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 76faea0..68d1eba 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -174,7 +174,8 @@ async def _repair_one( model: str | None, ) -> list[str]: """Repair one task; return the ``group.field`` paths that were accepted.""" - subset = _subset_spec(task.doc_spec, failures) + failing_keys = {(f.group, f.field) for f in failures} + subset = _subset_spec(task.doc_spec, failing_keys) repaired_groups = await self._extractor.extract_repair( document_bytes=task.slice_bytes, media_type=task.segment.media_type, @@ -200,7 +201,6 @@ async def _repair_one( model=model, ) - failing_keys = {(f.group, f.field) for f in failures} repaired_by_key: dict[tuple[str, str], ExtractedField] = { (group.name, field.name): field for group in repaired_groups for field in group.fields } @@ -237,9 +237,8 @@ def _is_clean(field: ExtractedField, *, require_judge_pass: bool) -> bool: return field.judge.status != JudgeStatus.FAIL and not field.judge.flag_for_review -def _subset_spec(doc: DocumentTypeSpec, failures: list[FailingField]) -> DocumentTypeSpec: +def _subset_spec(doc: DocumentTypeSpec, failing_keys: set[tuple[str, str]]) -> DocumentTypeSpec: """A copy of ``doc`` whose field_groups contain only the failing fields.""" - failing_keys = {(f.group, f.field) for f in failures} groups = [] for group in doc.field_groups: fields = [f for f in group.fields if (group.name, f.name) in failing_keys] From bcf500d1c6b8419bfeb9cbeca182659b395fd7c5 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:35:30 +0200 Subject: [PATCH 26/38] perf(pipeline): skip bbox re-grade when no repair candidate was accepted --- src/flydocs/core/services/pipeline/orchestrator.py | 10 ++++++---- tests/unit/test_field_repairer.py | 4 +--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/flydocs/core/services/pipeline/orchestrator.py b/src/flydocs/core/services/pipeline/orchestrator.py index 7f6fe28..e55a22d 100644 --- a/src/flydocs/core/services/pipeline/orchestrator.py +++ b/src/flydocs/core/services/pipeline/orchestrator.py @@ -800,10 +800,12 @@ async def _step_repair(self, ctx: PipelineContext, _inputs: dict[str, Any]) -> A return {"repair_triggered": False} ctx.metadata["repair"] = info # Repaired values carry fresh judge/validation verdicts but their - # bboxes came from a new LLM pass -- re-grade the geometry. - for task in ctx.metadata["tasks"]: - if task.extracted_groups: - self._bbox_validator.validate_groups(task.extracted_groups) + # bboxes came from a new LLM pass -- re-grade the geometry. Skip + # when every candidate was rejected (groups are unchanged). + if info.fields_repaired: + for task in ctx.metadata["tasks"]: + if task.extracted_groups: + self._bbox_validator.validate_groups(task.extracted_groups) return {"repair_triggered": True, "fields_repaired": info.fields_repaired} async def _step_judge_escalation(self, ctx: PipelineContext, _inputs: dict[str, Any]) -> Any: diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 72d335b..e43d887 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -121,9 +121,7 @@ def _ctx(tasks: list[Any], model_id: str = "base-model") -> Any: return SimpleNamespace(metadata={"tasks": tasks, "model_id": model_id}) -def _request( - *, repair: bool = True, judge: bool = True, field_validation: bool = True -) -> ExtractionRequest: +def _request(*, repair: bool = True, judge: bool = True, field_validation: bool = True) -> ExtractionRequest: return ExtractionRequest( intention="test", files=[FileInput(filename="doc.pdf", content_base64=_DUMMY, expected_type="passport")], From 3dcdf8ba2828ff2859dedac88fb81237d2e246c4 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 16:45:59 +0200 Subject: [PATCH 27/38] test: reconcile per-stage orchestrator harness with field_repairer param after merging #40 + #41 --- tests/unit/test_per_stage_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_per_stage_models.py b/tests/unit/test_per_stage_models.py index 676fd0b..11d5488 100644 --- a/tests/unit/test_per_stage_models.py +++ b/tests/unit/test_per_stage_models.py @@ -174,6 +174,7 @@ def _orchestrator(settings: IDPSettings, extractor: Any, judge: Any) -> Pipeline rule_engine=MagicMock(), judge_escalator=MagicMock(), transformation_engine=MagicMock(), + field_repairer=MagicMock(), settings=settings, default_model="default-model", ) From fbd61318ed4c8e60c17d320b08e287459bc743d1 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:39:12 +0200 Subject: [PATCH 28/38] feat: repair_include_flagged setting to restrict repair to hard failures Flagged-but-PASS fields are often ambiguous-but-correct; repairing them is spend the accept guard usually discards. FLYDOCS_REPAIR_INCLUDE_FLAGGED (default true, preserving current behaviour) lets cost-sensitive deployments limit repair to judge FAIL + validator errors. --- src/flydocs/config.py | 6 ++++ src/flydocs/core/configuration.py | 1 + .../core/services/repair/field_repairer.py | 22 +++++++----- tests/unit/test_field_repairer.py | 34 +++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 56db855..7dacb73 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -234,8 +234,14 @@ class IDPSettings(BaseSettings): # ``repair_model`` pins the pass to its own model (``None`` = the # request model). Runs BEFORE judge_escalation, so the full re-run # only fires when targeted repair was not enough. + # ``repair_include_flagged`` widens the failure predicate to judge + # ``flag_for_review`` fields (matching what the escalation counter + # counts). Flagged-but-PASS fields are often ambiguous-but-correct, + # so set it to false to restrict repair to hard failures (judge FAIL + # or a validator error) and save those extra passes. repair_model: str | None = None repair_timeout_s: int = 300 + repair_include_flagged: bool = True # -- Webhook -------------------------------------------------------- # The result webhook delivers the full extraction (split docs + fields + diff --git a/src/flydocs/core/configuration.py b/src/flydocs/core/configuration.py index ed2b9a5..2e6dac8 100644 --- a/src/flydocs/core/configuration.py +++ b/src/flydocs/core/configuration.py @@ -372,6 +372,7 @@ def field_repairer( judge=judge, field_validator=field_validator, default_model=settings.repair_model, + include_flagged=settings.repair_include_flagged, ) # ------------------------------------------------------------------ diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 68d1eba..43f974a 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -61,21 +61,25 @@ class FailingField: evidence: str -def collect_failing_fields(groups: list[ExtractedFieldGroup]) -> list[FailingField]: +def collect_failing_fields( + groups: list[ExtractedFieldGroup], *, include_flagged: bool = True +) -> list[FailingField]: """Top-level fields whose judge verdict or validation failed. A field fails when ``judge.status == FAIL``, ``judge.flag_for_review`` - is set, or ``validation.valid`` is false. The evidence string joins - the judge's reasoning with every validator error message. Array - fields carry ``valid=False`` with an empty parent error list (the - messages live on the row sub-fields), so the evidence for arrays is - harvested from the rows. + is set (unless ``include_flagged`` is false -- flagged-but-PASS fields + are usually ambiguous-but-correct, so cost-sensitive deployments can + restrict repair to hard failures), or ``validation.valid`` is false. + The evidence string joins the judge's reasoning with every validator + error message. Array fields carry ``valid=False`` with an empty parent + error list (the messages live on the row sub-fields), so the evidence + for arrays is harvested from the rows. """ failures: list[FailingField] = [] for group in groups: for field in group.fields: reasons: list[str] = [] - if field.judge.status == JudgeStatus.FAIL or field.judge.flag_for_review: + if field.judge.status == JudgeStatus.FAIL or (include_flagged and field.judge.flag_for_review): reasons.append(field.judge.evidence or field.judge.notes or "judge rejected the value") if not field.validation.valid: messages = [e.message for e in field.validation.errors] @@ -117,11 +121,13 @@ def __init__( judge: Judge, field_validator: FieldValidator, default_model: str | None, + include_flagged: bool = True, ) -> None: self._extractor = extractor self._judge = judge self._field_validator = field_validator self._default_model = default_model + self._include_flagged = include_flagged async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo | None: """Return a :class:`RepairInfo` when at least one field was flagged, else ``None``. @@ -136,7 +142,7 @@ async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo async def _repair_task(task: Any) -> None: nonlocal flagged - failures = collect_failing_fields(task.extracted_groups) + failures = collect_failing_fields(task.extracted_groups, include_flagged=self._include_flagged) if not failures: return flagged += len(failures) diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index e43d887..772419f 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -169,6 +169,38 @@ def test_collect_failing_fields_includes_flag_for_review() -> None: assert [(f.group, f.field) for f in failures] == [("identity", "number")] +def test_collect_failing_fields_can_exclude_flagged_pass_fields() -> None: + """include_flagged=False restricts collection to hard failures: a PASS + field that is merely flag_for_review (ambiguous-but-correct) is skipped.""" + flagged = ExtractedField( + name="number", + value="X123", + judge=JudgeOutcome(status=JudgeStatus.PASS, flag_for_review=True), + ) + hard_fail = _judge_failed_field("iban", "ES00", "checksum") + groups = [ExtractedFieldGroup(name="identity", fields=[flagged, hard_fail])] + failures = collect_failing_fields(groups, include_flagged=False) + assert [(f.group, f.field) for f in failures] == [("identity", "iban")] + + +@pytest.mark.asyncio +async def test_maybe_repair_skips_flagged_pass_fields_when_configured() -> None: + flagged = ExtractedField( + name="number", + value="X123", + judge=JudgeOutcome(status=JudgeStatus.PASS, flag_for_review=True), + ) + task = _task([ExtractedFieldGroup(name="identity", fields=[flagged])]) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), include_flagged=False) + info = await repairer.maybe_repair(_ctx([task]), _request()) + + assert info is None + extractor.extract_repair.assert_not_awaited() + + def test_collect_failing_fields_empty_when_all_clean() -> None: groups = [ExtractedFieldGroup(name="identity", fields=[_clean_field("name", "JOHN")])] assert collect_failing_fields(groups) == [] @@ -227,12 +259,14 @@ def _repairer( judge: Any, *, default_model: str | None = None, + include_flagged: bool = True, ) -> FieldRepairer: return FieldRepairer( extractor=extractor, judge=judge, field_validator=FieldValidator(), default_model=default_model, + include_flagged=include_flagged, ) From 74f4a6098d23139f28593da94ce898ab4d184787 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:41:10 +0200 Subject: [PATCH 29/38] feat: repair_max_failing_fraction scope cap on targeted repair Without a cap, an all-fail task degenerates into a full re-extract plus a full judge re-check on the repair model, with escalation still able to fire afterwards -- strictly costlier than judge->escalation alone. Above the cap (default 0.5) repair now steps aside for that task and leaves the FAIL verdicts for judge_escalation; RepairInfo.tasks_skipped records it. --- src/flydocs/config.py | 7 +++ src/flydocs/core/configuration.py | 1 + .../core/services/repair/field_repairer.py | 24 +++++++- src/flydocs/interfaces/dtos/extract.py | 3 + tests/unit/test_field_repairer.py | 59 +++++++++++++++++++ 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 7dacb73..1f754c3 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -239,9 +239,16 @@ class IDPSettings(BaseSettings): # counts). Flagged-but-PASS fields are often ambiguous-but-correct, # so set it to false to restrict repair to hard failures (judge FAIL # or a validator error) and save those extra passes. + # ``repair_max_failing_fraction`` caps the repair scope per task: + # above this failing-field fraction the extraction is globally + # untrustworthy and a focused pass would re-extract nearly everything + # on top of the eventual escalation re-run, so repair steps aside and + # judge_escalation (whose trigger rate is unchanged) takes over. + # 1.0 disables the cap. repair_model: str | None = None repair_timeout_s: int = 300 repair_include_flagged: bool = True + repair_max_failing_fraction: float = 0.5 # -- Webhook -------------------------------------------------------- # The result webhook delivers the full extraction (split docs + fields + diff --git a/src/flydocs/core/configuration.py b/src/flydocs/core/configuration.py index 2e6dac8..fdba01f 100644 --- a/src/flydocs/core/configuration.py +++ b/src/flydocs/core/configuration.py @@ -373,6 +373,7 @@ def field_repairer( field_validator=field_validator, default_model=settings.repair_model, include_flagged=settings.repair_include_flagged, + max_failing_fraction=settings.repair_max_failing_fraction, ) # ------------------------------------------------------------------ diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 43f974a..4664961 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -122,12 +122,14 @@ def __init__( field_validator: FieldValidator, default_model: str | None, include_flagged: bool = True, + max_failing_fraction: float = 0.5, ) -> None: self._extractor = extractor self._judge = judge self._field_validator = field_validator self._default_model = default_model self._include_flagged = include_flagged + self._max_failing_fraction = max_failing_fraction async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo | None: """Return a :class:`RepairInfo` when at least one field was flagged, else ``None``. @@ -138,14 +140,30 @@ async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo tasks: list[Any] = ctx.metadata.get("tasks", []) model = self._default_model or ctx.metadata.get("model_id") flagged = 0 + skipped_tasks = 0 repaired_paths: list[str] = [] async def _repair_task(task: Any) -> None: - nonlocal flagged + nonlocal flagged, skipped_tasks failures = collect_failing_fields(task.extracted_groups, include_flagged=self._include_flagged) if not failures: return flagged += len(failures) + total = sum(len(group.fields) for group in task.extracted_groups) + if total and len(failures) / total > self._max_failing_fraction: + # The extraction is globally untrustworthy: a focused pass + # would re-extract nearly everything on top of the eventual + # escalation re-run. Leave the FAIL verdicts in place so + # judge_escalation triggers as before. + skipped_tasks += 1 + logger.info( + "repair skipped for %s: %d/%d fields failing exceeds max fraction %.2f", + task.task_id, + len(failures), + total, + self._max_failing_fraction, + ) + return try: accepted = await self._repair_one(task, failures, request, model) except Exception as exc: # noqa: BLE001 -- repair must never break the pipeline @@ -163,11 +181,13 @@ async def _repair_task(task: Any) -> None: fields_flagged=flagged, fields_repaired=len(repaired_paths), repaired_fields=sorted(repaired_paths), + tasks_skipped=skipped_tasks, ) logger.info( - "repair flagged=%d repaired=%d model=%s", + "repair flagged=%d repaired=%d skipped_tasks=%d model=%s", info.fields_flagged, info.fields_repaired, + info.tasks_skipped, model, ) return info diff --git a/src/flydocs/interfaces/dtos/extract.py b/src/flydocs/interfaces/dtos/extract.py index 22ae2a3..26b9bec 100644 --- a/src/flydocs/interfaces/dtos/extract.py +++ b/src/flydocs/interfaces/dtos/extract.py @@ -260,6 +260,9 @@ class RepairInfo(BaseModel): fields_flagged: int = Field(default=0, ge=0) fields_repaired: int = Field(default=0, ge=0) repaired_fields: list[str] = Field(default_factory=list) + # Tasks whose repair was skipped because the failing-field fraction + # exceeded FLYDOCS_REPAIR_MAX_FAILING_FRACTION (escalation handles them). + tasks_skipped: int = Field(default=0, ge=0) class UsageBreakdown(BaseModel): diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 772419f..40235aa 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -260,6 +260,7 @@ def _repairer( *, default_model: str | None = None, include_flagged: bool = True, + max_failing_fraction: float = 1.0, ) -> FieldRepairer: return FieldRepairer( extractor=extractor, @@ -267,6 +268,7 @@ def _repairer( field_validator=FieldValidator(), default_model=default_model, include_flagged=include_flagged, + max_failing_fraction=max_failing_fraction, ) @@ -437,6 +439,63 @@ async def test_maybe_repair_falls_back_to_request_model() -> None: assert extractor.extract_repair.await_args.kwargs["model"] == "request-model" +# --------------------------------------------------------------------------- +# Scope cap +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_maybe_repair_skips_task_above_max_failing_fraction() -> None: + """When most fields failed the extraction is globally untrustworthy: + targeted repair is skipped and judge_escalation's full re-run (whose + trigger rate is unchanged) is the right tool.""" + task = _task( + [ + ExtractedFieldGroup( + name="identity", + fields=[ + _judge_failed_field("number", "X123", "misread"), + _judge_failed_field("name", "J0HN", "misread"), + ], + ) + ] + ) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), max_failing_fraction=0.5) + info = await repairer.maybe_repair(_ctx([task]), _request()) + + extractor.extract_repair.assert_not_awaited() + assert info is not None and info.triggered + assert info.fields_flagged == 2 + assert info.fields_repaired == 0 + assert info.tasks_skipped == 1 + + +@pytest.mark.asyncio +async def test_maybe_repair_runs_when_failing_fraction_at_or_below_cap() -> None: + task = _task( + [ + ExtractedFieldGroup( + name="identity", + fields=[ + _clean_field("name", "JOHN"), + _judge_failed_field("number", "X123", "misread"), + ], + ) + ] + ) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), max_failing_fraction=0.5) + info = await repairer.maybe_repair(_ctx([task]), _request()) + + extractor.extract_repair.assert_awaited_once() + assert info is not None and info.tasks_skipped == 0 + + # --------------------------------------------------------------------------- # Prompt catalog # --------------------------------------------------------------------------- From 92597ad75dc31d4d4457d906e9855a7134776cde Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:42:43 +0200 Subject: [PATCH 30/38] feat: repair_task_concurrency bounds parallel task repairs Repair fanned out one unbounded gather over all failing tasks; FLYDOCS_REPAIR_TASK_CONCURRENCY (default 4) now caps in-flight repair passes, mirroring bbox_refine_doc_concurrency for provider rate limits. --- src/flydocs/config.py | 5 +++ src/flydocs/core/configuration.py | 1 + .../core/services/repair/field_repairer.py | 8 ++++- tests/unit/test_field_repairer.py | 35 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/flydocs/config.py b/src/flydocs/config.py index 1f754c3..ab19a2f 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -245,10 +245,15 @@ class IDPSettings(BaseSettings): # on top of the eventual escalation re-run, so repair steps aside and # judge_escalation (whose trigger rate is unchanged) takes over. # 1.0 disables the cap. + # ``repair_task_concurrency`` bounds how many tasks are repaired at + # once (each repair is one extract call plus an optional judge + # re-check); lower it under provider rate limits, mirroring + # ``bbox_refine_doc_concurrency``. repair_model: str | None = None repair_timeout_s: int = 300 repair_include_flagged: bool = True repair_max_failing_fraction: float = 0.5 + repair_task_concurrency: int = 4 # -- Webhook -------------------------------------------------------- # The result webhook delivers the full extraction (split docs + fields + diff --git a/src/flydocs/core/configuration.py b/src/flydocs/core/configuration.py index fdba01f..2c4cc14 100644 --- a/src/flydocs/core/configuration.py +++ b/src/flydocs/core/configuration.py @@ -374,6 +374,7 @@ def field_repairer( default_model=settings.repair_model, include_flagged=settings.repair_include_flagged, max_failing_fraction=settings.repair_max_failing_fraction, + task_concurrency=settings.repair_task_concurrency, ) # ------------------------------------------------------------------ diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 4664961..dd9fa07 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -123,6 +123,7 @@ def __init__( default_model: str | None, include_flagged: bool = True, max_failing_fraction: float = 0.5, + task_concurrency: int = 4, ) -> None: self._extractor = extractor self._judge = judge @@ -130,6 +131,7 @@ def __init__( self._default_model = default_model self._include_flagged = include_flagged self._max_failing_fraction = max_failing_fraction + self._task_concurrency = task_concurrency async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo | None: """Return a :class:`RepairInfo` when at least one field was flagged, else ``None``. @@ -142,6 +144,9 @@ async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo flagged = 0 skipped_tasks = 0 repaired_paths: list[str] = [] + # Same rationale as bbox_refine_doc_concurrency: each repaired task + # multiplies in-flight LLM calls; lower it under provider rate limits. + semaphore = asyncio.Semaphore(max(1, self._task_concurrency)) async def _repair_task(task: Any) -> None: nonlocal flagged, skipped_tasks @@ -165,7 +170,8 @@ async def _repair_task(task: Any) -> None: ) return try: - accepted = await self._repair_one(task, failures, request, model) + async with semaphore: + accepted = await self._repair_one(task, failures, request, model) except Exception as exc: # noqa: BLE001 -- repair must never break the pipeline logger.warning("repair failed for %s: %s; keeping original fields", task.task_id, exc) return diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 40235aa..2bda1ba 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -22,6 +22,7 @@ from __future__ import annotations +import asyncio import base64 from types import SimpleNamespace from typing import Any @@ -261,6 +262,7 @@ def _repairer( default_model: str | None = None, include_flagged: bool = True, max_failing_fraction: float = 1.0, + task_concurrency: int = 4, ) -> FieldRepairer: return FieldRepairer( extractor=extractor, @@ -269,6 +271,7 @@ def _repairer( default_model=default_model, include_flagged=include_flagged, max_failing_fraction=max_failing_fraction, + task_concurrency=task_concurrency, ) @@ -496,6 +499,38 @@ async def test_maybe_repair_runs_when_failing_fraction_at_or_below_cap() -> None assert info is not None and info.tasks_skipped == 0 +# --------------------------------------------------------------------------- +# Concurrency bound +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_maybe_repair_bounds_concurrent_task_repairs() -> None: + tasks = [ + _task([ExtractedFieldGroup(name="identity", fields=[_judge_failed_field("number", "X", "bad")])]) + for _ in range(3) + ] + in_flight = 0 + peak = 0 + + async def _tracked_repair(**_: Any) -> list[ExtractedFieldGroup]: + nonlocal in_flight, peak + in_flight += 1 + peak = max(peak, in_flight) + await asyncio.sleep(0.01) + in_flight -= 1 + return [] + + extractor = MagicMock() + extractor.extract_repair = AsyncMock(side_effect=_tracked_repair) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock()), task_concurrency=1) + await repairer.maybe_repair(_ctx(tasks), _request()) + + assert extractor.extract_repair.await_count == 3 + assert peak == 1 + + # --------------------------------------------------------------------------- # Prompt catalog # --------------------------------------------------------------------------- From f09c2276a819bbfe5321ab1100ce34a0b6c9c5fd Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:44:40 +0200 Subject: [PATCH 31/38] feat: RepairInfo.rows_changed audits whole-array repairs Arrays are re-extracted and accepted whole, so a previously-correct row can be replaced by a new verifier-passing value -- monotonicity is field-level only. rows_changed records how many rows differ per repaired array so table repairs can be diffed in QA; the module docstring no longer overstates the guarantee. --- .../core/services/repair/field_repairer.py | 48 ++++++++++++++++--- src/flydocs/interfaces/dtos/extract.py | 4 ++ tests/unit/test_field_repairer.py | 37 ++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index dd9fa07..ba45a1c 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -25,7 +25,11 @@ re-check when the judge stage is enabled) and replace the originals only when they come back clean -- with the judge on, that means an explicit PASS, and a null candidate never replaces a value. The repair -is monotonic: it can fix fields, never degrade them. +is monotonic at the field level: a field is only replaced by a verified +candidate. Array fields are the caveat: they are re-extracted and +accepted whole, so a previously-correct row may be replaced by a new +verifier-passing value -- ``RepairInfo.rows_changed`` records how many +rows differ per repaired array so table repairs can be diffed. Opt-in via ``options.stages.repair``. Runs after judge and BEFORE judge_escalation, so the expensive full re-run only fires when targeted @@ -144,6 +148,7 @@ async def maybe_repair(self, ctx: Any, request: ExtractionRequest) -> RepairInfo flagged = 0 skipped_tasks = 0 repaired_paths: list[str] = [] + rows_changed: dict[str, int] = {} # Same rationale as bbox_refine_doc_concurrency: each repaired task # multiplies in-flight LLM calls; lower it under provider rate limits. semaphore = asyncio.Semaphore(max(1, self._task_concurrency)) @@ -171,11 +176,12 @@ async def _repair_task(task: Any) -> None: return try: async with semaphore: - accepted = await self._repair_one(task, failures, request, model) + accepted, task_rows_changed = await self._repair_one(task, failures, request, model) except Exception as exc: # noqa: BLE001 -- repair must never break the pipeline logger.warning("repair failed for %s: %s; keeping original fields", task.task_id, exc) return repaired_paths.extend(accepted) + rows_changed.update(task_rows_changed) await asyncio.gather(*(_repair_task(t) for t in tasks)) @@ -188,6 +194,7 @@ async def _repair_task(task: Any) -> None: fields_repaired=len(repaired_paths), repaired_fields=sorted(repaired_paths), tasks_skipped=skipped_tasks, + rows_changed=rows_changed, ) logger.info( "repair flagged=%d repaired=%d skipped_tasks=%d model=%s", @@ -204,8 +211,9 @@ async def _repair_one( failures: list[FailingField], request: ExtractionRequest, model: str | None, - ) -> list[str]: - """Repair one task; return the ``group.field`` paths that were accepted.""" + ) -> tuple[list[str], dict[str, int]]: + """Repair one task; return the accepted ``group.field`` paths and, + for accepted array fields, how many rows differ from the original.""" failing_keys = {(f.group, f.field) for f in failures} subset = _subset_spec(task.doc_spec, failing_keys) repaired_groups = await self._extractor.extract_repair( @@ -218,7 +226,7 @@ async def _repair_one( model=model, ) if not repaired_groups: - return [] + return [], {} # Re-verify: validators always (deterministic, free); judge subset # re-check only when the caller enabled the judge stage. @@ -238,6 +246,7 @@ async def _repair_one( } judged = request.options.stages.judge accepted: list[str] = [] + rows_changed: dict[str, int] = {} for group in task.extracted_groups: for index, field in enumerate(group.fields): key = (group.name, field.name) @@ -246,9 +255,14 @@ async def _repair_one( candidate = repaired_by_key.get(key) if candidate is None or not _is_clean(candidate, require_judge_pass=judged): continue + path = f"{group.name}.{field.name}" + if isinstance(field.value, list) and isinstance(candidate.value, list): + changed = _changed_row_count(field.value, candidate.value) + if changed: + rows_changed[path] = changed group.fields[index] = candidate - accepted.append(f"{group.name}.{field.name}") - return accepted + accepted.append(path) + return accepted, rows_changed def _is_clean(field: ExtractedField, *, require_judge_pass: bool) -> bool: @@ -269,6 +283,26 @@ def _is_clean(field: ExtractedField, *, require_judge_pass: bool) -> bool: return field.judge.status != JudgeStatus.FAIL and not field.judge.flag_for_review +def _changed_row_count(old_rows: list[Any], new_rows: list[Any]) -> int: + """Rows of a repaired array whose content differs from the original. + + Compared positionally by (name, value) signature -- rows have no + stable keys, so an inserted or dropped row also counts as a change. + """ + changed = sum( + 1 for old, new in zip(old_rows, new_rows) if _row_signature(old) != _row_signature(new) + ) + return changed + abs(len(old_rows) - len(new_rows)) + + +def _row_signature(row: Any) -> Any: + if isinstance(row, ExtractedField): + if isinstance(row.value, list): + return (row.name, tuple(_row_signature(sub) for sub in row.value)) + return (row.name, row.value) + return row + + def _subset_spec(doc: DocumentTypeSpec, failing_keys: set[tuple[str, str]]) -> DocumentTypeSpec: """A copy of ``doc`` whose field_groups contain only the failing fields.""" groups = [] diff --git a/src/flydocs/interfaces/dtos/extract.py b/src/flydocs/interfaces/dtos/extract.py index 26b9bec..340e512 100644 --- a/src/flydocs/interfaces/dtos/extract.py +++ b/src/flydocs/interfaces/dtos/extract.py @@ -263,6 +263,10 @@ class RepairInfo(BaseModel): # Tasks whose repair was skipped because the failing-field fraction # exceeded FLYDOCS_REPAIR_MAX_FAILING_FRACTION (escalation handles them). tasks_skipped: int = Field(default=0, ge=0) + # Array fields are accepted whole, so previously-correct rows may be + # replaced; per repaired ``group.field`` path, how many rows differ + # from the original (row count changes included). + rows_changed: dict[str, int] = Field(default_factory=dict) class UsageBreakdown(BaseModel): diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 2bda1ba..975bf54 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -442,6 +442,43 @@ async def test_maybe_repair_falls_back_to_request_model() -> None: assert extractor.extract_repair.await_args.kwargs["model"] == "request-model" +# --------------------------------------------------------------------------- +# Array audit +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_maybe_repair_records_changed_rows_for_arrays() -> None: + """Arrays are accepted whole, so previously-correct rows may be replaced; + rows_changed records how many rows differ so QA can diff table repairs.""" + old_rows = [ + ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]), + ExtractedField(name="row", value=[ExtractedField(name="b", value="-3")]), + ] + array_field = ExtractedField( + name="line_items", + value=old_rows, + validation=FieldValidation(valid=False, errors=[]), + ) + task = _task([ExtractedFieldGroup(name="items", fields=[array_field])]) + new_rows = [ + ExtractedField(name="row", value=[ExtractedField(name="a", value="1")]), # untouched + ExtractedField(name="row", value=[ExtractedField(name="b", value="3")]), # fixed + ] + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup(name="items", fields=[ExtractedField(name="line_items", value=new_rows)]) + ] + ) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + info = await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + assert info is not None and info.fields_repaired == 1 + assert info.rows_changed == {"items.line_items": 1} + + # --------------------------------------------------------------------------- # Scope cap # --------------------------------------------------------------------------- From 5c71443a40b56d5f8252dc26f71d41c7c9b34ca8 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:47:19 +0200 Subject: [PATCH 32/38] feat: slice the repair pass to the failing fields' pages The focused repair trimmed the schema subset but re-sent every page of the segment slice -- and pages, not fields, dominate multimodal input cost. When every failing field carries page provenance and the media is PDF, the repair (extract + judge re-check) now runs on a sub-slice spanning the failing pages plus one page of margin, and accepted candidates' slice-relative pages are remapped to segment coordinates. Falls back to the full slice for non-PDF media, unknown pages (nulled values lose theirs), non-shrinking spans, or slicing errors. --- .../core/services/repair/field_repairer.py | 62 +++++++++++++- tests/unit/test_field_repairer.py | 84 +++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index ba45a1c..0f9a194 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -45,6 +45,7 @@ from typing import Any from flydocs.core.services.extraction.extractor import MultimodalExtractor +from flydocs.core.services.extraction.pdf_slicer import PageRange, slice_pdf from flydocs.core.services.judge import Judge from flydocs.core.services.validation.field_validator import FieldValidator from flydocs.interfaces.dtos.document_type import DocumentTypeSpec @@ -63,6 +64,9 @@ class FailingField: field: str value: Any evidence: str + # 1-indexed slice-relative pages the value was reported on; empty when + # unknown (typical for null-value failures, whose page is discarded). + pages: tuple[int, ...] = () def collect_failing_fields( @@ -97,11 +101,22 @@ def collect_failing_fields( field=field.name, value=field.value, evidence="; ".join(reasons), + pages=_field_pages(field), ) ) return failures +def _field_pages(field: ExtractedField) -> tuple[int, ...]: + """Every page the field (or, for arrays, its rows) was reported on.""" + pages = set(field.pages) + if isinstance(field.value, list): + for sub in field.value: + if isinstance(sub, ExtractedField): + pages.update(_field_pages(sub)) + return tuple(sorted(pages)) + + def _row_error_messages(rows: list[Any]) -> list[str]: """Validator messages stamped on the row sub-fields of an array field.""" messages: list[str] = [] @@ -216,10 +231,11 @@ async def _repair_one( for accepted array fields, how many rows differ from the original.""" failing_keys = {(f.group, f.field) for f in failures} subset = _subset_spec(task.doc_spec, failing_keys) + doc_bytes, page_count, page_offset = _repair_slice(task, failures) repaired_groups = await self._extractor.extract_repair( - document_bytes=task.slice_bytes, + document_bytes=doc_bytes, media_type=task.segment.media_type, - page_count=task.slice_pages, + page_count=page_count, doc=subset, failing_fields_text=_failures_text(failures), language_hint=request.options.language_hint, @@ -233,7 +249,7 @@ async def _repair_one( self._field_validator.validate(subset.field_groups, repaired_groups) if request.options.stages.judge: await self._judge.judge( - document_bytes=task.slice_bytes, + document_bytes=doc_bytes, media_type=task.segment.media_type, doc=subset, extracted_groups=repaired_groups, @@ -255,6 +271,8 @@ async def _repair_one( candidate = repaired_by_key.get(key) if candidate is None or not _is_clean(candidate, require_judge_pass=judged): continue + if page_offset: + _shift_pages(candidate, page_offset) path = f"{group.name}.{field.name}" if isinstance(field.value, list) and isinstance(candidate.value, list): changed = _changed_row_count(field.value, candidate.value) @@ -283,6 +301,44 @@ def _is_clean(field: ExtractedField, *, require_judge_pass: bool) -> bool: return field.judge.status != JudgeStatus.FAIL and not field.judge.flag_for_review +def _repair_slice(task: Any, failures: list[FailingField]) -> tuple[bytes, int, int]: + """Bytes, page count and page offset for the focused repair pass. + + The dominant repair cost is the input pages, not the schema subset, + so when every failing field carries page provenance the segment PDF + is sliced to the failing span plus one page of margin. Falls back to + the full segment slice when the media is not PDF, any failing field + has no known pages (nulled values lose theirs), the span would not + actually shrink the document, or slicing fails. + """ + full = (task.slice_bytes, task.slice_pages, 0) + if task.segment.media_type != "application/pdf": + return full + if not failures or any(not f.pages for f in failures): + return full + pages = sorted({page for f in failures for page in f.pages}) + start = max(1, pages[0] - 1) + end = min(task.slice_pages, pages[-1] + 1) + if end - start + 1 >= task.slice_pages: + return full + try: + sliced = slice_pdf(task.slice_bytes, PageRange(start=start, end=end)) + except Exception as exc: # noqa: BLE001 -- slicing is an optimization, never fatal + logger.warning("repair page slicing failed for %s: %s; using full slice", task.task_id, exc) + return full + return sliced, end - start + 1, start - 1 + + +def _shift_pages(field: ExtractedField, offset: int) -> None: + """Map slice-relative page numbers of an accepted candidate back to + segment coordinates (recursing into array rows and sub-fields).""" + field.pages = [page + offset for page in field.pages] + if isinstance(field.value, list): + for sub in field.value: + if isinstance(sub, ExtractedField): + _shift_pages(sub, offset) + + def _changed_row_count(old_rows: list[Any], new_rows: list[Any]) -> int: """Rows of a repaired array whose content differs from the original. diff --git a/tests/unit/test_field_repairer.py b/tests/unit/test_field_repairer.py index 975bf54..84b90e7 100644 --- a/tests/unit/test_field_repairer.py +++ b/tests/unit/test_field_repairer.py @@ -24,10 +24,12 @@ import asyncio import base64 +import io from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock +import pypdf import pytest from flydocs.config import IDPSettings @@ -442,6 +444,88 @@ async def test_maybe_repair_falls_back_to_request_model() -> None: assert extractor.extract_repair.await_args.kwargs["model"] == "request-model" +# --------------------------------------------------------------------------- +# Page-sliced repair +# --------------------------------------------------------------------------- + + +def _pdf(pages: int) -> bytes: + writer = pypdf.PdfWriter() + for _ in range(pages): + writer.add_blank_page(width=72, height=72) + buf = io.BytesIO() + writer.write(buf) + return buf.getvalue() + + +def _paged_task(groups: list[ExtractedFieldGroup], *, pages: int) -> Any: + task = _task(groups) + task.slice_bytes = _pdf(pages) + task.slice_pages = pages + return task + + +@pytest.mark.asyncio +async def test_repair_slices_pdf_to_failing_pages_and_remaps_candidate() -> None: + """A failure localized on page 3 of 5 re-sends only pages 2-4 (one page + of margin); the accepted candidate's slice-relative pages are shifted + back to document coordinates.""" + failing = _judge_failed_field("number", "X123", "misread") + failing.pages = [3] + task = _paged_task([ExtractedFieldGroup(name="identity", fields=[failing])], pages=5) + extractor = MagicMock() + extractor.extract_repair = AsyncMock( + return_value=[ + ExtractedFieldGroup( + name="identity", fields=[ExtractedField(name="number", value="Y456", pages=[2])] + ) + ] + ) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + info = await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + kwargs = extractor.extract_repair.await_args.kwargs + assert kwargs["page_count"] == 3 + assert len(pypdf.PdfReader(io.BytesIO(kwargs["document_bytes"])).pages) == 3 + assert info is not None and info.fields_repaired == 1 + assert task.extracted_groups[0].fields[0].pages == [3] + + +@pytest.mark.asyncio +async def test_repair_sends_full_document_when_failing_pages_unknown() -> None: + """Null-value failures carry pages=[]; without provenance the repair + falls back to the full segment slice.""" + failing = _judge_failed_field("number", "X123", "misread") # pages=[] by default + task = _paged_task([ExtractedFieldGroup(name="identity", fields=[failing])], pages=5) + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + kwargs = extractor.extract_repair.await_args.kwargs + assert kwargs["document_bytes"] == task.slice_bytes + assert kwargs["page_count"] == 5 + + +@pytest.mark.asyncio +async def test_repair_sends_full_document_for_non_pdf_media() -> None: + failing = _judge_failed_field("number", "X123", "misread") + failing.pages = [3] + task = _paged_task([ExtractedFieldGroup(name="identity", fields=[failing])], pages=5) + task.segment = SimpleNamespace(media_type="image/png") + extractor = MagicMock() + extractor.extract_repair = AsyncMock(return_value=[]) + + repairer = _repairer(extractor, MagicMock(judge=AsyncMock())) + await repairer.maybe_repair(_ctx([task]), _request(judge=False)) + + kwargs = extractor.extract_repair.await_args.kwargs + assert kwargs["document_bytes"] == task.slice_bytes + assert kwargs["page_count"] == 5 + + # --------------------------------------------------------------------------- # Array audit # --------------------------------------------------------------------------- From 655292ac3ee598f7d47c60e5a4767a3feac5b678 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:49:13 +0200 Subject: [PATCH 33/38] feat: repair_sync_latency warning for repair on synchronous requests The repair pass eats into the hard FLYDOCS_SYNC_TIMEOUT_S wall (the pipeline is cancelled at the ceiling with a deterministic 408), so enabling stages.repair on the sync channel raises 408 likelihood. RequestValidator.validate() gains a sync flag, set by the sync controller and the /extract:validate dry-run; the async submit path is unchanged. --- .../services/validation/request_validator.py | 26 ++++++++++++++++--- .../web/controllers/extract_controller.py | 4 +-- tests/unit/test_request_validator.py | 18 +++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/flydocs/core/services/validation/request_validator.py b/src/flydocs/core/services/validation/request_validator.py index 96fc60c..cccfc33 100644 --- a/src/flydocs/core/services/validation/request_validator.py +++ b/src/flydocs/core/services/validation/request_validator.py @@ -100,13 +100,13 @@ class RequestValidator: share one instance. """ - def validate(self, request: ExtractionRequest) -> ValidationReport: + def validate(self, request: ExtractionRequest, *, sync: bool = False) -> ValidationReport: report = ValidationReport() self._check_files(request, report) self._check_document_types(request, report) self._check_rule_references(request, report) self._check_rule_dag(request, report) - self._check_stage_consistency(request, report) + self._check_stage_consistency(request, report, sync=sync) return report # -- file-level checks (multi-file shape) ---------------------------- @@ -338,7 +338,9 @@ def _check_rule_dag(self, request: ExtractionRequest, report: ValidationReport) # -- stage / toggle consistency -------------------------------------- - def _check_stage_consistency(self, request: ExtractionRequest, report: ValidationReport) -> None: + def _check_stage_consistency( + self, request: ExtractionRequest, report: ValidationReport, *, sync: bool = False + ) -> None: stages = request.options.stages # rule_engine on but no rules => the stage is a no-op. Warn. @@ -385,6 +387,24 @@ def _check_stage_consistency(self, request: ExtractionRequest, report: Validatio ) ) + # repair on a synchronous request => the extra LLM pass eats into + # the hard sync wall (FLYDOCS_SYNC_TIMEOUT_S) and raises the odds + # of a 408. Warn; the async API has the budget for it. + if sync and stages.repair: + report.issues.append( + ValidationIssue( + severity="warning", + code="repair_sync_latency", + message=( + "stages.repair adds an extra LLM pass per failing " + "field; synchronous requests are hard-capped at " + "FLYDOCS_SYNC_TIMEOUT_S and are likely to return " + "408 -- prefer the async POST /api/v1/extractions." + ), + path="options.stages.repair", + ) + ) + # repair on but no stage produces failure signals => the node is # never added to the DAG. Warn. if stages.repair and not (stages.judge or stages.field_validation): diff --git a/src/flydocs/web/controllers/extract_controller.py b/src/flydocs/web/controllers/extract_controller.py index 4e4a56b..32600de 100644 --- a/src/flydocs/web/controllers/extract_controller.py +++ b/src/flydocs/web/controllers/extract_controller.py @@ -99,7 +99,7 @@ async def validate(self, request: Valid[Body[ExtractionRequest]]) -> ValidationR whether to proceed. Identical schema for both errors and warnings: ``[{severity, code, message, path}]``. """ - report = self._validator.validate(request) + report = self._validator.validate(request, sync=True) return ValidationResponse( ok=not report.has_errors, error_count=len(report.errors), @@ -163,7 +163,7 @@ def _enforce_size_limits(request: ExtractionRequest, *, max_bytes: int) -> None: def _enforce_semantic_validation(request: ExtractionRequest, validator: RequestValidator) -> None: """Reject the request with a 422 when the semantic validator finds errors.""" - report: ValidationReport = validator.validate(request) + report: ValidationReport = validator.validate(request, sync=True) if report.has_errors: raise _http_problem_with_payload( status_code=422, diff --git a/tests/unit/test_request_validator.py b/tests/unit/test_request_validator.py index 50846e3..5e6774c 100644 --- a/tests/unit/test_request_validator.py +++ b/tests/unit/test_request_validator.py @@ -267,6 +267,24 @@ def test_repair_with_judge_produces_no_warning(validator: RequestValidator) -> N assert "repair_no_verification_stage" not in codes +# -- warning: repair on a synchronous request --------------------------------- + + +def test_repair_on_sync_request_warns_about_latency(validator: RequestValidator) -> None: + options = ExtractionOptions(stages=StageToggles(repair=True, judge=True)) + report = validator.validate(_request(options=options), sync=True) + assert not report.has_errors + codes = [i.code for i in report.warnings] + assert "repair_sync_latency" in codes + + +def test_repair_on_async_request_has_no_sync_warning(validator: RequestValidator) -> None: + options = ExtractionOptions(stages=StageToggles(repair=True, judge=True)) + report = validator.validate(_request(options=options)) + codes = [i.code for i in report.warnings] + assert "repair_sync_latency" not in codes + + # -- warning: visual_authenticity on but no visual checks -------------------- From 723f65b2362dbe57dfb49f1beeb1f0e4b82d191d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:50:36 +0200 Subject: [PATCH 34/38] docs: tier mechanical stages to haiku and document repair knobs in env_template Splitter, classifier and the bbox value matcher are mechanical work and run fine on haiku; extraction/reasoning stay on sonnet and the judge on opus, so the per-stage lever cuts cost instead of only raising it. Also state that a pinned stage wins over options.model, correct the repair predicate wording (flag_for_review included), and document the three new repair settings plus the sync-cap note. --- env_template | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/env_template b/env_template index 1b8e74f..bf59c1a 100644 --- a/env_template +++ b/env_template @@ -55,16 +55,19 @@ FLYDOCS_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_FALLBACK_MODEL=openai:gpt-4o # Default models for each pipeline stage. Any stage left unset falls back -# to the global FLYDOCS_MODEL (or the request's ``options.model``). -FLYDOCS_SPLITTER_MODEL=anthropic:claude-sonnet-4-6 -FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-sonnet-4-6 +# to the global FLYDOCS_MODEL (or the request's ``options.model``); a set +# stage wins over the request's ``options.model``. Tiering: mechanical +# stages (splitting, classification, bbox value matching) run on haiku, +# extraction and reasoning on sonnet, the judge on opus. +FLYDOCS_SPLITTER_MODEL=anthropic:claude-haiku-4-5 +FLYDOCS_CLASSIFIER_MODEL=anthropic:claude-haiku-4-5 FLYDOCS_EXTRACT_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_VISUAL_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_CONTENT_AUTHENTICITY_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_JUDGE_MODEL=anthropic:claude-opus-4-8 FLYDOCS_RULE_ENGINE_MODEL=anthropic:claude-sonnet-4-6 FLYDOCS_TRANSFORM_MODEL=anthropic:claude-sonnet-4-6 -FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-sonnet-4-6 +FLYDOCS_BBOX_MATCHER_MODEL=anthropic:claude-haiku-4-5 # Maximum pages per single extraction request. Documents above this go through # the async API only. The document bytes are sent directly to the multimodal @@ -75,15 +78,25 @@ FLYDOCS_MAX_SYNC_PAGES=10 FLYDOCS_MAX_BYTES=33554432 # Closed-loop targeted repair (stage is opt-in per request via -# ``options.stages.repair``). Fields that failed the judge or a validator -# are re-extracted once with the failure evidence quoted back to the -# model. Default model for the repair pass; unset falls back to the -# global FLYDOCS_MODEL. A stronger model than the extractor is -# recommended -- it is fixing what the extractor got wrong. Note the -# repair pass consumes part of the FLYDOCS_ASYNC_TIMEOUT_S budget on -# async jobs -- raise that budget if you also raise this timeout. +# ``options.stages.repair``). Fields the judge marked FAIL or +# flag_for_review, or that failed a validator, are re-extracted once +# with the failure evidence quoted back to the model. Default model for +# the repair pass; unset falls back to the global FLYDOCS_MODEL. A +# stronger model than the extractor is recommended -- it is fixing what +# the extractor got wrong. Note the repair pass consumes part of the +# FLYDOCS_ASYNC_TIMEOUT_S budget on async jobs -- raise that budget if +# you also raise this timeout. Sync requests stay capped by +# FLYDOCS_SYNC_TIMEOUT_S, so repair usually warrants the async API. FLYDOCS_REPAIR_MODEL=anthropic:claude-opus-4-8 FLYDOCS_REPAIR_TIMEOUT_S=300 +# Set to false to repair only hard failures (judge FAIL or a validator +# error), skipping flagged-but-PASS fields that are usually correct. +FLYDOCS_REPAIR_INCLUDE_FLAGGED=true +# Above this failing-field fraction per task, targeted repair steps +# aside and leaves the full re-run to judge_escalation (1.0 = no cap). +FLYDOCS_REPAIR_MAX_FAILING_FRACTION=0.5 +# Max tasks repaired concurrently; lower it under provider rate limits. +FLYDOCS_REPAIR_TASK_CONCURRENCY=4 # Per-call timeouts (seconds). Sync requests get the shorter one; async jobs # retry up to ``JOB_MAX_ATTEMPTS`` times on timeout. From 662bd7c32cd8389cdcb1b1db7050cd9b3fd58f69 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 17:51:05 +0200 Subject: [PATCH 35/38] docs: stage-pin precedence over options.model + repair scope/page-slicing notes payload-reference and deployment now state that a pinned FLYDOCS__MODEL wins over the request's options.model (previously only a code comment). pipeline.md documents the repair page sub-slice, the failing-fraction scope cap, the concurrency bound, the include-flagged knob and the sync-latency warning. --- docs/deployment.md | 7 ++++--- docs/payload-reference.md | 2 +- docs/pipeline.md | 12 +++++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 4ac4671..a588b2d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -354,9 +354,10 @@ to lean on: content_authenticity for high-volume bulk runs; reserve them for high-risk paths. 2. **Model choice.** `FLYDOCS_MODEL` is the default; override per - request via `options.model`. Lighter models (haiku, gpt-4o-mini) - are fine for high-volume extraction; reserve the heavier ones for - adversarial or low-confidence cases. + request via `options.model` (stages pinned via a + `FLYDOCS__MODEL` keep their model regardless). Lighter + models (haiku, gpt-4o-mini) are fine for high-volume extraction; + reserve the heavier ones for adversarial or low-confidence cases. 3. **Fallback.** `FLYDOCS_FALLBACK_MODEL` is used when the primary errors out. Setting it to a cheaper model avoids double-paying for a single failed call — and gives you a graceful degradation path. diff --git a/docs/payload-reference.md b/docs/payload-reference.md index 9c1585c..807d079 100644 --- a/docs/payload-reference.md +++ b/docs/payload-reference.md @@ -437,7 +437,7 @@ for validators to fire. |------------------------|----------|-----------------------|--------------------------------------------------------------------| | `return_bboxes` | boolean | `true` | `false` strips bboxes from the response (cheaper to transfer). | | `language_hint` | string? | `null` | ISO 639-1, ≤ 16 chars. Guides multilingual OCR / extraction. | -| `model` | string? | `null` (env default) | Per-request primary model id (`anthropic:claude-sonnet-4-6`, …). | +| `model` | string? | `null` (env default) | Per-request primary model id (`anthropic:claude-sonnet-4-6`, …). Applies only to stages without a pinned `FLYDOCS__MODEL`; a pinned stage keeps its model. | | `declared_media_type` | string? | `null` | Override sniffing; rare. | | `stages` | object | server defaults | See `StageToggles` below. | | `escalation` | object? | `null` | `{ threshold, model }`. Required when `stages.judge_escalation=true`. | diff --git a/docs/pipeline.md b/docs/pipeline.md index b8ad801..9f65d0e 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -55,7 +55,7 @@ reflect exactly what executed. | 9 | `visual_authenticity` | no | LLM evaluates caller-defined visual validators (signature present, stamp present, …). | 180 s | | 10 | `content_authenticity` | no | LLM audit: dates consistent, totals add up, expected boilerplate, tampering signals. | 180 s | | 11 | `judge` | no | Second LLM pass re-grades every extracted value against the source. | 300 s | -| 12 | `repair` | no | Targeted repair: re-extract ONLY the fields that failed the judge or a validator, quoting the failure evidence; accept per field only when the re-check passes. | 300 s | +| 12 | `repair` | no | Targeted repair: re-extract ONLY the fields the judge marked FAIL or flag_for_review, or that failed a validator, quoting the failure evidence; accept per field only when the re-check passes. For PDFs with known failing pages the pass runs on a sub-slice of those pages (plus one page of margin). | 300 s | | 13 | `judge_escalation` | no | When the judge's failure rate exceeds `escalation_threshold`, re-run extract + judge with `escalation_model` and keep the better result. | 600 s | | 14 | `transform` | no | Caller-declared post-extraction transformations: declarative entity resolution + free-form LLM transformations. See [transformations.md](transformations.md). | 600 s | | 15 | `rules` | no | LLM evaluates the business-rule DAG, level by level. | 180 s | @@ -79,6 +79,16 @@ Other short-circuits: - `repair` is silently skipped when both `judge` and `field_validation` are off (no failure signals to act on); the request validator emits a `repair_no_verification_stage` warning. + Per task, repair also steps aside when more than + `FLYDOCS_REPAIR_MAX_FAILING_FRACTION` of the fields are failing -- + the extraction is globally untrustworthy and `judge_escalation`'s + full re-run is the right tool (`RepairInfo.tasks_skipped` records + it). Concurrent task repairs are bounded by + `FLYDOCS_REPAIR_TASK_CONCURRENCY`, and + `FLYDOCS_REPAIR_INCLUDE_FLAGGED=false` restricts repair to hard + failures. On the sync channel the validator emits a + `repair_sync_latency` warning -- the pass eats into the + `FLYDOCS_SYNC_TIMEOUT_S` wall. - `judge_escalation` is silently skipped when `judge` is off. - `transform` is silently a no-op when `options.transformations` is empty even with the toggle on. From 3e8f5fffb907a3ba54876c65d31e9c93a654c64c Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 18:31:48 +0200 Subject: [PATCH 36/38] style: explicit strict=False on _changed_row_count zip (B905) --- src/flydocs/core/services/repair/field_repairer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index 0f9a194..a4cd980 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -346,7 +346,9 @@ def _changed_row_count(old_rows: list[Any], new_rows: list[Any]) -> int: stable keys, so an inserted or dropped row also counts as a change. """ changed = sum( - 1 for old, new in zip(old_rows, new_rows) if _row_signature(old) != _row_signature(new) + 1 + for old, new in zip(old_rows, new_rows, strict=False) + if _row_signature(old) != _row_signature(new) ) return changed + abs(len(old_rows) - len(new_rows)) From 553b6d1b3e420a7a4d1d2cbe10d9b28a6968b951 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 18:38:37 +0200 Subject: [PATCH 37/38] docs: thin repair config comments duplicated in env_template --- src/flydocs/config.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/flydocs/config.py b/src/flydocs/config.py index ab19a2f..5a56878 100644 --- a/src/flydocs/config.py +++ b/src/flydocs/config.py @@ -234,21 +234,12 @@ class IDPSettings(BaseSettings): # ``repair_model`` pins the pass to its own model (``None`` = the # request model). Runs BEFORE judge_escalation, so the full re-run # only fires when targeted repair was not enough. - # ``repair_include_flagged`` widens the failure predicate to judge - # ``flag_for_review`` fields (matching what the escalation counter - # counts). Flagged-but-PASS fields are often ambiguous-but-correct, - # so set it to false to restrict repair to hard failures (judge FAIL - # or a validator error) and save those extra passes. - # ``repair_max_failing_fraction`` caps the repair scope per task: - # above this failing-field fraction the extraction is globally - # untrustworthy and a focused pass would re-extract nearly everything - # on top of the eventual escalation re-run, so repair steps aside and - # judge_escalation (whose trigger rate is unchanged) takes over. - # 1.0 disables the cap. - # ``repair_task_concurrency`` bounds how many tasks are repaired at - # once (each repair is one extract call plus an optional judge - # re-check); lower it under provider rate limits, mirroring - # ``bbox_refine_doc_concurrency``. + # ``repair_include_flagged=false`` restricts repair to hard failures + # (judge FAIL or a validator error), skipping ambiguous flagged-PASS + # fields. ``repair_max_failing_fraction`` caps scope: above it the doc + # is globally untrustworthy, so repair steps aside for judge_escalation + # (1.0 disables). ``repair_task_concurrency`` bounds parallel repairs + # under provider rate limits, mirroring ``bbox_refine_doc_concurrency``. repair_model: str | None = None repair_timeout_s: int = 300 repair_include_flagged: bool = True From 63543f8b711e9575e30400a260c8ecf9df81e3e5 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Jul 2026 18:50:11 +0200 Subject: [PATCH 38/38] style: ruff format _changed_row_count (CI lint fix) --- src/flydocs/core/services/repair/field_repairer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/flydocs/core/services/repair/field_repairer.py b/src/flydocs/core/services/repair/field_repairer.py index a4cd980..8dfa61e 100644 --- a/src/flydocs/core/services/repair/field_repairer.py +++ b/src/flydocs/core/services/repair/field_repairer.py @@ -346,9 +346,7 @@ def _changed_row_count(old_rows: list[Any], new_rows: list[Any]) -> int: stable keys, so an inserted or dropped row also counts as a change. """ changed = sum( - 1 - for old, new in zip(old_rows, new_rows, strict=False) - if _row_signature(old) != _row_signature(new) + 1 for old, new in zip(old_rows, new_rows, strict=False) if _row_signature(old) != _row_signature(new) ) return changed + abs(len(old_rows) - len(new_rows))