diff --git a/src/parse_bench/evaluation/layout_adapters/adapters.py b/src/parse_bench/evaluation/layout_adapters/adapters.py index b9d2297..0d1dcd5 100644 --- a/src/parse_bench/evaluation/layout_adapters/adapters.py +++ b/src/parse_bench/evaluation/layout_adapters/adapters.py @@ -2792,8 +2792,6 @@ def to_layout_output( class PyMuPDF4LLMLayoutAdapter(LayoutAdapter): """Extract canonical layout predictions from PyMuPDF4LLM parse output.""" - _SCALE = 1000 - @classmethod def matches(cls, inference_result: InferenceResult) -> bool: if not isinstance(inference_result.output, ParseOutput) or not inference_result.output.layout_pages: @@ -2819,29 +2817,44 @@ def to_layout_output( if not isinstance(inference_result.output, ParseOutput): raise ValueError("PyMuPDF4LLMLayoutAdapter requires ParseOutput or LayoutOutput") + layout_pages = inference_result.output.layout_pages + # Segments are normalized to [0, 1] against their own page. Scale each by + # its real page dimensions and expose the (reference) page's real + # dimensions as image_width/height so the metric's normalize_bbox_xyxy + # round-trips back to [0, 1] (a numeric no-op vs. the old synthetic 1000 + # scale). Mirrors DoclingParseLayoutAdapter's real-dimension convention. + selected_pages = [lp for lp in layout_pages if page_filter is None or lp.page_number == page_filter] + reference_page = selected_pages[0] if selected_pages else (layout_pages[0] if layout_pages else None) + output_width = int(reference_page.width or 1) if reference_page is not None else 1 + output_height = int(reference_page.height or 1) if reference_page is not None else 1 + predictions: list[LayoutPrediction] = [] - for page in inference_result.output.layout_pages: + for page in layout_pages: if page_filter is not None and page.page_number != page_filter: continue + page_width = float(page.width or output_width) + page_height = float(page.height or output_height) for item in page.items: segments = item.layout_segments or ([item.bbox] if item.bbox is not None else []) for segment in segments: if segment is None: continue + # Emit the raw provider label; canonicalization happens in the + # label-mapper layer during projection. label = segment.label or item.type or "Text" - content_text = item.html if label == "Table" and item.html else item.md or item.value + is_table = item.type == "table" + content_text = item.html if is_table and item.html else item.md or item.value content = _build_docling_parse_content( - "table" if label == "Table" else "text", + "table" if is_table else "text", content_text, ) - scale = self._SCALE predictions.append( LayoutPrediction( bbox=[ - segment.x * scale, - segment.y * scale, - (segment.x + segment.w) * scale, - (segment.y + segment.h) * scale, + segment.x * page_width, + segment.y * page_height, + (segment.x + segment.w) * page_width, + (segment.y + segment.h) * page_height, ], score=segment.confidence if segment.confidence is not None else 1.0, label=label, @@ -2861,8 +2874,8 @@ def to_layout_output( example_id=inference_result.request.example_id, pipeline_name=inference_result.pipeline_name, model=LayoutDetectionModel.PYMUPDF4LLM_LAYOUT, - image_width=self._SCALE, - image_height=self._SCALE, + image_width=max(output_width, 1), + image_height=max(output_height, 1), predictions=predictions, markdown=inference_result.output.markdown, ) diff --git a/src/parse_bench/evaluation/layout_label_mappers/mappers.py b/src/parse_bench/evaluation/layout_label_mappers/mappers.py index 819e05d..4d23d1d 100644 --- a/src/parse_bench/evaluation/layout_label_mappers/mappers.py +++ b/src/parse_bench/evaluation/layout_label_mappers/mappers.py @@ -109,6 +109,64 @@ def to_canonical( return canonical +@register_layout_label_mapper( + "pymupdf4llm", + "model:pymupdf4llm_layout", + priority=95, +) +class PyMuPDF4LLMLabelMapper(LayoutLabelMapper): + """Mapper for raw PyMuPDF4LLM boxclass labels emitted in layout_pages. + + The ``pymupdf4llm`` PARSE provider forwards the raw pymupdf4llm boxclass + string on each layout segment; this mapper owns the canonical semantics and + raises on any genuinely unknown class so a new upstream label cannot silently + become incorrect benchmark ground truth. + """ + + _MAPPING: dict[str, CanonicalLabel] = { + "caption": CanonicalLabel.CAPTION, + "footnote": CanonicalLabel.FOOTNOTE, + "formula": CanonicalLabel.FORMULA, + "list-item": CanonicalLabel.LIST_ITEM, + "listitem": CanonicalLabel.LIST_ITEM, + "page-footer": CanonicalLabel.PAGE_FOOTER, + "pagefooter": CanonicalLabel.PAGE_FOOTER, + "page-header": CanonicalLabel.PAGE_HEADER, + "pageheader": CanonicalLabel.PAGE_HEADER, + "picture": CanonicalLabel.PICTURE, + "image": CanonicalLabel.PICTURE, + "section-header": CanonicalLabel.SECTION_HEADER, + "sectionheader": CanonicalLabel.SECTION_HEADER, + "heading": CanonicalLabel.SECTION_HEADER, + "table": CanonicalLabel.TABLE, + "text": CanonicalLabel.TEXT, + "title": CanonicalLabel.TITLE, + "code": CanonicalLabel.CODE, + "document-index": CanonicalLabel.DOCUMENT_INDEX, + "documentindex": CanonicalLabel.DOCUMENT_INDEX, + "form": CanonicalLabel.FORM, + "key-value-region": CanonicalLabel.KEY_VALUE_REGION, + "keyvalueregion": CanonicalLabel.KEY_VALUE_REGION, + "checkbox-selected": CanonicalLabel.CHECKBOX_SELECTED, + "checkboxselected": CanonicalLabel.CHECKBOX_SELECTED, + "checkbox-unselected": CanonicalLabel.CHECKBOX_UNSELECTED, + "checkboxunselected": CanonicalLabel.CHECKBOX_UNSELECTED, + } + + def to_canonical( + self, + label: str, + prediction: LayoutPrediction, + context: MappingContext, + ) -> CanonicalLabel: + del prediction, context + normalized = label.strip().lower().replace("_", "-").replace(" ", "-") + mapped = self._MAPPING.get(normalized) + if mapped is None: + raise UnknownRawLayoutLabelError(f"Unknown PyMuPDF4LLM raw layout label '{label}'") + return mapped + + @register_layout_label_mapper( "model:yolo_doclaynet", "model:docling_layout_old", diff --git a/src/parse_bench/inference/providers/parse/pymupdf4llm.py b/src/parse_bench/inference/providers/parse/pymupdf4llm.py index 70cd85b..c459bca 100644 --- a/src/parse_bench/inference/providers/parse/pymupdf4llm.py +++ b/src/parse_bench/inference/providers/parse/pymupdf4llm.py @@ -31,24 +31,6 @@ logger = logging.getLogger(__name__) -# PyMuPDF Layout 1.28 emits exactly the DocLayNet/Core11 classes. Keep the -# mapping strict so a new upstream class does not silently become incorrect -# benchmark ground truth. -_PYMUPDF_CLASS_TO_CANONICAL = { - "caption": "Caption", - "footnote": "Footnote", - "formula": "Formula", - "list-item": "List-item", - "page-footer": "Page-footer", - "page-header": "Page-header", - "picture": "Picture", - "section-header": "Section-header", - "table": "Table", - "text": "Text", - "title": "Title", -} - - @register_provider("pymupdf4llm") class PyMuPDF4LLMProvider(Provider): """Provider for PyMuPDF4LLM (markdown). AGPL — runtime dep only.""" @@ -309,17 +291,18 @@ def _build_layout_page( return None items: list[LayoutItemIR] = [] - unknown_classes: set[str] = set() for page_box in page_data.get("page_boxes", []): if not isinstance(page_box, dict): continue - raw_class = str(page_box.get("class", "")).strip().lower().replace("_", "-") - canonical_label = _PYMUPDF_CLASS_TO_CANONICAL.get(raw_class) - if canonical_label is None: - if raw_class: - unknown_classes.add(raw_class) + # Emit the raw pymupdf4llm boxclass label untouched. Canonicalization + # and failing loud on genuinely unknown classes are owned by the + # evaluation label-mapper layer (PyMuPDF4LLMLabelMapper), not the + # provider, so no class is silently dropped here. + raw_label = str(page_box.get("class", "")).strip() + if not raw_label: continue + normalized_class = raw_label.lower().replace("_", "-") bbox = cls._coerce_bbox( page_box.get("bbox"), @@ -354,20 +337,23 @@ def _build_layout_page( w=bbox[2], h=bbox[3], confidence=confidence, - label=canonical_label, + label=raw_label, start_index=start_index, end_index=end_index, ) - if canonical_label == "Table": + if normalized_class == "table": item_type = "table" - item_html = cls._convert_md_tables_to_html(content) - elif canonical_label == "Picture": + # If the sliced content is already native HTML (e.g. a pipeline + # opted into table_output="html"), keep it verbatim; otherwise + # convert Markdown pipe tables via markdown2. + item_html = content if " MappingContext: + layout_output = LayoutOutput( + example_id="ex", + pipeline_name="pymupdf4llm_markdown", + model=LayoutDetectionModel.PYMUPDF4LLM_LAYOUT, + image_width=100, + image_height=100, + predictions=[], + ) + return MappingContext( + provider_name=provider_name, + pipeline_name="pymupdf4llm_markdown", + model=LayoutDetectionModel.PYMUPDF4LLM_LAYOUT, + raw_output={}, + layout_output=layout_output, + ) + + +def test_mapper_registered_for_provider_and_model_keys() -> None: + keys = list_layout_label_mappers() + assert "pymupdf4llm" in keys + assert "model:pymupdf4llm_layout" in keys + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("table", CanonicalLabel.TABLE), + ("text", CanonicalLabel.TEXT), + ("title", CanonicalLabel.TITLE), + ("section-header", CanonicalLabel.SECTION_HEADER), + ("Section_Header", CanonicalLabel.SECTION_HEADER), + ("list-item", CanonicalLabel.LIST_ITEM), + ("list_item", CanonicalLabel.LIST_ITEM), + ("picture", CanonicalLabel.PICTURE), + ("image", CanonicalLabel.PICTURE), + ("caption", CanonicalLabel.CAPTION), + ], +) +def test_maps_known_raw_labels(raw: str, expected: CanonicalLabel) -> None: + assert PyMuPDF4LLMLabelMapper().to_canonical(raw, None, None) == expected # type: ignore[arg-type] + + +def test_unknown_label_raises() -> None: + with pytest.raises(UnknownRawLayoutLabelError): + PyMuPDF4LLMLabelMapper().to_canonical("totally-unknown-class", None, None) # type: ignore[arg-type] + + +def test_resolver_selects_mapper_by_provider_name() -> None: + mapper = resolve_layout_label_mapper(_context(provider_name="pymupdf4llm")) + assert isinstance(mapper, PyMuPDF4LLMLabelMapper) + + +def test_resolver_selects_mapper_by_model_key() -> None: + # With no provider name, resolution must fall back to the model: key. + mapper = resolve_layout_label_mapper(_context(provider_name=None)) + assert isinstance(mapper, PyMuPDF4LLMLabelMapper) diff --git a/tests/parse_bench/inference/providers/parse/test_pymupdf4llm.py b/tests/parse_bench/inference/providers/parse/test_pymupdf4llm.py index 2973c91..d6c6bc5 100644 --- a/tests/parse_bench/inference/providers/parse/test_pymupdf4llm.py +++ b/tests/parse_bench/inference/providers/parse/test_pymupdf4llm.py @@ -1,31 +1,86 @@ """Tests for PyMuPDF4LLM layout normalization helpers.""" -from parse_bench.inference.providers.parse.pymupdf4llm import ( - _PYMUPDF_CLASS_TO_CANONICAL, - PyMuPDF4LLMProvider, -) +from parse_bench.inference.providers.parse.pymupdf4llm import PyMuPDF4LLMProvider -def test_build_layout_page_maps_all_pymupdf_classes() -> None: +def _build_page(page_boxes: list[dict], *, raw_markdown: str): + return PyMuPDF4LLMProvider._build_layout_page( + { + "page_number": 1, + "width": 100, + "height": 100, + "page_boxes": page_boxes, + }, + raw_markdown=raw_markdown, + ) + + +def test_build_layout_page_emits_raw_boxclass_labels() -> None: + """Provider forwards raw boxclass labels verbatim and never drops unknowns. + + Canonicalization (and failing loud on unknown classes) is owned by the + evaluation label-mapper layer, so an unrecognized class must survive here. + """ markdown = "grounded content" + raw_classes = [ + "caption", + "table", + "section-header", + "text", + "picture", + "totally-unknown-class", + ] page_boxes = [ { "class": raw_class, "bbox": [10, 10 + index * 5, 90, 14 + index * 5], "pos": [0, len(markdown)], } - for index, raw_class in enumerate(_PYMUPDF_CLASS_TO_CANONICAL) + for index, raw_class in enumerate(raw_classes) ] - page = PyMuPDF4LLMProvider._build_layout_page( - { - "page_number": 1, - "width": 100, - "height": 100, - "page_boxes": page_boxes, - }, - raw_markdown=markdown, - ) + page = _build_page(page_boxes, raw_markdown=markdown) + + assert page is not None + assert [item.layout_segments[0].label for item in page.items] == raw_classes + + +def test_build_layout_page_sets_item_type_from_raw_class() -> None: + markdown = "x" + page_boxes = [ + {"class": "Table", "bbox": [10, 10, 90, 20], "pos": [0, 1]}, + {"class": "picture", "bbox": [10, 25, 90, 40], "pos": [0, 1]}, + {"class": "section_header", "bbox": [10, 45, 90, 60], "pos": [0, 1]}, + ] + + page = _build_page(page_boxes, raw_markdown=markdown) + + assert page is not None + assert [item.type for item in page.items] == ["table", "image", "text"] + + +def test_table_native_html_content_is_passed_through() -> None: + """A native in the sliced content is preserved untouched.""" + markdown = "
ab
" + page_boxes = [{"class": "table", "bbox": [10, 10, 90, 90], "pos": [0, len(markdown)]}] + + page = _build_page(page_boxes, raw_markdown=markdown) + + assert page is not None + item = page.items[0] + assert item.type == "table" + assert item.html == markdown + + +def test_table_pipe_markdown_is_converted_to_html() -> None: + """Markdown pipe tables still fall back to the markdown2 conversion.""" + markdown = "| a | b |\n| --- | --- |\n| 1 | 2 |" + page_boxes = [{"class": "table", "bbox": [10, 10, 90, 90], "pos": [0, len(markdown)]}] + + page = _build_page(page_boxes, raw_markdown=markdown) assert page is not None - assert [item.layout_segments[0].label for item in page.items] == list(_PYMUPDF_CLASS_TO_CANONICAL.values()) + item = page.items[0] + assert item.type == "table" + assert item.html != markdown + assert "" in item.html.lower()