From 5cb254e2cb24a8811aef82386394c177d2377aab Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Thu, 9 Jul 2026 15:36:24 +0200 Subject: [PATCH 1/5] fixing the word extraction with spaces as blockers Signed-off-by: Peter Staar --- scripts/export_dclx.py | 32 ++++++-- src/parse/page_item_sanitators/cells.h | 102 +++++++++++++++---------- src/parse/pdf_decoders/page.h | 3 +- 3 files changed, 89 insertions(+), 48 deletions(-) diff --git a/scripts/export_dclx.py b/scripts/export_dclx.py index a2c06de2..9ea19275 100644 --- a/scripts/export_dclx.py +++ b/scripts/export_dclx.py @@ -67,6 +67,8 @@ def parse_args(argv=None): help="Page-image raster scale in pixels-per-point (default: 2.0).") ap.add_argument("--threads", type=int, default=4, help="Worker threads for the parser (default: 4).") + ap.add_argument("--log-text", action="store_true", + help="Print a table of each exported text cell and its bounding box.") return ap.parse_args(argv) @@ -83,7 +85,19 @@ def _content_config_for(mode: str) -> ContentConfig: ) -def _add_page_to_doc(doc, page_no, page, image, dpi, unit): +def _format_bbox_coord(value: float) -> str: + return f"{value:.2f}" + + +def _log_text_cell(page_no: int, text: str, bbox) -> None: + print( + f"{page_no}\t{text!r}\t{_format_bbox_coord(bbox.l)}\t" + f"{_format_bbox_coord(bbox.t)}\t{_format_bbox_coord(bbox.r)}\t" + f"{_format_bbox_coord(bbox.b)}" + ) + + +def _add_page_to_doc(doc, page_no, page, image, dpi, unit, log_text=False): """Attach one page (image + text cells + picture bitmaps) to the doc. Returns (n_text_cells, n_pictures). @@ -99,9 +113,12 @@ def _add_page_to_doc(doc, page_no, page, image, dpi, unit): text = cell.text if not text: continue + bbox = cell.rect.to_bounding_box() + if log_text: + _log_text_cell(page_no, text, bbox) prov = ProvenanceItem( page_no=page_no, - bbox=cell.rect.to_bounding_box(), + bbox=bbox, charspan=(0, len(text)), ) doc.add_text(label=DocItemLabel.TEXT, text=text, orig=text, prov=prov) @@ -123,7 +140,7 @@ def _add_page_to_doc(doc, page_no, page, image, dpi, unit): def export(input_path: Path, mode: str, scale: float, threads: int, - page: int | None = None) -> DoclingDocument: + page: int | None = None, log_text: bool = False) -> DoclingDocument: content_config = _content_config_for(mode) page_numbers = [page] if page is not None else None @@ -162,9 +179,13 @@ def export(input_path: Path, mode: str, scale: float, threads: int, total_cells = 0 total_pictures = 0 + if log_text: + print("page\ttext\tx0\ty0\tx1\ty1") + for page_no in sorted(pages): page, image = pages[page_no] - n_cells, n_pics = _add_page_to_doc(doc, page_no, page, image, dpi, unit) + n_cells, n_pics = _add_page_to_doc( + doc, page_no, page, image, dpi, unit, log_text=log_text) total_cells += n_cells total_pictures += n_pics print(f" page {page_no}: {n_cells} {mode} cell(s), {n_pics} picture(s)") @@ -187,7 +208,8 @@ def main(argv=None): page_label = f", page={args.page}" if args.page is not None else "" print(f"Exporting '{args.input}' (mode={args.mode}, scale={args.scale}{page_label}) ...") - doc = export(args.input, args.mode, args.scale, args.threads, args.page) + doc = export(args.input, args.mode, args.scale, args.threads, args.page, + log_text=args.log_text) default_suffix = ( f".{args.mode}.p{args.page}.dclx" diff --git a/src/parse/page_item_sanitators/cells.h b/src/parse/page_item_sanitators/cells.h index 9efb4ae4..556e98c9 100644 --- a/src/parse/page_item_sanitators/cells.h +++ b/src/parse/page_item_sanitators/cells.h @@ -31,7 +31,8 @@ namespace pdflib double horizontal_cell_tolerance, //=1.0, bool enforce_same_font, //=true, double space_width_factor_for_merge, //=1.5, - double space_width_factor_for_merge_with_space); //=0.33); + double space_width_factor_for_merge_with_space, //=0.33, + bool block_spaces); // when true, space cells act as hard merge barriers void sanitize_text(page_item& cells); @@ -41,27 +42,31 @@ namespace pdflib bool applicable_for_merge(page_item& cell_i, page_item& cell_j, - bool enforce_same_font); + bool enforce_same_font, + bool block_spaces); void contract_cells_into_lines_right_to_left(page_item& cells, double horizontal_cell_tolerance, bool enforce_same_font, double space_width_factor_for_merge, - double space_width_factor_for_merge_with_space); + double space_width_factor_for_merge_with_space, + bool block_spaces); void contract_cells_into_lines_left_to_right(page_item& cells, double horizontal_cell_tolerance, bool enforce_same_font, double space_width_factor_for_merge, double space_width_factor_for_merge_with_space, + bool block_spaces, bool allow_reverse); - + // linear void contract_cells_into_lines_v1(page_item& cells, double horizontal_cell_tolerance=1.0, bool enforce_same_font=true, double space_width_factor_for_merge=1.5, - double space_width_factor_for_merge_with_space=0.33); + double space_width_factor_for_merge_with_space=0.33, + bool block_spaces=false); // quadratic void contract_cells_into_lines_v2(page_item& cells, @@ -140,7 +145,23 @@ namespace pdflib LOG_S(INFO) << "#-char cells: " << word_cells.size(); - // remove all spaces + // Keep the space cells in place and let sanitize_bbox treat them as hard + // word-boundary barriers (block_spaces=true). An explicit space glyph is a + // far more reliable word separator than the geometric-gap heuristic, which + // is ambiguous for tightly-set fonts with narrow spaces. The spaces are + // erased afterwards, once the words have been contracted. + + // > space_width_factor_for_merge, so nothing gets merged with a space + double space_width_factor_for_merge_with_space = 2.0*config.word_space_width_factor_for_merge; + + sanitize_bbox(word_cells, + config.horizontal_cell_tolerance, + config.enforce_same_font, + config.word_space_width_factor_for_merge, + space_width_factor_for_merge_with_space, + true); + + // remove the space cells that acted as word-boundary barriers auto itr = word_cells.begin(); while(itr!=word_cells.end()) { @@ -154,17 +175,6 @@ namespace pdflib } } - LOG_S(INFO) << "#-char cells (without spaces): " << word_cells.size(); - - // > space_width_factor_for_merge, so nothing gets merged with a space - double space_width_factor_for_merge_with_space = 2.0*config.word_space_width_factor_for_merge; - - sanitize_bbox(word_cells, - config.horizontal_cell_tolerance, - config.enforce_same_font, - config.word_space_width_factor_for_merge, - space_width_factor_for_merge_with_space); - LOG_S(INFO) << "#-word cells: " << word_cells.size(); //return to_records(word_cells); @@ -184,12 +194,14 @@ namespace pdflib LOG_S(INFO) << "# char-cells: " << line_cells.size(); + // lines keep their internal spaces, so spaces are merged normally (block_spaces=false) sanitize_bbox(line_cells, config.horizontal_cell_tolerance, config.enforce_same_font, config.line_space_width_factor_for_merge, - config.line_space_width_factor_for_merge_with_space); - + config.line_space_width_factor_for_merge_with_space, + false); + LOG_S(INFO) << "# line-cells: " << line_cells.size(); //return to_records(line_cells); @@ -373,37 +385,40 @@ namespace pdflib double horizontal_cell_tolerance, bool enforce_same_font, double space_width_factor_for_merge, - double space_width_factor_for_merge_with_space) + double space_width_factor_for_merge_with_space, + bool block_spaces) { contract_cells_into_lines_v1(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, - space_width_factor_for_merge_with_space); - - /* - contract_cells_into_lines_v2(cells, - horizontal_cell_tolerance, - enforce_same_font, - space_width_factor_for_merge, - space_width_factor_for_merge_with_space); - */ + space_width_factor_for_merge_with_space, + block_spaces); } bool page_item_sanitator::applicable_for_merge(page_item& cell_i, page_item& cell_j, - bool enforce_same_font) + bool enforce_same_font, + bool block_spaces) { if(not cell_i.active) { return false; } - + if(not cell_j.active) { return false; } - + + // An explicit space glyph is a hard word boundary: never merge a space cell + // with anything, nor across one. The space cells are removed afterwards. + if(block_spaces and + (utils::string::is_space(cell_i.text) or utils::string::is_space(cell_j.text))) + { + return false; + } + if(enforce_same_font and cell_i.font_name!=cell_j.font_name) { // Exception: ligature glyphs are often encoded in a different font than @@ -427,13 +442,14 @@ namespace pdflib double horizontal_cell_tolerance, bool enforce_same_font, double space_width_factor_for_merge, - double space_width_factor_for_merge_with_space) + double space_width_factor_for_merge_with_space, + bool block_spaces) { - contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, false); - - contract_cells_into_lines_right_to_left(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space); - - contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, true); + contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces, false); + + contract_cells_into_lines_right_to_left(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces); + + contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces, true); } void page_item_sanitator::contract_cells_into_lines_left_to_right(page_item& cells, @@ -441,6 +457,7 @@ namespace pdflib bool enforce_same_font, double space_width_factor_for_merge, double space_width_factor_for_merge_with_space, + bool block_spaces, bool allow_reverse) { // take care for left to right printing @@ -454,7 +471,7 @@ namespace pdflib for(int j=i+1; j=0; i--) { @@ -541,7 +559,7 @@ namespace pdflib for(int j=i-1; j>=0; j--) { - if(not applicable_for_merge(cells[i], cells[j], enforce_same_font)) + if(not applicable_for_merge(cells[i], cells[j], enforce_same_font, block_spaces)) { break; } diff --git a/src/parse/pdf_decoders/page.h b/src/parse/pdf_decoders/page.h index dddbec15..8e90b00e 100644 --- a/src/parse/pdf_decoders/page.h +++ b/src/parse/pdf_decoders/page.h @@ -1369,7 +1369,8 @@ namespace pdflib horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, - space_width_factor_for_merge_with_space); + space_width_factor_for_merge_with_space, + false); //sanitator.sanitize_text(cells); From d2e19f7c86621de202c336275e857b04f65ad321 Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Fri, 10 Jul 2026 07:02:47 +0200 Subject: [PATCH 2/5] fixed the tests data Signed-off-by: Peter Staar --- scripts/check_rendering_regression.py | 3 ++- tests/constants.py | 13 +++++++++++++ tests/data_utils.py | 4 ++-- tests/test_parse.py | 10 +--------- tests/test_threaded_parse.py | 16 +++++----------- tests/test_threaded_render.py | 17 +++++------------ 6 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 tests/constants.py diff --git a/scripts/check_rendering_regression.py b/scripts/check_rendering_regression.py index 97305ab2..c69e8427 100644 --- a/scripts/check_rendering_regression.py +++ b/scripts/check_rendering_regression.py @@ -24,7 +24,8 @@ ) from tests.data_utils import ensure_test_data_downloaded # noqa: E402 from tests.rendering_regression import renderer_image_path # noqa: E402 -from tests.test_parse import PARSER_PAGE_RESTRICTIONS, REGRESSION_FOLDER # noqa: E402 +from tests.constants import PARSER_PAGE_RESTRICTIONS # noqa: E402 +from tests.test_parse import REGRESSION_FOLDER # noqa: E402 DEFAULT_DELTA_DIR = Path("tests/data/render_deltas") DEFAULT_PIXEL_THRESHOLD = 12 diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 00000000..a2a672d2 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,13 @@ +HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" +HF_DATASET_REVISION = "17980e180c394d7941cc6b47860a7d24570fceaf" + +PARSER_PAGE_RESTRICTIONS = { + "deep-mediabox-inheritance.pdf": [2], + "font_06.pdf": [1], + "font_07.pdf": [1], + "font_08.pdf": [1], + "font_09.pdf": [1], + "font_10.pdf": [1], + "font_11.pdf": [1, 2], + "2508.13113v2.pdf": [2, 9, 17], +} diff --git a/tests/data_utils.py b/tests/data_utils.py index 2a64e108..1d42f73d 100644 --- a/tests/data_utils.py +++ b/tests/data_utils.py @@ -4,8 +4,8 @@ from huggingface_hub import snapshot_download -HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "0bf5447051628cf0c75aca9aa992ed5d7837f627" +from tests.constants import HF_DATASET_REPO_ID, HF_DATASET_REVISION + TESTS_DIR = Path(__file__).resolve().parent TEST_DATA_DIR = TESTS_DIR / "data" TEST_DATA_GROUNDTRUTH_DIR = TEST_DATA_DIR / "groundtruth" diff --git a/tests/test_parse.py b/tests/test_parse.py index 55e03595..dd5e9e77 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -28,6 +28,7 @@ DoclingPdfParser, PdfDocument, ) +from tests.constants import PARSER_PAGE_RESTRICTIONS from tests.data_utils import PARSER_GROUNDTRUTH_DIR @@ -61,15 +62,6 @@ def save_as_json_rounded(page: SegmentedPdfPage, filename, indent=2, ndigits=3): GROUNDTRUTH_FOLDER = os.fspath(PARSER_GROUNDTRUTH_DIR) REGRESSION_FOLDER = "tests/data/regression/*.pdf" -PARSER_PAGE_RESTRICTIONS = { - "deep-mediabox-inheritance.pdf": [2], - "font_06.pdf": [1], - "font_07.pdf": [1], - "font_08.pdf": [1], - "font_09.pdf": [1], - "font_10.pdf": [1], - "2508.13113v2.pdf": [2, 9, 17], -} def _truncate_data_uri(uri, max_chars: int = 64): diff --git a/tests/test_threaded_parse.py b/tests/test_threaded_parse.py index cbabf464..94155ff4 100644 --- a/tests/test_threaded_parse.py +++ b/tests/test_threaded_parse.py @@ -15,6 +15,7 @@ DoclingThreadedPdfParser, ThreadedPdfParserConfig, ) +from tests.constants import PARSER_PAGE_RESTRICTIONS from tests.test_parse import ( GROUNDTRUTH_FOLDER, REGRESSION_FOLDER, @@ -48,16 +49,6 @@ def test_threaded_reference_documents_from_filenames(): decode_config=_make_decode_config(), ) - page_restrictions = { - "deep-mediabox-inheritance.pdf": [2], - "font_06.pdf": [1], - "font_07.pdf": [1], - "font_08.pdf": [1], - "font_09.pdf": [1], - "font_10.pdf": [1], - "2508.13113v2.pdf": [2, 9, 17], - } - # Each entry: (doc_name, page_no_str, mode, success, error_msg) test_results: list[tuple[str, str, str, bool, str]] = [] first_failure: tuple[BaseException, object] | None = None @@ -118,7 +109,10 @@ def test_threaded_reference_documents_from_filenames(): for page_no, pred_page in sorted(results.get(key, {}).items()): print(f" -> Page {page_no} has {len(pred_page.textline_cells)} cells.") - if rname in page_restrictions and page_no not in page_restrictions[rname]: + if ( + rname in PARSER_PAGE_RESTRICTIONS + and page_no not in PARSER_PAGE_RESTRICTIONS[rname] + ): continue fname = os.path.join( diff --git a/tests/test_threaded_render.py b/tests/test_threaded_render.py index 2ee9f884..0e155955 100644 --- a/tests/test_threaded_render.py +++ b/tests/test_threaded_render.py @@ -28,9 +28,9 @@ measure_image_comparison, write_renderer_groundtruth, ) +from tests.constants import PARSER_PAGE_RESTRICTIONS from tests.test_parse import ( GROUNDTRUTH_FOLDER, - PARSER_PAGE_RESTRICTIONS, REGRESSION_FOLDER, verify_SegmentedPdfPage, ) @@ -424,16 +424,6 @@ def test_render_reference_documents_from_filenames(): parser = _make_parser(threads=4, max_concurrent=32) - page_restrictions = { - "deep-mediabox-inheritance.pdf": [2], - "font_06.pdf": [1], - "font_07.pdf": [1], - "font_08.pdf": [1], - "font_09.pdf": [1], - "font_10.pdf": [1], - "2508.13113v2.pdf": [2, 9, 17], - } - test_results: list[tuple[str, str, str, bool, str]] = [] first_failure: tuple[BaseException, TracebackType | None] | None = None doc_keys: dict[str, str] = {} @@ -491,7 +481,10 @@ def test_render_reference_documents_from_filenames(): rname = os.path.basename(pdf_doc_path) for page_no, pred_page in sorted(results[key].items()): - if rname in page_restrictions and page_no not in page_restrictions[rname]: + if ( + rname in PARSER_PAGE_RESTRICTIONS + and page_no not in PARSER_PAGE_RESTRICTIONS[rname] + ): continue fname = os.path.join( From 252007144bbaf563394c31f9867f38401d57fe6c Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Fri, 10 Jul 2026 10:17:01 +0200 Subject: [PATCH 3/5] fixed some fonts/glyph extraction Signed-off-by: Peter Staar --- scripts/export_dclx.py | 31 +++++-- src/parse/pdf_resources/page_font.h | 131 ++++++++++++++-------------- 2 files changed, 88 insertions(+), 74 deletions(-) diff --git a/scripts/export_dclx.py b/scripts/export_dclx.py index 9ea19275..4e1045d5 100644 --- a/scripts/export_dclx.py +++ b/scripts/export_dclx.py @@ -48,6 +48,9 @@ "text": TextCellUnit.LINE, } +LOG_COORD_WIDTH = 8 +LOG_COORD_PRECISION = 2 + def parse_args(argv=None): ap = argparse.ArgumentParser( @@ -67,6 +70,10 @@ def parse_args(argv=None): help="Page-image raster scale in pixels-per-point (default: 2.0).") ap.add_argument("--threads", type=int, default=4, help="Worker threads for the parser (default: 4).") + ap.add_argument("-l", "--loglevel", + choices=["fatal", "error", "warn", "warning", "info"], + default="fatal", + help="C++ parser log level (default: fatal).") ap.add_argument("--log-text", action="store_true", help="Print a table of each exported text cell and its bounding box.") return ap.parse_args(argv) @@ -86,14 +93,16 @@ def _content_config_for(mode: str) -> ContentConfig: def _format_bbox_coord(value: float) -> str: - return f"{value:.2f}" + return f"{value:{LOG_COORD_WIDTH}.{LOG_COORD_PRECISION}f}" def _log_text_cell(page_no: int, text: str, bbox) -> None: print( - f"{page_no}\t{text!r}\t{_format_bbox_coord(bbox.l)}\t" - f"{_format_bbox_coord(bbox.t)}\t{_format_bbox_coord(bbox.r)}\t" - f"{_format_bbox_coord(bbox.b)}" + f"{_format_bbox_coord(bbox.l)} " + f"{_format_bbox_coord(bbox.t)} " + f"{_format_bbox_coord(bbox.r)} " + f"{_format_bbox_coord(bbox.b)} " + f"{page_no:>4} {text!r}" ) @@ -140,7 +149,8 @@ def _add_page_to_doc(doc, page_no, page, image, dpi, unit, log_text=False): def export(input_path: Path, mode: str, scale: float, threads: int, - page: int | None = None, log_text: bool = False) -> DoclingDocument: + page: int | None = None, log_text: bool = False, + loglevel: str = "fatal") -> DoclingDocument: content_config = _content_config_for(mode) page_numbers = [page] if page is not None else None @@ -151,6 +161,7 @@ def export(input_path: Path, mode: str, scale: float, threads: int, parser = DoclingThreadedPdfParser( parser_config=ThreadedPdfParserConfig( threads=threads, + loglevel=loglevel, render_config=render_config, page_content_config=content_config, ), @@ -180,7 +191,13 @@ def export(input_path: Path, mode: str, scale: float, threads: int, total_cells = 0 total_pictures = 0 if log_text: - print("page\ttext\tx0\ty0\tx1\ty1") + print( + f"{'x0':>{LOG_COORD_WIDTH}} " + f"{'y0':>{LOG_COORD_WIDTH}} " + f"{'x1':>{LOG_COORD_WIDTH}} " + f"{'y1':>{LOG_COORD_WIDTH}} " + f"{'page':>4} text" + ) for page_no in sorted(pages): page, image = pages[page_no] @@ -209,7 +226,7 @@ def main(argv=None): page_label = f", page={args.page}" if args.page is not None else "" print(f"Exporting '{args.input}' (mode={args.mode}, scale={args.scale}{page_label}) ...") doc = export(args.input, args.mode, args.scale, args.threads, args.page, - log_text=args.log_text) + log_text=args.log_text, loglevel=args.loglevel) default_suffix = ( f".{args.mode}.p{args.page}.dclx" diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index b4ff9727..cedd1071 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -542,23 +542,12 @@ namespace pdflib std::string pdf_resource::get_correct_character(uint32_t c) { - // Sometimes, a font has differences-map and a cmap - // defined at the same time. So far, it seems that the - // differences should take precedent over the cmap. This - // is however not really clear (eg p 292). Notice also that - // we init the cmap before we init the difference and that the - // difference inherits the content of a the cmap. It is a bit - // messy and unclear her. + // For codes covered by /Encoding/Differences, diff_numb_to_char + // already encodes the precedence of PDF 32000-1 section 9.10.2: + // init_differences() resolves each code via the /ToUnicode cmap + // first and only falls back to glyph-name based methods. For codes + // not covered by /Differences, the cmap is consulted directly below. - /* - if(diff_numb_to_char.count(c)>0 and cmap_numb_to_char.count(c)>0) - { - LOG_S(WARNING) << "there might be some confusion here: " - << "diff["<0) { return diff_numb_to_char.at(c); @@ -2209,9 +2198,43 @@ namespace pdflib // Create a regex object std::regex re_01(R"(\/(.+)\.(.+))"); std::regex re_02(R"((\/)?(uni|UNI)([0-9A-Fa-f]{4}))"); - std::regex re_03(R"((\/)(g|G)\d+)"); std::regex re_04(R"((\/)(C)(\d+))"); - + + // The unicode replacement character U+FFFD in utf8: a /ToUnicode + // mapping to this value means 'unknown character' and is treated as + // no mapping at all, so the glyph-name based methods can still + // recover the code (eg /bullet.003 mapped to U+FFFD by the cmap). + const std::string replacement_char = "\xEF\xBF\xBD"; + + // PDF 32000-1 (section 9.10.2) defines the /ToUnicode cmap as the + // first and most authoritative method to map a character-code to + // unicode; the glyph-name based methods only apply when no (valid) + // cmap entry exists for the code. + auto has_to_unicode = [&](int numb) + { + return cmap_initialized + and cmap_numb_to_char.count(numb)==1 + and cmap_numb_to_char.at(numb).size()>0 + and cmap_numb_to_char.at(numb)!=replacement_char; + }; + + // Last-resort for glyph-names that neither the /ToUnicode cmap nor + // any glyph-table could resolve (eg custom ligatures like /Th, /ft + // or /tt in a font without cmap): keep the glyph-name itself without + // the leading '/' and any '.suffix' as the reading text. + auto resolve_unknown_name = [](const std::string& name, const std::string& name_) + { + std::string result = name_; + if(result.empty()) + { + result = (name.size()>0 and name[0]=='/')? name.substr(1) : name; + } + + LOG_S(WARNING) << "unknown glyph-name " << name + << ": falling back to '" << result << "'"; + return result; + }; + if(utils::json::has(keys, json_font)) { auto diffs = utils::json::get(keys, json_font); @@ -2260,28 +2283,13 @@ namespace pdflib LOG_S(INFO) << name << ", in cmap: " << cmap_numb_to_char.count(numb) << ", #-names: " << name_to_descr.size() << ", type: " << subtype; - if(subtype==TYPE_3 and //name_to_descr.count(name)==1 and // only for TYPE_3 fonts - cmap_numb_to_char.count(numb)==1) - { - LOG_S(WARNING) << "overloading difference from cmap"; - diff_numb_to_char[numb] = cmap_numb_to_char.at(numb); - } - - // FIXME: might need to be commented out or fixed - /* - else if(name_to_descr.count(name)==1 and - cmap_numb_to_char.count(numb)==0) - { - //assert(subtype==TYPE_3); - - LOG_S(WARNING) << "could not resolve the character (name="< " << name @@ -2293,13 +2301,6 @@ namespace pdflib LOG_S(INFO) << "differences[" << numb << "] -> " << name << " -> " << diff_numb_to_char[numb]; } - else if(glyphs.has(name)) - { - diff_numb_to_char[numb] = glyphs[name]; - LOG_S(INFO) << "differences[" << numb << "] -> " << name - << " -> " << diff_numb_to_char[numb]; - } - else if(glyphs.has(name_) and font_subname=="sups") { diff_numb_to_char[numb] = "$^{" + glyphs[name_] + "}"; @@ -2311,7 +2312,20 @@ namespace pdflib diff_numb_to_char[numb] = "$_{" + glyphs[name_] + "}"; LOG_S(INFO) << "differences[" << numb << "] -> " << name_ << " -> " << diff_numb_to_char[numb]; - } + } + else if(has_to_unicode(numb)) // method 1 of PDF 32000-1 section 9.10.2 + { + diff_numb_to_char[numb] = cmap_numb_to_char.at(numb); + LOG_S(INFO) << "differences[" << numb << "] -> " << name + << " -> " << diff_numb_to_char[numb] + << " (/ToUnicode)"; + } + else if(glyphs.has(name)) // method 2: glyph-name -> unicode via the glyph-tables (AGL) + { + diff_numb_to_char[numb] = glyphs[name]; + LOG_S(INFO) << "differences[" << numb << "] -> " << name + << " -> " << diff_numb_to_char[numb]; + } else if(glyphs.has(name_)) { diff_numb_to_char[numb] = glyphs[name_]; @@ -2360,19 +2374,10 @@ namespace pdflib } else { - diff_numb_to_char[numb] = name; - LOG_S(WARNING) << "differences[" << numb << "] -> " << name - << " (unresolved ligature)"; + diff_numb_to_char[numb] = resolve_unknown_name(name, joined); } } } - /* - else if(name_.size()>0) - { - diff_numb_to_char[numb] = name_; - LOG_S(WARNING) << "differences["< " << name_; - } - */ else if(std::regex_search(name, match, re_02)) { std::string unicode_hex = match[3].str(); @@ -2393,13 +2398,6 @@ namespace pdflib << diff_numb_to_char[numb] << " (from " << name << ")"; } - else if(std::regex_match(name, match, re_03) and cmap_numb_to_char.count(numb)==1) // if the name is of type /g23 of /G23 and we have a match in the cmap - { - LOG_S(WARNING) << "overloading difference from cmap"; - diff_numb_to_char[numb] = cmap_numb_to_char.at(numb); - //diff_numb_to_char[numb] = name; - //LOG_S(ERROR) << "weird differences["< " << name; - } else if(std::regex_match(name, match, re_04)) // if the name is of type /C treat the number as a Unicode code point { uint32_t codepoint = static_cast(std::stoul(match[3].str())); @@ -2411,8 +2409,7 @@ namespace pdflib } else { - diff_numb_to_char[numb] = name; - LOG_S(WARNING) << "differences["< " << name; + diff_numb_to_char[numb] = resolve_unknown_name(name, name_); } LOG_S(INFO) << font_name << ": differences["< " << name << " -> " << diff_numb_to_char[numb]; From aa207cdb6c5a10d2455c397a6e5415c5ac8dc4f4 Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Fri, 10 Jul 2026 10:17:58 +0200 Subject: [PATCH 4/5] pinned the latest tests commit Signed-off-by: Peter Staar --- tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index a2a672d2..91823b01 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,5 +1,5 @@ HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "17980e180c394d7941cc6b47860a7d24570fceaf" +HF_DATASET_REVISION = "1048d569d49064b613f5ef5c886baa10a6bd616c" PARSER_PAGE_RESTRICTIONS = { "deep-mediabox-inheritance.pdf": [2], From ca0075b5385e0135ccae24ec55d45d6e7b1d1871 Mon Sep 17 00:00:00 2001 From: Peter Staar Date: Fri, 10 Jul 2026 10:30:26 +0200 Subject: [PATCH 5/5] ran pre-commit Signed-off-by: Peter Staar --- tests/test_threaded_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_threaded_render.py b/tests/test_threaded_render.py index 0e155955..58a518ad 100644 --- a/tests/test_threaded_render.py +++ b/tests/test_threaded_render.py @@ -18,6 +18,7 @@ RenderConfig, ThreadedPdfParserConfig, ) +from tests.constants import PARSER_PAGE_RESTRICTIONS from tests.rendering_regression import ( ImageTolerance, compare_bitmap_artifacts, @@ -28,7 +29,6 @@ measure_image_comparison, write_renderer_groundtruth, ) -from tests.constants import PARSER_PAGE_RESTRICTIONS from tests.test_parse import ( GROUNDTRUTH_FOLDER, REGRESSION_FOLDER,