fix: fixing the word extraction with spaces as blockers

scripts/check_rendering_regression.py

@@ -24,7 +24,8 @@
 )
 from tests.data_utils import ensure_test_data_downloaded  # noqa: E402
 from tests.rendering_regression import renderer_image_path  # noqa: E402
-from tests.test_parse import PARSER_PAGE_RESTRICTIONS, REGRESSION_FOLDER  # noqa: E402
+from tests.constants import PARSER_PAGE_RESTRICTIONS  # noqa: E402
+from tests.test_parse import REGRESSION_FOLDER  # noqa: E402
 
 DEFAULT_DELTA_DIR = Path("tests/data/render_deltas")
 DEFAULT_PIXEL_THRESHOLD = 12

scripts/export_dclx.py

@@ -48,6 +48,9 @@
     "text": TextCellUnit.LINE,
 }
 
+LOG_COORD_WIDTH = 8
+LOG_COORD_PRECISION = 2
+
 
 def parse_args(argv=None):
     ap = argparse.ArgumentParser(
@@ -67,6 +70,12 @@ def parse_args(argv=None):
                     help="Page-image raster scale in pixels-per-point (default: 2.0).")
     ap.add_argument("--threads", type=int, default=4,
                     help="Worker threads for the parser (default: 4).")
+    ap.add_argument("-l", "--loglevel",
+                    choices=["fatal", "error", "warn", "warning", "info"],
+                    default="fatal",
+                    help="C++ parser log level (default: fatal).")
+    ap.add_argument("--log-text", action="store_true",
+                    help="Print a table of each exported text cell and its bounding box.")
     return ap.parse_args(argv)
 
 
@@ -83,7 +92,21 @@ def _content_config_for(mode: str) -> ContentConfig:
     )
 
 
-def _add_page_to_doc(doc, page_no, page, image, dpi, unit):
+def _format_bbox_coord(value: float) -> str:
+    return f"{value:{LOG_COORD_WIDTH}.{LOG_COORD_PRECISION}f}"
+
+
+def _log_text_cell(page_no: int, text: str, bbox) -> None:
+    print(
+        f"{_format_bbox_coord(bbox.l)} "
+        f"{_format_bbox_coord(bbox.t)} "
+        f"{_format_bbox_coord(bbox.r)} "
+        f"{_format_bbox_coord(bbox.b)} "
+        f"{page_no:>4}  {text!r}"
+    )
+
+
+def _add_page_to_doc(doc, page_no, page, image, dpi, unit, log_text=False):
     """Attach one page (image + text cells + picture bitmaps) to the doc.
 
     Returns (n_text_cells, n_pictures).
@@ -99,9 +122,12 @@ def _add_page_to_doc(doc, page_no, page, image, dpi, unit):
         text = cell.text
         if not text:
             continue
+        bbox = cell.rect.to_bounding_box()
+        if log_text:
+            _log_text_cell(page_no, text, bbox)
         prov = ProvenanceItem(
             page_no=page_no,
-            bbox=cell.rect.to_bounding_box(),
+            bbox=bbox,
             charspan=(0, len(text)),
         )
         doc.add_text(label=DocItemLabel.TEXT, text=text, orig=text, prov=prov)
@@ -123,7 +149,8 @@ def _add_page_to_doc(doc, page_no, page, image, dpi, unit):
 
 
 def export(input_path: Path, mode: str, scale: float, threads: int,
-           page: int | None = None) -> DoclingDocument:
+           page: int | None = None, log_text: bool = False,
+           loglevel: str = "fatal") -> DoclingDocument:
     content_config = _content_config_for(mode)
     page_numbers = [page] if page is not None else None
 
@@ -134,6 +161,7 @@ def export(input_path: Path, mode: str, scale: float, threads: int,
     parser = DoclingThreadedPdfParser(
         parser_config=ThreadedPdfParserConfig(
             threads=threads,
+            loglevel=loglevel,
             render_config=render_config,
             page_content_config=content_config,
         ),
@@ -162,9 +190,19 @@ def export(input_path: Path, mode: str, scale: float, threads: int,
 
     total_cells = 0
     total_pictures = 0
+    if log_text:
+        print(
+            f"{'x0':>{LOG_COORD_WIDTH}} "
+            f"{'y0':>{LOG_COORD_WIDTH}} "
+            f"{'x1':>{LOG_COORD_WIDTH}} "
+            f"{'y1':>{LOG_COORD_WIDTH}} "
+            f"{'page':>4}  text"
+        )
+
     for page_no in sorted(pages):
         page, image = pages[page_no]
-        n_cells, n_pics = _add_page_to_doc(doc, page_no, page, image, dpi, unit)
+        n_cells, n_pics = _add_page_to_doc(
+            doc, page_no, page, image, dpi, unit, log_text=log_text)
         total_cells += n_cells
         total_pictures += n_pics
         print(f"  page {page_no}: {n_cells} {mode} cell(s), {n_pics} picture(s)")
@@ -187,7 +225,8 @@ def main(argv=None):
 
     page_label = f", page={args.page}" if args.page is not None else ""
     print(f"Exporting '{args.input}' (mode={args.mode}, scale={args.scale}{page_label}) ...")
-    doc = export(args.input, args.mode, args.scale, args.threads, args.page)
+    doc = export(args.input, args.mode, args.scale, args.threads, args.page,
+                 log_text=args.log_text, loglevel=args.loglevel)
 
     default_suffix = (
         f".{args.mode}.p{args.page}.dclx"

src/parse/page_item_sanitators/cells.h

@@ -31,7 +31,8 @@ namespace pdflib
 		       double horizontal_cell_tolerance, //=1.0,
 		       bool enforce_same_font, //=true,
 		       double space_width_factor_for_merge, //=1.5,
-		       double space_width_factor_for_merge_with_space); //=0.33);
+		       double space_width_factor_for_merge_with_space, //=0.33,
+		       bool block_spaces); // when true, space cells act as hard merge barriers
 
     void sanitize_text(page_item<PAGE_CELLS>& cells);
 
@@ -41,27 +42,31 @@ namespace pdflib
 
     bool applicable_for_merge(page_item<PAGE_CELL>& cell_i,
 			      page_item<PAGE_CELL>& cell_j,
-			      bool enforce_same_font);
+			      bool enforce_same_font,
+			      bool block_spaces);
 
     void contract_cells_into_lines_right_to_left(page_item<PAGE_CELLS>& cells,
 						 double horizontal_cell_tolerance,
 						 bool enforce_same_font,
 						 double space_width_factor_for_merge,
-						 double space_width_factor_for_merge_with_space);
+						 double space_width_factor_for_merge_with_space,
+						 bool block_spaces);
 
     void contract_cells_into_lines_left_to_right(page_item<PAGE_CELLS>& cells,
 						 double horizontal_cell_tolerance,
 						 bool enforce_same_font,
 						 double space_width_factor_for_merge,
 						 double space_width_factor_for_merge_with_space,
+						 bool block_spaces,
 						 bool allow_reverse);
-    
+
     // linear
     void contract_cells_into_lines_v1(page_item<PAGE_CELLS>& cells,
 				      double horizontal_cell_tolerance=1.0,
 				      bool enforce_same_font=true,
 				      double space_width_factor_for_merge=1.5,
-				      double space_width_factor_for_merge_with_space=0.33);
+				      double space_width_factor_for_merge_with_space=0.33,
+				      bool block_spaces=false);
 
     // quadratic
     void contract_cells_into_lines_v2(page_item<PAGE_CELLS>& cells,
@@ -140,7 +145,23 @@ namespace pdflib
 
     LOG_S(INFO) << "#-char cells: " << word_cells.size();
 
-    // remove all spaces
+    // Keep the space cells in place and let sanitize_bbox treat them as hard
+    // word-boundary barriers (block_spaces=true). An explicit space glyph is a
+    // far more reliable word separator than the geometric-gap heuristic, which
+    // is ambiguous for tightly-set fonts with narrow spaces. The spaces are
+    // erased afterwards, once the words have been contracted.
+
+    // > space_width_factor_for_merge, so nothing gets merged with a space
+    double space_width_factor_for_merge_with_space = 2.0*config.word_space_width_factor_for_merge;
+
+    sanitize_bbox(word_cells,
+		  config.horizontal_cell_tolerance,
+		  config.enforce_same_font,
+		  config.word_space_width_factor_for_merge,
+		  space_width_factor_for_merge_with_space,
+		  true);
+
+    // remove the space cells that acted as word-boundary barriers
     auto itr = word_cells.begin();
     while(itr!=word_cells.end())
       {
@@ -154,17 +175,6 @@ namespace pdflib
 	  }
       }
 
-    LOG_S(INFO) << "#-char cells (without spaces): " << word_cells.size();
-
-    // > space_width_factor_for_merge, so nothing gets merged with a space
-    double space_width_factor_for_merge_with_space = 2.0*config.word_space_width_factor_for_merge;
-
-    sanitize_bbox(word_cells,
-		  config.horizontal_cell_tolerance,
-		  config.enforce_same_font,
-		  config.word_space_width_factor_for_merge,
-		  space_width_factor_for_merge_with_space);
-
     LOG_S(INFO) << "#-word cells: " << word_cells.size();
 
     //return to_records(word_cells);
@@ -184,12 +194,14 @@ namespace pdflib
 
     LOG_S(INFO) << "# char-cells: " << line_cells.size();
 
+    // lines keep their internal spaces, so spaces are merged normally (block_spaces=false)
     sanitize_bbox(line_cells,
 		  config.horizontal_cell_tolerance,
 		  config.enforce_same_font,
 		  config.line_space_width_factor_for_merge,
-		  config.line_space_width_factor_for_merge_with_space);
-
+		  config.line_space_width_factor_for_merge_with_space,
+		  false);
+
     LOG_S(INFO) << "# line-cells: " << line_cells.size();
 
     //return to_records(line_cells);
@@ -373,37 +385,40 @@ namespace pdflib
 						double horizontal_cell_tolerance,
 						bool enforce_same_font,
 						double space_width_factor_for_merge,
-						double space_width_factor_for_merge_with_space)
+						double space_width_factor_for_merge_with_space,
+						bool block_spaces)
   {
     contract_cells_into_lines_v1(cells,
 				 horizontal_cell_tolerance,
 				 enforce_same_font,
 				 space_width_factor_for_merge,
-				 space_width_factor_for_merge_with_space);
-
-    /*
-    contract_cells_into_lines_v2(cells,
-				 horizontal_cell_tolerance,
-				 enforce_same_font,
-				 space_width_factor_for_merge,
-				 space_width_factor_for_merge_with_space);
-    */
+				 space_width_factor_for_merge_with_space,
+				 block_spaces);
   }
 
   bool page_item_sanitator<PAGE_CELLS>::applicable_for_merge(page_item<PAGE_CELL>& cell_i,
 						       page_item<PAGE_CELL>& cell_j,
-						       bool enforce_same_font)
+						       bool enforce_same_font,
+						       bool block_spaces)
   {
     if(not cell_i.active)
       {
 	return false;
       }
-    
+
     if(not cell_j.active)
       {
 	return false;
       }
-
+
+    // An explicit space glyph is a hard word boundary: never merge a space cell
+    // with anything, nor across one. The space cells are removed afterwards.
+    if(block_spaces and
+       (utils::string::is_space(cell_i.text) or utils::string::is_space(cell_j.text)))
+      {
+	return false;
+      }
+
     if(enforce_same_font and cell_i.font_name!=cell_j.font_name)
       {
 	// Exception: ligature glyphs are often encoded in a different font than
@@ -427,20 +442,22 @@ namespace pdflib
 							       double horizontal_cell_tolerance,
 							       bool enforce_same_font,
 							       double space_width_factor_for_merge,
-							       double space_width_factor_for_merge_with_space)
+							       double space_width_factor_for_merge_with_space,
+							       bool block_spaces)
   {
-    contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, false);
-    
-    contract_cells_into_lines_right_to_left(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space);
-    
-    contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, true);
+    contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces, false);
+
+    contract_cells_into_lines_right_to_left(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces);
+
+    contract_cells_into_lines_left_to_right(cells, horizontal_cell_tolerance, enforce_same_font, space_width_factor_for_merge, space_width_factor_for_merge_with_space, block_spaces, true);
   }
 
   void page_item_sanitator<PAGE_CELLS>::contract_cells_into_lines_left_to_right(page_item<PAGE_CELLS>& cells,
 									  double horizontal_cell_tolerance,
 									  bool enforce_same_font,
 									  double space_width_factor_for_merge,
 									  double space_width_factor_for_merge_with_space,
+									  bool block_spaces,
 									  bool allow_reverse)
   {
     // take care for left to right printing
@@ -454,7 +471,7 @@ namespace pdflib
 
 	for(int j=i+1; j<cells.size(); j++)
 	  {
-	    if(not applicable_for_merge(cells[i], cells[j], enforce_same_font))
+	    if(not applicable_for_merge(cells[i], cells[j], enforce_same_font, block_spaces))
 	      {
 		break;
 	      }
@@ -528,8 +545,9 @@ namespace pdflib
 									  double horizontal_cell_tolerance,
 									  bool enforce_same_font,
 									  double space_width_factor_for_merge,
-									  double space_width_factor_for_merge_with_space)
-  {    
+									  double space_width_factor_for_merge_with_space,
+									  bool block_spaces)
+  {
     // take care for right to left printing
     for(int i=cells.size()-1; i>=0; i--)
       {
@@ -541,7 +559,7 @@ namespace pdflib
 
 	for(int j=i-1; j>=0; j--)
 	  {
-	    if(not applicable_for_merge(cells[i], cells[j], enforce_same_font))
+	    if(not applicable_for_merge(cells[i], cells[j], enforce_same_font, block_spaces))
 	      {
 		break;
 	      }

src/parse/pdf_decoders/page.h

@@ -1369,7 +1369,8 @@ namespace pdflib
                               horizontal_cell_tolerance,
                               enforce_same_font,
                               space_width_factor_for_merge,
-                              space_width_factor_for_merge_with_space);
+                              space_width_factor_for_merge_with_space,
+                              false);
 
       //sanitator.sanitize_text(cells);