diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 191720b..dc954fe 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -98,7 +98,13 @@ "Bash(cargo search *)", "Bash(cargo add *)", "Bash(cargo tree *)", - "Bash(curl -s https://crates.io/api/v1/crates/arrow/__TRACKED_VAR__.0.0)" + "Bash(curl -s https://crates.io/api/v1/crates/arrow/__TRACKED_VAR__.0.0)", + "Bash(.venv/Scripts/xenon src/ --max-average B --max-modules C --max-absolute D)", + "PowerShell(.venv/Scripts/python -m pytest tests/output/test_emit_render_block.py -x -q 2>&1)", + "PowerShell(.venv/Scripts/python -m pytest tests/dictionary/test_schema.py -x -q 2>&1 | Select-Object -Last 5)", + "PowerShell(.venv/Scripts/python -m pytest -q --tb=short 2>&1 | Select-Object -Last 5)", + "PowerShell(.venv/Scripts/python -m radon cc src/cifflow/dictionary/schema.py --show-complexity --min C 2>&1)", + "PowerShell(.venv/Scripts/python -m xenon src/ --max-average B --max-modules C --max-absolute D 2>&1)" ] } } diff --git a/CLAUDE.md b/CLAUDE.md index 98b7727..97c4040 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -222,6 +222,18 @@ Ingest a known-good file, emit, re-ingest, compare. `second.cif` is the primary Load `cif_pow.dic` via `DictionaryLoader`, call `generate_schema`, call `apply_schema`. See `tests/ingestion/test_integration.py` for the reference pattern. +### Complexity analysis +``` +# Show all functions rated C or worse (the refactoring backlog) +.venv/Scripts/python -m radon cc src/ --show-complexity --min C + +# Enforce thresholds (exits non-zero if any module averages above B, +# any function exceeds C, or any block exceeds D) +.venv/Scripts/xenon src/ --max-average B --max-modules C --max-absolute D +``` +Grade scale: A ≤5, B 6–10, C 11–15, D 16–25, E 26–50, F 51+. +Target: no F or E functions; D functions should have a documented reason. + --- ## Checklist Before Any Change diff --git a/pyproject.toml b/pyproject.toml index bce8dc3..4a7c37f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,8 @@ dev = [ "bump-my-version", "ruff>=0.4", "pydoclint[flake8]>=0.4", + "radon>=6.0", + "xenon>=0.9", ] docs = [ "mkdocs>=1.6", @@ -97,3 +99,6 @@ convention = "numpy" [tool.pydoclint] style = "numpy" arg-type-hints-in-docstring = false + +[tool.radon] +exclude = "tests/*" diff --git a/src/cifflow/database/component_intensities.py b/src/cifflow/database/component_intensities.py index ab00695..8ec82a4 100644 --- a/src/cifflow/database/component_intensities.py +++ b/src/cifflow/database/component_intensities.py @@ -38,6 +38,182 @@ def _all_dot(stored: str) -> bool: return False +def _consolidate_within_transaction( + connection: duckdb.DuckDBPyConnection, + diff_ids: list[str], + clear_source: bool, +) -> tuple[int, bool, bool]: + """Run Steps 1-3 of consolidation inside an open transaction. + + Returns (total_rows_updated, any_real_net, any_real_total). + """ + # Step 1 — Presentation order: sorted distinct phase_ids per diffractogram. + # DuckDB produces plain JSON; _prefix_sql() re-adds _CONTAINER_PREFIX at + # DuckDB runtime (via chr(0)||) so quote() renders it as a CIF list. + raw_pres = connection.execute(""" + SELECT "diffractogram_id", + to_json(list("phase_id" ORDER BY "phase_id"))::VARCHAR + FROM (SELECT DISTINCT "diffractogram_id", "phase_id" + FROM pd_calc_component + WHERE "diffractogram_id" IS NOT NULL AND "phase_id" IS NOT NULL) + GROUP BY "diffractogram_id" + """).fetchall() + + existing_diffs = { + r[0] for r in connection.execute( + 'SELECT "diffractogram_id" FROM pd_calc_overall' + ' WHERE "diffractogram_id" IS NOT NULL' + ).fetchall() + } + to_update = [(pres, d) for d, pres in raw_pres if d in existing_diffs] + to_insert = [(d, pres) for d, pres in raw_pres if d not in existing_diffs] + + if to_update: + vals_sql = ', '.join( + f'({_prefix_sql(pres)}, {_sl(d)})' for pres, d in to_update + ) + connection.execute(f""" + UPDATE pd_calc_overall + SET "component_presentation_order" = t.pres + FROM (VALUES {vals_sql}) AS t(pres, "diffractogram_id") + WHERE pd_calc_overall."diffractogram_id" = t."diffractogram_id" + """) + if to_insert: + vals_sql = ', '.join( + f'({_sl(d)}, {_prefix_sql(pres)})' for d, pres in to_insert + ) + connection.execute(f""" + INSERT INTO pd_calc_overall ("diffractogram_id", "component_presentation_order") + SELECT "diffractogram_id", pres + FROM (VALUES {vals_sql}) AS t("diffractogram_id", pres) + """) + + # Step 2 — Ensure pd_calc rows exist for every (point_id, diffractogram_id) in component. + # MIN(_cifflow_block_id) carries a valid block ID so the emit layer can locate the rows. + connection.execute(""" + INSERT INTO pd_calc ("point_id", "diffractogram_id", "_cifflow_block_id") + SELECT c."point_id", c."diffractogram_id", MIN(c."_cifflow_block_id") + FROM pd_calc_component c + WHERE c."point_id" IS NOT NULL + AND c."diffractogram_id" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM pd_calc p + WHERE p."point_id" = c."point_id" + AND p."diffractogram_id" = c."diffractogram_id" + ) + GROUP BY c."point_id", c."diffractogram_id" + """) + + # Step 3 — Assemble and write intensity lists, per diffractogram. + # CROSS JOIN the full phase list against each point so every phase slot is + # always present (LEFT JOIN fills absent phases with NULL → COALESCE → '.'). + # One batch VALUES UPDATE per diffractogram — avoids per-row execute() calls + # which are slow on Windows due to AV scanning overhead. + pres_by_diff = dict(raw_pres) # diff_id -> plain JSON string (no prefix) + total = 0 + any_real_net = False + any_real_total = False + + for diff_id in diff_ids: + pres_json = pres_by_diff.get(diff_id) + if not pres_json: + continue + + rows = connection.execute(f""" + WITH phases AS ( + SELECT unnest(from_json({_sl(pres_json)}, '["VARCHAR"]')) AS phase_id, + generate_subscripts(from_json({_sl(pres_json)}, '["VARCHAR"]'), 1) AS ord + ), + points AS ( + SELECT DISTINCT "point_id" + FROM pd_calc_component + WHERE "diffractogram_id" = {_sl(diff_id)} + AND "point_id" IS NOT NULL + ) + SELECT + p."point_id", + to_json(list(COALESCE(c."intensity_net", '.') ORDER BY ph.ord))::VARCHAR, + to_json(list(COALESCE(c."intensity_total", '.') ORDER BY ph.ord))::VARCHAR + FROM points p + CROSS JOIN phases ph + LEFT JOIN pd_calc_component c + ON c."point_id" = p."point_id" + AND c."diffractogram_id" = {_sl(diff_id)} + AND c."phase_id" = ph.phase_id + GROUP BY p."point_id" + """).fetchall() + + if rows: + vals_sql = ', '.join( + f'({_sl(r[0])}, {_prefix_sql(r[1])}, {_prefix_sql(r[2])})' for r in rows + ) + n = connection.execute(f""" + UPDATE pd_calc + SET "component_intensities_net" = t.net_list, + "component_intensities_total" = t.total_list + FROM (VALUES {vals_sql}) AS t("point_id", net_list, total_list) + WHERE pd_calc."point_id" = t."point_id" + AND pd_calc."diffractogram_id" = {_sl(diff_id)} + AND pd_calc."component_intensities_net" IS NULL + """).fetchone()[0] + total += n + any_real_net = any_real_net or not all(_all_dot(r[1]) for r in rows) + any_real_total = any_real_total or not all(_all_dot(r[2]) for r in rows) + + # Fill pd_calc rows that have no component data at all for this diffractogram + # (the CROSS JOIN only covers points present in pd_calc_component). + n_phases = len(json.loads(pres_json)) + all_dot = json.dumps(['.'] * n_phases) + connection.execute(f""" + UPDATE pd_calc + SET "component_intensities_net" = {_prefix_sql(all_dot)}, + "component_intensities_total" = {_prefix_sql(all_dot)} + WHERE "diffractogram_id" = {_sl(diff_id)} + AND "component_intensities_net" IS NULL + """) + + if clear_source: + connection.execute('DELETE FROM pd_calc_component') + + return total, any_real_net, any_real_total + + +def _drop_all_dot_columns( + connection: duckdb.DuckDBPyConnection, + any_real_net: bool, + any_real_total: bool, +) -> None: + """Drop all-'.' intensity columns from pd_calc (DDL outside the transaction).""" + if not any_real_net: + net_vals = connection.execute( + 'SELECT "component_intensities_net" FROM pd_calc' + ' WHERE "component_intensities_net" IS NOT NULL' + ).fetchall() + if all(_all_dot(r[0]) for r in net_vals): + connection.execute( + 'ALTER TABLE pd_calc DROP COLUMN "component_intensities_net"' + ) + else: + any_real_net = True + + if not any_real_total: + total_vals = connection.execute( + 'SELECT "component_intensities_total" FROM pd_calc' + ' WHERE "component_intensities_total" IS NOT NULL' + ).fetchall() + if all(_all_dot(r[0]) for r in total_vals): + connection.execute( + 'ALTER TABLE pd_calc DROP COLUMN "component_intensities_total"' + ) + else: + any_real_total = True + + if not any_real_net and not any_real_total: + connection.execute( + 'ALTER TABLE pd_calc_overall DROP COLUMN "component_presentation_order"' + ) + + def consolidate_component_intensities( connection: duckdb.DuckDBPyConnection, schema: SchemaSpec, @@ -95,136 +271,13 @@ def consolidate_component_intensities( connection.execute('BEGIN TRANSACTION') total = 0 - ok = False any_real_net = False any_real_total = False + ok = False try: - # Step 1 — Presentation order: sorted distinct phase_ids per diffractogram. - # DuckDB produces plain JSON; _prefix_sql() re-adds _CONTAINER_PREFIX at - # DuckDB runtime (via chr(0)||) so quote() renders it as a CIF list. - raw_pres = connection.execute(""" - SELECT "diffractogram_id", - to_json(list("phase_id" ORDER BY "phase_id"))::VARCHAR - FROM (SELECT DISTINCT "diffractogram_id", "phase_id" - FROM pd_calc_component - WHERE "diffractogram_id" IS NOT NULL AND "phase_id" IS NOT NULL) - GROUP BY "diffractogram_id" - """).fetchall() - - existing_diffs = { - r[0] for r in connection.execute( - 'SELECT "diffractogram_id" FROM pd_calc_overall' - ' WHERE "diffractogram_id" IS NOT NULL' - ).fetchall() - } - to_update = [(pres, d) for d, pres in raw_pres if d in existing_diffs] - to_insert = [(d, pres) for d, pres in raw_pres if d not in existing_diffs] - - if to_update: - vals_sql = ', '.join( - f'({_prefix_sql(pres)}, {_sl(d)})' for pres, d in to_update - ) - connection.execute(f""" - UPDATE pd_calc_overall - SET "component_presentation_order" = t.pres - FROM (VALUES {vals_sql}) AS t(pres, "diffractogram_id") - WHERE pd_calc_overall."diffractogram_id" = t."diffractogram_id" - """) - if to_insert: - vals_sql = ', '.join( - f'({_sl(d)}, {_prefix_sql(pres)})' for d, pres in to_insert - ) - connection.execute(f""" - INSERT INTO pd_calc_overall ("diffractogram_id", "component_presentation_order") - SELECT "diffractogram_id", pres - FROM (VALUES {vals_sql}) AS t("diffractogram_id", pres) - """) - - # Step 2 — Ensure pd_calc rows exist for every (point_id, diffractogram_id) in component. - # MIN(_cifflow_block_id) carries a valid block ID so the emit layer can locate the rows. - connection.execute(""" - INSERT INTO pd_calc ("point_id", "diffractogram_id", "_cifflow_block_id") - SELECT c."point_id", c."diffractogram_id", MIN(c."_cifflow_block_id") - FROM pd_calc_component c - WHERE c."point_id" IS NOT NULL - AND c."diffractogram_id" IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM pd_calc p - WHERE p."point_id" = c."point_id" - AND p."diffractogram_id" = c."diffractogram_id" - ) - GROUP BY c."point_id", c."diffractogram_id" - """) - - # Step 3 — Assemble and write intensity lists, per diffractogram. - # CROSS JOIN the full phase list against each point so every phase slot is - # always present (LEFT JOIN fills absent phases with NULL → COALESCE → '.'). - # One batch VALUES UPDATE per diffractogram — avoids per-row execute() calls - # which are slow on Windows due to AV scanning overhead. - pres_by_diff = dict(raw_pres) # diff_id -> plain JSON string (no prefix) - - for diff_id in diff_ids: - pres_json = pres_by_diff.get(diff_id) - if not pres_json: - continue - - rows = connection.execute(f""" - WITH phases AS ( - SELECT unnest(from_json({_sl(pres_json)}, '["VARCHAR"]')) AS phase_id, - generate_subscripts(from_json({_sl(pres_json)}, '["VARCHAR"]'), 1) AS ord - ), - points AS ( - SELECT DISTINCT "point_id" - FROM pd_calc_component - WHERE "diffractogram_id" = {_sl(diff_id)} - AND "point_id" IS NOT NULL - ) - SELECT - p."point_id", - to_json(list(COALESCE(c."intensity_net", '.') ORDER BY ph.ord))::VARCHAR, - to_json(list(COALESCE(c."intensity_total", '.') ORDER BY ph.ord))::VARCHAR - FROM points p - CROSS JOIN phases ph - LEFT JOIN pd_calc_component c - ON c."point_id" = p."point_id" - AND c."diffractogram_id" = {_sl(diff_id)} - AND c."phase_id" = ph.phase_id - GROUP BY p."point_id" - """).fetchall() - - if rows: - vals_sql = ', '.join( - f'({_sl(r[0])}, {_prefix_sql(r[1])}, {_prefix_sql(r[2])})' for r in rows - ) - n = connection.execute(f""" - UPDATE pd_calc - SET "component_intensities_net" = t.net_list, - "component_intensities_total" = t.total_list - FROM (VALUES {vals_sql}) AS t("point_id", net_list, total_list) - WHERE pd_calc."point_id" = t."point_id" - AND pd_calc."diffractogram_id" = {_sl(diff_id)} - AND pd_calc."component_intensities_net" IS NULL - """).fetchone()[0] - total += n - any_real_net = any_real_net or not all(_all_dot(r[1]) for r in rows) - any_real_total = any_real_total or not all(_all_dot(r[2]) for r in rows) - - # Fill pd_calc rows that have no component data at all for this diffractogram - # (the CROSS JOIN only covers points present in pd_calc_component). - n_phases = len(json.loads(pres_json)) - all_dot = json.dumps(['.'] * n_phases) - connection.execute(f""" - UPDATE pd_calc - SET "component_intensities_net" = {_prefix_sql(all_dot)}, - "component_intensities_total" = {_prefix_sql(all_dot)} - WHERE "diffractogram_id" = {_sl(diff_id)} - AND "component_intensities_net" IS NULL - """) - - # Step 5 — Clear source values (optional). - if clear_source: - connection.execute('DELETE FROM pd_calc_component') - + total, any_real_net, any_real_total = _consolidate_within_transaction( + connection, diff_ids, clear_source + ) ok = True except Exception: raise @@ -237,34 +290,5 @@ def consolidate_component_intensities( except duckdb.Error: pass - # Step 4 — Drop all-'.' columns (DDL runs outside the transaction). - if not any_real_net: - net_vals = connection.execute( - 'SELECT "component_intensities_net" FROM pd_calc' - ' WHERE "component_intensities_net" IS NOT NULL' - ).fetchall() - if all(_all_dot(r[0]) for r in net_vals): - connection.execute( - 'ALTER TABLE pd_calc DROP COLUMN "component_intensities_net"' - ) - else: - any_real_net = True - - if not any_real_total: - total_vals = connection.execute( - 'SELECT "component_intensities_total" FROM pd_calc' - ' WHERE "component_intensities_total" IS NOT NULL' - ).fetchall() - if all(_all_dot(r[0]) for r in total_vals): - connection.execute( - 'ALTER TABLE pd_calc DROP COLUMN "component_intensities_total"' - ) - else: - any_real_total = True - - if not any_real_net and not any_real_total: - connection.execute( - 'ALTER TABLE pd_calc_overall DROP COLUMN "component_presentation_order"' - ) - + _drop_all_dot_columns(connection, any_real_net, any_real_total) return total diff --git a/src/cifflow/dictionary/schema.py b/src/cifflow/dictionary/schema.py index eeee52c..797447a 100644 --- a/src/cifflow/dictionary/schema.py +++ b/src/cifflow/dictionary/schema.py @@ -479,197 +479,123 @@ def _find_transitive_bridge( # --------------------------------------------------------------------------- -# Public functions +# generate_schema helpers # --------------------------------------------------------------------------- -def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: - """ - Derive a :class:`SchemaSpec` from a loaded ``DdlmDictionary``. - - Iterates over all categories in *dictionary*, creating one - :class:`TableDef` for each ``Set`` or ``Loop`` category. ``Head`` and - ``Functions`` categories are silently skipped (they never appear in data - instance files); any other unrecognised class emits a warning and is also - skipped. - - Foreign-key constraints are built in a second pass over all items whose - ``type_purpose`` is ``"Link"``. ``SU`` items populate - :attr:`ColumnDef.linked_item_id` but do not produce - :class:`ForeignKeyDef` entries. - - ``alias_to_definition_id`` and ``deprecated_ids`` are copied directly from - *dictionary* so that ``ingest()`` can perform alias resolution and - deprecation checking without retaining a reference to the dictionary. - - Parameters - ---------- - dictionary: - The loaded dictionary returned by - :meth:`~cifflow.dictionary.loader.DictionaryLoader.load`. - - Returns - ------- - SchemaSpec - The complete schema specification including all tables, column - definitions, primary keys, foreign keys, the reverse - ``column_to_tag`` mapping, and alias/deprecation metadata. - """ - warnings: list[str] = [] - tables: dict[str, TableDef] = {} - column_to_tag: dict[tuple[str, str], str] = {} - partial_links: list[PartialLinkDef] = [] - - for cat_id, cat_item in dictionary.categories.items(): - cat_class = cat_item.definition_class - if cat_class not in ('Set', 'Loop'): - if cat_class not in ('Head', 'Functions'): - warnings.append( - f"category {cat_id!r} has unsupported class {cat_class!r} -- skipped" - ) +def _determine_primary_keys( + cat_id: str, + cat_class: str, + cat_item: DdlmItem, + dictionary: DdlmDictionary, + warnings: list[str], +) -> 'tuple[list[str], list[str], bool]': + """Return (primary_keys, non_synthetic_pks, use_fallback_pk).""" + non_synthetic_pks: list[str] = [] + for key_tag in cat_item.category_keys: + key_item = dictionary.tag_to_item.get(key_tag) + if key_item is None: + warnings.append( + f"category {cat_id!r}: category key {key_tag!r} not found " + f"in dictionary -- skipped" + ) continue - - # Table name is derived from the category's own definition_id. - tbl_name = _table_name(cat_item.definition_id) - - # Domain items: those whose _name.category_id points to this category. - domain_items: dict[str, DdlmItem] = { - item.object_id: item - for item in dictionary.items.values() - if item.category_id == cat_item.definition_id - and item.object_id is not None - } - - # --- Determine primary key column names --- - non_synthetic_pks: list[str] = [] - for key_tag in cat_item.category_keys: - key_item = dictionary.tag_to_item.get(key_tag) - if key_item is None: - warnings.append( - f"category {cat_id!r}: category key {key_tag!r} not found " - f"in dictionary -- skipped" - ) - continue - if key_item.object_id is None: - warnings.append( - f"category {cat_id!r}: category key {key_tag!r} has no " - f"object_id -- skipped" - ) - continue - non_synthetic_pks.append(key_item.object_id) - - use_fallback_pk = not non_synthetic_pks - if use_fallback_pk: - if cat_class == 'Set': - warnings.append( - f"category {cat_id!r} (Set) has no _category_key.name -- " - f"using _cifflow_id as primary key" - ) - primary_keys = ['_cifflow_id'] - else: # Loop - warnings.append( - f"category {cat_id!r} (Loop) has no _category_key.name -- " - f"using _cifflow_block_id + _cifflow_row_id as primary key" - ) - primary_keys = ['_cifflow_block_id', '_cifflow_row_id'] + if key_item.object_id is None: + warnings.append( + f"category {cat_id!r}: category key {key_tag!r} has no " + f"object_id -- skipped" + ) + continue + non_synthetic_pks.append(key_item.object_id) + use_fallback_pk = not non_synthetic_pks + if use_fallback_pk: + if cat_class == 'Set': + warnings.append( + f"category {cat_id!r} (Set) has no _category_key.name -- " + f"using _cifflow_id as primary key" + ) + primary_keys: list[str] = ['_cifflow_id'] else: - primary_keys = list(non_synthetic_pks) - - # --- Build columns in specified order --- - columns: list[ColumnDef] = [] - - # 1. _cifflow_block_id (always first; informational only for keyed tables) - block_id_is_pk = '_cifflow_block_id' in primary_keys + warnings.append( + f"category {cat_id!r} (Loop) has no _category_key.name -- " + f"using _cifflow_block_id + _cifflow_row_id as primary key" + ) + primary_keys = ['_cifflow_block_id', '_cifflow_row_id'] + else: + primary_keys = list(non_synthetic_pks) + return primary_keys, non_synthetic_pks, use_fallback_pk + + +def _build_table_columns( + cat_class: str, + tbl_name: str, + domain_items: 'dict[str, DdlmItem]', + non_synthetic_pks: list[str], + use_fallback_pk: bool, + primary_keys: list[str], + warnings: list[str], + column_to_tag: 'dict[tuple[str, str], str]', +) -> list[ColumnDef]: + columns: list[ColumnDef] = [] + block_id_is_pk = '_cifflow_block_id' in primary_keys + columns.append(ColumnDef( + name='_cifflow_block_id', + definition_id='', + type_contents=None, + type_container=None, + nullable=False, + is_primary_key=block_id_is_pk, + is_synthetic=True, + linked_item_id=None, + )) + if use_fallback_pk and cat_class == 'Set': columns.append(ColumnDef( - name='_cifflow_block_id', + name='_cifflow_id', definition_id='', type_contents=None, type_container=None, nullable=False, - is_primary_key=block_id_is_pk, + is_primary_key=True, is_synthetic=True, linked_item_id=None, )) - - # 2. _cifflow_id (keyless Set tables only) - if use_fallback_pk and cat_class == 'Set': - columns.append(ColumnDef( - name='_cifflow_id', + row_id_is_pk = '_cifflow_row_id' in primary_keys + columns.append(ColumnDef( + name='_cifflow_row_id', + definition_id='', + type_contents=None, + type_container=None, + nullable=False, + is_primary_key=row_id_is_pk, + is_synthetic=True, + linked_item_id=None, + )) + for obj_id in non_synthetic_pks: + item = domain_items.get(obj_id) + if item is None: + warnings.append( + f"table {tbl_name!r}: primary key column {obj_id!r} not " + f"found in category items -- using TEXT" + ) + col = ColumnDef( + name=obj_id, definition_id='', type_contents=None, type_container=None, nullable=False, is_primary_key=True, - is_synthetic=True, + is_synthetic=False, linked_item_id=None, - )) - - # 3. _cifflow_row_id (all Set and Loop tables) - row_id_is_pk = '_cifflow_row_id' in primary_keys - columns.append(ColumnDef( - name='_cifflow_row_id', - definition_id='', - type_contents=None, - type_container=None, - nullable=False, - is_primary_key=row_id_is_pk, - is_synthetic=True, - linked_item_id=None, - )) - - # 4. Non-synthetic primary-key columns (in category_keys order) - for obj_id in non_synthetic_pks: - item = domain_items.get(obj_id) - if item is None: - warnings.append( - f"table {tbl_name!r}: primary key column {obj_id!r} not " - f"found in category items -- using TEXT" - ) - col = ColumnDef( - name=obj_id, - definition_id='', - type_contents=None, - type_container=None, - nullable=False, - is_primary_key=True, - is_synthetic=False, - linked_item_id=None, - ) - else: - col = ColumnDef( - name=obj_id, - definition_id=item.definition_id, - type_contents=item.type_contents or 'Text', - type_container=item.type_container or 'Single', - nullable=False, - is_primary_key=True, - is_synthetic=False, - linked_item_id=item.linked_item_id, - enumeration_states=item.enumeration_states, - enumeration_range=item.enumeration_range, - type_dimension=item.type_dimension, - enumeration_default=item.enumeration_default, - enumeration_def_index_ids=item.enumeration_def_index_ids, - enumeration_defaults=item.enumeration_defaults, - ) - column_to_tag[(tbl_name, obj_id)] = item.definition_id - columns.append(col) - - # 5. Remaining domain columns (alphabetically, excluding PKs) - pk_set = set(non_synthetic_pks) - for obj_id, item in sorted(domain_items.items()): - if obj_id in pk_set: - continue + ) + else: col = ColumnDef( name=obj_id, definition_id=item.definition_id, type_contents=item.type_contents or 'Text', type_container=item.type_container or 'Single', - nullable=True, - is_primary_key=False, + nullable=False, + is_primary_key=True, is_synthetic=False, - linked_item_id=( - item.linked_item_id if item.type_purpose == 'SU' else None - ), + linked_item_id=item.linked_item_id, enumeration_states=item.enumeration_states, enumeration_range=item.enumeration_range, type_dimension=item.type_dimension, @@ -677,9 +603,63 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: enumeration_def_index_ids=item.enumeration_def_index_ids, enumeration_defaults=item.enumeration_defaults, ) - columns.append(col) column_to_tag[(tbl_name, obj_id)] = item.definition_id + columns.append(col) + pk_set = set(non_synthetic_pks) + for obj_id, item in sorted(domain_items.items()): + if obj_id in pk_set: + continue + col = ColumnDef( + name=obj_id, + definition_id=item.definition_id, + type_contents=item.type_contents or 'Text', + type_container=item.type_container or 'Single', + nullable=True, + is_primary_key=False, + is_synthetic=False, + linked_item_id=( + item.linked_item_id if item.type_purpose == 'SU' else None + ), + enumeration_states=item.enumeration_states, + enumeration_range=item.enumeration_range, + type_dimension=item.type_dimension, + enumeration_default=item.enumeration_default, + enumeration_def_index_ids=item.enumeration_def_index_ids, + enumeration_defaults=item.enumeration_defaults, + ) + columns.append(col) + column_to_tag[(tbl_name, obj_id)] = item.definition_id + return columns + +def _build_tables( + dictionary: DdlmDictionary, + warnings: list[str], +) -> 'tuple[dict[str, TableDef], dict[tuple[str, str], str]]': + tables: dict[str, TableDef] = {} + column_to_tag: dict[tuple[str, str], str] = {} + for cat_id, cat_item in dictionary.categories.items(): + cat_class = cat_item.definition_class + if cat_class not in ('Set', 'Loop'): + if cat_class not in ('Head', 'Functions'): + warnings.append( + f"category {cat_id!r} has unsupported class {cat_class!r} -- skipped" + ) + continue + tbl_name = _table_name(cat_item.definition_id) + domain_items: dict[str, DdlmItem] = { + item.object_id: item + for item in dictionary.items.values() + if item.category_id == cat_item.definition_id + and item.object_id is not None + } + primary_keys, non_synthetic_pks, use_fallback_pk = _determine_primary_keys( + cat_id, cat_class, cat_item, dictionary, warnings, + ) + columns = _build_table_columns( + cat_class, tbl_name, domain_items, non_synthetic_pks, + use_fallback_pk, primary_keys, warnings, column_to_tag, + ) tables[tbl_name] = TableDef( name=tbl_name, definition_id=cat_item.definition_id, @@ -688,29 +668,195 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: primary_keys=primary_keys, foreign_keys=[], ) + return tables, column_to_tag - # --- Second pass: foreign-key detection --- - # Collect all Link items grouped by (src_tbl, tgt_tbl). When multiple - # source columns all link to columns that together cover the target table's - # full composite PK, emit one composite FOREIGN KEY constraint. Single- - # column FKs targeting a sole PK are handled as the degenerate case. - # - # SQLite requires the FK target to have a UNIQUE index. For a sole-PK - # table SQLite creates one automatically; for a composite PK it does NOT - # create per-column UNIQUE indices. Therefore a valid FK must reference - # EITHER the sole PK (single-column FK) OR the full composite PK (multi- - # column FK). Partial or non-PK references are warned and skipped. - bridge_columns: list[BridgeColumnDef] = [] +def _resolve_fk_group( + src_tbl: str, + tgt_tbl: str, + pairs: 'list[tuple[str, str, DdlmItem]]', + tgt_pks: list[str], + tables: 'dict[str, TableDef]', + dictionary: DdlmDictionary, + link_groups: dict, + warnings: list[str], + bridge_columns: list[BridgeColumnDef], + partial_links: list[PartialLinkDef], +) -> None: + """Resolve one (src_tbl, tgt_tbl) link group into FK constraints.""" + tgt_pks_set = set(tgt_pks) + pk_pairs = [] + for src_col, tgt_col, item in pairs: + if tgt_col not in tgt_pks_set: + warnings.append( + f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " + f"target column '{tgt_col}' is not a PK of " + f"'{tgt_tbl}' (PKs={tgt_pks}) -- skipping FK constraint" + ) + partial_links.append(PartialLinkDef( + source_table=src_tbl, + source_column=src_col, + target_table=tgt_tbl, + target_column=tgt_col, + covered_pk_cols=[], + missing_pk_cols=list(tgt_pks), + reason=f"target column '{tgt_col}' is not a PK of '{tgt_tbl}'", + )) + else: + pk_pairs.append((src_col, tgt_col, item)) + if not pk_pairs: + return + pairs = pk_pairs + tgt_to_srcs: dict[str, list[str]] = defaultdict(list) + for src_col, tgt_col, _ in pairs: + tgt_to_srcs[tgt_col].append(src_col) + tgt_cols_covered = set(tgt_to_srcs.keys()) + missing_pk_cols = tgt_pks_set - tgt_cols_covered + has_conflicts = any(len(v) > 1 for v in tgt_to_srcs.values()) + if has_conflicts and not missing_pk_cols: + for tgt_col, src_list in tgt_to_srcs.items(): + for src_col in src_list: + tables[src_tbl].foreign_keys.append(ForeignKeyDef( + source_table=src_tbl, + source_columns=[src_col], + target_table=tgt_tbl, + target_columns=[tgt_col], + )) + elif len(missing_pk_cols) == 1: + [missing_pk_col] = missing_pk_cols + src_col_names = {c.name for c in tables[src_tbl].columns} + bridge_col_in_src: str | None = ( + missing_pk_col if missing_pk_col in src_col_names else None + ) + if bridge_col_in_src is None: + found = _find_transitive_bridge( + src_tbl, tgt_tbl, missing_pk_col, tables, dictionary, link_groups, + ) + if found is not None: + primary = found[0] + hops = [(vc, bt, bp) for vc, bt, bp, _ in primary] + bridge_val_col = primary[-1][3] + fallback_chains = [ + ([(vc, bt, bp) for vc, bt, bp, _ in alt], alt[-1][3]) + for alt in found[1:] + ] + tables[src_tbl].columns.append(ColumnDef( + name=missing_pk_col, + definition_id='', + type_contents=None, + type_container=None, + nullable=True, + is_primary_key=False, + is_synthetic=True, + linked_item_id=None, + )) + bridge_columns.append(BridgeColumnDef( + table_name=src_tbl, + column_name=missing_pk_col, + hops=hops, + bridge_value_column=bridge_val_col, + fallback_chains=fallback_chains, + )) + bridge_col_in_src = missing_pk_col + if bridge_col_in_src is not None: + if has_conflicts: + for tgt_col, src_list in tgt_to_srcs.items(): + for src_col in src_list: + src_ordered = [ + src_col if pk == tgt_col else bridge_col_in_src + for pk in tgt_pks + ] + tables[src_tbl].foreign_keys.append(ForeignKeyDef( + source_table=src_tbl, + source_columns=src_ordered, + target_table=tgt_tbl, + target_columns=list(tgt_pks), + )) + else: + src_ordered = [ + tgt_to_srcs[pk][0] if pk in tgt_to_srcs else bridge_col_in_src + for pk in tgt_pks + ] + tables[src_tbl].foreign_keys.append(ForeignKeyDef( + source_table=src_tbl, + source_columns=src_ordered, + target_table=tgt_tbl, + target_columns=list(tgt_pks), + )) + else: + for src_col, tgt_col, item in pairs: + warnings.append( + f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " + f"partial FK to '{tgt_tbl}' -- covers " + f"{sorted(tgt_cols_covered)} of PKs={tgt_pks}, " + f"no transitive bridge found -- skipping FK constraint" + ) + partial_links.append(PartialLinkDef( + source_table=src_tbl, + source_column=src_col, + target_table=tgt_tbl, + target_column=tgt_col, + covered_pk_cols=sorted(tgt_cols_covered), + missing_pk_cols=sorted(missing_pk_cols), + reason=f"covers {sorted(tgt_cols_covered)} of PKs={tgt_pks}, no bridge found", + )) + elif missing_pk_cols or has_conflicts: + for src_col, tgt_col, item in pairs: + if len(tgt_to_srcs.get(tgt_col, [])) > 1: + msg = ( + f"ambiguous composite FK -- multiple source columns " + f"link to '{tgt_tbl}'.'{tgt_col}'" + ) + reason = f"ambiguous: multiple source columns link to '{tgt_tbl}'.'{tgt_col}'" + elif len(missing_pk_cols) > 1: + msg = ( + f"partial FK to '{tgt_tbl}' -- covers " + f"['{tgt_col}'] of PKs={tgt_pks} " + f"({len(missing_pk_cols)} missing PKs, bridge search skipped)" + ) + reason = f"covers ['{tgt_col}'] of PKs={tgt_pks}, {len(missing_pk_cols)} missing (bridge search skipped)" + else: + msg = ( + f"partial FK to '{tgt_tbl}' -- covers " + f"['{tgt_col}'] of PKs={tgt_pks}" + ) + reason = f"covers ['{tgt_col}'] of PKs={tgt_pks}" + warnings.append( + f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " + f"{msg} -- skipping FK constraint" + ) + partial_links.append(PartialLinkDef( + source_table=src_tbl, + source_column=src_col, + target_table=tgt_tbl, + target_column=tgt_col, + covered_pk_cols=sorted(tgt_cols_covered), + missing_pk_cols=sorted(missing_pk_cols), + reason=reason, + )) + else: + src_ordered = [tgt_to_srcs[tc][0] for tc in tgt_pks] + tables[src_tbl].foreign_keys.append(ForeignKeyDef( + source_table=src_tbl, + source_columns=src_ordered, + target_table=tgt_tbl, + target_columns=list(tgt_pks), + )) + +def _build_foreign_keys( + dictionary: DdlmDictionary, + tables: 'dict[str, TableDef]', + warnings: list[str], + partial_links: list[PartialLinkDef], +) -> list[BridgeColumnDef]: + bridge_columns: list[BridgeColumnDef] = [] _link_groups: dict[ tuple[str, str], list[tuple[str, str, DdlmItem]] - ] = defaultdict(list) # (src_tbl, tgt_tbl) → [(src_col, tgt_col, item)] - + ] = defaultdict(list) for item in dictionary.items.values(): if item.type_purpose != 'Link' or item.linked_item_id is None: continue - target_item = dictionary.tag_to_item.get(item.linked_item_id) if target_item is None: warnings.append( @@ -718,25 +864,20 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: f"{item.definition_id!r} not found in dictionary -- skipped" ) continue - if item.category_id is None or item.object_id is None: continue if target_item.category_id is None or target_item.object_id is None: continue - src_tbl = _table_name(item.category_id) tgt_tbl = _table_name(target_item.category_id) - if src_tbl not in tables: - continue # source category not schema-generating (Head etc.) + continue if tgt_tbl not in tables: warnings.append( f"FK: target table {tgt_tbl!r} for {item.definition_id!r} " f"not in schema -- skipped" ) continue - - # Warn if linked item is not a category key of the target. tgt_cat = dictionary.categories.get(target_item.category_id) if tgt_cat and item.linked_item_id not in tgt_cat.category_keys: warnings.append( @@ -745,212 +886,60 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: f"{target_item.category_id!r} " f"(PKs={sorted(dictionary.tag_to_item[k].object_id for k in tgt_cat.category_keys if k in dictionary.tag_to_item)}) -- attempting FK resolution" ) - _link_groups[(src_tbl, tgt_tbl)].append( (item.object_id, target_item.object_id, item) ) - for (src_tbl, tgt_tbl), pairs in sorted(_link_groups.items()): tgt_pks: list[str] = tables[tgt_tbl].primary_keys - tgt_pks_set = set(tgt_pks) + _resolve_fk_group( + src_tbl, tgt_tbl, pairs, tgt_pks, + tables, dictionary, _link_groups, + warnings, bridge_columns, partial_links, + ) + return bridge_columns + + +def _build_tag_metadata( + dictionary: DdlmDictionary, +) -> 'tuple[dict[str, str], dict[str, list[str]]]': + tag_to_category_class: dict[str, str] = {} + deprecated_replacements: dict[str, list[str]] = {} + for defn_id, item in dictionary.tag_to_item.items(): + if item.category_id: + cat = dictionary.categories.get(item.category_id) + if cat and cat.definition_class in ('Set', 'Loop'): + tag_to_category_class[defn_id] = cat.definition_class + if item.is_deprecated: + deprecated_replacements[defn_id] = item.replaced_by + return tag_to_category_class, deprecated_replacements - # Strip pairs that target non-PK columns and warn about each one. - # A mixed group must not prevent valid PK-targeting pairs from forming FKs. - pk_pairs = [] - for src_col, tgt_col, item in pairs: - if tgt_col not in tgt_pks_set: - warnings.append( - f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " - f"target column '{tgt_col}' is not a PK of " - f"'{tgt_tbl}' (PKs={tgt_pks}) -- skipping FK constraint" - ) - partial_links.append(PartialLinkDef( - source_table=src_tbl, - source_column=src_col, - target_table=tgt_tbl, - target_column=tgt_col, - covered_pk_cols=[], - missing_pk_cols=list(tgt_pks), - reason=f"target column '{tgt_col}' is not a PK of '{tgt_tbl}'", - )) - else: - pk_pairs.append((src_col, tgt_col, item)) - if not pk_pairs: +def _build_category_parent( + dictionary: DdlmDictionary, + tables: 'dict[str, TableDef]', +) -> 'dict[str, str | None]': + category_parent: dict[str, str | None] = {} + for _cat_id, cat_item in dictionary.categories.items(): + if cat_item.definition_class not in ('Set', 'Loop'): + continue + tbl_name = _table_name(cat_item.definition_id) + if tbl_name not in tables: continue - pairs = pk_pairs - - # tgt_col → [src_col, ...]: detect full coverage and duplicate targets - tgt_to_srcs: dict[str, list[str]] = defaultdict(list) - for src_col, tgt_col, _ in pairs: - tgt_to_srcs[tgt_col].append(src_col) - - tgt_cols_covered = set(tgt_to_srcs.keys()) - missing_pk_cols = tgt_pks_set - tgt_cols_covered - has_conflicts = any(len(v) > 1 for v in tgt_to_srcs.values()) - - if has_conflicts and not missing_pk_cols: - # Multiple source columns each independently reference the full PK - # (e.g. bond.atom_1 and bond.atom_2 both → atom.number). - # Emit one separate single/composite FK per source column. - for tgt_col, src_list in tgt_to_srcs.items(): - for src_col in src_list: - tables[src_tbl].foreign_keys.append(ForeignKeyDef( - source_table=src_tbl, - source_columns=[src_col], - target_table=tgt_tbl, - target_columns=[tgt_col], - )) - elif len(missing_pk_cols) == 1: - # All covered columns are PKs; exactly one PK column is missing. - # Sub-case A: the missing column already exists in src_tbl (self-ref - # or previously bridged) -- use it directly. - # Sub-case B: try to derive it via a transitive bridge table. - [missing_pk_col] = missing_pk_cols - src_col_names = {c.name for c in tables[src_tbl].columns} - bridge_col_in_src: str | None = ( - missing_pk_col if missing_pk_col in src_col_names else None + parent_id = cat_item.category_id + if parent_id: + parent_tbl = _table_name(parent_id) + category_parent[tbl_name] = ( + parent_tbl if parent_tbl in tables and parent_tbl != tbl_name else None ) - - if bridge_col_in_src is None: - found = _find_transitive_bridge( - src_tbl, tgt_tbl, missing_pk_col, - tables, dictionary, _link_groups, - ) - if found is not None: - # found is a list of paths; each path is a list of - # (via_col, bridge_tbl, bridge_pk, val_col_or_None) tuples. - # Intermediate entries carry None; the last entry carries - # the real value column. Use the first path as primary and - # carry the rest as fallback chains so ingest can try them - # in order when the primary yields None for a given row. - primary = found[0] - hops = [(vc, bt, bp) for vc, bt, bp, _ in primary] - bridge_val_col = primary[-1][3] - fallback_chains = [ - ([(vc, bt, bp) for vc, bt, bp, _ in alt], alt[-1][3]) - for alt in found[1:] - ] - # Add derived column once per (src_tbl, col) pair - tables[src_tbl].columns.append(ColumnDef( - name=missing_pk_col, - definition_id='', - type_contents=None, - type_container=None, - nullable=True, - is_primary_key=False, - is_synthetic=True, # transitive bridge -- no CIF tag - linked_item_id=None, - )) - bridge_columns.append(BridgeColumnDef( - table_name=src_tbl, - column_name=missing_pk_col, - hops=hops, - bridge_value_column=bridge_val_col, - fallback_chains=fallback_chains, - )) - bridge_col_in_src = missing_pk_col - - if bridge_col_in_src is not None: - # Emit one composite FK per conflicting src column (or one if - # no conflicts), with tgt_pks ordering throughout. - if has_conflicts: - for tgt_col, src_list in tgt_to_srcs.items(): - for src_col in src_list: - src_ordered = [ - src_col if pk == tgt_col else bridge_col_in_src - for pk in tgt_pks - ] - tables[src_tbl].foreign_keys.append(ForeignKeyDef( - source_table=src_tbl, - source_columns=src_ordered, - target_table=tgt_tbl, - target_columns=list(tgt_pks), - )) - else: - src_ordered = [ - tgt_to_srcs[pk][0] if pk in tgt_to_srcs else bridge_col_in_src - for pk in tgt_pks - ] - tables[src_tbl].foreign_keys.append(ForeignKeyDef( - source_table=src_tbl, - source_columns=src_ordered, - target_table=tgt_tbl, - target_columns=list(tgt_pks), - )) - else: - # No bridge found -- warn per pair - for src_col, tgt_col, item in pairs: - warnings.append( - f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " - f"partial FK to '{tgt_tbl}' -- covers " - f"{sorted(tgt_cols_covered)} of PKs={tgt_pks}, " - f"no transitive bridge found -- skipping FK constraint" - ) - partial_links.append(PartialLinkDef( - source_table=src_tbl, - source_column=src_col, - target_table=tgt_tbl, - target_column=tgt_col, - covered_pk_cols=sorted(tgt_cols_covered), - missing_pk_cols=sorted(missing_pk_cols), - reason=f"covers {sorted(tgt_cols_covered)} of PKs={tgt_pks}, no bridge found", - )) - elif missing_pk_cols or has_conflicts: - # Cannot form a complete, unambiguous (composite) FK. - # Emit one warning per failing pair so each source item is named. - for src_col, tgt_col, item in pairs: - if len(tgt_to_srcs.get(tgt_col, [])) > 1: - msg = ( - f"ambiguous composite FK -- multiple source columns " - f"link to '{tgt_tbl}'.'{tgt_col}'" - ) - reason = f"ambiguous: multiple source columns link to '{tgt_tbl}'.'{tgt_col}'" - elif len(missing_pk_cols) > 1: - msg = ( - f"partial FK to '{tgt_tbl}' -- covers " - f"['{tgt_col}'] of PKs={tgt_pks} " - f"({len(missing_pk_cols)} missing PKs, bridge search skipped)" - ) - reason = f"covers ['{tgt_col}'] of PKs={tgt_pks}, {len(missing_pk_cols)} missing (bridge search skipped)" - else: - msg = ( - f"partial FK to '{tgt_tbl}' -- covers " - f"['{tgt_col}'] of PKs={tgt_pks}" - ) - reason = f"covers ['{tgt_col}'] of PKs={tgt_pks}" - warnings.append( - f"FK: {item.definition_id!r} -> {item.linked_item_id!r}: " - f"{msg} -- skipping FK constraint" - ) - partial_links.append(PartialLinkDef( - source_table=src_tbl, - source_column=src_col, - target_table=tgt_tbl, - target_column=tgt_col, - covered_pk_cols=sorted(tgt_cols_covered), - missing_pk_cols=sorted(missing_pk_cols), - reason=reason, - )) else: - # All PKs covered, no non-PK targets, no duplicate targets. - # Order source columns to match the target PK column order. - src_ordered = [tgt_to_srcs[tc][0] for tc in tgt_pks] - tables[src_tbl].foreign_keys.append(ForeignKeyDef( - source_table=src_tbl, - source_columns=src_ordered, - target_table=tgt_tbl, - target_columns=list(tgt_pks), - )) + category_parent[tbl_name] = None + return category_parent + - # --- Third pass: propagation links --- - # For every PK column that is a Link item, record the target definition_id - # so that _apply_fk can still fill the column from the fk_accumulator or - # loop values even when no formal FK constraint was emitted. - # - # Additionally, PK Link columns with skipped FKs are made nullable: the - # database cannot enforce referential integrity for them, and NULL is the - # correct representation of an absent/default value. +def _build_propagation_links( + dictionary: DdlmDictionary, + tables: 'dict[str, TableDef]', +) -> 'dict[str, list[tuple[str, str, str | None]]]': propagation_links: dict[str, list[tuple[str, str, str | None]]] = {} _seen_prop: set[tuple[str, str]] = set() for item in dictionary.items.values(): @@ -968,8 +957,6 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: if src_col_def is None: continue is_pk = src_col_def.is_primary_key - # Non-PK items: only include when they carry an enumeration_default that - # should be applied to absent columns. if not is_pk and item.enumeration_default is None: continue key = (src_tbl, item.object_id) @@ -980,38 +967,54 @@ def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: (item.object_id, item.linked_item_id, item.enumeration_default) ) if is_pk: - # Make PK column nullable: FK was skipped, so NULL is valid here. src_col_def.nullable = True + return propagation_links - # Build category parent map: table_name → parent table_name (or None). - # Used by the output layer for wildcard category expansion. - category_parent: dict[str, str | None] = {} - for cat_id, cat_item in dictionary.categories.items(): - if cat_item.definition_class not in ('Set', 'Loop'): - continue - tbl_name = _table_name(cat_item.definition_id) - if tbl_name not in tables: - continue - parent_id = cat_item.category_id - if parent_id: - parent_tbl = _table_name(parent_id) - # Exclude self-references (top-level categories often have - # _name.category_id pointing to themselves). - category_parent[tbl_name] = ( - parent_tbl if parent_tbl in tables and parent_tbl != tbl_name else None - ) - else: - category_parent[tbl_name] = None - tag_to_category_class: dict[str, str] = {} - deprecated_replacements: dict[str, list[str]] = {} - for defn_id, item in dictionary.tag_to_item.items(): - if item.category_id: - cat = dictionary.categories.get(item.category_id) - if cat and cat.definition_class in ('Set', 'Loop'): - tag_to_category_class[defn_id] = cat.definition_class - if item.is_deprecated: - deprecated_replacements[defn_id] = item.replaced_by +# --------------------------------------------------------------------------- +# Public functions +# --------------------------------------------------------------------------- + +def generate_schema(dictionary: DdlmDictionary) -> SchemaSpec: + """ + Derive a :class:`SchemaSpec` from a loaded ``DdlmDictionary``. + + Iterates over all categories in *dictionary*, creating one + :class:`TableDef` for each ``Set`` or ``Loop`` category. ``Head`` and + ``Functions`` categories are silently skipped (they never appear in data + instance files); any other unrecognised class emits a warning and is also + skipped. + + Foreign-key constraints are built in a second pass over all items whose + ``type_purpose`` is ``"Link"``. ``SU`` items populate + :attr:`ColumnDef.linked_item_id` but do not produce + :class:`ForeignKeyDef` entries. + + ``alias_to_definition_id`` and ``deprecated_ids`` are copied directly from + *dictionary* so that ``ingest()`` can perform alias resolution and + deprecation checking without retaining a reference to the dictionary. + + Parameters + ---------- + dictionary: + The loaded dictionary returned by + :meth:`~cifflow.dictionary.loader.DictionaryLoader.load`. + + Returns + ------- + SchemaSpec + The complete schema specification including all tables, column + definitions, primary keys, foreign keys, the reverse + ``column_to_tag`` mapping, and alias/deprecation metadata. + """ + warnings: list[str] = [] + partial_links: list[PartialLinkDef] = [] + + tables, column_to_tag = _build_tables(dictionary, warnings) + bridge_columns = _build_foreign_keys(dictionary, tables, warnings, partial_links) + propagation_links = _build_propagation_links(dictionary, tables) + category_parent = _build_category_parent(dictionary, tables) + tag_to_category_class, deprecated_replacements = _build_tag_metadata(dictionary) return SchemaSpec( tables=tables, diff --git a/src/cifflow/dictionary/visualise.py b/src/cifflow/dictionary/visualise.py index 841f8dc..b1d4d61 100644 --- a/src/cifflow/dictionary/visualise.py +++ b/src/cifflow/dictionary/visualise.py @@ -442,6 +442,180 @@ def _component_label(component: frozenset[str]) -> str: return min(component) +def _emit_clustered_nodes( + lines: 'list[str]', + schema: SchemaSpec, + real_tables: 'set[str]', + ghost_tables: 'set[str]', + bridge_only: 'set[str]', + orphans: 'set[str]', + pass1_components: 'list[frozenset[str]]', + highlight_orphans: bool, + show_columns: 'Literal["all", "sparse", "none"]', + deprecated_ids: 'frozenset[str]', +) -> None: + """Emit subgraph cluster blocks for all connected components (highlight_components=True).""" + def _connectivity(name: str) -> str: + if name in bridge_only: + return 'bridge_only' + if name in orphans: + return 'orphan' + return 'connected' + + sorted_components = sorted(pass1_components, key=_component_label) + real_structural_components = [ + c for c in sorted_components if len(c) >= 2 and c.issubset(real_tables) + ] + partial_structural_components = [ + c for c in sorted_components + if len(c) >= 2 and not c.issubset(real_tables) and any(t in real_tables for t in c) + ] + all_structural = real_structural_components + partial_structural_components + + for i, component in enumerate(all_structural): + visible_members = sorted(t for t in component if t in real_tables) + if not visible_members: + continue + rep = _component_label(component) + lines.append(f' subgraph cluster_{i} {{') + lines.append(f' label="{_escape(rep)}" style=filled fillcolor="#f5f5f5"') + for tbl_name in visible_members: + tbl = schema.tables[tbl_name] + for node_line in _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids): + lines.append(' ' + node_line) + lines.append(' }') + lines.append('') + + visible_orphans = sorted(orphans & real_tables) + visible_bridge_only = sorted(bridge_only & real_tables) + if visible_orphans or visible_bridge_only: + lines.append(' subgraph cluster_orphans {') + lines.append(' label="Isolated tables" style=filled fillcolor="#fff8f8"') + for tbl_name in visible_orphans + visible_bridge_only: + if tbl_name not in real_tables: + continue + tbl = schema.tables[tbl_name] + for node_line in _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids): + lines.append(' ' + node_line) + lines.append(' }') + lines.append('') + + if ghost_tables: + lines.append(' subgraph cluster_missing {') + lines.append(' label="Missing tables" style=filled fillcolor="#ffe8e8"') + for ghost in sorted(ghost_tables): + for node_line in _ghost_node_dot(ghost): + lines.append(' ' + node_line) + lines.append(' }') + lines.append('') + + placed = set() + for c in all_structural: + placed.update(c) + placed.update(orphans) + placed.update(bridge_only) + for tbl_name in sorted(real_tables - placed): + tbl = schema.tables[tbl_name] + lines += _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids) + lines.append('') + + +def _emit_fk_edges( + lines: 'list[str]', + schema: SchemaSpec, + real_tables: 'set[str]', + ghost_tables: 'set[str]', + show_columns: str, + deprecated_ids: 'frozenset[str]', +) -> None: + for tbl_name in sorted(real_tables): + tbl = schema.tables[tbl_name] + vis_cols = _visible_columns(tbl, schema, show_columns, deprecated_ids) + for fk in tbl.foreign_keys: + target = fk.target_table + if target not in ghost_tables and target not in real_tables: + continue + label = _fk_label(fk, vis_cols, show_columns) + attr = f' [label="{label}"]' if label else '' + lines.append(f' {_dot_id(fk.source_table)} -> {_dot_id(target)}{attr}') + + +def _emit_bridge_edges( + lines: 'list[str]', + schema: SchemaSpec, + real_tables: 'set[str]', + ghost_tables: 'set[str]', + bridge_only: 'set[str]', + show_bridge: bool, +) -> None: + bridge_col_by_table: dict[str, list[BridgeColumnDef]] = {} + for bc in schema.bridge_columns: + bridge_col_by_table.setdefault(bc.table_name, []).append(bc) + for tbl_name in sorted(real_tables): + if tbl_name not in bridge_col_by_table: + continue + is_bridge_only_node = tbl_name in bridge_only + for bc in bridge_col_by_table[tbl_name]: + bridge_target = bc.bridge_table + target_is_ghost = bridge_target in ghost_tables + target_in_real = bridge_target in real_tables + if not target_is_ghost and not target_in_real: + continue + if show_bridge or is_bridge_only_node or target_is_ghost: + label = _escape(f'{bc.column_name} via {bc.via_column}') + lines.append( + f' {_dot_id(tbl_name)} -> {_dot_id(bridge_target)}' + f' [label="{label}" style=dashed color="#888888"]' + ) + + +def _emit_parent_edges( + lines: 'list[str]', + schema: SchemaSpec, + real_tables: 'set[str]', + ghost_tables: 'set[str]', + show_parent_edges: bool, +) -> None: + for child, parent in sorted(schema.category_parent.items()): + if not parent: + continue + child_in_real = child in real_tables + parent_is_ghost = parent in ghost_tables + parent_in_real = parent in real_tables + if not child_in_real: + continue + if not parent_is_ghost and not parent_in_real: + continue + if show_parent_edges or parent_is_ghost: + lines.append( + f' {_dot_id(child)} -> {_dot_id(parent)}' + f' [style=dotted arrowhead=open color="#aaaaaa"]' + ) + + +def _build_vis_context( + schema: SchemaSpec, + hide_deprecated: bool, + show_orphans: bool, + bridge_only: set[str], + orphans: set[str], + ghost_tables: set[str], +) -> 'tuple[frozenset[str], set[str], set[str]]': + """Return (deprecated_ids, real_tables, ghost_tables) after filtering.""" + deprecated_ids: frozenset[str] = ( + frozenset(schema.deprecated_ids) if hide_deprecated else frozenset() + ) + hidden_deprecated: set[str] = ( + _deprecated_table_names(schema) if hide_deprecated else set() + ) + if show_orphans: + real_tables = set(schema.tables) - hidden_deprecated + else: + real_tables = set(schema.tables) - bridge_only - orphans - hidden_deprecated + filtered_ghosts = ghost_tables - hidden_deprecated + return deprecated_ids, real_tables, filtered_ghosts + + # --------------------------------------------------------------------------- # Public: visualise_schema # --------------------------------------------------------------------------- @@ -524,23 +698,9 @@ def visualise_schema( """ ghost_tables = _collect_ghost_tables(schema) bridge_only, orphans, pass1_components = _classify_tables(schema) - - # Deprecated filtering - deprecated_ids: frozenset[str] = ( - frozenset(schema.deprecated_ids) if hide_deprecated else frozenset() + deprecated_ids, real_tables, ghost_tables = _build_vis_context( + schema, hide_deprecated, show_orphans, bridge_only, orphans, ghost_tables ) - hidden_deprecated: set[str] = ( - _deprecated_table_names(schema) if hide_deprecated else set() - ) - - # Determine which real tables to emit - if show_orphans: - real_tables = set(schema.tables) - hidden_deprecated - else: - real_tables = set(schema.tables) - bridge_only - orphans - hidden_deprecated - - # Ghost tables must not include tables we deliberately hid as deprecated - ghost_tables -= hidden_deprecated concentrate_attr = ' concentrate=true' if concentrate else '' lines: list[str] = [ @@ -553,7 +713,7 @@ def visualise_schema( '', ] - # --- Connectivity lookup --- + # --- Connectivity lookup (used by flat path) --- def _connectivity(name: str) -> str: if name in bridge_only: return 'bridge_only' @@ -569,70 +729,10 @@ def _connectivity(name: str) -> str: # --- Real table nodes (possibly clustered) --- if highlight_components: - # Sort components by their representative name for stability - sorted_components = sorted(pass1_components, key=_component_label) - - # Collect component nodes (only real, non-orphan/bridge tables) - real_structural_components = [ - c for c in sorted_components if len(c) >= 2 and c.issubset(real_tables) - ] - # Partial components (some members hidden by show_orphans=False) - partial_structural_components = [ - c for c in sorted_components - if len(c) >= 2 and not c.issubset(real_tables) and any(t in real_tables for t in c) - ] - # Include partially-visible components too - all_structural = real_structural_components + partial_structural_components - - for i, component in enumerate(all_structural): - visible_members = sorted(t for t in component if t in real_tables) - if not visible_members: - continue - rep = _component_label(component) - lines.append(f' subgraph cluster_{i} {{') - lines.append(f' label="{_escape(rep)}" style=filled fillcolor="#f5f5f5"') - for tbl_name in visible_members: - tbl = schema.tables[tbl_name] - for node_line in _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids): - lines.append(' ' + node_line) - lines.append(' }') - lines.append('') - - # Orphans cluster - visible_orphans = sorted(orphans & real_tables) - visible_bridge_only = sorted(bridge_only & real_tables) - if visible_orphans or visible_bridge_only: - lines.append(' subgraph cluster_orphans {') - lines.append(' label="Isolated tables" style=filled fillcolor="#fff8f8"') - for tbl_name in visible_orphans + visible_bridge_only: - if tbl_name not in real_tables: - continue - tbl = schema.tables[tbl_name] - for node_line in _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids): - lines.append(' ' + node_line) - lines.append(' }') - lines.append('') - - # Ghost node cluster - if ghost_tables: - lines.append(' subgraph cluster_missing {') - lines.append(' label="Missing tables" style=filled fillcolor="#ffe8e8"') - for ghost in sorted(ghost_tables): - for node_line in _ghost_node_dot(ghost): - lines.append(' ' + node_line) - lines.append(' }') - lines.append('') - - # Singleton real-table nodes not yet placed - placed = set() - for c in all_structural: - placed.update(c) - placed.update(orphans) - placed.update(bridge_only) - for tbl_name in sorted(real_tables - placed): - tbl = schema.tables[tbl_name] - lines += _table_node_dot(tbl, _connectivity(tbl_name), highlight_orphans, show_columns, schema, deprecated_ids) - lines.append('') + _emit_clustered_nodes( + lines, schema, real_tables, ghost_tables, bridge_only, orphans, + pass1_components, highlight_orphans, show_columns, deprecated_ids, + ) else: for tbl_name in sorted(real_tables): tbl = schema.tables[tbl_name] @@ -646,60 +746,9 @@ def _connectivity(name: str) -> str: # --- Edges --- lines.append('') - - # FK edges - for tbl_name in sorted(real_tables): - tbl = schema.tables[tbl_name] - vis_cols = _visible_columns(tbl, schema, show_columns, deprecated_ids) - for fk in tbl.foreign_keys: - target = fk.target_table - # Skip if target is a real table that's been hidden - if target not in ghost_tables and target not in real_tables: - continue - label = _fk_label(fk, vis_cols, show_columns) - attr = f' [label="{label}"]' if label else '' - lines.append(f' {_dot_id(fk.source_table)} -> {_dot_id(target)}{attr}') - - # Bridge edges - bridge_col_by_table: dict[str, list[BridgeColumnDef]] = {} - for bc in schema.bridge_columns: - bridge_col_by_table.setdefault(bc.table_name, []).append(bc) - - for tbl_name in sorted(real_tables): - if tbl_name not in bridge_col_by_table: - continue - is_bridge_only_node = tbl_name in bridge_only - for bc in bridge_col_by_table[tbl_name]: - bridge_target = bc.bridge_table - target_is_ghost = bridge_target in ghost_tables - target_in_real = bridge_target in real_tables - if not target_is_ghost and not target_in_real: - continue - # Show bridge edge if: show_bridge is True, OR the node is bridge_only, OR target is ghost - if show_bridge or is_bridge_only_node or target_is_ghost: - label = _escape(f'{bc.column_name} via {bc.via_column}') - lines.append( - f' {_dot_id(tbl_name)} -> {_dot_id(bridge_target)}' - f' [label="{label}" style=dashed color="#888888"]' - ) - - # Parent-hierarchy edges - for child, parent in sorted(schema.category_parent.items()): - if not parent: - continue - child_in_real = child in real_tables - parent_is_ghost = parent in ghost_tables - parent_in_real = parent in real_tables - if not child_in_real: - continue - if not parent_is_ghost and not parent_in_real: - continue - # Show parent edge if: show_parent_edges is True, OR target is ghost - if show_parent_edges or parent_is_ghost: - lines.append( - f' {_dot_id(child)} -> {_dot_id(parent)}' - f' [style=dotted arrowhead=open color="#aaaaaa"]' - ) + _emit_fk_edges(lines, schema, real_tables, ghost_tables, show_columns, deprecated_ids) + _emit_bridge_edges(lines, schema, real_tables, ghost_tables, bridge_only, show_bridge) + _emit_parent_edges(lines, schema, real_tables, ghost_tables, show_parent_edges) lines.append('}') return '\n'.join(lines) diff --git a/src/cifflow/ingestion/duckdb_ingest.py b/src/cifflow/ingestion/duckdb_ingest.py index 792ccb9..4c1f968 100644 --- a/src/cifflow/ingestion/duckdb_ingest.py +++ b/src/cifflow/ingestion/duckdb_ingest.py @@ -535,44 +535,83 @@ def flush_table_batches( # FK propagation (SQL) # --------------------------------------------------------------------------- -def _run_fk_fill_pass( +def _fill_single_fk( db: duckdb.DuckDBPyConnection, + tbl_name: str, + table: 'TableDef', + col_by_name: dict, schema: SchemaSpec, - topo: list[str], - tag_to_column: dict[str, tuple[str, str]], propagate_fk: bool, emit: Callable[..., None], - populated: set[str] | None = None, ) -> None: - """One pass of FK fill: propagate parent values into child FK columns.""" - for tbl_name in topo: - if populated is not None and tbl_name not in populated: + """Propagate single-column FK values from parent staging tables.""" + for fk in table.foreign_keys: + if len(fk.source_columns) != 1: continue - table = schema.tables[tbl_name] - col_by_name = {c.name: c for c in table.columns if not c.is_synthetic} - - # --- Single-column FKs --- - for fk in table.foreign_keys: - if len(fk.source_columns) != 1: - continue - src_col = fk.source_columns[0] - tgt_col = fk.target_columns[0] - col = col_by_name.get(src_col) - if col is None: - continue - is_key_fk = col.is_primary_key - if not is_key_fk and not propagate_fk: - continue + src_col = fk.source_columns[0] + tgt_col = fk.target_columns[0] + col = col_by_name.get(src_col) + if col is None: + continue + is_key_fk = col.is_primary_key + if not is_key_fk and not propagate_fk: + continue + tgt_tbl = fk.target_table + if tgt_tbl not in schema.tables: + if is_key_fk: + emit( + f"FK target '{tgt_tbl}'.'{tgt_col}' not in structured schema; " + f"leaving NULL" + ) + continue + db.execute(f""" + UPDATE "_raw_{tbl_name}" c + SET "{src_col}" = COALESCE( + (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p + WHERE p._cifflow_block_id = c._cifflow_block_id + AND p._loop_id = c._loop_id + AND p._iter_idx = c._iter_idx + AND p."{tgt_col}" IS NOT NULL + LIMIT 1), + (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p + WHERE p._cifflow_block_id = c._cifflow_block_id + AND p._loop_id = '{_SCALARS_LOOP_ID}' + AND p."{tgt_col}" IS NOT NULL + LIMIT 1), + (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p + WHERE p._cifflow_block_id = c._cifflow_block_id + AND p."{tgt_col}" IS NOT NULL + ORDER BY (p._loop_id = '{_SCALARS_LOOP_ID}') DESC, p._iter_idx + LIMIT 1) + ) + WHERE c."{src_col}" IS NULL + """) - tgt_tbl = fk.target_table - if tgt_tbl not in schema.tables: - if is_key_fk: - emit( - f"FK target '{tgt_tbl}'.'{tgt_col}' not in structured schema; " - f"leaving NULL" - ) - continue +def _fill_composite_fk( + db: duckdb.DuckDBPyConnection, + tbl_name: str, + table: 'TableDef', + col_by_name: dict, + schema: SchemaSpec, + propagate_fk: bool, +) -> None: + """Propagate composite FK values from parent staging tables.""" + for fk in table.foreign_keys: + if len(fk.source_columns) <= 1: + continue + is_key_fk = all( + col_by_name.get(sc) is not None and col_by_name[sc].is_primary_key + for sc in fk.source_columns + ) + if not is_key_fk and not propagate_fk: + continue + tgt_tbl = fk.target_table + if tgt_tbl not in schema.tables: + continue + if not all(sc in col_by_name for sc in fk.source_columns): + continue + for src_col, tgt_col in zip(fk.source_columns, fk.target_columns): db.execute(f""" UPDATE "_raw_{tbl_name}" c SET "{src_col}" = COALESCE( @@ -596,116 +635,103 @@ def _run_fk_fill_pass( WHERE c."{src_col}" IS NULL """) - # --- Composite FKs --- - for fk in table.foreign_keys: - if len(fk.source_columns) <= 1: - continue - is_key_fk = all( - col_by_name.get(sc) is not None and col_by_name[sc].is_primary_key - for sc in fk.source_columns - ) - if not is_key_fk and not propagate_fk: - continue - tgt_tbl = fk.target_table - if tgt_tbl not in schema.tables: - continue - if not all(sc in col_by_name for sc in fk.source_columns): - continue +def _fill_propagation_links( + db: duckdb.DuckDBPyConnection, + tbl_name: str, + table: 'TableDef', + col_by_name: dict, + schema: SchemaSpec, + propagate_fk: bool, + tag_to_column: dict[str, tuple[str, str]], +) -> None: + """Propagate values via schema propagation links and apply enumeration defaults.""" + for col_name, target_def_id, default_val in schema.propagation_links.get(tbl_name, []): + col = col_by_name.get(col_name) + if col is None: + continue + do_fk_propagate = col.is_primary_key or propagate_fk + if not do_fk_propagate and default_val is None: + continue - for src_col, tgt_col in zip(fk.source_columns, fk.target_columns): - db.execute(f""" - UPDATE "_raw_{tbl_name}" c - SET "{src_col}" = COALESCE( - (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p - WHERE p._cifflow_block_id = c._cifflow_block_id - AND p._loop_id = c._loop_id - AND p._iter_idx = c._iter_idx - AND p."{tgt_col}" IS NOT NULL - LIMIT 1), - (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p - WHERE p._cifflow_block_id = c._cifflow_block_id - AND p._loop_id = '{_SCALARS_LOOP_ID}' - AND p."{tgt_col}" IS NOT NULL - LIMIT 1), - (SELECT p."{tgt_col}" FROM "_raw_{tgt_tbl}" p - WHERE p._cifflow_block_id = c._cifflow_block_id - AND p."{tgt_col}" IS NOT NULL - ORDER BY (p._loop_id = '{_SCALARS_LOOP_ID}') DESC, p._iter_idx - LIMIT 1) - ) - WHERE c."{src_col}" IS NULL - """) + if do_fk_propagate: + # Follow the propagation chain transitively so that across-block + # scenarios (ALL_BLOCKS re-ingest) can fall back to deeper ancestors. + lookup_chain: list[tuple[str, str]] = [] + current_def_id = target_def_id + visited_defs: set[str] = set() + for _ in range(8): + if current_def_id in visited_defs: + break + visited_defs.add(current_def_id) + loc = tag_to_column.get(current_def_id) + if not loc: + break + tgt_tbl_c, tgt_col_c = loc + if tgt_tbl_c not in schema.tables: + break + lookup_chain.append((tgt_tbl_c, tgt_col_c)) + next_link = next( + (lnk for lnk in schema.propagation_links.get(tgt_tbl_c, []) + if lnk[0] == tgt_col_c), + None, + ) + if next_link is None: + break + current_def_id = next_link[1] + else: + lookup_chain = [] + + if lookup_chain: + subs: list[str] = [] + for tgt_tbl_c, tgt_col_c in lookup_chain: + subs += [ + f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' + f' WHERE p._cifflow_block_id = c._cifflow_block_id' + f' AND p._loop_id = c._loop_id AND p._iter_idx = c._iter_idx' + f' AND p."{tgt_col_c}" IS NOT NULL LIMIT 1)', + f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' + f' WHERE p._cifflow_block_id = c._cifflow_block_id' + f" AND p._loop_id = '{_SCALARS_LOOP_ID}'" + f' AND p."{tgt_col_c}" IS NOT NULL LIMIT 1)', + f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' + f' WHERE p._cifflow_block_id = c._cifflow_block_id' + f' AND p."{tgt_col_c}" IS NOT NULL' + f" ORDER BY (p._loop_id = '{_SCALARS_LOOP_ID}') DESC, p._iter_idx" + f' LIMIT 1)', + ] + db.execute(f""" + UPDATE "_raw_{tbl_name}" c + SET "{col_name}" = COALESCE({", ".join(subs)}) + WHERE c."{col_name}" IS NULL + """) - # --- Propagation links --- - for col_name, target_def_id, default_val in schema.propagation_links.get(tbl_name, []): - col = col_by_name.get(col_name) - if col is None: - continue - do_fk_propagate = col.is_primary_key or propagate_fk - # Non-PK items without a default have nothing to do here. - if not do_fk_propagate and default_val is None: - continue + if default_val is not None: + db.execute(f""" + UPDATE "_raw_{tbl_name}" + SET "{col_name}" = ? + WHERE "{col_name}" IS NULL + """, [default_val]) - if do_fk_propagate: - # Follow the propagation chain transitively so that across-block - # scenarios (ALL_BLOCKS re-ingest) can fall back to deeper ancestors - # that share the current block's _cifflow_block_id. - lookup_chain: list[tuple[str, str]] = [] - current_def_id = target_def_id - visited_defs: set[str] = set() - for _ in range(8): - if current_def_id in visited_defs: - break - visited_defs.add(current_def_id) - loc = tag_to_column.get(current_def_id) - if not loc: - break - tgt_tbl_c, tgt_col_c = loc - if tgt_tbl_c not in schema.tables: - break - lookup_chain.append((tgt_tbl_c, tgt_col_c)) - next_link = next( - (lnk for lnk in schema.propagation_links.get(tgt_tbl_c, []) - if lnk[0] == tgt_col_c), - None, - ) - if next_link is None: - break - current_def_id = next_link[1] - else: - lookup_chain = [] - - if lookup_chain: - subs: list[str] = [] - for tgt_tbl_c, tgt_col_c in lookup_chain: - subs += [ - f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' - f' WHERE p._cifflow_block_id = c._cifflow_block_id' - f' AND p._loop_id = c._loop_id AND p._iter_idx = c._iter_idx' - f' AND p."{tgt_col_c}" IS NOT NULL LIMIT 1)', - f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' - f' WHERE p._cifflow_block_id = c._cifflow_block_id' - f" AND p._loop_id = '{_SCALARS_LOOP_ID}'" - f' AND p."{tgt_col_c}" IS NOT NULL LIMIT 1)', - f'(SELECT p."{tgt_col_c}" FROM "_raw_{tgt_tbl_c}" p' - f' WHERE p._cifflow_block_id = c._cifflow_block_id' - f' AND p."{tgt_col_c}" IS NOT NULL' - f" ORDER BY (p._loop_id = '{_SCALARS_LOOP_ID}') DESC, p._iter_idx" - f' LIMIT 1)', - ] - db.execute(f""" - UPDATE "_raw_{tbl_name}" c - SET "{col_name}" = COALESCE({", ".join(subs)}) - WHERE c."{col_name}" IS NULL - """) - if default_val is not None: - db.execute(f""" - UPDATE "_raw_{tbl_name}" - SET "{col_name}" = ? - WHERE "{col_name}" IS NULL - """, [default_val]) +def _run_fk_fill_pass( + db: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + topo: list[str], + tag_to_column: dict[str, tuple[str, str]], + propagate_fk: bool, + emit: Callable[..., None], + populated: set[str] | None = None, +) -> None: + """One pass of FK fill: propagate parent values into child FK columns.""" + for tbl_name in topo: + if populated is not None and tbl_name not in populated: + continue + table = schema.tables[tbl_name] + col_by_name = {c.name: c for c in table.columns if not c.is_synthetic} + _fill_single_fk(db, tbl_name, table, col_by_name, schema, propagate_fk, emit) + _fill_composite_fk(db, tbl_name, table, col_by_name, schema, propagate_fk) + _fill_propagation_links(db, tbl_name, table, col_by_name, schema, propagate_fk, tag_to_column) def _insert_key_fk_stubs( @@ -747,25 +773,14 @@ def _insert_key_fk_stubs( populated.add(tgt_tbl) -def propagate_fk_sql( +def _generate_uuid_pks( db: duckdb.DuckDBPyConnection, schema: SchemaSpec, - tag_to_column: dict[str, tuple[str, str]], - propagate_fk: bool, - emit: Callable[..., None], - populated: set[str] | None = None, + populated: set[str] | None, ) -> None: - """Fill missing FK/PK columns in DuckDB staging tables.""" - topo = _topo_order(schema) - - _run_fk_fill_pass(db, schema, topo, tag_to_column, propagate_fk, emit, populated) - _insert_key_fk_stubs(db, schema, topo, populated) - _run_fk_fill_pass(db, schema, topo, tag_to_column, propagate_fk, emit, populated) - - # --- UUID generation for remaining NULL non-synthetic PKs --- - sibling_groups = _sibling_groups(schema) + """Assign UUIDs to NULL non-synthetic PK columns not already filled by FK propagation.""" sibling_canonicals: dict[str, str] = {} - for group in sibling_groups: + for group in _sibling_groups(schema): canonical = group[0] for t in group: sibling_canonicals[t] = canonical @@ -780,7 +795,6 @@ def propagate_fk_sql( ) if has_single_fk: continue - canonical_tbl = sibling_canonicals.get(tbl_name, tbl_name) if canonical_tbl == tbl_name: db.execute(f""" @@ -808,7 +822,14 @@ def propagate_fk_sql( WHERE "{pk_col.name}" IS NULL """) - # --- Create stub parent rows for non-null FK values --- + +def _create_composite_fk_stub_parents( + db: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + topo: list[str], + populated: set[str] | None, +) -> None: + """Insert missing parent rows for composite FKs in topological order.""" for tbl_name in topo: if populated is not None and tbl_name not in populated: continue @@ -843,6 +864,14 @@ def propagate_fk_sql( if populated is not None: populated.add(tgt_tbl) + +def _create_single_fk_stub_parents( + db: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + topo: list[str], + populated: set[str] | None, +) -> None: + """Insert missing parent rows for single-column FKs in topological order.""" for tbl_name in topo: if populated is not None and tbl_name not in populated: continue @@ -871,6 +900,24 @@ def propagate_fk_sql( populated.add(tgt_tbl) +def propagate_fk_sql( + db: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + tag_to_column: dict[str, tuple[str, str]], + propagate_fk: bool, + emit: Callable[..., None], + populated: set[str] | None = None, +) -> None: + """Fill missing FK/PK columns in DuckDB staging tables.""" + topo = _topo_order(schema) + _run_fk_fill_pass(db, schema, topo, tag_to_column, propagate_fk, emit, populated) + _insert_key_fk_stubs(db, schema, topo, populated) + _run_fk_fill_pass(db, schema, topo, tag_to_column, propagate_fk, emit, populated) + _generate_uuid_pks(db, schema, populated) + _create_composite_fk_stub_parents(db, schema, topo, populated) + _create_single_fk_stub_parents(db, schema, topo, populated) + + # --------------------------------------------------------------------------- # Infrastructure + final-table creation # --------------------------------------------------------------------------- diff --git a/src/cifflow/inspect/_schema.py b/src/cifflow/inspect/_schema.py index f09aa79..e8eca27 100644 --- a/src/cifflow/inspect/_schema.py +++ b/src/cifflow/inspect/_schema.py @@ -9,6 +9,32 @@ ) +def _load_schema(source: 'Union[str, pathlib.Path, SchemaSpec, DdlmDictionary]') -> 'SchemaSpec': + """Resolve *source* to a :class:`~cifflow.dictionary.schema.SchemaSpec`.""" + from cifflow.dictionary.loader import DictionaryLoader, directory_resolver + from cifflow.dictionary.schema import SchemaSpec, generate_schema + + try: + from cifflow.dictionary.loader import DdlmDictionary + except ImportError: + DdlmDictionary = None + + if isinstance(source, SchemaSpec): + return source + if DdlmDictionary is not None and isinstance(source, DdlmDictionary): + return generate_schema(source) + if isinstance(source, pathlib.Path) or ( + isinstance(source, str) and not source.lstrip().startswith('#') + and '\n' not in source.strip() + ): + path = pathlib.Path(source) + raw = path.read_text(encoding='utf-8') + loader = DictionaryLoader(resolver=directory_resolver(path.parent)) + return generate_schema(loader.load(raw)) + loader = DictionaryLoader(resolver=None) + return generate_schema(loader.load(source)) + + def inspect_schema( source: 'Union[str, pathlib.Path, SchemaSpec, DdlmDictionary]', *, @@ -38,138 +64,135 @@ def inspect_schema( file: Output stream. Default ``sys.stdout``. """ - from cifflow.dictionary.loader import DictionaryLoader, directory_resolver - from cifflow.dictionary.schema import SchemaSpec, generate_schema, emit_create_statements - - try: - from cifflow.dictionary.loader import DdlmDictionary - except ImportError: - DdlmDictionary = None - - if isinstance(source, SchemaSpec): - schema = source - elif DdlmDictionary is not None and isinstance(source, DdlmDictionary): - schema = generate_schema(source) - elif isinstance(source, pathlib.Path) or ( - isinstance(source, str) and not source.lstrip().startswith('#') - and '\n' not in source.strip() - ): - path = pathlib.Path(source) - raw = path.read_text(encoding='utf-8') - loader = DictionaryLoader(resolver=directory_resolver(path.parent)) - dictionary = loader.load(raw) - schema = generate_schema(dictionary) - else: - loader = DictionaryLoader(resolver=None) - dictionary = loader.load(source) - schema = generate_schema(dictionary) - - n_tables = len(schema.tables) - n_set = sum(1 for t in schema.tables.values() if t.category_class == 'Set') - n_loop = sum(1 for t in schema.tables.values() if t.category_class == 'Loop') - n_fk = sum(len(t.foreign_keys) for t in schema.tables.values()) - n_warn = len(schema.warnings) - - summary = ( - f'{n_tables} table{"s" if n_tables != 1 else ""}' - f' ({n_set} Set, {n_loop} Loop)' - f' {n_fk} FK{"s" if n_fk != 1 else ""}' - f' {n_warn} warning{"s" if n_warn != 1 else ""}' + schema = _load_schema(source) + _print_schema(schema, show_ddl=show_ddl, file=file) + + +def _col_display_type(col: 'ColumnDef') -> str: + """Return the display type string for a column.""" + if col.name == '_cifflow_row_id': + return 'INTEGER' + return col.type_contents or 'TEXT' + + +def _depr_suffix(definition_id: str, schema: 'SchemaSpec', file: TextIO) -> str: + """Return a coloured deprecation annotation string, or empty string.""" + if definition_id not in schema.deprecated_ids: + return '' + replacements = [r for r in schema.deprecated_replacements.get(definition_id, []) if r] + if replacements: + return ' ' + c('DEPRECATED -> ' + ', '.join(replacements), RED, file=file) + return ' ' + c('DEPRECATED', RED, file=file) + + +def _column_flags(col: 'ColumnDef', file: TextIO) -> list[str]: + """Return the ordered list of flag strings for one column.""" + flags: list[str] = [] + if not col.nullable: + flags.append(c('NOT NULL', DIM, file=file)) + if col.is_synthetic and col.name == '_cifflow_row_id': + flags.append(c('UNIQUE', DIM, file=file)) + if col.is_primary_key: + flags.append(c('PK', YELLOW, file=file)) + if col.is_synthetic: + flags.append(c('synthetic', DIM, file=file)) + return flags + + +def _column_tag_part(col: 'ColumnDef', schema: 'SchemaSpec', file: TextIO) -> str: + """Return the definition-id / SU annotation / deprecation suffix for a column.""" + tag_part = '' + if not col.is_synthetic: + tag_part = ' ' + c(col.definition_id, DIM, file=file) + if col.linked_item_id and not col.is_primary_key: + tag_part += ' ' + c(f'->su {col.linked_item_id}', MAGENTA, file=file) + tag_part += _depr_suffix(col.definition_id, schema, file) + return tag_part + + +def _print_table( + table: 'TableDef', + *, + schema: 'SchemaSpec', + ddl_stmt: 'str | None', + file: TextIO, +) -> None: + """Render one table block (header, PK, columns, FKs, optional DDL).""" + cls_colour = CYAN if table.category_class == 'Loop' else BLUE + header = ( + c(table.name, BOLD, file=file) + + ' ' + + c(f'[{table.category_class}]', cls_colour, file=file) + + _depr_suffix(table.definition_id, schema, file) ) - print(c('-- schema --', BOLD, DIM, file=file), file=file) - print(c(summary, DIM, file=file), file=file) - print(file=file) + print(header, file=file) + + pk_str = ', '.join(c(k, YELLOW, file=file) for k in table.primary_keys) + print(f' PK {pk_str}', file=file) + + col_name_w = max((len(col.name) for col in table.columns), default=8) + type_w = max((len(_col_display_type(col)) for col in table.columns), default=4) + + print(f' {c("columns", DIM, file=file)}', file=file) + for col in table.columns: + name_part = c(col.name.ljust(col_name_w), YELLOW, file=file) + type_part = c(_col_display_type(col).ljust(type_w), GREEN, file=file) + flag_str = ' '.join(_column_flags(col, file)) + tag_part = _column_tag_part(col, schema, file) + print(f' {name_part} {type_part} {flag_str}{tag_part}', file=file) + + if table.foreign_keys: + print(f' {c("foreign keys", DIM, file=file)}', file=file) + for fk in table.foreign_keys: + if len(fk.source_columns) == 1: + src = c(fk.source_columns[0], YELLOW, file=file) + tgt = c(f'{fk.target_table}.{fk.target_columns[0]}', CYAN, file=file) + else: + src = c('(' + ', '.join(fk.source_columns) + ')', YELLOW, file=file) + tgt = c( + f'{fk.target_table}.(' + ', '.join(fk.target_columns) + ')', + CYAN, file=file, + ) + print(f' {src} -> {tgt} DEFERRABLE', file=file) - ddl_stmts = emit_create_statements(schema) if show_ddl else [] - ddl_by_table: dict[str, str] = {} - if show_ddl: - for stmt, table in zip(ddl_stmts, schema.tables.values()): - ddl_by_table[table.name] = stmt + if ddl_stmt is not None: + print(f' {c("ddl", DIM, file=file)}', file=file) + for ddl_line in ddl_stmt.splitlines(): + print(f' {c(ddl_line, DIM, file=file)}', file=file) - def _depr_suffix(definition_id: str) -> str: - if definition_id not in schema.deprecated_ids: - return '' - replacements = [r for r in schema.deprecated_replacements.get(definition_id, []) if r] - if replacements: - return ' ' + c('DEPRECATED -> ' + ', '.join(replacements), RED, file=file) - return ' ' + c('DEPRECATED', RED, file=file) + print(file=file) - for table in sorted(schema.tables.values(), key=lambda t: t.name): - cls_colour = CYAN if table.category_class == 'Loop' else BLUE - header = ( - c(table.name, BOLD, file=file) - + ' ' - + c(f'[{table.category_class}]', cls_colour, file=file) - + _depr_suffix(table.definition_id) - ) - print(header, file=file) - - pk_str = ', '.join(c(k, YELLOW, file=file) for k in table.primary_keys) - print(f' PK {pk_str}', file=file) - - def _col_display_type(col) -> str: - if col.name == '_cifflow_row_id': - return 'INTEGER' - return col.type_contents or 'TEXT' - - col_name_w = max((len(col.name) for col in table.columns), default=8) - type_w = max((len(_col_display_type(col)) for col in table.columns), default=4) - - print(f' {c("columns", DIM, file=file)}', file=file) - for col in table.columns: - name_part = c(col.name.ljust(col_name_w), YELLOW, file=file) - type_part = c(_col_display_type(col).ljust(type_w), GREEN, file=file) - - flags: list[str] = [] - if not col.nullable: - flags.append(c('NOT NULL', DIM, file=file)) - if col.is_synthetic and col.name == '_cifflow_row_id': - flags.append(c('UNIQUE', DIM, file=file)) - if col.is_primary_key: - flags.append(c('PK', YELLOW, file=file)) - if col.is_synthetic: - flags.append(c('synthetic', DIM, file=file)) - - tag_part = '' - if not col.is_synthetic: - tag_part = ' ' + c(col.definition_id, DIM, file=file) - if col.linked_item_id and not col.is_primary_key: - tag_part += ' ' + c(f'->su {col.linked_item_id}', MAGENTA, file=file) - tag_part += _depr_suffix(col.definition_id) - - flag_str = ' '.join(flags) - print(f' {name_part} {type_part} {flag_str}{tag_part}', file=file) - - if table.foreign_keys: - print(f' {c("foreign keys", DIM, file=file)}', file=file) - for fk in table.foreign_keys: - if len(fk.source_columns) == 1: - src = c(fk.source_columns[0], YELLOW, file=file) - tgt = c( - f'{fk.target_table}.{fk.target_columns[0]}', - CYAN, file=file, - ) - else: - src = c( - '(' + ', '.join(fk.source_columns) + ')', - YELLOW, file=file, - ) - tgt = c( - f'{fk.target_table}.(' + ', '.join(fk.target_columns) + ')', - CYAN, file=file, - ) - print(f' {src} -> {tgt} DEFERRABLE', file=file) - if show_ddl and table.name in ddl_by_table: - print(f' {c("ddl", DIM, file=file)}', file=file) - for ddl_line in ddl_by_table[table.name].splitlines(): - print(f' {c(ddl_line, DIM, file=file)}', file=file) +def _resolves_to_set( + linked_item_id: str, + schema: 'SchemaSpec', + tag_to_table_col: dict, + col_by_key: dict, + visited: set, +) -> bool: + """Return True if *linked_item_id* transitively reaches a Set category.""" + if not linked_item_id or linked_item_id in visited: + return False + visited.add(linked_item_id) + canonical = schema.alias_to_definition_id.get(linked_item_id, linked_item_id) + cls = schema.tag_to_category_class.get(canonical) + if cls == 'Set': + return True + if cls != 'Loop': + return False + entry = tag_to_table_col.get(canonical) + if entry is None: + return False + target_col = col_by_key.get(entry) + if target_col is not None and target_col.linked_item_id: + return _resolves_to_set(target_col.linked_item_id, schema, tag_to_table_col, col_by_key, visited) + return False - print(file=file) +def _print_floating_loops(schema: 'SchemaSpec', *, file: TextIO) -> None: + """Print the floating-loop section (Loop tables with no Set-derived PK).""" set_tables = {name for name, t in schema.tables.items() if t.category_class == 'Set'} - # Reverse map: definition_id → (table_name, col_name), for transitive chain-following. tag_to_table_col: dict[str, tuple[str, str]] = { defn_id: (tbl, col_name) for (tbl, col_name), defn_id in schema.column_to_tag.items() @@ -180,25 +203,6 @@ def _col_display_type(col) -> str: for col in tbl_def.columns } - def _resolves_to_set(linked_item_id: str, visited: set) -> bool: - """Return True if linked_item_id transitively reaches a Set category.""" - if not linked_item_id or linked_item_id in visited: - return False - visited.add(linked_item_id) - canonical = schema.alias_to_definition_id.get(linked_item_id, linked_item_id) - cls = schema.tag_to_category_class.get(canonical) - if cls == 'Set': - return True - if cls != 'Loop': - return False - entry = tag_to_table_col.get(canonical) - if entry is None: - return False - target_col = col_by_key.get(entry) - if target_col is not None and target_col.linked_item_id: - return _resolves_to_set(target_col.linked_item_id, visited) - return False - bridge_by_table: dict[str, list] = {} for bc in schema.bridge_columns: bridge_by_table.setdefault(bc.table_name, []).append(bc) @@ -210,11 +214,10 @@ def _resolves_to_set(linked_item_id: str, visited: set) -> bool: pk_set = set(table.primary_keys) has_set_link = any( - _resolves_to_set(col.linked_item_id, set()) + _resolves_to_set(col.linked_item_id, schema, tag_to_table_col, col_by_key, set()) for col in table.columns if col.is_primary_key and not col.is_synthetic and col.linked_item_id ) - has_set_bridge = any( bc.column_name in pk_set and bc.hops[-1][1] in set_tables for bc in bridge_by_table.get(table.name, []) @@ -230,6 +233,38 @@ def _resolves_to_set(linked_item_id: str, visited: set) -> bool: print(f' {c(table.name, BOLD, file=file)} PK: {pk_str}', file=file) print(file=file) + +def _print_schema(schema: 'SchemaSpec', *, show_ddl: bool, file: TextIO) -> None: + """Render the full schema summary to *file*.""" + from cifflow.dictionary.schema import emit_create_statements + + n_tables = len(schema.tables) + n_set = sum(1 for t in schema.tables.values() if t.category_class == 'Set') + n_loop = sum(1 for t in schema.tables.values() if t.category_class == 'Loop') + n_fk = sum(len(t.foreign_keys) for t in schema.tables.values()) + n_warn = len(schema.warnings) + + summary = ( + f'{n_tables} table{"s" if n_tables != 1 else ""}' + f' ({n_set} Set, {n_loop} Loop)' + f' {n_fk} FK{"s" if n_fk != 1 else ""}' + f' {n_warn} warning{"s" if n_warn != 1 else ""}' + ) + print(c('-- schema --', BOLD, DIM, file=file), file=file) + print(c(summary, DIM, file=file), file=file) + print(file=file) + + ddl_stmts = emit_create_statements(schema) if show_ddl else [] + ddl_by_table: dict[str, str] = {} + if show_ddl: + for stmt, table in zip(ddl_stmts, schema.tables.values()): + ddl_by_table[table.name] = stmt + + for table in sorted(schema.tables.values(), key=lambda t: t.name): + _print_table(table, schema=schema, ddl_stmt=ddl_by_table.get(table.name), file=file) + + _print_floating_loops(schema, file=file) + if schema.warnings: print(c('-- schema warnings --', BOLD, DIM, file=file), file=file) for w in schema.warnings: diff --git a/src/cifflow/output/emit.py b/src/cifflow/output/emit.py index 88f408f..c5c2268 100644 --- a/src/cifflow/output/emit.py +++ b/src/cifflow/output/emit.py @@ -537,234 +537,151 @@ def _expand_with_child_sets(base: frozenset[str], schema: SchemaSpec) -> frozens return frozenset(expanded) -def _collect_grouped( - conn: duckdb.DuckDBPyConnection, - schema: SchemaSpec, - version: CifVersion, -) -> list[_BlockData]: - """Group source blocks by Set-identity fingerprint (GROUPED mode). +# --------------------------------------------------------------------------- +# _collect_grouped helpers +# --------------------------------------------------------------------------- - The fingerprint of a source ``_cifflow_block_id`` is the frozenset of - ``(table_name, sorted_pk_value_tuples)`` for every keyed Set-class table - that *directly owns* rows in that block (winner ``_cifflow_block_id`` match). - Source blocks with identical fingerprints are merged into one output block. - Blocks with an empty fingerprint (no owned keyed Set rows) are passed - through unchanged as pure-loop blocks, one per source ``_cifflow_block_id``. +def _collect_set_pk_vals( + cache: '_EmitCache', + t: str, + td: TableDef, + bid: str, +) -> set[tuple]: + """PK value tuples present in *bid* for a keyed Set table (rows + tag_presence).""" + domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] + pk_vals: set[tuple] = set() + for r in cache.rows_for_block(t, bid): + tup = tuple(str(r.get(pk)) if r.get(pk) is not None else '' for pk in domain_pks) + if any(tup): + pk_vals.add(tup) + for _col, pk_json in cache.tag_presence(bid, t): + try: + vals = json.loads(pk_json) + tup = tuple(str(v) if v is not None else '' for v in vals) + if any(tup): + pk_vals.add(tup) + except Exception: + pass + return pk_vals + + +def _fp_entries_for_expanded( + cache: '_EmitCache', + bid: str, + keyed_set_tables: dict, + expanded: frozenset, +) -> frozenset: + """Fingerprint entries for keyed Set tables in *expanded* for source block *bid*.""" + entries: list[tuple] = [] + for t, td in keyed_set_tables.items(): + if t not in expanded: + continue + pk_vals_set = _collect_set_pk_vals(cache, t, td, bid) + if pk_vals_set: + entries.append((t, tuple(sorted(pk_vals_set)))) + return frozenset(entries) - Row collection is by ``_cifflow_block_id`` membership only — no FK-graph - traversal is required. ``anchor_frozenset`` reflects all keyed Set tables - in the fingerprint, enabling multi-anchor predicates such as - ``all_of('pd_diffractogram', 'pd_phase')`` to match bridge blocks. - Set-class tables that are present in a source block but NOT reachable from - any Loop table via PK-only FK chains are treated as *incidental* — they do - not widen the main fingerprint and are instead emitted as their own - dedicated output blocks with a single-table anchor frozenset. +def _drop_incidental_child_sets( + fp_incidental_entries: list[tuple], + keyed_set_tables: dict, +) -> list[tuple]: + """Remove child-Set tables from incidental entries. + + A child is a table whose ALL domain PKs are FK source columns pointing to + other incidental tables. Child data is collected by the BFS in the parent's + incidental block; a separate entry would produce a duplicate block. """ - # GROUPED mode propagates existing dataset IDs but does not generate new UUIDs. - # Blocks without a source _audit_dataset.id will not have one injected. - fallback_id: str | None = None - cache = _EmitCache(conn, schema) - all_table_names = list(schema.tables.keys()) + incidental_set = frozenset(t for t, _ in fp_incidental_entries) + to_drop: set[str] = set() + for inc_t, _ in fp_incidental_entries: + inc_td = keyed_set_tables[inc_t] + domain_pks_t = [pk for pk in inc_td.primary_keys if pk not in _SYNTHETIC] + fk_to_incidental: set[str] = set() + for fk in inc_td.foreign_keys: + if fk.target_table in incidental_set and fk.target_table != inc_t: + fk_to_incidental.update(fk.source_columns) + if domain_pks_t and all(pk in fk_to_incidental for pk in domain_pks_t): + to_drop.add(inc_t) + return [(t, vals) for t, vals in fp_incidental_entries if t not in to_drop] + + +def _compute_block_fingerprint( + bid: str, + schema: SchemaSpec, + cache: '_EmitCache', + keyed_set_tables: dict, + pk_reachable_sets: dict, +) -> tuple[list[frozenset], frozenset]: + """Compute (main_fps, incidental_fp) for one source block in GROUPED mode. + + Each distinct non-empty pk_reachable frozenset among Loop tables produces one + fingerprint entry (anchor group). Keyed Set tables whose PK is not reachable + from any Loop PK FK chain are treated as incidental. + """ + loop_tables_present = [ + t for t in schema.tables + if schema.tables[t].category_class != 'Set' + and cache.rows_for_block(t, bid) + ] - all_block_ids = sorted(set( - _all_cifflow_block_ids_for_tables(conn, all_table_names) - ) | { - r.get('_cifflow_block_id') for r in cache.all_fallback() - if r.get('_cifflow_block_id') - }) + pkreach_to_expanded: dict[frozenset, frozenset] = {} + for lt in loop_tables_present: + pr = pk_reachable_sets.get(lt, frozenset()) + if pr and pr not in pkreach_to_expanded: + pkreach_to_expanded[pr] = _expand_with_child_sets(pr, schema) - # Keyed Set tables: Set-class with at least one non-synthetic PK column. - keyed_set_tables = { - t: td - for t, td in schema.tables.items() - if td.category_class == 'Set' - and any(pk not in _SYNTHETIC for pk in td.primary_keys) - } + if not loop_tables_present: + fp = _fp_entries_for_expanded(cache, bid, keyed_set_tables, frozenset(keyed_set_tables.keys())) + return ([fp] if fp else []), frozenset() - def _block_fingerprint(bid: str) -> tuple[list[frozenset], frozenset]: - # Each distinct non-empty pkreach frozenset among loop tables produces one - # fingerprint entry (anchor group). This ensures that co-located but - # independently-anchored Sets (e.g. structure, model, space_group from - # atom_site / geom_angle / space_group_symop respectively) produce separate - # output blocks, while Sets jointly anchored by a single Loop table - # (e.g. pd_diffractogram + pd_phase via refln) stay together. - # - # When no loop tables are present, all keyed Sets go into one main fp. - loop_tables_present = [ - t for t in schema.tables - if schema.tables[t].category_class != 'Set' - and cache.rows_for_block(t, bid) - ] - - def _fp_entries_for_expanded(expanded: frozenset) -> frozenset: - entries: list[tuple] = [] - for t, td in keyed_set_tables.items(): - if t not in expanded: - continue - domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] - pk_vals_set: set[tuple] = set() - for r in cache.rows_for_block(t, bid): - tup = tuple(str(r.get(pk)) if r.get(pk) is not None else '' for pk in domain_pks) - if any(tup): - pk_vals_set.add(tup) - for _col, pk_json in cache.tag_presence(bid, t): - try: - vals = json.loads(pk_json) - tup = tuple(str(v) if v is not None else '' for v in vals) - if any(tup): - pk_vals_set.add(tup) - except Exception: - pass - if pk_vals_set: - entries.append((t, tuple(sorted(pk_vals_set)))) - return frozenset(entries) - - # Group loop tables by their pk_reachable frozenset (distinct non-empty values). - pkreach_to_expanded: dict[frozenset, frozenset] = {} - for lt in loop_tables_present: - pr = pk_reachable_sets.get(lt, frozenset()) - if pr and pr not in pkreach_to_expanded: - pkreach_to_expanded[pr] = _expand_with_child_sets(pr, schema) - - # No loop tables at all: one main fp with all keyed Sets (Set-only source block). - if not loop_tables_present: - fp = _fp_entries_for_expanded(frozenset(keyed_set_tables.keys())) - return ([fp] if fp else []), frozenset() - - # Loop tables present but none have PK-FK chains to any Set: treat like - # original pure-loop case — return empty main fps so the block routes to - # pure_loop_block_ids, with incidental entries for any keyed Sets present. - if not pkreach_to_expanded: - # All keyed Sets become incidental (same as when pk_reach_expanded=∅). - incidental: list[tuple] = [] - for t, td in keyed_set_tables.items(): - domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] - pk_vals_set: set[tuple] = set() - for r in cache.rows_for_block(t, bid): - tup = tuple(str(r.get(pk)) if r.get(pk) is not None else '' for pk in domain_pks) - if any(tup): - pk_vals_set.add(tup) - for _col, pk_json in cache.tag_presence(bid, t): - try: - vals = json.loads(pk_json) - tup = tuple(str(v) if v is not None else '' for v in vals) - if any(tup): - pk_vals_set.add(tup) - except Exception: - pass - if pk_vals_set: - incidental.append((t, tuple(sorted(pk_vals_set)))) - return [], frozenset(incidental) - - # One fp per distinct pkreach group. - main_fps: list[frozenset] = [] - accounted_sets: set[str] = set() - for pr, expanded in pkreach_to_expanded.items(): - fp = _fp_entries_for_expanded(expanded) - if fp: - main_fps.append(fp) - accounted_sets.update(expanded) - - # Incidental Sets: keyed Sets present but not in any pkreach group's expanded set. - fp_incidental_entries: list[tuple] = [] + if not pkreach_to_expanded: + incidental: list[tuple] = [] for t, td in keyed_set_tables.items(): - if t in accounted_sets: - continue - domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] - pk_vals_set: set[tuple] = set() - for r in cache.rows_for_block(t, bid): - tup = tuple(str(r.get(pk)) if r.get(pk) is not None else '' for pk in domain_pks) - if any(tup): - pk_vals_set.add(tup) - for _col, pk_json in cache.tag_presence(bid, t): - try: - vals = json.loads(pk_json) - tup = tuple(str(v) if v is not None else '' for v in vals) - if any(tup): - pk_vals_set.add(tup) - except Exception: - pass + pk_vals_set = _collect_set_pk_vals(cache, t, td, bid) if pk_vals_set: - fp_incidental_entries.append((t, tuple(sorted(pk_vals_set)))) - - # Drop child-Set tables from incidental entries: they will be collected - # inside the parent's incidental block by the child-Set BFS below. - # A child is an incidental table whose ALL domain PKs are FK columns - # pointing to other tables in the incidental set. - incidental_set = frozenset(t for t, _ in fp_incidental_entries) - to_drop: set[str] = set() - for inc_t, _ in fp_incidental_entries: - inc_td = keyed_set_tables[inc_t] - domain_pks_t = [pk for pk in inc_td.primary_keys if pk not in _SYNTHETIC] - fk_to_incidental: set[str] = set() - for fk in inc_td.foreign_keys: - if fk.target_table in incidental_set and fk.target_table != inc_t: - fk_to_incidental.update(fk.source_columns) - if domain_pks_t and all(pk in fk_to_incidental for pk in domain_pks_t): - to_drop.add(inc_t) - fp_incidental_entries = [ - (t, vals) for t, vals in fp_incidental_entries - if t not in to_drop - ] - - return main_fps, frozenset(fp_incidental_entries) - - # Precompute Set-class tables reachable from each non-Set table via FK chain. - # Used to route Loop rows that have no FK path to a fingerprint anchor into a - # separate orphan block rather than absorbing them into an unrelated anchor block. - reachable_sets: dict[str, frozenset[str]] = { - t: _reachable_set_tables(t, schema) - for t, td in schema.tables.items() - if td.category_class != 'Set' - } - - # Precompute Set-class tables reachable via PK FK columns only. Used to - # strip incidental Set anchors (e.g. instrument/diffractometer tables that - # are co-located in source blocks but whose identity is independent of the - # Loop tables' primary keys) from the anchor frozenset used for spec matching. - pk_reachable_sets: dict[str, frozenset[str]] = { - t: _pk_reachable_set_tables(t, schema) - for t, td in schema.tables.items() - if td.category_class != 'Set' - } - - fingerprint_to_block_ids: dict[frozenset, list[str]] = {} - pure_loop_block_ids: list[str] = [] - # incidental_groups: (table_name, pk_val_tuple) → set of source block_ids - incidental_groups: dict[tuple, set[str]] = {} + incidental.append((t, tuple(sorted(pk_vals_set)))) + return [], frozenset(incidental) + + main_fps: list[frozenset] = [] + accounted_sets: set[str] = set() + for pr, expanded in pkreach_to_expanded.items(): + fp = _fp_entries_for_expanded(cache, bid, keyed_set_tables, expanded) + if fp: + main_fps.append(fp) + accounted_sets.update(expanded) + + fp_incidental_entries: list[tuple] = [] + for t, td in keyed_set_tables.items(): + if t in accounted_sets: + continue + pk_vals_set = _collect_set_pk_vals(cache, t, td, bid) + if pk_vals_set: + fp_incidental_entries.append((t, tuple(sorted(pk_vals_set)))) - for bid in all_block_ids: - main_fps, incidental_fp = _block_fingerprint(bid) - if not main_fps: - pure_loop_block_ids.append(bid) - else: - for fp in main_fps: - fingerprint_to_block_ids.setdefault(fp, []).append(bid) - for t, pk_vals_tuple in incidental_fp: - for pk_val in pk_vals_tuple: - incidental_groups.setdefault((t, pk_val), set()).add(bid) + fp_incidental_entries = _drop_incidental_child_sets(fp_incidental_entries, keyed_set_tables) + return main_fps, frozenset(fp_incidental_entries) - # For non-Set tables with no FK path to any Set: decide whether to include - # in a fingerprint block or emit as a shared orphan block. - # - # Two strategies depending on whether T has reverse FKs (children that point to it): - # - HAS reverse FKs: use reverse-FK reachability — T is "needed by" FP if any child - # table has rows in FP's source blocks. This handles deduplication correctly - # (e.g. atom_type rows deduplicated to one block but atom_site spans many groups). - # - NO reverse FKs (leaf tables): use direct row ownership — T belongs to FP if - # any of FP's source block_ids directly own rows of T. Leaf tables like - # space_group_symop (no children because space_group is keyless) need this path. - no_set_fk_tables = {t for t in reachable_sets if not reachable_sets[t]} - # reverse_fk: T → set of tables R that have a FK pointing to T +def _compute_no_set_fk_routing( + no_set_fk_tables: set[str], + schema: SchemaSpec, + fingerprint_to_block_ids: dict[frozenset, list[str]], + cache: '_EmitCache', + pk_reachable_sets: dict, +) -> tuple[dict[str, frozenset], set[str]]: + """Classify Loop tables with no Set FK path as single-fp or orphan. + + Returns ``(single_fp_tables, orphan_tables)``. single_fp_tables maps a table + to the one fingerprint whose source blocks own all of its rows. orphan_tables + contains tables whose rows span two or more fingerprint groups. + """ reverse_fk: dict[str, set[str]] = {} for r_name, r_def in schema.tables.items(): for fk in r_def.foreign_keys: if fk.target_table in no_set_fk_tables: reverse_fk.setdefault(fk.target_table, set()).add(r_name) - # For each T, which fingerprint groups "need" it? table_to_needed_by: dict[str, set[frozenset]] = {t: set() for t in no_set_fk_tables} for fp, block_ids in fingerprint_to_block_ids.items(): fp_ts = frozenset(t for t, _ in fp) @@ -772,13 +689,10 @@ def _fp_entries_for_expanded(expanded: frozenset) -> frozenset: for t in no_set_fk_tables: refs = reverse_fk.get(t, set()) if refs: - # Reverse-FK strategy: only count child r as belonging to fp if - # r's pkreach is a subset of fp's expanded set (pkreach=∅ children - # pass through unchecked since they have no Set anchor constraint). for r in refs: pr_r = pk_reachable_sets.get(r, frozenset()) if pr_r and not pr_r.issubset(fp_ts_exp): - continue # r belongs to a different anchor group + continue found = False for bid in block_ids: if cache.rows_for_block(r, bid): @@ -788,179 +702,352 @@ def _fp_entries_for_expanded(expanded: frozenset) -> frozenset: table_to_needed_by[t].add(fp) break else: - # Leaf strategy: check direct row ownership. for bid in block_ids: if cache.rows_for_block(t, bid): table_to_needed_by[t].add(fp) break - single_fp_tables: dict[str, frozenset] = {} # table → the one fingerprint it maps to + single_fp_tables: dict[str, frozenset] = {} orphan_tables: set[str] = set() for t, fps in table_to_needed_by.items(): if len(fps) == 1: single_fp_tables[t] = next(iter(fps)) elif len(fps) > 1: orphan_tables.add(t) + return single_fp_tables, orphan_tables - # Sets that have at least one dedicated single-anchor fingerprint group (anchor_fs == {t}) - # OR appear as their own incidental block. Only these Sets are stripped to PK-only in - # multi-anchor bridge blocks; Sets that appear exclusively in bridge blocks keep their - # full data. - sets_with_own_block: set[str] = set() - for fp in fingerprint_to_block_ids: - afs = _fingerprint_anchor_fs(fp, schema) - if len(afs) == 1: - sets_with_own_block.add(next(iter(afs))) - for (t, _pk_val) in incidental_groups: - sets_with_own_block.add(t) - result: list[_BlockData] = [] - - # Orphan Loop rows: Loop tables with no FK path to any Set anchor whose rows - # span multiple fingerprint groups (shared reference data). Deduplicated by PK - # and emitted as a single block at the end. - orphan_by_table: dict[str, dict[tuple, dict]] = {} - orphan_block_ids: set[str] = set() - - # Union of all Set-class tables present in any main fingerprint. Used in - # incidental block processing to exclude Loop tables that are PK-reachable - # to a main-fingerprint Set (those belong in the main block, not incidental). - main_fp_set_tables = frozenset(t for fp in fingerprint_to_block_ids for t, _ in fp) - - for fp, block_ids in sorted(fingerprint_to_block_ids.items(), key=lambda x: sorted(x[1])): - table_rows: dict[str, list[dict]] = {} - # PK-reachable Set tables in this fingerprint, expanded to include their - # child-Set descendants. Used to restrict Set-row collection so that - # incidental Sets (present in source blocks but not linked to Loop data's PKs) - # are excluded and emitted as their own dedicated blocks. - fp_tables = frozenset(t for t, _ in fp) - fp_tables_expanded = _expand_with_child_sets(fp_tables, schema) - for t, td in schema.tables.items(): - if td.category_class == 'Set': - if t not in fp_tables_expanded: - continue # incidental Set — handled separately - # Set tables: collect via _fetch_rows_for_block so that non-winner - # contributions (tag_presence) are included. Deduplicate by PK, - # preferring the first occurrence (which is the winner row when - # the winning block comes first in sorted order). - by_pk: dict[tuple, dict] = {} +def _collect_fp_table_rows( + conn: duckdb.DuckDBPyConnection, + fp: frozenset, + block_ids: list[str], + schema: SchemaSpec, + cache: '_EmitCache', + pk_reachable_sets: dict, + fp_tables_expanded: frozenset, + orphan_tables: set[str], + orphan_by_table: dict[str, dict[tuple, dict]], + orphan_block_ids: set[str], + single_fp_tables: dict[str, frozenset], + reachable_sets: dict, +) -> dict[str, list[dict]]: + """Collect table rows for one fingerprint group. + + Mutates *orphan_by_table* and *orphan_block_ids* in place to accumulate + rows for tables that span multiple fingerprint groups. + """ + table_rows: dict[str, list[dict]] = {} + for t, td in schema.tables.items(): + if td.category_class == 'Set': + if t not in fp_tables_expanded: + continue + by_pk: dict[tuple, dict] = {} + for bid in sorted(block_ids): + for r in _fetch_rows_for_block(conn, bid, t, td, cache=cache): + pk_key = tuple(r.get(pk) for pk in td.primary_keys) + if pk_key not in by_pk: + by_pk[pk_key] = r + if by_pk: + table_rows[t] = sorted( + by_pk.values(), + key=lambda r: r.get('_cifflow_row_id', 0), + ) + else: + pr = pk_reachable_sets.get(t, frozenset()) + if pr and pr.issubset(fp_tables_expanded): + rows = [] for bid in sorted(block_ids): - for r in _fetch_rows_for_block(conn, bid, t, td, cache=cache): - pk_key = tuple(r.get(pk) for pk in td.primary_keys) - if pk_key not in by_pk: - by_pk[pk_key] = r - if by_pk: - table_rows[t] = sorted( - by_pk.values(), - key=lambda r: r.get('_cifflow_row_id', 0), - ) - else: - pr = pk_reachable_sets.get(t, frozenset()) - if pr and pr.issubset(fp_tables_expanded): - # PK-FK chain resolves entirely to this anchor group. + rows.extend(cache.rows_for_block(t, bid)) + if rows: + table_rows[t] = rows + elif not pr: + if t in orphan_tables: + domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] + tbl_orphan = orphan_by_table.setdefault(t, {}) + for bid in sorted(block_ids): + for r in cache.rows_for_block(t, bid): + pk_key = tuple(str(r.get(pk, '')) for pk in domain_pks) + if pk_key not in tbl_orphan: + tbl_orphan[pk_key] = r + orphan_block_ids.add(bid) + elif single_fp_tables.get(t) == fp: + rows = [] + for bid in sorted(block_ids): + rows.extend(cache.rows_for_block(t, bid)) + if rows: + table_rows[t] = rows + elif reachable_sets.get(t, frozenset()) & fp_tables_expanded: rows = [] for bid in sorted(block_ids): rows.extend(cache.rows_for_block(t, bid)) if rows: table_rows[t] = rows - elif not pr: - # No PK-FK to any Set: use orphan/single_fp routing or - # fall back to non-PK FK reachability. - if t in orphan_tables: - domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] - tbl_orphan = orphan_by_table.setdefault(t, {}) - for bid in sorted(block_ids): - for r in cache.rows_for_block(t, bid): - pk_key = tuple(str(r.get(pk, '')) for pk in domain_pks) - if pk_key not in tbl_orphan: - tbl_orphan[pk_key] = r - orphan_block_ids.add(bid) - elif single_fp_tables.get(t) == fp: - rows = [] - for bid in sorted(block_ids): - rows.extend(cache.rows_for_block(t, bid)) - if rows: - table_rows[t] = rows - elif reachable_sets.get(t, frozenset()) & fp_tables_expanded: - # Non-PK FK to a Set in this anchor: include directly. - rows = [] - for bid in sorted(block_ids): - rows.extend(cache.rows_for_block(t, bid)) - if rows: - table_rows[t] = rows + return table_rows - fallback: list[dict] = [] - for bid in sorted(block_ids): - fallback.extend(cache.fallback_for_block(bid)) - anchor_fs = frozenset(t for t, _ in fp) +def _compute_fp_anchor( + fp: frozenset, + schema: SchemaSpec, + table_rows: dict[str, list[dict]], + sets_with_own_block: set[str], +) -> tuple[frozenset, dict[str, list[str]]]: + """Compute anchor_fs (child-Set stripped) and anchor_kd; PK-strip bridge Set rows. + + Mutates *table_rows* in place: Set rows that have a dedicated single-anchor + block elsewhere are reduced to PK columns only in multi-anchor (bridge) blocks. + Returns ``(anchor_frozenset, anchor_key_dict)``. + """ + raw_fp_tables = frozenset(t for t, _ in fp) + anchor_fk_cols: set[str] = set() + for t in raw_fp_tables: + for fk in schema.tables[t].foreign_keys: + if fk.target_table in raw_fp_tables: + anchor_fk_cols.update(fk.source_columns) + + anchor_fs = frozenset( + t for t in raw_fp_tables + if not ( + (domain_pks := [pk for pk in schema.tables[t].primary_keys if pk not in _SYNTHETIC]) + and all(pk in anchor_fk_cols for pk in domain_pks) + ) + ) - # Compute FK columns within the fingerprint: for every anchor table, - # collect FK source columns that point to another anchor. These are the - # columns that identify child-Set tables (tables whose domain PKs are - # entirely composed of such FK columns). - anchor_fk_cols: set[str] = set() + if len(anchor_fs) > 1: for t in anchor_fs: - td = schema.tables[t] - for fk in td.foreign_keys: - if fk.target_table in anchor_fs: - anchor_fk_cols.update(fk.source_columns) - - # Strip child-Set tables from anchor_frozenset so that plan predicates - # such as only('diffrn_radiation') are not confused by co-located - # child-Set tables (e.g. diffrn_source whose sole PK is a FK to - # diffrn_radiation). Same exclusion rule as anchor_kd below. - anchor_fs = frozenset( - t for t in anchor_fs - if not ( - (domain_pks := [pk for pk in schema.tables[t].primary_keys if pk not in _SYNTHETIC]) - and all(pk in anchor_fk_cols for pk in domain_pks) - ) + if t in sets_with_own_block and t in table_rows: + pk_set = set(schema.tables[t].primary_keys) + table_rows[t] = [ + {k: v for k, v in r.items() if k in pk_set} + for r in table_rows[t] + ] + + anchor_kd: dict[str, list[str]] = {} + for t, pk_vals_tuple in sorted(fp, key=lambda x: x[0]): + domain_pks = [pk for pk in schema.tables[t].primary_keys if pk not in _SYNTHETIC] + if domain_pks and all(pk in anchor_fk_cols for pk in domain_pks): + continue + for pk_val_row in pk_vals_tuple: + for pk_col, val in zip(domain_pks, pk_val_row): + if val: + key = f'{t}.{pk_col}' + if val not in anchor_kd.setdefault(key, []): + anchor_kd[key].append(val) + + return anchor_fs, anchor_kd + + +def _compute_primary_fp_anchors( + fingerprint_to_block_ids: dict[frozenset, list[str]], + schema: SchemaSpec, +) -> set[tuple]: + """Return (table, pk_val_tuple) pairs that are primary (non-child-Set) fp anchors.""" + primary: set[tuple] = set() + for fp in fingerprint_to_block_ids: + fp_tables_set = frozenset(t for t, _ in fp) + fp_anchor_fk_cols: set[str] = set() + for fp_t in fp_tables_set: + for fk in schema.tables[fp_t].foreign_keys: + if fk.target_table in fp_tables_set: + fp_anchor_fk_cols.update(fk.source_columns) + for fp_t, pk_vals_tuple in fp: + fp_td = schema.tables[fp_t] + domain_pks = [pk for pk in fp_td.primary_keys if pk not in _SYNTHETIC] + if domain_pks and all(pk in fp_anchor_fk_cols for pk in domain_pks): + continue + for pv in pk_vals_tuple: + primary.add((fp_t, pv)) + return primary + + +def _collect_incidental_block_rows( + conn: duckdb.DuckDBPyConnection, + t: str, + td: TableDef, + pk_val_str: tuple, + block_ids_set: set[str], + inc_tables_expanded: frozenset, + schema: SchemaSpec, + cache: '_EmitCache', + main_fp_set_tables: frozenset, + pk_reachable_sets: dict, + reachable_sets: dict, +) -> dict[str, list[dict]]: + """Collect rows for one incidental Set block: anchor + reachable Loops + child-Sets via BFS. + + Returns an empty dict when no anchor rows match *pk_val_str* exactly. + """ + domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] + + by_pk: dict[tuple, dict] = {} + for bid in sorted(block_ids_set): + for r in _fetch_rows_for_block(conn, bid, t, td, cache=cache): + this_pk = tuple(str(r.get(pk, '')) if r.get(pk) is not None else '' for pk in domain_pks) + if this_pk == pk_val_str and this_pk not in by_pk: + by_pk[this_pk] = r + if not by_pk: + return {} + + inc_table_rows: dict[str, list[dict]] = { + t: sorted(by_pk.values(), key=lambda r: r.get('_cifflow_row_id', 0)) + } + + for loop_t, loop_td in schema.tables.items(): + if loop_td.category_class == 'Set': + continue + if not (reachable_sets.get(loop_t, frozenset()) & inc_tables_expanded): + continue + if pk_reachable_sets.get(loop_t, frozenset()) & main_fp_set_tables: + continue + rows = [] + for bid in sorted(block_ids_set): + rows.extend(cache.rows_for_block(loop_t, bid)) + if rows: + inc_table_rows[loop_t] = rows + + collected_set_rows: dict[str, dict[tuple, dict]] = {t: by_pk} + pending_child_sets = [ct for ct in sorted(inc_tables_expanded) if ct != t] + inc_table_rows.update( + _bfs_collect_child_sets( + pending_child_sets, block_ids_set, inc_tables_expanded, + schema, cache, conn, collected_set_rows, ) + ) + return inc_table_rows - # In multi-anchor (bridge) blocks, reduce anchor Set rows to PK columns only — - # but ONLY for Sets that also have a dedicated single-anchor block elsewhere. - # Sets that appear exclusively in bridge blocks keep their full data here. - if len(anchor_fs) > 1: - for t in anchor_fs: - if t in sets_with_own_block and t in table_rows: - td = schema.tables[t] - pk_set = set(td.primary_keys) - table_rows[t] = [ - {k: v for k, v in r.items() if k in pk_set} - for r in table_rows[t] - ] - - anchor_kd: dict[str, list[str]] = {} - for t, pk_vals_tuple in sorted(fp, key=lambda x: x[0]): - td = schema.tables[t] - domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] - if domain_pks and all(pk in anchor_fk_cols for pk in domain_pks): - continue # child-Set table: skip, its identity is derived from parent - for pk_val_row in pk_vals_tuple: - for pk_col, val in zip(domain_pks, pk_val_row): - if val: - key = f'{t}.{pk_col}' - if val not in anchor_kd.setdefault(key, []): - anchor_kd[key].append(val) - default_name = _default_block_name(anchor_kd) if anchor_kd else sorted(block_ids)[0] - fallback_name = _sanitize_block_name(default_name) or 'block' +def _bfs_collect_child_sets( + pending_child_sets: list[str], + block_ids_set: set[str], + inc_tables_expanded: frozenset, + schema: SchemaSpec, + cache: '_EmitCache', + conn: duckdb.DuckDBPyConnection, + collected_set_rows: dict[str, dict[tuple, dict]], +) -> dict[str, list[dict]]: + """Collect child-Set rows via BFS, filtering by parent FK values. + + Iterates until the pending list stops shrinking (fixed-point). Mutates + *collected_set_rows* in place so each BFS pass can reference already-collected + parents. Returns child rows keyed by table name. + """ + child_rows: dict[str, list[dict]] = {} + prev_pending_count = -1 + while pending_child_sets and len(pending_child_sets) != prev_pending_count: + prev_pending_count = len(pending_child_sets) + still_pending: list[str] = [] + for child_t in pending_child_sets: + child_td = schema.tables.get(child_t) + if child_td is None: + continue + child_domain_pks = [pk for pk in child_td.primary_keys if pk not in _SYNTHETIC] + fk_filter: dict[str, set[str]] = {} + all_parents_ready = True + for fk in child_td.foreign_keys: + if fk.target_table not in inc_tables_expanded: + continue + if fk.target_table not in collected_set_rows: + all_parents_ready = False + break + parent_rows = collected_set_rows[fk.target_table] + for src_col, tgt_col in zip(fk.source_columns, fk.target_columns): + if src_col in child_domain_pks: + fk_filter[src_col] = { + str(r.get(tgt_col, '')) for r in parent_rows.values() + } + if not all_parents_ready: + still_pending.append(child_t) + continue + by_child_pk: dict[tuple, dict] = {} + for bid in sorted(block_ids_set): + for r in _fetch_rows_for_block(conn, bid, child_t, child_td, cache=cache): + if fk_filter and not all( + str(r.get(sc, '')) in allowed_vals + for sc, allowed_vals in fk_filter.items() + ): + continue + child_pk_key = tuple(r.get(pk) for pk in child_td.primary_keys) + if child_pk_key not in by_child_pk: + by_child_pk[child_pk_key] = r + if by_child_pk: + child_rows[child_t] = sorted( + by_child_pk.values(), key=lambda r: r.get('_cifflow_row_id', 0) + ) + collected_set_rows[child_t] = by_child_pk + pending_child_sets = still_pending + return child_rows - result.append(_BlockData( - name=fallback_name, - table_rows=table_rows, - fallback_rows=fallback, - anchor_frozenset=anchor_fs, - anchor_key_dict=anchor_kd, - suppress_fk_pk=True, - suppress_all_fk_to_set=True, - dataset_id=_resolve_dataset_id(conn, set(block_ids), fallback_id), - )) - # Pure-loop blocks — one output block per source _cifflow_block_id. +def _build_grouped_state( + all_block_ids: list[str], + schema: SchemaSpec, + cache: '_EmitCache', +) -> tuple: + """Compute keyed-Set tables, reachability maps, and fingerprint groups for GROUPED mode. + + Returns a 9-tuple: + ``(keyed_set_tables, reachable_sets, pk_reachable_sets, + fingerprint_to_block_ids, pure_loop_block_ids, incidental_groups, + no_set_fk_tables, sets_with_own_block, main_fp_set_tables)`` + """ + keyed_set_tables = { + t: td + for t, td in schema.tables.items() + if td.category_class == 'Set' + and any(pk not in _SYNTHETIC for pk in td.primary_keys) + } + reachable_sets = { + t: _reachable_set_tables(t, schema) + for t, td in schema.tables.items() + if td.category_class != 'Set' + } + pk_reachable_sets = { + t: _pk_reachable_set_tables(t, schema) + for t, td in schema.tables.items() + if td.category_class != 'Set' + } + fingerprint_to_block_ids: dict[frozenset, list[str]] = {} + pure_loop_block_ids: list[str] = [] + incidental_groups: dict[tuple, set[str]] = {} + for bid in all_block_ids: + main_fps, incidental_fp = _compute_block_fingerprint( + bid, schema, cache, keyed_set_tables, pk_reachable_sets, + ) + if not main_fps: + pure_loop_block_ids.append(bid) + else: + for fp in main_fps: + fingerprint_to_block_ids.setdefault(fp, []).append(bid) + for t, pk_vals_tuple in incidental_fp: + for pk_val in pk_vals_tuple: + incidental_groups.setdefault((t, pk_val), set()).add(bid) + no_set_fk_tables = {t for t in reachable_sets if not reachable_sets[t]} + sets_with_own_block: set[str] = set() + for fp in fingerprint_to_block_ids: + afs = _fingerprint_anchor_fs(fp, schema) + if len(afs) == 1: + sets_with_own_block.add(next(iter(afs))) + for (t, _pk_val) in incidental_groups: + sets_with_own_block.add(t) + main_fp_set_tables = frozenset(t for fp in fingerprint_to_block_ids for t, _ in fp) + return ( + keyed_set_tables, reachable_sets, pk_reachable_sets, + fingerprint_to_block_ids, pure_loop_block_ids, incidental_groups, + no_set_fk_tables, sets_with_own_block, main_fp_set_tables, + ) + + +def _emit_pure_loop_blocks( + pure_loop_block_ids: list[str], + all_table_names: list[str], + cache: '_EmitCache', + conn: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + fallback_id: str | None, +) -> list['_BlockData']: + """Emit one output block per pure-loop source block (no keyed Set anchor).""" + result: list[_BlockData] = [] for bid in sorted(pure_loop_block_ids): - table_rows = {} + table_rows: dict[str, list[dict]] = {} for t in all_table_names: rows = cache.rows_for_block(t, bid) if rows: @@ -973,9 +1060,85 @@ def _fp_entries_for_expanded(expanded: frozenset) -> frozenset: suppress_all_fk_to_set=True, dataset_id=_resolve_dataset_id(conn, {bid}, fallback_id), )) + return result + + +def _collect_grouped( + conn: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + version: CifVersion, +) -> list[_BlockData]: + """Group source blocks by Set-identity fingerprint (GROUPED mode). + + The fingerprint of a source ``_cifflow_block_id`` is the frozenset of + ``(table_name, sorted_pk_value_tuples)`` for every keyed Set-class table + that *directly owns* rows in that block (winner ``_cifflow_block_id`` match). + Source blocks with identical fingerprints are merged into one output block. + Blocks with an empty fingerprint (no owned keyed Set rows) are passed + through unchanged as pure-loop blocks, one per source ``_cifflow_block_id``. + + Row collection is by ``_cifflow_block_id`` membership only — no FK-graph + traversal is required. ``anchor_frozenset`` reflects all keyed Set tables + in the fingerprint, enabling multi-anchor predicates such as + ``all_of('pd_diffractogram', 'pd_phase')`` to match bridge blocks. + + Set-class tables that are present in a source block but NOT reachable from + any Loop table via PK-only FK chains are treated as *incidental* — they do + not widen the main fingerprint and are instead emitted as their own + dedicated output blocks with a single-table anchor frozenset. + """ + fallback_id: str | None = None + cache = _EmitCache(conn, schema) + all_table_names = list(schema.tables.keys()) + + all_block_ids = sorted(set( + _all_cifflow_block_ids_for_tables(conn, all_table_names) + ) | { + r.get('_cifflow_block_id') for r in cache.all_fallback() + if r.get('_cifflow_block_id') + }) + + ( + keyed_set_tables, reachable_sets, pk_reachable_sets, + fingerprint_to_block_ids, pure_loop_block_ids, incidental_groups, + no_set_fk_tables, sets_with_own_block, main_fp_set_tables, + ) = _build_grouped_state(all_block_ids, schema, cache) + + single_fp_tables, orphan_tables = _compute_no_set_fk_routing( + no_set_fk_tables, schema, fingerprint_to_block_ids, cache, pk_reachable_sets, + ) + + result: list[_BlockData] = [] + orphan_by_table: dict[str, dict[tuple, dict]] = {} + orphan_block_ids: set[str] = set() + + for fp, block_ids in sorted(fingerprint_to_block_ids.items(), key=lambda x: sorted(x[1])): + fp_tables_expanded = _expand_with_child_sets(frozenset(t for t, _ in fp), schema) + table_rows = _collect_fp_table_rows( + conn, fp, block_ids, schema, cache, pk_reachable_sets, + fp_tables_expanded, orphan_tables, orphan_by_table, orphan_block_ids, + single_fp_tables, reachable_sets, + ) + fallback: list[dict] = [] + for bid in sorted(block_ids): + fallback.extend(cache.fallback_for_block(bid)) + anchor_fs, anchor_kd = _compute_fp_anchor(fp, schema, table_rows, sets_with_own_block) + default_name = _default_block_name(anchor_kd) if anchor_kd else sorted(block_ids)[0] + result.append(_BlockData( + name=_sanitize_block_name(default_name) or 'block', + table_rows=table_rows, + fallback_rows=fallback, + anchor_frozenset=anchor_fs, + anchor_key_dict=anchor_kd, + suppress_fk_pk=True, + suppress_all_fk_to_set=True, + dataset_id=_resolve_dataset_id(conn, set(block_ids), fallback_id), + )) + + result.extend(_emit_pure_loop_blocks( + pure_loop_block_ids, all_table_names, cache, conn, schema, fallback_id, + )) - # Orphan Loop block — Loop tables with no FK path to any fingerprint anchor. - # Their rows were excluded from fingerprint group blocks; emit them together here. if orphan_by_table: orphan_table_rows: dict[str, list[dict]] = {} for t in sorted(orphan_by_table): @@ -996,141 +1159,36 @@ def _fp_entries_for_expanded(expanded: frozenset) -> frozenset: dataset_id=_resolve_dataset_id(conn, orphan_block_ids, fallback_id), )) - # Sets that are PRIMARY anchors of main fingerprint blocks — these have already - # been fully emitted there. An incidental block for such a Set is redundant - # unless it is needed to carry child-Set data (e.g. chemical_formula keyed on - # pd_phase). A table is "primary" if its domain PKs are NOT all FK source - # columns pointing to other tables within the same fingerprint. - primary_fp_anchors: set[tuple] = set() - for fp in fingerprint_to_block_ids: - fp_tables_set = frozenset(t for t, _ in fp) - fp_anchor_fk_cols: set[str] = set() - for fp_t in fp_tables_set: - for fk in schema.tables[fp_t].foreign_keys: - if fk.target_table in fp_tables_set: - fp_anchor_fk_cols.update(fk.source_columns) - for fp_t, pk_vals_tuple in fp: - fp_td = schema.tables[fp_t] - domain_pks = [pk for pk in fp_td.primary_keys if pk not in _SYNTHETIC] - if domain_pks and all(pk in fp_anchor_fk_cols for pk in domain_pks): - continue # child Set — not a primary anchor - for pv in pk_vals_tuple: - primary_fp_anchors.add((fp_t, pv)) - - # Incidental Set blocks — one block per unique (table, pk_val) combination. - # These are Set tables that were present in source blocks but are not - # PK-reachable from any Loop table, so they are emitted separately rather - # than being absorbed into the main fingerprint blocks. - # Loop tables included here are those that FK-reach the incidental Set but - # have no PK-reachable path to any main-fingerprint Set. + primary_fp_anchors = _compute_primary_fp_anchors(fingerprint_to_block_ids, schema) + for (t, pk_val), block_ids_set in sorted( incidental_groups.items(), key=lambda x: (x[0][0], x[0][1]), ): - # Skip if already emitted as a primary anchor AND it has no child Sets - # whose data is only reachable via this incidental block. if (t, pk_val) in primary_fp_anchors: if _expand_with_child_sets(frozenset({t}), schema) == frozenset({t}): continue td = schema.tables[t] domain_pks = [pk for pk in td.primary_keys if pk not in _SYNTHETIC] pk_val_str = tuple(str(v) if v is not None else '' for v in pk_val) - - # Collect Set rows for this exact pk_val. - by_pk: dict[tuple, dict] = {} - for bid in sorted(block_ids_set): - for r in _fetch_rows_for_block(conn, bid, t, td, cache=cache): - this_pk = tuple(str(r.get(pk, '')) if r.get(pk) is not None else '' for pk in domain_pks) - if this_pk == pk_val_str and this_pk not in by_pk: - by_pk[this_pk] = r - if not by_pk: - continue - - inc_table_rows: dict[str, list[dict]] = { - t: sorted(by_pk.values(), key=lambda r: r.get('_cifflow_row_id', 0)) - } - - # Include Loop tables that FK-reach this incidental Set but have no - # PK-reachable path to any main-fingerprint Set table. inc_tables_expanded = _expand_with_child_sets(frozenset({t}), schema) - for loop_t, loop_td in schema.tables.items(): - if loop_td.category_class == 'Set': - continue - if not (reachable_sets.get(loop_t, frozenset()) & inc_tables_expanded): - continue - if pk_reachable_sets.get(loop_t, frozenset()) & main_fp_set_tables: - continue # belongs in a main fingerprint block - rows = [] - for bid in sorted(block_ids_set): - rows.extend(cache.rows_for_block(loop_t, bid)) - if rows: - inc_table_rows[loop_t] = rows - - # Include child-Set tables (e.g. chemical_formula keyed on pd_phase). - # Process BFS so parents are always collected before their children. - # collected_set_rows: table → {pk_key: row} for FK-value filtering. - collected_set_rows: dict[str, dict[tuple, dict]] = {t: by_pk} - pending_child_sets = [ct for ct in sorted(inc_tables_expanded) if ct != t] - prev_pending_count = -1 - while pending_child_sets and len(pending_child_sets) != prev_pending_count: - prev_pending_count = len(pending_child_sets) - still_pending: list[str] = [] - for child_t in pending_child_sets: - child_td = schema.tables.get(child_t) - if child_td is None: - continue - child_domain_pks = [pk for pk in child_td.primary_keys if pk not in _SYNTHETIC] - # Build allowed-value sets per FK src column from already-collected parents. - fk_filter: dict[str, set[str]] = {} - all_parents_ready = True - for fk in child_td.foreign_keys: - if fk.target_table not in inc_tables_expanded: - continue - if fk.target_table not in collected_set_rows: - all_parents_ready = False - break - parent_rows = collected_set_rows[fk.target_table] - for src_col, tgt_col in zip(fk.source_columns, fk.target_columns): - if src_col in child_domain_pks: - fk_filter[src_col] = { - str(r.get(tgt_col, '')) for r in parent_rows.values() - } - if not all_parents_ready: - still_pending.append(child_t) - continue - by_child_pk: dict[tuple, dict] = {} - for bid in sorted(block_ids_set): - for r in _fetch_rows_for_block(conn, bid, child_t, child_td, cache=cache): - if fk_filter and not all( - str(r.get(sc, '')) in allowed_vals - for sc, allowed_vals in fk_filter.items() - ): - continue - child_pk_key = tuple(r.get(pk) for pk in child_td.primary_keys) - if child_pk_key not in by_child_pk: - by_child_pk[child_pk_key] = r - if by_child_pk: - inc_table_rows[child_t] = sorted( - by_child_pk.values(), key=lambda r: r.get('_cifflow_row_id', 0) - ) - collected_set_rows[child_t] = by_child_pk - pending_child_sets = still_pending - - inc_anchor_fs = frozenset({t}) + inc_table_rows = _collect_incidental_block_rows( + conn, t, td, pk_val_str, block_ids_set, inc_tables_expanded, + schema, cache, main_fp_set_tables, pk_reachable_sets, reachable_sets, + ) + if not inc_table_rows: + continue inc_anchor_kd: dict[str, list[str]] = {} for pk_col, val in zip(domain_pks, pk_val): v = str(val) if val is not None else '' if v: inc_anchor_kd[f'{t}.{pk_col}'] = [v] - default_name = _default_block_name(inc_anchor_kd) if inc_anchor_kd else t - fallback_name = _sanitize_block_name(default_name) or t - result.append(_BlockData( - name=fallback_name, + name=_sanitize_block_name(default_name) or t, table_rows=inc_table_rows, fallback_rows=[], - anchor_frozenset=inc_anchor_fs, + anchor_frozenset=frozenset({t}), anchor_key_dict=inc_anchor_kd, suppress_fk_pk=True, suppress_all_fk_to_set=True, @@ -1208,24 +1266,20 @@ def _replace_anchor( ) -def _collect_structure( - conn: duckdb.DuckDBPyConnection, - schema: SchemaSpec, - version: CifVersion, -) -> list[_BlockData]: - """STRUCTURE mode: GROUPED plus absorption of pd_phase/space_group/model into structure blocks. - - Satellite blocks (pd_phase, space_group, model with a single structure referent) whose - data is absorbed are removed from the top-level output. Unreferenced satellites and - models with multiple structure referents are emitted unchanged. - """ - grouped = _collect_grouped(conn, schema, version) - - # --- Segregate --- +def _segregate_structure_blocks( + grouped: list['_BlockData'], + schema: 'SchemaSpec | None' = None, +) -> tuple[ + list['_BlockData'], + dict[str, '_BlockData'], + dict[str, '_BlockData'], + dict[str, list['_BlockData']], + list['_BlockData'], +]: + """Partition grouped blocks into structure, pd_phase, space_group, model, and other buckets.""" structure_blocks: list[_BlockData] = [] - pd_phase_blocks: dict[str, _BlockData] = {} # phase_id → block - space_group_blocks: dict[str, _BlockData] = {} # sg_id → block - # structure_id → list of model blocks whose model.structure_id matches + pd_phase_blocks: dict[str, _BlockData] = {} + space_group_blocks: dict[str, _BlockData] = {} model_blocks_by_struct: dict[str, list[_BlockData]] = {} other_blocks: list[_BlockData] = [] @@ -1235,13 +1289,13 @@ def _collect_structure( structure_blocks.append(block) elif afs == frozenset({'pd_phase'}): for phase_id in block.anchor_key_dict.get('pd_phase.id', []): - if phase_id in pd_phase_blocks: + if phase_id in pd_phase_blocks and schema is not None: pd_phase_blocks[phase_id] = _merge_blocks_into(pd_phase_blocks[phase_id], [block], schema) else: pd_phase_blocks[phase_id] = block elif afs == frozenset({'space_group'}): for sg_id in block.anchor_key_dict.get('space_group.id', []): - if sg_id in space_group_blocks: + if sg_id in space_group_blocks and schema is not None: space_group_blocks[sg_id] = _merge_blocks_into(space_group_blocks[sg_id], [block], schema) else: space_group_blocks[sg_id] = block @@ -1259,51 +1313,82 @@ def _collect_structure( else: other_blocks.append(block) - # --- Merge satellites into structure blocks --- - consumed_block_ids: set[int] = set() # id() of absorbed _BlockData objects + return structure_blocks, pd_phase_blocks, space_group_blocks, model_blocks_by_struct, other_blocks + + +def _collect_satellite_merges( + block: '_BlockData', + pd_phase_blocks: dict[str, '_BlockData'], + space_group_blocks: dict[str, '_BlockData'], + model_blocks_by_struct: dict[str, list['_BlockData']], + consumed_block_ids: set[int], +) -> list['_BlockData']: + """Return satellite blocks (pd_phase, space_group, single model) to merge into a structure block. + + Mutates consumed_block_ids with the id()s of selected satellites. + """ + to_merge: list[_BlockData] = [] + seen_src_ids: set[int] = set() + + for row in block.table_rows.get('structure', []): + phase_id = row.get('phase_id') + if phase_id and phase_id not in ('.', '?'): + src = pd_phase_blocks.get(str(phase_id)) + if src is not None and id(src) not in seen_src_ids: + to_merge.append(src) + seen_src_ids.add(id(src)) + consumed_block_ids.add(id(src)) + + sg_id = row.get('space_group_id') + if sg_id and sg_id not in ('.', '?'): + src = space_group_blocks.get(str(sg_id)) + if src is not None and id(src) not in seen_src_ids: + to_merge.append(src) + seen_src_ids.add(id(src)) + consumed_block_ids.add(id(src)) + + struct_id = row.get('id') + if struct_id is not None: + models = model_blocks_by_struct.get(str(struct_id), []) + if len(models) == 1 and id(models[0]) not in seen_src_ids: + to_merge.append(models[0]) + seen_src_ids.add(id(models[0])) + consumed_block_ids.add(id(models[0])) + + return to_merge + + +def _collect_structure( + conn: duckdb.DuckDBPyConnection, + schema: SchemaSpec, + version: CifVersion, +) -> list[_BlockData]: + """STRUCTURE mode: GROUPED plus absorption of pd_phase/space_group/model into structure blocks. + + Satellite blocks (pd_phase, space_group, model with a single structure referent) whose + data is absorbed are removed from the top-level output. Unreferenced satellites and + models with multiple structure referents are emitted unchanged. + """ + grouped = _collect_grouped(conn, schema, version) + structure_blocks, pd_phase_blocks, space_group_blocks, model_blocks_by_struct, other_blocks = ( + _segregate_structure_blocks(grouped, schema) + ) + + consumed_block_ids: set[int] = set() result: list[_BlockData] = [] for block in structure_blocks: - struct_rows = block.table_rows.get('structure', []) - to_merge: list[_BlockData] = [] - seen_src_ids: set[int] = set() - - for row in struct_rows: - phase_id = row.get('phase_id') - if phase_id and phase_id not in ('.', '?'): - src = pd_phase_blocks.get(str(phase_id)) - if src is not None and id(src) not in seen_src_ids: - to_merge.append(src) - seen_src_ids.add(id(src)) - consumed_block_ids.add(id(src)) - - sg_id = row.get('space_group_id') - if sg_id and sg_id not in ('.', '?'): - src = space_group_blocks.get(str(sg_id)) - if src is not None and id(src) not in seen_src_ids: - to_merge.append(src) - seen_src_ids.add(id(src)) - consumed_block_ids.add(id(src)) - - struct_id = row.get('id') - if struct_id is not None: - models = model_blocks_by_struct.get(str(struct_id), []) - if len(models) == 1 and id(models[0]) not in seen_src_ids: - to_merge.append(models[0]) - seen_src_ids.add(id(models[0])) - consumed_block_ids.add(id(models[0])) - + to_merge = _collect_satellite_merges( + block, pd_phase_blocks, space_group_blocks, model_blocks_by_struct, consumed_block_ids, + ) if to_merge: original_anchor_fs = block.anchor_frozenset original_anchor_kd = block.anchor_key_dict block = _merge_blocks_into(block, to_merge, schema) - # Satellites are absorbed data passengers, not co-equal anchors. - # Keep the structure block's anchor identity so plan predicates - # such as only('structure') still match in STRUCTURE mode. + # Satellites are absorbed data passengers; keep structure block's anchor identity. block = _replace_anchor(block, original_anchor_fs, original_anchor_kd) result.append(block) - # --- Emit unconsumed satellite blocks --- seen_output_ids: set[int] = set() all_satellites = ( list(pd_phase_blocks.values()) @@ -1501,6 +1586,162 @@ def _resolve_dataset_id( return fallback +def _validate_all_blocks_preconditions( + conn: duckdb.DuckDBPyConnection, + schema: SchemaSpec, +) -> None: + """Raise ValueError if the database cannot support ALL_BLOCKS emission.""" + fallback_count = conn.execute('SELECT COUNT(*) FROM "_cif_fallback"').fetchone()[0] + if fallback_count: + raise ValueError( + f"ALL_BLOCKS requires all tags to be known to the dictionary, but " + f"{fallback_count} fallback row(s) are present in _cif_fallback. " + f"Unknown tags cannot be reliably assigned to a dictionary-split block." + ) + + keyless_problems: list[str] = [] + for table_name, tdef in schema.tables.items(): + if tdef.category_class != 'Set': + continue + domain_pks = [pk for pk in tdef.primary_keys if pk not in _SYNTHETIC] + if domain_pks: + continue + count = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()[0] + if count: + keyless_problems.append(f"{table_name} ({count} row(s))") + if keyless_problems: + raise ValueError( + f"ALL_BLOCKS requires every Set category to have a domain primary key, " + f"but the following keyless Set table(s) contain data: " + f"{', '.join(keyless_problems)}. " + f"Rows in keyless Sets cannot be unambiguously associated with a " + f"dictionary-split block." + ) + + +def _inject_set_parents( + block_table_rows: 'dict[str, list[dict]]', + parent_tables: 'list[str]', + set_key_cols: 'list[tuple]', + vals: list, +) -> None: + """Inject synthetic single-row Set parent entries into *block_table_rows*. + + For each (set_table, set_col, val) triple, inserts a minimal row so that + the rendering layer can suppress the FK column and emit the parent tag as + a scalar above the data. Populates *parent_tables* as a side-effect. + """ + for (_col, _parent_tag, set_table, set_col), val in zip(set_key_cols, vals): + if set_table and val is not None: + block_table_rows[set_table] = [ + {'_cifflow_block_id': '', '_cifflow_row_id': 0, set_col: val} + ] + parent_tables.append(set_table) + + +def _collect_set_table_blocks( + table_name: str, + rows: 'list[dict]', + tdef: 'TableDef', + domain_pks: 'list[str]', + schema: SchemaSpec, + conn: duckdb.DuckDBPyConnection, + fallback_id: 'str | None', +) -> 'list[_BlockData]': + """Build one _BlockData per Set-category row.""" + col_info = _classify_pk_cols(tdef, schema) + set_key_cols = [(col, tag, st, sc) for col, is_set, tag, st, sc in col_info if is_set] + result: list[_BlockData] = [] + + for row in sorted(rows, key=lambda r: tuple(r.get(pk) or '' for pk in domain_pks)): + pk_vals = [str(row.get(pk) or '') for pk in domain_pks] + block_name = _sanitize_block_name('_'.join([table_name] + pk_vals)) or table_name + + block_table_rows: dict[str, list[dict]] = {table_name: [row]} + parent_tables: list[str] = [] + if set_key_cols: + _inject_set_parents( + block_table_rows, parent_tables, set_key_cols, + [row.get(col) for col, _, _, _ in set_key_cols], + ) + + cat_order = sorted(parent_tables) + [table_name] if parent_tables else None + did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id')}, fallback_id) + result.append(_BlockData( + name=block_name, + table_rows=block_table_rows, + fallback_rows=[], + anchor_frozenset=frozenset(), + anchor_key_dict={}, + suppress_fk_pk=bool(set_key_cols), + dataset_id=did, + preferred_category_order=cat_order, + )) + return result + + +def _collect_loop_table_blocks( + table_name: str, + rows: 'list[dict]', + tdef: 'TableDef', + schema: SchemaSpec, + conn: duckdb.DuckDBPyConnection, + fallback_id: 'str | None', +) -> 'list[_BlockData]': + """Build _BlockData entries for a Loop-category table. + + Pure Loop (no Set FK in PK) → one block for all rows. + Loop with Set FK(s) in PK → one block per unique Set-key combination. + """ + col_info = _classify_pk_cols(tdef, schema) + set_key_cols = [(col, tag, st, sc) for col, is_set, tag, st, sc in col_info if is_set] + result: list[_BlockData] = [] + + if not set_key_cols: + block_name = _sanitize_block_name(table_name) or table_name + did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id') for row in rows}, fallback_id) + result.append(_BlockData( + name=block_name, + table_rows={table_name: rows}, + fallback_rows=[], + anchor_frozenset=frozenset(), + anchor_key_dict={}, + suppress_fk_pk=False, + dataset_id=did, + )) + else: + groups: dict[tuple, list[dict]] = {} + for row in rows: + key = tuple(row.get(col) for col, _, _, _ in set_key_cols) + groups.setdefault(key, []).append(row) + + for set_vals in sorted(groups, key=lambda t: tuple(v or '' for v in t)): + group_rows = groups[set_vals] + val_strs = [str(v or '') for v in set_vals] + block_name = _sanitize_block_name('_'.join([table_name] + val_strs)) or table_name + + block_table_rows: dict[str, list[dict]] = {table_name: group_rows} + parent_tables: list[str] = [] + _inject_set_parents( + block_table_rows, parent_tables, set_key_cols, list(set_vals), + ) + + cat_order = sorted(parent_tables) + [table_name] + did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id') for row in group_rows}, fallback_id) + result.append(_BlockData( + name=block_name, + table_rows=block_table_rows, + fallback_rows=[], + anchor_frozenset=frozenset(), + anchor_key_dict={}, + suppress_fk_pk=True, + suppress_loop_fk_pk=True, + dataset_id=did, + preferred_category_order=cat_order, + )) + return result + + def _collect_all_blocks( conn: duckdb.DuckDBPyConnection, schema: SchemaSpec, @@ -1529,34 +1770,7 @@ def _collect_all_blocks( keyless Set tables — neither can be unambiguously assigned to a dictionary-split block. """ - # Guard: fallback rows - fallback_count = conn.execute('SELECT COUNT(*) FROM "_cif_fallback"').fetchone()[0] - if fallback_count: - raise ValueError( - f"ALL_BLOCKS requires all tags to be known to the dictionary, but " - f"{fallback_count} fallback row(s) are present in _cif_fallback. " - f"Unknown tags cannot be reliably assigned to a dictionary-split block." - ) - - # Guard: keyless Set tables (Set tables with no domain primary key) - keyless_problems: list[str] = [] - for table_name, tdef in schema.tables.items(): - if tdef.category_class != 'Set': - continue - domain_pks = [pk for pk in tdef.primary_keys if pk not in _SYNTHETIC] - if domain_pks: - continue - count = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()[0] - if count: - keyless_problems.append(f"{table_name} ({count} row(s))") - if keyless_problems: - raise ValueError( - f"ALL_BLOCKS requires every Set category to have a domain primary key, " - f"but the following keyless Set table(s) contain data: " - f"{', '.join(keyless_problems)}. " - f"Rows in keyless Sets cannot be unambiguously associated with a " - f"dictionary-split block." - ) + _validate_all_blocks_preconditions(conn, schema) fallback_id: str | None = str(uuid.uuid4()) if version == CifVersion.CIF_2_0 else None result: list[_BlockData] = [] @@ -1570,96 +1784,13 @@ def _collect_all_blocks( domain_pks = [pk for pk in tdef.primary_keys if pk not in _SYNTHETIC] if tdef.category_class == 'Set': - # One block per row. - # Classify PK columns: some may FK to a parent Set category. - col_info = _classify_pk_cols(tdef, schema) - set_key_cols = [(col, tag, st, sc) for col, is_set, tag, st, sc in col_info if is_set] - - for row in sorted(rows, key=lambda r: tuple(r.get(pk) or '' for pk in domain_pks)): - pk_vals = [str(row.get(pk) or '') for pk in domain_pks] - block_name = _sanitize_block_name('_'.join([table_name] + pk_vals)) or table_name - - block_table_rows: dict[str, list[dict]] = {table_name: [row]} - parent_tables: list[str] = [] - if set_key_cols: - # Inject synthetic parent rows so _suppressed_fk_pk_cols - # suppresses the FK column and the parent tag is emitted as a scalar. - for (col, _parent_tag, set_table, set_col) in set_key_cols: - val = row.get(col) - if set_table and val is not None: - block_table_rows[set_table] = [ - {'_cifflow_block_id': '', '_cifflow_row_id': 0, set_col: val} - ] - parent_tables.append(set_table) - - cat_order = sorted(parent_tables) + [table_name] if parent_tables else None - did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id')}, fallback_id) - result.append(_BlockData( - name=block_name, - table_rows=block_table_rows, - fallback_rows=[], - anchor_frozenset=frozenset(), - anchor_key_dict={}, - suppress_fk_pk=bool(set_key_cols), - dataset_id=did, - preferred_category_order=cat_order, - )) + result.extend(_collect_set_table_blocks( + table_name, rows, tdef, domain_pks, schema, conn, fallback_id, + )) else: - # Loop category: classify PK columns. - col_info = _classify_pk_cols(tdef, schema) - set_key_cols = [(col, tag, st, sc) for col, is_set, tag, st, sc in col_info if is_set] - - if not set_key_cols: - # Pure Loop — one block for all rows. - block_name = _sanitize_block_name(table_name) or table_name - did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id') for row in rows}, fallback_id) - result.append(_BlockData( - name=block_name, - table_rows={table_name: rows}, - fallback_rows=[], - anchor_frozenset=frozenset(), - anchor_key_dict={}, - suppress_fk_pk=False, - dataset_id=did, - )) - else: - # Group rows by Set-key tuple. - groups: dict[tuple, list[dict]] = {} - for row in rows: - key = tuple(row.get(col) for col, _, _, _ in set_key_cols) - groups.setdefault(key, []).append(row) - - for set_vals in sorted(groups, key=lambda t: tuple(v or '' for v in t)): - group_rows = groups[set_vals] - val_strs = [str(v or '') for v in set_vals] - block_name = _sanitize_block_name('_'.join([table_name] + val_strs)) or table_name - - # Inject synthetic single-row Set parent entries so that - # _suppressed_fk_pk_cols can find them (suppressing the FK - # columns from the loop) and _render_set_category emits them - # as scalar tag-value pairs above the loop. - block_table_rows = {table_name: group_rows} - parent_tables = [] - for (col, _parent_tag, set_table, set_col), val in zip(set_key_cols, set_vals): - if set_table and val is not None: - block_table_rows[set_table] = [ - {'_cifflow_block_id': '', '_cifflow_row_id': 0, set_col: val} - ] - parent_tables.append(set_table) - - cat_order = sorted(parent_tables) + [table_name] - did = _resolve_dataset_id(conn, {row.get('_cifflow_block_id') for row in group_rows}, fallback_id) - result.append(_BlockData( - name=block_name, - table_rows=block_table_rows, - fallback_rows=[], - anchor_frozenset=frozenset(), - anchor_key_dict={}, - suppress_fk_pk=True, - suppress_loop_fk_pk=True, - dataset_id=did, - preferred_category_order=cat_order, - )) + result.extend(_collect_loop_table_blocks( + table_name, rows, tdef, schema, conn, fallback_id, + )) return result @@ -1668,35 +1799,171 @@ def _collect_all_blocks( # Block renderer # --------------------------------------------------------------------------- -def _render_block( - block_name: str, - data: _BlockData, +def _render_merge_group_item( + item: list[str], + data: '_BlockData', schema: SchemaSpec, version: CifVersion, - spec: BlockSpec | None, + spec: 'BlockSpec | None', reconstruct_su: bool, pretty: bool, - line_limit: int | None = None, + line_limit: int | None, + extra_cols_for: dict, ) -> list[str]: - """Render a single CIF block as a flat list of output lines.""" - if version == CifVersion.CIF_1_1 and len(block_name) > 75: - raise ValueError( - f"CIF 1.1 block code {block_name!r} exceeds the 75-character identifier " - f"limit (length {len(block_name)})" + """Render one merge-group item from _ordered_categories.""" + if data.suppress_loop_fk_pk: + # ORIGINAL mode: join positionally by _loop_id + _iter_idx + return _render_original_loop_group( + item, data.table_rows, data, schema, version, spec, + reconstruct_su, pretty, line_limit, extra_cols_for, ) - lines: list[str] = [f'data_{block_name}'] - first_category = True + suppress_pkg = None + if data.suppress_fk_pk and data.suppress_all_fk_to_set: + first_present = next((c for c in item if data.table_rows.get(c)), None) + if first_present: + ftdef = schema.tables.get(first_present) + frows = data.table_rows.get(first_present, []) + if ftdef and frows: + suppress_pkg = _suppressed_fk_pk_cols(ftdef, frows, data.table_rows, schema) + return _render_merge_group( + item, data.table_rows, schema, version, spec, + reconstruct_su, pretty, line_limit, + suppress_pk_cols=suppress_pkg, + suppress_all_fk_to_set=data.suppress_all_fk_to_set, + ) + + +def _render_single_table_item( + table_name: str, + data: '_BlockData', + schema: SchemaSpec, + version: CifVersion, + spec: 'BlockSpec | None', + reconstruct_su: bool, + pretty: bool, + line_limit: int | None, + extra_cols_for: dict, +) -> list[str]: + """Render one single-table item from _ordered_categories. + + Returns [] to signal the table should be skipped. + """ + rows = data.table_rows.get(table_name) + if not rows: + return [] + table_def = schema.tables[table_name] + cols = _active_cols(table_def, rows, spec, reconstruct_su) + if not cols: + return [] + + if data.suppress_fk_pk and ( + (table_def.category_class == 'Set' and len(rows) == 1) + or data.suppress_loop_fk_pk + or data.suppress_all_fk_to_set + ): + suppressed = _suppressed_fk_pk_cols( + table_def, rows, data.table_rows, schema, + suppress_all_to_set=data.suppress_all_fk_to_set, + ) + cols = [c for c in cols if c not in suppressed] + if not cols: + return [] + + if data.suppress_all_fk_to_set: + # GROUPED mode: suppress columns whose every value is '.' (inapplicable). + cols = [c for c in cols if not all(row.get(c) == '.' for row in rows)] + if not cols: + return [] + + extra = extra_cols_for.get(table_name) + if table_def.category_class == 'Set' and len(rows) == 1: + return _render_set_category( + rows[0], cols, table_name, schema, version, table_def, + reconstruct_su, pretty, line_limit, + ) + return _render_loop_category( + rows, cols, table_name, schema, version, table_def, + reconstruct_su, pretty, line_limit, extra_fallback_cols=extra, + ) + + +def _collect_remnant_rows( + extra_cols_for: dict, + table_rows: dict, + initial_rows: list[dict], +) -> list[dict]: + """Collect fallback rows not rendered as part of any table.""" + result = initial_rows + for ref, cols_list in extra_cols_for.items(): + if ref not in table_rows: + for tag, _ci, row_vals in cols_list: + for _rid, (val, vtype) in row_vals.items(): + result.append({'tag': tag, 'value': val, 'value_type': vtype}) + return result + + +def _inject_header_items( + lines: list[str], + data: '_BlockData', + schema: SchemaSpec, + version: CifVersion, + remnant_rows: list[dict], + audit_id_tag: str, +) -> bool: + """Inject conformance tags and _audit_dataset.id before main block content. + + Mutates *lines* in place. Returns True if anything was injected + (caller should set first_category = False). + """ + injected = False + + if data.conformance_tags: + for ctag, cval in data.conformance_tags: + lines.append(f'{ctag} {quote(cval, version)}') + injected = True + + if data.dataset_id is not None: + # If audit_dataset is a Loop category in table_rows it would render as loop_. + # Remove it so the scalar injection below always controls the output format. + audit_td = schema.tables.get('audit_dataset') + if (audit_td is not None + and audit_td.category_class != 'Set' + and 'audit_dataset' in data.table_rows): + data.table_rows.pop('audit_dataset') + + audit_in_table = 'audit_dataset' in data.table_rows + # Check only remnant (scalar) fallback rows — loop-form rows were stripped above. + audit_in_fallback = any( + (r.get('tag') or '').lower() == audit_id_tag + for r in remnant_rows + ) + if not audit_in_table and not audit_in_fallback: + audit_tag = schema.column_to_tag.get(('audit_dataset', 'id'), '_audit_dataset.id') + if isinstance(data.dataset_id, list): + lines.append('loop_') + lines.append(f' {audit_tag}') + for did in data.dataset_id: + lines.append(f' {quote(did, version)}') + else: + lines.append(f'{audit_tag} {quote(data.dataset_id, version)}') + injected = True + + return injected - # Partition fallback rows into three groups: - # mixed_fallback — unknown tags that were in a loop alongside known tags; - # keyed ref_table -> loop_id -> col_index -> {row_id: (value, vtype)} - # pure_loop_rows — unknown tags in a loop with no known tags; keyed by loop_id - # remnant_rows — scalar fallback (loop_id is None) and anything not injected + +def _organise_fallback_rows( + fallback_rows: list[dict], + audit_id_tag: str, +) -> 'tuple[dict, list[dict], dict]': + """Partition fallback rows into pure-loop, remnant, and per-table extra-column structures. + + *audit_id_tag* rows are stripped from pure-loop groups so scalar injection controls output. + """ mixed_fallback: dict[str, dict[int, dict[int, dict[int, tuple]]]] = {} pure_loop_rows: dict[int, list[dict]] = {} remnant_rows: list[dict] = [] - for r in data.fallback_rows: + for r in fallback_rows: lid = r.get('loop_id') ref = r.get('ref_table') if lid is None: @@ -1714,66 +1981,55 @@ def _render_block( else: pure_loop_rows.setdefault(lid, []).append(r) - # _audit_dataset.id must always be emitted as a scalar first-line item. - # Strip it from pure_loop_rows so the scalar injection below takes over. - # (FK propagation can carry a loop-form _audit_dataset.id from a publication - # block into sibling blocks, producing a spurious single-value loop_.) - _audit_id_tag = schema.column_to_tag.get(('audit_dataset', 'id'), '_audit_dataset.id').lower() + # Strip _audit_dataset.id from pure-loop groups — scalar injection takes over. pure_loop_rows = { - lid: [r for r in rows if (r.get('tag') or '').lower() != _audit_id_tag] + lid: [r for r in rows if (r.get('tag') or '').lower() != audit_id_tag] for lid, rows in pure_loop_rows.items() } pure_loop_rows = {lid: rows for lid, rows in pure_loop_rows.items() if rows} - # Build per-table extra-column list: - # ref_table -> list of (tag, col_index, {row_id: (value, vtype)}) - # ordered by (loop_id, col_index) to preserve original column ordering. + # Build per-table extra-column list ordered by (loop_id, col_index). extra_cols_for: dict[str, list[tuple[str, int, dict[int, tuple]]]] = {} for ref, loop_dict in mixed_fallback.items(): cols_list: list[tuple[str, int, dict[int, tuple]]] = [] for lid in sorted(loop_dict): for col_idx in sorted(loop_dict[lid]): cell_map = loop_dict[lid][col_idx] - # All cells for this (loop_id, col_idx) share the same tag. sample = next(iter(cell_map.values())) tag = sample[0] - # row_id -> (value, vtype) row_vals = {rid: (v, vt) for rid, (_, v, vt) in cell_map.items()} cols_list.append((tag, col_idx, row_vals)) extra_cols_for[ref] = cols_list - # Inject conformance tags (ONE_BLOCK) before all other content. - if data.conformance_tags: - for ctag, cval in data.conformance_tags: - lines.append(f'{ctag} {quote(cval, version)}') - first_category = False + return pure_loop_rows, remnant_rows, extra_cols_for - # Inject _audit_dataset.id when requested. - if data.dataset_id is not None: - # If audit_dataset is a Loop category in table_rows it would render as loop_. - # Remove it so the scalar injection below always controls the output format. - audit_td = schema.tables.get('audit_dataset') - if (audit_td is not None - and audit_td.category_class != 'Set' - and 'audit_dataset' in data.table_rows): - data.table_rows.pop('audit_dataset') - audit_in_table = 'audit_dataset' in data.table_rows - # Check only remnant (scalar) fallback rows — loop-form rows were stripped above. - audit_in_fallback = any( - (r.get('tag') or '').lower() == _audit_id_tag - for r in remnant_rows +def _render_block( + block_name: str, + data: _BlockData, + schema: SchemaSpec, + version: CifVersion, + spec: BlockSpec | None, + reconstruct_su: bool, + pretty: bool, + line_limit: int | None = None, +) -> list[str]: + """Render a single CIF block as a flat list of output lines.""" + if version == CifVersion.CIF_1_1 and len(block_name) > 75: + raise ValueError( + f"CIF 1.1 block code {block_name!r} exceeds the 75-character identifier " + f"limit (length {len(block_name)})" ) - if not audit_in_table and not audit_in_fallback: - audit_tag = schema.column_to_tag.get(('audit_dataset', 'id'), '_audit_dataset.id') - if isinstance(data.dataset_id, list): - lines.append('loop_') - lines.append(f' {audit_tag}') - for did in data.dataset_id: - lines.append(f' {quote(did, version)}') - else: - lines.append(f'{audit_tag} {quote(data.dataset_id, version)}') - first_category = False + lines: list[str] = [f'data_{block_name}'] + first_category = True + + _audit_id_tag = schema.column_to_tag.get(('audit_dataset', 'id'), '_audit_dataset.id').lower() + pure_loop_rows, remnant_rows, extra_cols_for = _organise_fallback_rows( + data.fallback_rows, _audit_id_tag, + ) + + if _inject_header_items(lines, data, schema, version, remnant_rows, _audit_id_tag): + first_category = False effective_spec = spec if data.preferred_category_order and spec is None: @@ -1784,72 +2040,20 @@ def _render_block( for item in _ordered_categories(schema, effective_spec, data.table_rows): if isinstance(item, list): - # Merge group - if data.suppress_loop_fk_pk: - # ORIGINAL mode: join positionally by _loop_id + _iter_idx - cat_lines = _render_original_loop_group( - item, data.table_rows, data, schema, version, spec, - reconstruct_su, pretty, line_limit, extra_cols_for, - ) - else: - suppress_pkg = None - if data.suppress_fk_pk and data.suppress_all_fk_to_set: - first_present = next((c for c in item if data.table_rows.get(c)), None) - if first_present: - ftdef = schema.tables.get(first_present) - frows = data.table_rows.get(first_present, []) - if ftdef and frows: - suppress_pkg = _suppressed_fk_pk_cols(ftdef, frows, data.table_rows, schema) - cat_lines = _render_merge_group( - item, data.table_rows, schema, version, spec, - reconstruct_su, pretty, line_limit, - suppress_pk_cols=suppress_pkg, - suppress_all_fk_to_set=data.suppress_all_fk_to_set, - ) - if cat_lines: - if not first_category: - lines.append('') - first_category = False - lines.extend(cat_lines) + cat_lines = _render_merge_group_item( + item, data, schema, version, spec, + reconstruct_su, pretty, line_limit, extra_cols_for, + ) else: - table_name = item - rows = data.table_rows.get(table_name) - if not rows: - continue - table_def = schema.tables[table_name] - cols = _active_cols(table_def, rows, spec, reconstruct_su) - if not cols: - continue - - if data.suppress_fk_pk and ( - (table_def.category_class == 'Set' and len(rows) == 1) - or data.suppress_loop_fk_pk - or data.suppress_all_fk_to_set - ): - suppressed = _suppressed_fk_pk_cols( - table_def, rows, data.table_rows, schema, - suppress_all_to_set=data.suppress_all_fk_to_set, - ) - cols = [c for c in cols if c not in suppressed] - if not cols: - continue - - if data.suppress_all_fk_to_set: - # GROUPED mode: suppress columns whose every value is '.' (inapplicable). - cols = [c for c in cols if not all(row.get(c) == '.' for row in rows)] - if not cols: - continue - + cat_lines = _render_single_table_item( + item, data, schema, version, spec, + reconstruct_su, pretty, line_limit, extra_cols_for, + ) + if cat_lines: if not first_category: lines.append('') first_category = False - - - extra = extra_cols_for.get(table_name) - if table_def.category_class == 'Set' and len(rows) == 1: - lines.extend(_render_set_category(rows[0], cols, table_name, schema, version, table_def, reconstruct_su, pretty, line_limit)) - else: - lines.extend(_render_loop_category(rows, cols, table_name, schema, version, table_def, reconstruct_su, pretty, line_limit, extra_fallback_cols=extra)) + lines.extend(cat_lines) # Pure-fallback loops: emit each loop_id group as a standalone loop_. for lid in sorted(pure_loop_rows): @@ -1859,15 +2063,7 @@ def _render_block( first_category = False lines.extend(_render_pure_fallback_loop(loop_rows, version, pretty, line_limit)) - # Scalar fallback and any remnant rows (scalars, or loop rows whose ref_table - # is not present in this block's table_rows — treated as plain fallback). - actual_remnant = remnant_rows - for ref, cols_list in extra_cols_for.items(): - if ref not in data.table_rows: - # ref_table not rendered in this block: fall back to plain fallback. - for tag, _ci, row_vals in cols_list: - for _rid, (val, vtype) in row_vals.items(): - actual_remnant.append({'tag': tag, 'value': val, 'value_type': vtype}) + actual_remnant = _collect_remnant_rows(extra_cols_for, data.table_rows, remnant_rows) if actual_remnant: if not first_category: lines.append('') @@ -2066,12 +2262,142 @@ def _expand_wildcard(pattern: str, schema: SchemaSpec) -> list[str]: found.add(child) queue.append(child) - return sorted(found) + return sorted(found) + + +# --------------------------------------------------------------------------- +# Merge group renderer +# --------------------------------------------------------------------------- + +def _build_merge_token_matrix( + merged_cols: 'list[tuple[str, str]]', + all_pk_vals: 'list[tuple]', + table_index: 'dict[str, dict[tuple, dict]]', + su_maps: 'dict[str, dict[str, str]]', + reconstruct_su: bool, + version: CifVersion, + line_limit: 'int | None', +) -> 'list[list[str]]': + """Build a token matrix (rows × columns) from the indexed table data.""" + matrix: list[list[str]] = [] + for pk_vals in all_pk_vals: + tokens = [] + for cat, col in merged_cols: + row = table_index[cat].get(pk_vals, {}) + value = row.get(col) + if value is None: + token = '.' + else: + su_map = su_maps[cat] + if reconstruct_su and col in su_map: + su_val = row.get(su_map[col]) + if su_val is not None: + value = _merge_su(value, su_val) + token = quote(value, version) + if line_limit is not None: + token = _apply_line_limit(value, token, line_limit) + tokens.append(token) + matrix.append(tokens) + return matrix + + +def _build_merge_merged_cols( + present: list[str], + cat_active: 'dict[str, list[str]]', + first_cat: str, + table_index: 'dict[str, dict[tuple, dict]]', + shared_pks: list[str], + schema: SchemaSpec, + spec: 'BlockSpec | None', + reconstruct_su: bool, + suppress_pk_cols: 'set[str] | None', +) -> 'list[tuple[str, str]]': + """Build the merged column list: shared PKs from first category, then per-table non-PK cols.""" + first_tdef = schema.tables[first_cat] + first_active = _active_cols(first_tdef, list(table_index[first_cat].values()), spec, reconstruct_su) + _suppress = suppress_pk_cols or set() + pk_in_first = [pk for pk in shared_pks if pk in set(first_active) and pk not in _suppress] + + merged_cols: list[tuple[str, str]] = [(first_cat, pk) for pk in pk_in_first] + pk_set = set(shared_pks) + for cat in present: + for col in cat_active.get(cat, []): + if col not in pk_set: + merged_cols.append((cat, col)) + return merged_cols + + +def _compute_merge_cat_active( + present: list[str], + table_index: 'dict[str, dict[tuple, dict]]', + pk_set: 'frozenset[str]', + schema: SchemaSpec, + spec: 'BlockSpec | None', + reconstruct_su: bool, + effective_suppressed: 'dict[str, set[str]]', + suppress_all_fk_to_set: bool, +) -> 'dict[str, list[str]]': + """Compute non-PK active columns per category for a merge group.""" + cat_active: dict[str, list[str]] = {} + for cat in present: + tdef = schema.tables[cat] + all_rows = list(table_index[cat].values()) + cols = _active_cols(tdef, all_rows, spec, reconstruct_su) + non_pk_cols = [c for c in cols if c not in pk_set] + if suppress_all_fk_to_set: + suppressed = effective_suppressed.get(cat, set()) + non_pk_cols = [c for c in non_pk_cols if c not in suppressed] + non_pk_cols = [c for c in non_pk_cols if not all(row.get(c) == '.' for row in all_rows)] + if non_pk_cols or cols: + cat_active[cat] = non_pk_cols + return cat_active + + +def _build_merge_table_index( + present: list[str], + table_rows: 'dict[str, list[dict]]', + shared_pks: list[str], +) -> 'tuple[list[tuple], dict[str, dict[tuple, dict]]]': + """Index each table by its PK tuple; collect unique PK tuples in encounter order.""" + all_pk_vals: list[tuple] = [] + seen_pk: set[tuple] = set() + table_index: dict[str, dict[tuple, dict]] = {} + for cat in present: + table_index[cat] = {} + for row in table_rows[cat]: + pk_tuple = tuple(row.get(pk) for pk in shared_pks) + if pk_tuple not in seen_pk: + seen_pk.add(pk_tuple) + all_pk_vals.append(pk_tuple) + table_index[cat][pk_tuple] = row + return all_pk_vals, table_index + +def _render_merge_group_incompatible( + present: list[str], + table_rows: 'dict[str, list[dict]]', + schema: SchemaSpec, + version: CifVersion, + spec: 'BlockSpec | None', + reconstruct_su: bool, + pretty: bool, + line_limit: 'int | None', +) -> list[str]: + """Render PK-incompatible categories as plain separate loops.""" + lines: list[str] = [] + first = True + for cat in present: + rows = table_rows[cat] + tdef = schema.tables[cat] + cols = _active_cols(tdef, rows, spec, reconstruct_su) + if not cols: + continue + if not first: + lines.append('') + first = False + lines.extend(_render_loop_category(rows, cols, cat, schema, version, tdef, reconstruct_su, pretty, line_limit)) + return lines -# --------------------------------------------------------------------------- -# Merge group renderer -# --------------------------------------------------------------------------- def _render_merge_group( group: list[str], @@ -2122,65 +2448,24 @@ def _render_merge_group( if not compatible: print(f"MERGE FAIL: {group}, present={present}, pk_sets={dict(zip(present, pk_sets))}") - # Fall back to plain loops in listed order. - lines: list[str] = [] - first = True - for cat in present: - rows = table_rows[cat] - tdef = schema.tables[cat] - cols = _active_cols(tdef, rows, spec, reconstruct_su) - if not cols: - continue - if not first: - lines.append('') - first = False - lines.extend(_render_loop_category(rows, cols, cat, schema, version, tdef, reconstruct_su, pretty, line_limit)) - return lines + return _render_merge_group_incompatible( + present, table_rows, schema, version, spec, reconstruct_su, pretty, line_limit, + ) # Key-compatible: FULL OUTER JOIN in Python. shared_pks = sorted(pk_sets[0]) + all_pk_vals, table_index = _build_merge_table_index(present, table_rows, shared_pks) - # Index each table by PK tuple; collect all unique PK tuples in encounter order. - all_pk_vals: list[tuple] = [] - seen_pk: set[tuple] = set() - table_index: dict[str, dict[tuple, dict]] = {} - for cat in present: - table_index[cat] = {} - for row in table_rows[cat]: - pk_tuple = tuple(row.get(pk) for pk in shared_pks) - if pk_tuple not in seen_pk: - seen_pk.add(pk_tuple) - all_pk_vals.append(pk_tuple) - table_index[cat][pk_tuple] = row - - # Determine active (non-PK) columns per table. - cat_active: dict[str, list[str]] = {} - for cat in present: - tdef = schema.tables[cat] - all_rows = list(table_index[cat].values()) - cols = _active_cols(tdef, all_rows, spec, reconstruct_su) - # Exclude shared PKs; they appear once at the start. - non_pk_cols = [c for c in cols if c not in pk_sets[0]] - if suppress_all_fk_to_set: - suppressed = effective_suppressed.get(cat, set()) - non_pk_cols = [c for c in non_pk_cols if c not in suppressed] - non_pk_cols = [c for c in non_pk_cols if not all(row.get(c) == '.' for row in all_rows)] - if non_pk_cols or cols: - cat_active[cat] = non_pk_cols + cat_active = _compute_merge_cat_active( + present, table_index, pk_sets[0], schema, spec, reconstruct_su, + effective_suppressed, suppress_all_fk_to_set, + ) - # Build merged column list: shared PKs (from first present cat), then each table's non-PK cols. first_cat = present[0] - first_tdef = schema.tables[first_cat] - first_active = _active_cols(first_tdef, list(table_index[first_cat].values()), spec, reconstruct_su) - _suppress = suppress_pk_cols or set() - pk_in_first = [pk for pk in shared_pks if pk in set(first_active) and pk not in _suppress] - - merged_cols: list[tuple[str, str]] = [(first_cat, pk) for pk in pk_in_first] - pk_set = set(shared_pks) - for cat in present: - for col in cat_active.get(cat, []): - if col not in pk_set: - merged_cols.append((cat, col)) + merged_cols = _build_merge_merged_cols( + present, cat_active, first_cat, table_index, shared_pks, + schema, spec, reconstruct_su, suppress_pk_cols, + ) if not merged_cols: return [] @@ -2190,35 +2475,82 @@ def _render_merge_group( lines.append(f' {_col_tag(cat, col, schema)}') su_maps = {cat: (_su_col_map(schema.tables[cat]) if reconstruct_su else {}) for cat in present} + matrix = _build_merge_token_matrix( + merged_cols, all_pk_vals, table_index, su_maps, reconstruct_su, version, line_limit, + ) + + if pretty: + real_idx = _real_col_indices_merged(merged_cols, schema) + if real_idx: + matrix = _apply_decimal_align(matrix, real_idx) + + col_widths = _col_widths(matrix) if pretty else None + + for tokens in matrix: + lines.extend(_format_row(tokens, col_widths, line_limit)) + + return lines + + +def _render_positional_join( + per_table: list[tuple[str, list[str], list[dict]]], + schema: 'SchemaSpec', + version: 'CifVersion', + reconstruct_su: bool, + pretty: bool, + line_limit: int | None, +) -> list[str]: + """Positional join: zip rows from multiple Loop tables by sorted _cifflow_row_id.""" + sorted_sets: list[tuple[str, list[str], list[dict]]] = [ + (t, cols, sorted(rows, key=lambda r: r.get('_cifflow_row_id') or 0)) + for t, cols, rows in per_table + ] + num_rows = max((len(rows) for _, _, rows in sorted_sets), default=0) + if num_rows == 0: + return [] + + su_maps = { + t: (_su_col_map(schema.tables[t]) if reconstruct_su else {}) + for t, _, _ in sorted_sets + } + + lines = ['loop_'] + for table_name, cols, _ in sorted_sets: + for col in cols: + lines.append(f' {_col_tag(table_name, col, schema)}') - # Build token matrix. matrix: list[list[str]] = [] - for pk_vals in all_pk_vals: - tokens = [] - for cat, col in merged_cols: - row = table_index[cat].get(pk_vals, {}) - value = row.get(col) - if value is None: - token = '.' - else: - su_map = su_maps[cat] - if reconstruct_su and col in su_map: - su_val = row.get(su_map[col]) - if su_val is not None: - value = _merge_su(value, su_val) - token = quote(value, version) - if line_limit is not None: - token = _apply_line_limit(value, token, line_limit) - tokens.append(token) + for i in range(num_rows): + tokens: list[str] = [] + for table_name, cols, sorted_rows in sorted_sets: + row = sorted_rows[i] if i < len(sorted_rows) else {} + su_map = su_maps[table_name] + for col in cols: + value = row.get(col) + if value is None: + tokens.append('.') + else: + if reconstruct_su and col in su_map: + su_val = row.get(su_map[col]) + if su_val is not None: + value = _merge_su(value, su_val) + token = quote(value, version) + if line_limit is not None: + token = _apply_line_limit(value, token, line_limit) + tokens.append(token) matrix.append(tokens) if pretty: - real_idx = _real_col_indices_merged(merged_cols, schema) + real_idx: list[int] = [] + offset = 0 + for table_name, cols, _ in sorted_sets: + for i in _real_col_indices(cols, schema.tables[table_name]): + real_idx.append(i + offset) + offset += len(cols) if real_idx: matrix = _apply_decimal_align(matrix, real_idx) col_widths = _col_widths(matrix) if pretty else None - for tokens in matrix: lines.extend(_format_row(tokens, col_widths, line_limit)) @@ -2285,85 +2617,24 @@ def _render_original_loop_group( reconstruct_su, pretty, line_limit, suppress_pk_cols=suppress_pks) - # Positional join: sort each table's rows by _cifflow_row_id and zip by index. - # (_loop_id/_iter_idx are not copied to the final tables; positional order is equivalent.) - sorted_sets: list[tuple[str, list[str], list[dict]]] = [ - (t, cols, sorted(rows, key=lambda r: r.get('_cifflow_row_id') or 0)) - for t, cols, rows in per_table - ] - - num_rows = max((len(rows) for _, _, rows in sorted_sets), default=0) - if num_rows == 0: - return [] - - su_maps = { - t: (_su_col_map(schema.tables[t]) if reconstruct_su else {}) - for t, _, _ in sorted_sets - } - - lines = ['loop_'] - for table_name, cols, _ in sorted_sets: - for col in cols: - lines.append(f' {_col_tag(table_name, col, schema)}') - - matrix: list[list[str]] = [] - for i in range(num_rows): - tokens: list[str] = [] - for table_name, cols, sorted_rows in sorted_sets: - row = sorted_rows[i] if i < len(sorted_rows) else {} - su_map = su_maps[table_name] - for col in cols: - value = row.get(col) - if value is None: - tokens.append('.') - else: - if reconstruct_su and col in su_map: - su_val = row.get(su_map[col]) - if su_val is not None: - value = _merge_su(value, su_val) - token = quote(value, version) - if line_limit is not None: - token = _apply_line_limit(value, token, line_limit) - tokens.append(token) - matrix.append(tokens) - - if pretty: - real_idx: list[int] = [] - offset = 0 - for table_name, cols, _ in sorted_sets: - for i in _real_col_indices(cols, schema.tables[table_name]): - real_idx.append(i + offset) - offset += len(cols) - if real_idx: - matrix = _apply_decimal_align(matrix, real_idx) - - col_widths = _col_widths(matrix) if pretty else None - for tokens in matrix: - lines.extend(_format_row(tokens, col_widths, line_limit)) - - return lines + return _render_positional_join(per_table, schema, version, reconstruct_su, pretty, line_limit) # --------------------------------------------------------------------------- # Category renderers # --------------------------------------------------------------------------- -def _render_set_category( - row: dict, +def _build_set_quads( cols: list[str], + row: dict, table_name: str, - schema: SchemaSpec, - version: CifVersion, - table_def: TableDef, + schema: 'SchemaSpec', + version: 'CifVersion', reconstruct_su: bool, - pretty: bool, - line_limit: int | None = None, -) -> list[str]: - """Emit a Set-class category as scalar tag–value pairs.""" - lines = [] - su_map = _su_col_map(table_def) if reconstruct_su else {} - - # Build (tag, col, value, token) quads; apply folding to any multiline tokens. + su_map: dict[str, str], + line_limit: int | None, +) -> list[tuple[str, str, str, str]]: + """Build (tag, col, value, token) quads for a Set row, skipping NULL values.""" quads: list[tuple[str, str, str, str]] = [] for col in cols: tag = _col_tag(table_name, col, schema) @@ -2379,47 +2650,79 @@ def _render_set_category( if line_limit is not None and token.startswith('\n'): token = make_text_field(value, line_limit) quads.append((tag, col, value, token)) + return quads + +def _requote_set_quads( + quads: list[tuple[str, str, str, str]], + tag_width: int, + line_limit: int, + pretty: bool, +) -> tuple[list[tuple[str, str, str, str]], int]: + """Re-quote inline tokens that exceed line_limit; return updated quads and tag_width.""" + new_quads: list[tuple[str, str, str, str]] = [] + for tag, col, value, token in quads: + if not token.startswith('\n'): + line_str = f'{tag:<{tag_width}} {token}' if pretty else f'{tag} {token}' + if len(line_str) > line_limit: + token = make_text_field(value, line_limit) + new_quads.append((tag, col, value, token)) if pretty: tag_width = max( - (len(tag) for tag, _c, _v, token in quads if not token.startswith('\n')), + (len(tag) for tag, _c, _v, token in new_quads if not token.startswith('\n')), default=0, ) - else: - tag_width = 0 + return new_quads, tag_width + + +def _decimal_align_set_quads( + quads: list[tuple[str, str, str, str]], + table_def: 'TableDef', +) -> list[tuple[str, str, str, str]]: + """Decimal-align all inline Real/Float tokens within the quads list.""" + col_type = {c.name: c.type_contents for c in table_def.columns} + real_positions = [ + i for i, (tag, col, _v, token) in enumerate(quads) + if col_type.get(col) in ('Real', 'Float') and not token.startswith('\n') + ] + if not real_positions: + return quads + real_tokens = [quads[i][3] for i in real_positions] + aligned = _decimal_align_column(real_tokens) + quads = list(quads) + for pos, new_tok in zip(real_positions, aligned): + tag, col, val, _old = quads[pos] + quads[pos] = (tag, col, val, new_tok) + return quads + + +def _render_set_category( + row: dict, + cols: list[str], + table_name: str, + schema: SchemaSpec, + version: CifVersion, + table_def: TableDef, + reconstruct_su: bool, + pretty: bool, + line_limit: int | None = None, +) -> list[str]: + """Emit a Set-class category as scalar tag–value pairs.""" + su_map = _su_col_map(table_def) if reconstruct_su else {} + quads = _build_set_quads(cols, row, table_name, schema, version, reconstruct_su, su_map, line_limit) + + tag_width = ( + max((len(tag) for tag, _c, _v, token in quads if not token.startswith('\n')), default=0) + if pretty else 0 + ) - # Re-quote inline tokens whose formatted line would exceed line_limit. if line_limit is not None: - new_quads: list[tuple[str, str, str, str]] = [] - for tag, col, value, token in quads: - if not token.startswith('\n'): - line_str = f'{tag:<{tag_width}} {token}' if pretty else f'{tag} {token}' - if len(line_str) > line_limit: - token = make_text_field(value, line_limit) - new_quads.append((tag, col, value, token)) - quads = new_quads - # Recompute tag_width now that some inline tokens may have become multiline. - if pretty: - tag_width = max( - (len(tag) for tag, _c, _v, token in quads if not token.startswith('\n')), - default=0, - ) + quads, tag_width = _requote_set_quads(quads, tag_width, line_limit, pretty) - # Decimal-align all inline Real/Float tokens within this Set category. if pretty: - col_type = {c.name: c.type_contents for c in table_def.columns} - real_positions = [ - i for i, (tag, col, _v, token) in enumerate(quads) - if col_type.get(col) in ('Real', 'Float') and not token.startswith('\n') - ] - if real_positions: - real_tokens = [quads[i][3] for i in real_positions] - aligned = _decimal_align_column(real_tokens) - quads = list(quads) - for pos, new_tok in zip(real_positions, aligned): - tag, col, val, _old = quads[pos] - quads[pos] = (tag, col, val, new_tok) + quads = _decimal_align_set_quads(quads, table_def) + lines = [] for tag, _col, _value, token in quads: if token.startswith('\n'): lines.append(tag) diff --git a/tasks/lessons.md b/tasks/lessons.md index dcb36eb..9267ec2 100644 --- a/tasks/lessons.md +++ b/tasks/lessons.md @@ -5,8 +5,8 @@ - **Arrow / PyO3 / Rust:** 103, 104, 105, 106, 107, 150 - **CIF model / builder:** 5, 6, 7, 8, 88, 89, 90 - **DuckDB ingest:** 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 123, 145 -- **Dictionary / schema:** 12, 14, 15, 16, 17, 27, 31, 36, 38, 40, 41, 42, 64, 151, 152, 153 -- **Emit / output:** 48b, 50, 51, 52, 53, 54, 55, 56, 57, 58, 61, 66, 67, 68, 69, 70, 71, 72, 73, 74, 120, 121, 122, 124, 125, 126, 127, 128, 129, 130, 131, 132, 136, 137, 138, 139, 140, 141, 146, 147, 148 +- **Dictionary / schema:** 12, 14, 15, 16, 17, 27, 31, 36, 38, 40, 41, 42, 64, 151, 152, 153, 157 +- **Emit / output:** 48b, 50, 51, 52, 53, 54, 55, 56, 57, 58, 61, 66, 67, 68, 69, 70, 71, 72, 73, 74, 120, 121, 122, 124, 125, 126, 127, 128, 129, 130, 131, 132, 136, 137, 138, 139, 140, 141, 146, 147, 148, 156, 158, 159 - **Known gaps:** 124 - **Fidelity:** 59, 60, 62, 63, 77 - **FK propagation / ingest:** 21, 22, 23, 24, 25, 26, 28, 29, 30, 32, 34, 35, 37, 39, 43, 44, 45, 46, 47, 83, 84, 85, 86 @@ -1537,6 +1537,68 @@ --- +## Lesson 154 — Refactoring a function with shared injection logic: extract vals as a list (2026-06-12) + +**Context:** `_collect_all_blocks` had two nearly-identical Set-parent-injection blocks. The Set path iterated `set_key_cols` and called `row.get(col)` per entry; the Loop path iterated `zip(set_key_cols, set_vals)` with pre-computed vals. Extracting a `_inject_set_parents(block_table_rows, parent_tables, set_key_cols, vals)` helper required a unified interface. + +**Observation:** The asymmetry — one path builds `vals` lazily per iteration, the other has them pre-built — disappears when you materialise `vals` before the call in both cases. The extra list comprehension is negligible overhead and makes both call sites identical in structure. + +**Rule:** When extracting shared iteration logic where one caller builds values lazily (per-iteration `row.get`) and another has them pre-built, materialise the lazy side into a list at the call site. The helper receives a plain list and uses `zip(set_key_cols, vals)`. + +--- + +## Lesson 155 — Extracting helpers from a high-CC render function: keep debug prints at the dispatch site (2026-06-12) + +**Context:** `_render_merge_group` (CC F/54) was refactored by extracting five helpers: `_build_merge_table_index`, `_compute_merge_cat_active`, `_build_merge_merged_cols`, `_build_merge_token_matrix`, and `_render_merge_group_incompatible`. The incompatible-path helper is called when PK sets are non-uniform, and a debug `print(f"MERGE FAIL: {group}, present={present}, pk_sets=...")` precedes it. + +**Observation:** The print references both `group` (a parameter of `_render_merge_group`) and `pk_sets` (a local computed just before the branch). Passing these into the helper solely to keep the print inside it would bloat the helper's signature for no gain. + +**Rule:** When a debug or diagnostic print references variables from the caller's local scope, keep the print at the call site (the dispatch point), not inside the extracted helper. The helper's job is the logic, not the diagnosis. + +--- + +## Lesson 157 — FK resolution is irreducibly complex; extract by pass, not by branch (2026-06-14) + +**Context:** Refactoring `generate_schema` (CC 98) in `dictionary/schema.py`. + +**Observation:** The FK group resolution logic (`_resolve_fk_group`) landed at E/33 after extraction. The four resolution arms (dual-endpoint, one-missing-bridged, one-missing-no-bridge, >1-missing/ambiguous) cannot be sensibly separated because they share `tgt_to_srcs`, `missing_pk_cols`, `has_conflicts`, and `bridge_col_in_src` state. Splitting them would force awkward parameter passing or a class. + +**Rule:** When refactoring a high-CC function, extract clean passes (table-building, FK detection, propagation, metadata) first. Accept residual complexity in a tightly-coupled decision core rather than forcing a split that destroys readability. Document the residual E/D grade with a reason in the complexity table rather than chasing further decomposition. + +--- + +## Lesson 159 — BFS child-Set collection requires a dedicated helper, not inline expansion (2026-06-14) + +**Context:** Extracting helpers from `_collect_grouped` (CC 170) in `output/emit.py`. + +**Observation:** The child-Set BFS loop inside `_collect_incidental_block_rows` accounted for ~20 of its ~39 CC points. It could not be removed by simplification — the BFS convergence logic, FK-filter construction, and row deduplication are all genuinely interdependent. Extracting the loop body to `_bfs_collect_child_sets` reduced `_collect_incidental_block_rows` from E/39 to C/18. + +**Rule:** When a nested iteration pattern (BFS, DFS, fixed-point loop) accounts for the majority of a function's CC, extract it as a named helper even if the signature is wide. The caller becomes linear; the helper documents the algorithm. Do not inline-expand the loop to reduce CC — that just distributes the branches without naming the concept. + +--- + +## Lesson 158 — Nested functions inflate the outer function's CC; extract to module level before measuring (2026-06-14) + +**Context:** Analysing `_collect_grouped` (reported CC 170) in `output/emit.py`. + +**Observation:** Radon attributed the CC of the nested `_block_fingerprint` function (and its own nested `_fp_entries_for_expanded`) to the outer `_collect_grouped`. The CC appeared to be 170 for the outer function, but after lifting both nested functions to module level the outer function's true CC became measurable. The core body was approximately D/26 after helper extraction. + +**Rule:** Before planning refactors of high-CC functions, check for nested function definitions — radon rolls their CC into the enclosing function. Lift nested functions to module level first, re-measure, then decide how many additional helpers are needed. + +--- + +## Lesson 156 — `_suppressed_fk_pk_cols` only suppresses composite PK FK columns (2026-06-13) + +**Context:** Writing tests for `_render_block` / `_render_single_table_item` in emit.py. + +**Mistake:** Assumed a plain FK column (FK to a parent table, but not part of the child's own PK) would be suppressed by `_suppressed_fk_pk_cols`. Wrote tests asserting the column was absent, but it remained. + +**Fix:** `_suppressed_fk_pk_cols` checks `is_fk_pk = all(c in pk_cols for c in fk.source_columns)` — a FK is only suppressed when its source columns are a subset of the child table's own PK columns. A plain FK column that is not in the child PK is never suppressed. + +**How to apply:** When writing tests for FK suppression, verify that the FK column is also declared in the child table's `primary_key`. Plain FK-only columns are never suppressed, regardless of mode. + +--- + ## Lesson 153 — `all_items` in `_load_recursive` can contain duplicate `definition_id`s (2026-06-06) **Context:** `_load_recursive` builds `all_items = list(pool.values()) + primary_items`. `pool` holds items from constituent imports; `primary_items` holds items from the current file's own frames. When a constituent and the current file both define the same item (e.g., `_exptl_crystal.id` in both `multi_block_core.dic` and `cif_core.dic` which it imports), both `DdlmItem` objects end up in `all_items` with the same `definition_id`. `_build_lookup_tables` handles the duplicate definition_id by overwriting in its first loop (primary wins), but then processes both items' alias lists — causing spurious "alias X collides with existing entry Y" warnings for the second item. diff --git a/tasks/todo.md b/tasks/todo.md index 0c4d759..0c644e7 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,128 +2,26 @@ --- -## ▶ RESUME FROM HERE - -## What was done (2026-06-05/06) - -**`inspect_fk_path` partial links (Lesson 151):** -- Added `PartialLinkDef` dataclass to `schema.py`; `generate_schema` now records every DDLm Link item skipped due to incomplete PK coverage, non-PK target, or ambiguity -- `inspect_fk_path` displays these as `~>` partial connections (with covered/missing PK columns and reason) when no complete FK or bridge path is found -- `PartialLinkDef` exported from `cifflow.dictionary` - -**`parser/__init__.py` restored (Lesson 152):** -- Deleting it caused CI docs build failure (`cifflow.parser.version` unreachable) -- Restored as a stub; `detect_version` remains available for `inspect_lexer` and `test_version.py` - -**Loader bug fix — spurious alias collision warnings (Lesson 153):** -- `_load_recursive` built `all_items = list(pool.values()) + primary_items` without deduplicating by `definition_id`. When a constituent and the current file both define the same item (e.g. `_exptl_crystal.id` in both `multi_block_core.dic` and imported `cif_core.dic`), `_build_lookup_tables` received two copies and fired a false alias collision warning for the second. -- Fixed by deduplicating via a dict merge (primary overwrites constituent) before calling `_build_lookup_tables` - -Test count: 1858 (all passing). - ---- - -## What was done (2026-06-03/04, multiple branches) - -**Dictionary / schema additions:** -- `DdlmItem.source_file` — each item now records the file path it came from; serialises into JSON cache automatically (old caches load fine with `None` default) -- `merge_dictionaries(*dicts, dupl='Ignore'|'Replace'|'Exit')` — public API to combine multiple loaded dictionaries; `Exit` raises `ValueError` listing all duplicate definition IDs -- `DictionaryLoader(block_constituent_imports=True)` — skips `mode="Full"` Head-target imports (whole-dictionary constituent pulls) while allowing frame-level imports to proceed - -**Inspect additions:** -- `inspect_fk_path(schema, source, target)` — prints all direct FK edge chains and bridge-column chains between two tables; annotates synthetic source columns with their bridge derivation and fallback chains - -**Lexer bug fix (both Python and Rust):** -- Mid-word quote characters no longer terminate bare words (Lesson 149). `hello"world` and `here"` are now single tokens. Updated 8 parser tests whose old assertions relied on the broken behaviour. - -**Code cleanup — Python lexer/parser removed:** -- `inspect_lexer`, `inspect_parse`, `inspect_model`, `test_lexer.py`, `test_parser.py` all migrated to `cifflow_core.lex_cif` / `cifflow_core.parse` (Rust) -- Deleted `lexer/lexer.py`, `lexer/tokens.py`, `parser/parser.py`, their `__init__.py` files -- Kept `parser/version.py` (`detect_version`) for `inspect_lexer` version-error display and `test_version.py` -- Added `lex_cif(source, mode)` to `cifflow_core` Rust extension (Lesson 150) - -Test count: 1858 (all passing). Suite time dropped from ~5 min to ~2.5 min. - ---- - -## What was done (2026-06-02, consolidate-calc-component + duplicate-spacegroup-block-bug branches) - -Completed `consolidate_component_intensities` (post-processing: assembles `pd_calc_component` rows into `pd_calc.component_intensities_net/total` and `pd_calc_overall.component_presentation_order`). Fixed four bugs found during integration testing: -- Step 2 INSERT missing `_cifflow_block_id` → `sorted()` crash in `_all_cifflow_block_ids_for_tables` (Lesson 145) -- `_all_cifflow_block_ids_for_tables` / `_all_cifflow_block_ids` now skip NULL `_cifflow_block_id` values defensively -- `clear_source=True` simplified to `DELETE FROM pd_calc_component` (was NULL-column approach with NOT NULL / synthetic-column pitfalls) -- GROUPED duplicate space_group block: FK propagation deposits Set rows into sibling blocks → incidental blocks emit empty PK-only duplicates; fixed by skipping incidental blocks for leaf-Set primary anchors (Lesson 146) - -Also fixed `release_patch.bat` quoting bug (Lesson 147) and released v0.1.12. All 1850 tests pass. - ---- - -## What was done (2026-06-01, debug-output-using-topas-source branch) — STRUCTURE mode fixes - -Fixed three bugs in `EmitMode.STRUCTURE`, plus the specificity-ranked routing system in `OutputPlan`. All 1850 tests pass. - -- **Bug 1 — Bridge blocks absorbed as structure targets**: `_collect_structure` used `if 'structure' in afs` to identify pure structure blocks, which also caught refln bridge blocks with anchor = {pd_diffractogram, pd_phase, structure}. Fixed to `if afs == frozenset({'structure'})`. -- **Bug 2 — `space_group_symop` (and child tables) silently dropped**: `space_group_blocks[sg_id] = block` overwrote the first (richest) block with later blocks that lacked symop rows. GROUPED emits one block per source-block-id per fingerprint; only the original source block has the loop rows. Fixed by merging (`_merge_blocks_into`) instead of overwriting for both `space_group_blocks` and `pd_phase_blocks`. -- **Bug 3 — `_replace_anchor` preserves structure block identity after satellite absorption**: `_merge_blocks_into` unions anchor_frozenset; structure blocks would gain anchor = {structure, pd_phase, space_group, model} after absorption, breaking `only('structure')`. `_replace_anchor` helper (added last session) restores the original anchor. -- **Specificity system**: `_Matcher.specificity` attribute added; `only` = 10000+len, `all_of` = len, `any_of`/`has` = 1. `plan.match` now picks highest-specificity match regardless of plan order. Plan ORDER controls output emission order; specificity controls routing. -- Lessons updated/added: 139 (updated rule), 140 (exact-anchor check), 141 (merge not overwrite in satellite accumulation). - ---- - -## What was done (2026-06-01, debug-output-using-topas-source branch) — per-pkreach-group fingerprinting - -Fixed fundamental GROUPED emit correctness bug where co-located but independently-anchored Set tables (e.g. `atom_site`→structure, `geom_angle`→model, `space_group_symop`→space_group in a single source block) were merged into one output block with union anchor `{structure,model,space_group}`, causing `only("structure")` to match nothing. All 1850 tests pass. - -- **Root cause**: `_block_fingerprint` unioned all PK-FK-reachable Set tables across all loop tables into a single fingerprint. Fixed by computing one fingerprint per *distinct* non-empty pkreach frozenset among loop tables — co-located independent Sets become separate output blocks; bridge blocks (one loop table's PK spans multiple Sets simultaneously) remain correctly multi-anchor. -- **Supporting fixes**: Updated `table_to_needed_by` to filter reverse-FK children by pkreach subset (atom_type correctly assigned to structure group only); updated `sets_with_own_block` to include incidental tables (pd_phase stripped to PK-only in bridge blocks); fixed edge case where loop tables with pkreach=∅ (core_schema) now route to pure_loop_block_ids preserving all data. -- **Test updated**: `test_all_of_structure_and_model_routes_structure_blocks` → `test_only_structure_routes_structure_blocks` (verifies structure blocks exist and do NOT contain model data). -- Lessons added: 138 (GROUPED fingerprints must be per-distinct-pkreach-group, not unioned). - ---- - -## What was done (2026-06-01, debug-output-using-topas-source branch) — reconstruct_su + merge-group fixes - -Fixed two bugs in GROUPED emit with `reconstruct_su=True`, discovered while testing against a real TOPAS powder diffraction CIF via `scripts/topas/pdcif2.py`. All 1850 tests pass. - -- **Bug 1 — FK-PK columns suppressed when `reconstruct_su=True`**: `_active_cols` used `col.linked_item_id is not None` to identify SU columns, but FK-PK Link columns (e.g. `pd_meas.point_id`) also carry `linked_item_id` and were incorrectly suppressed. Fixed by replacing the check with `set(_su_col_map(table_def).values())`, which only returns genuine within-table SU columns. Added 3 new tests in `TestReconstructSU`. -- **Bug 2 — Merge group `['pd_data', 'pd_meas', 'pd_proc', 'pd_calc']` not combining**: `_render_merge_group` PK-compatibility check used raw schema PKs (`{point_id, diffractogram_id}`), leaving FK-PK columns in the join key even though they are suppressed in GROUPED output. Fixed by pre-computing `effective_suppressed` per table before the compatibility check, so effective PKs (`{point_id}`) are used for both compatibility and join-key selection. -- Lessons added: 136 (`_active_cols` must use `_su_col_map`), 137 (`_render_merge_group` PK-compatibility must account for FK-PK suppression). -## What was done (2026-05-30, main branch) — STRUCTURE mode + release pipeline - -Implemented `EmitMode.STRUCTURE` (absorbs `pd_phase`, `space_group`, single-model `model` blocks into their parent `structure` block), wrote 14 tests covering merge / orphan / multi-model cases (1847 tests pass), and updated `docs/outputspec.md` with full STRUCTURE documentation including the `any_of('structure')` anchor-frozenset caveat. Also overhauled the release pipeline: `release_patch.bat` now creates a `release/vX.Y.Z` branch and opens a PR instead of pushing directly to `main`; `release.yml` now triggers on `push: branches:[main] + paths:[pyproject.toml]` instead of on tag push (so PyPI publish only fires after CI passes); `[skip ci]` removed from `pyproject.toml` commit_message (was silently suppressing the release workflow). - -**Immediate actions needed on restart:** -1. `git tag -d v0.1.8 v0.1.9 v0.1.10` — delete dangling local tags that point to orphaned amended commits -2. Commit the `pyproject.toml` `[skip ci]` removal (currently an unstaged working-tree change) -3. Create `release/v0.1.10` branch from current local `main` (already has the bump commit), push and open PR manually (`gh` not installed yet — do it on GitHub or install with `winget install GitHub.cli`) -4. After PR merges, verify `release.yml` fires and publishes v0.1.10 to PyPI - -**Current local state:** -- `pyproject.toml` version = 0.1.10 (bump commit 8afb598 already on local main, not yet pushed to origin) -- `pyproject.toml` commit_message `[skip ci]` removed in working tree (unstaged) -- `release.yml`, `release_patch.bat`, `release_patch_dry.bat` all updated and already on origin/main via PRs #58/#59 - ---- - -## What was done (2026-05-13, main branch) — auto-generated docs - -Completed the full MkDocs + mkdocstrings documentation pipeline (Phases 1–5 of `prompts/autogenerate docs.md`). All docstrings across `src/cifflow/` converted to NumPy style, `ruff check src/` and `pydoclint src/` pass clean, and `mkdocs build --strict` succeeds. All 1835 tests pass. - -- Converted docstrings in all modules (`dictionary/`, `ingestion/`, `output/`, `validation/`, `fidelity/`, `inspect/`, `database/`, `cifmodel/`, `lexer/`, `parser/`, root `__init__.py`). Fixed ruff D-rules and pydoclint DOC-rules file by file. -- Created `docs/api/` pages for all modules using mkdocstrings `:::` directives; filled in `parser.md` and `model.md` stubs. -- Added CI `docs` job (build + deploy) and `release.yml` `docs` job; fixed maturin venv requirement and pydoclint console-script invocation. -- Completed Phase 5: created `CONTRIBUTING.md`, deleted `docs/api.md`, updated cross-references, added GitHub Pages links to `README.md` and `docs/index.md`. -- Audited all public functions for transitive raises; updated `Raises` sections in `writer.py` and `plan.py`; documented `KeyError` from `_find_loop_index` in parameter descriptions (pydoclint DOC502 workaround). -- Added complete `OutputPlan` example to `docs/outputspec.md`; moved non-example root scripts to `scripts/`. -- Lessons added: 133 (pydoclint console script), 134 (transitive raises in parameter descriptions), 135 (maturin CI venv). - ---- - -## Previous work (summary) - -- **2026-05-12, debug_grouped branch**: Fixed seven cascading GROUPED emit correctness bugs — hybrid orphan routing for no-FK-to-Set tables, non-PK FK suppression removed, `fallback_id = None`, bridge-block PK-stripping restricted to sets with own blocks. 1959 tests passing. -- **2026-05-06, debug_grouped branch**: Redesigned GROUPED mode with Set-identity fingerprint approach replacing FK-graph BFS; `all_of` multi-anchor matching now works correctly. 1900 tests passing. -- **2026-05-05, debug-original-output branch**: Implemented `OutputPlan`/`BlockSpec` enhancements (`only`, `any_of`, `all_of`, `has`, `attach_to`, `SchemaSpec.descendants`); fixed ORIGINAL mode category ordering with `_loop_groups` event positions; ORIGINAL mode now ignores `OutputPlan` with a warning. 1813 tests passing. +## What was done (2026-06-13/14, complexity branch) + +Decomposed all F-grade and all 6 reducible E-grade functions into private helpers, writing branch-coverage tests before each refactor. Test count: 1858 → 2076 (+218). No remaining F-grade or reducible E-grade; only `_resolve_fk_group` (E/33) stays as an irreducible xenon exception. + +**F-grade (all resolved):** +- **`inspect_schema` (61 → 1)**: 7 helpers; 16 tests. +- **`_collect_all_blocks` (45 → B/7)**: 4 helpers; 11 tests. +- **`_render_merge_group` (54 → D/21)**: 5 helpers; 11 tests. +- **`visualise_schema` (55 → B/7)**: 5 helpers; 6 tests. +- **`_render_block` (69 → C/14)**: 5 helpers; 24 tests. +- **`generate_schema` (98 → A/2)**: 7 helpers; 19 tests. +- **`_collect_grouped` (170 → D/26)**: 12 helpers; 29 tests. + +**E-grade (all reducible resolved):** +- **`propagate_fk_sql` (36 → A)**: 3 helpers; 17 tests. +- **`_run_fk_fill_pass` (39 → A)**: 3 helpers; 11 tests. +- **`_collect_structure` (32 → A)**: 2 helpers; 20 tests. +- **`_render_original_loop_group` (38 → C/15)**: extracted `_render_positional_join` D/24; 8 tests. +- **`_render_set_category` (31 → A)**: 3 helpers; 19 tests. +- **`consolidate_component_intensities` (33 → B/7)**: 2 helpers; 21 tests. --- @@ -141,7 +39,6 @@ Completed the full MkDocs + mkdocstrings documentation pipeline (Phases 1–5 of bottleneck is `ROW_NUMBER()` sort for large tables. Not in scope unless a specific use case requires it. - --- ## What's Next (priority order) @@ -167,7 +64,6 @@ Completed the full MkDocs + mkdocstrings documentation pipeline (Phases 1–5 of ## Remaining Items (unscheduled) - - **`_validation_result` table** — see Open Decision 2. - **Scope `ddl.dic` defaults** — load `ddl.dic` at schema-generation time as authoritative diff --git a/tests/database/test_component_intensities.py b/tests/database/test_component_intensities.py new file mode 100644 index 0000000..288c4b2 --- /dev/null +++ b/tests/database/test_component_intensities.py @@ -0,0 +1,311 @@ +"""Branch-coverage tests for consolidate_component_intensities. + +Tests lock down current behaviour before refactoring the E-grade function. +All tests run against an in-memory DuckDB connection — no schema or IR needed. +""" +import json + +import duckdb +import pytest + +from cifflow.database.component_intensities import ( + _all_dot, + consolidate_component_intensities, +) +from cifflow.ingestion.ingest import _CONTAINER_PREFIX + + +# --------------------------------------------------------------------------- +# DB helpers +# --------------------------------------------------------------------------- + +def _make_db() -> duckdb.DuckDBPyConnection: + """Create an in-memory DuckDB with the three tables the function requires.""" + db = duckdb.connect() + db.execute(""" + CREATE TABLE pd_calc_component ( + "_cifflow_block_id" TEXT, + "point_id" TEXT, + "diffractogram_id" TEXT, + "phase_id" TEXT, + "intensity_net" TEXT, + "intensity_total" TEXT + ) + """) + db.execute(""" + CREATE TABLE pd_calc ( + "_cifflow_block_id" TEXT, + "point_id" TEXT, + "diffractogram_id" TEXT, + "component_intensities_net" TEXT, + "component_intensities_total" TEXT + ) + """) + db.execute(""" + CREATE TABLE pd_calc_overall ( + "diffractogram_id" TEXT, + "component_presentation_order" TEXT + ) + """) + return db + + +def _insert_component(db, point_id, diff_id, phase_id, + net=None, total=None, block_id='blk1'): + db.execute( + 'INSERT INTO pd_calc_component VALUES (?, ?, ?, ?, ?, ?)', + [block_id, point_id, diff_id, phase_id, net, total], + ) + + +def _insert_pd_calc(db, point_id, diff_id, block_id='blk1'): + db.execute( + 'INSERT INTO pd_calc ("_cifflow_block_id", "point_id", "diffractogram_id")' + ' VALUES (?, ?, ?)', + [block_id, point_id, diff_id], + ) + + +def _insert_overall(db, diff_id, pres_order=None): + db.execute( + 'INSERT INTO pd_calc_overall VALUES (?, ?)', + [diff_id, pres_order], + ) + + +def _get_net(db, point_id, diff_id): + row = db.execute( + 'SELECT "component_intensities_net" FROM pd_calc' + ' WHERE "point_id"=? AND "diffractogram_id"=?', + [point_id, diff_id], + ).fetchone() + return row[0] if row else None + + +def _get_total(db, point_id, diff_id): + row = db.execute( + 'SELECT "component_intensities_total" FROM pd_calc' + ' WHERE "point_id"=? AND "diffractogram_id"=?', + [point_id, diff_id], + ).fetchone() + return row[0] if row else None + + +def _get_pres(db, diff_id): + row = db.execute( + 'SELECT "component_presentation_order" FROM pd_calc_overall' + ' WHERE "diffractogram_id"=?', + [diff_id], + ).fetchone() + return row[0] if row else None + + +def _has_column(db, table, col) -> bool: + try: + db.execute(f'SELECT "{col}" FROM {table} LIMIT 1') + return True + except duckdb.Error: + return False + + +def _decode(val: str) -> list: + """Strip chr(0) prefix and json.loads.""" + if val and val.startswith(_CONTAINER_PREFIX): + val = val[1:] + return json.loads(val) + + +# --------------------------------------------------------------------------- +# Tests: _all_dot +# --------------------------------------------------------------------------- + +class TestAllDot: + def test_all_dot_true_for_all_dot_list(self): + stored = _CONTAINER_PREFIX + '[".", "."]' + assert _all_dot(stored) + + def test_all_dot_false_for_mixed_list(self): + stored = _CONTAINER_PREFIX + '[".", "1.0"]' + assert not _all_dot(stored) + + def test_all_dot_false_for_empty_list(self): + stored = _CONTAINER_PREFIX + '[]' + assert not _all_dot(stored) + + def test_all_dot_false_for_unprefixed_raw_dot(self): + # Raw to_json() output (no prefix) — used during assembly check + assert _all_dot('["."]') + + def test_all_dot_false_for_invalid_json(self): + assert not _all_dot('not json') + + +# --------------------------------------------------------------------------- +# Tests: consolidate_component_intensities +# --------------------------------------------------------------------------- + +class TestConsolidateComponentIntensities: + def test_no_table_returns_zero(self): + """Missing pd_calc_component → duckdb.Error caught, return 0.""" + db = duckdb.connect() + result = consolidate_component_intensities(db, None) + assert result == 0 + + def test_empty_component_table_returns_zero(self): + db = _make_db() + result = consolidate_component_intensities(db, None) + assert result == 0 + + def test_single_phase_single_point_assembled(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='100.0', total='110.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + n = consolidate_component_intensities(db, None) + assert n == 1 + assert _decode(_get_net(db, 'p1', 'd1')) == ['100.0'] + assert _decode(_get_total(db, 'p1', 'd1')) == ['110.0'] + + def test_phase_ids_sorted_alphabetically_in_list(self): + db = _make_db() + # Insert in reverse order — presentation order must be sorted + _insert_component(db, 'p1', 'd1', 'phZ', net='10.0') + _insert_component(db, 'p1', 'd1', 'phA', net='20.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + pres = _get_pres(db, 'd1') + assert _decode(pres) == ['phA', 'phZ'] + # phA=20.0 first, phZ=10.0 second + assert _decode(_get_net(db, 'p1', 'd1')) == ['20.0', '10.0'] + + def test_missing_phase_for_point_filled_with_dot(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='10.0') + _insert_component(db, 'p1', 'd1', 'phB', net='20.0') + _insert_component(db, 'p2', 'd1', 'phA', net='30.0') + # p2 has no phB row → slot must be '.' + _insert_pd_calc(db, 'p1', 'd1') + _insert_pd_calc(db, 'p2', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + vals = _decode(_get_net(db, 'p2', 'd1')) + assert vals[1] == '.' # phB slot + + def test_all_dot_net_column_dropped(self): + """All net values are NULL → assembled to '.' → column dropped.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net=None, total='10.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + assert not _has_column(db, 'pd_calc', 'component_intensities_net') + assert _has_column(db, 'pd_calc', 'component_intensities_total') + + def test_all_dot_total_column_dropped(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='10.0', total=None) + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + assert _has_column(db, 'pd_calc', 'component_intensities_net') + assert not _has_column(db, 'pd_calc', 'component_intensities_total') + + def test_both_all_dot_drops_presentation_order_column(self): + """When both intensity columns dropped, component_presentation_order is too.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net=None, total=None) + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + assert not _has_column(db, 'pd_calc_overall', 'component_presentation_order') + + def test_presentation_order_updated_for_existing_diff(self): + """Existing pd_calc_overall row is UPDATEd, not duplicated.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1', pres_order='old_value') + consolidate_component_intensities(db, None) + pres = _get_pres(db, 'd1') + assert _decode(pres) == ['phA'] + count = db.execute( + 'SELECT COUNT(*) FROM pd_calc_overall WHERE "diffractogram_id"=?', ['d1'] + ).fetchone()[0] + assert count == 1 # no duplicate inserted + + def test_presentation_order_inserted_for_absent_diff(self): + """No pd_calc_overall row → new row INSERTed.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + _insert_pd_calc(db, 'p1', 'd1') + # Intentionally no _insert_overall + consolidate_component_intensities(db, None) + pres = _get_pres(db, 'd1') + assert _decode(pres) == ['phA'] + + def test_clear_source_deletes_component_rows(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None, clear_source=True) + count = db.execute('SELECT COUNT(*) FROM pd_calc_component').fetchone()[0] + assert count == 0 + + def test_pd_calc_row_auto_created_when_absent(self): + """pd_calc row missing for a component point → auto-inserted.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + # No _insert_pd_calc + _insert_overall(db, 'd1') + n = consolidate_component_intensities(db, None) + assert n == 1 + assert _get_net(db, 'p1', 'd1') is not None + + def test_pd_calc_rows_without_component_data_get_all_dot(self): + """pd_calc rows for a diffractogram with no component entries get all-'.' lists.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + _insert_pd_calc(db, 'p1', 'd1') + # p2 exists in pd_calc but has no component row + _insert_pd_calc(db, 'p2', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + assert _decode(_get_net(db, 'p2', 'd1')) == ['.'] + + def test_multiple_diffractograms_handled_independently(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='5.0') + _insert_component(db, 'p2', 'd2', 'phB', net='10.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_pd_calc(db, 'p2', 'd2') + _insert_overall(db, 'd1') + _insert_overall(db, 'd2') + n = consolidate_component_intensities(db, None) + assert n == 2 + assert _decode(_get_net(db, 'p1', 'd1')) == ['5.0'] + assert _decode(_get_net(db, 'p2', 'd2')) == ['10.0'] + + def test_returns_count_of_pd_calc_rows_updated(self): + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net='1.0') + _insert_component(db, 'p2', 'd1', 'phA', net='2.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_pd_calc(db, 'p2', 'd1') + _insert_overall(db, 'd1') + n = consolidate_component_intensities(db, None) + assert n == 2 + + def test_null_net_coalesced_to_dot_in_list(self): + """NULL intensity_net in component → '.' in assembled list.""" + db = _make_db() + _insert_component(db, 'p1', 'd1', 'phA', net=None, total='5.0') + _insert_component(db, 'p1', 'd1', 'phB', net='3.0', total='4.0') + _insert_pd_calc(db, 'p1', 'd1') + _insert_overall(db, 'd1') + consolidate_component_intensities(db, None) + vals = _decode(_get_net(db, 'p1', 'd1')) + # phA slot should be '.' + idx_a = _decode(_get_pres(db, 'd1')).index('phA') + assert vals[idx_a] == '.' diff --git a/tests/dictionary/test_schema.py b/tests/dictionary/test_schema.py index d2485f5..340d64e 100644 --- a/tests/dictionary/test_schema.py +++ b/tests/dictionary/test_schema.py @@ -9,6 +9,7 @@ from cifflow.dictionary.schema import ( ColumnDef, ForeignKeyDef, + PartialLinkDef, SchemaSpec, TableDef, emit_create_statements, @@ -992,4 +993,386 @@ def test_unknown_root_returns_empty(self, schema): assert schema.descendants('does_not_exist') == frozenset() +# --------------------------------------------------------------------------- +# Dual links to the same PK (bond-endpoint pattern) +# --------------------------------------------------------------------------- + +class TestDualLinkToSamePK: + """has_conflicts=True, missing_pk_cols={}: two source columns reference the same sole PK. + Expect one independent FK per source column (not a composite FK).""" + + @pytest.fixture + def schema(self): + cats = [ + _cat('atom', 'atom', 'Loop', ['_atom.number']), + _cat('bond', 'bond', 'Loop', ['_bond.id']), + ] + items = [ + _item('_atom.number', 'atom', 'number', type_purpose='Key', type_contents='Text'), + _item('_bond.id', 'bond', 'id', type_purpose='Key', type_contents='Text'), + _item('_bond.atom_1', 'bond', 'atom_1', type_purpose='Link', + linked_item_id='_atom.number', type_contents='Text'), + _item('_bond.atom_2', 'bond', 'atom_2', type_purpose='Link', + linked_item_id='_atom.number', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + def test_two_fks_produced(self, schema): + assert len(schema.tables['bond'].foreign_keys) == 2 + + def test_each_fk_targets_atom_number(self, schema): + for fk in schema.tables['bond'].foreign_keys: + assert fk.target_table == 'atom' + assert fk.target_columns == ['number'] + + def test_both_endpoint_columns_present_as_sources(self, schema): + src_cols = {fk.source_columns[0] for fk in schema.tables['bond'].foreign_keys} + assert src_cols == {'atom_1', 'atom_2'} + + +# --------------------------------------------------------------------------- +# Missing one PK column already present in source (no bridge lookup needed) +# --------------------------------------------------------------------------- + +class TestMissingOnePKAlreadyInSrc: + """missing_pk_cols==1 and the missing column already exists in src. + Expect a composite FK formed without a transitive bridge lookup.""" + + def test_composite_fk_formed_from_existing_column(self): + # parent PKs: (a, b); child has 'b' in its own PK; child.c → parent.a + # missing_pk_col='b' is found directly in child's columns → FK formed. + cats = [ + _cat('parent', 'parent', 'Loop', ['_parent.a', '_parent.b']), + _cat('child', 'child', 'Loop', ['_child.b', '_child.x']), + ] + items = [ + _item('_parent.a', 'parent', 'a', type_purpose='Key', type_contents='Text'), + _item('_parent.b', 'parent', 'b', type_purpose='Key', type_contents='Text'), + _item('_child.b', 'child', 'b', type_purpose='Key', type_contents='Text'), + _item('_child.x', 'child', 'x', type_purpose='Key', type_contents='Text'), + _item('_child.c', 'child', 'c', type_purpose='Link', + linked_item_id='_parent.a', type_contents='Text'), + ] + d = _make_dict(cats, items) + schema = generate_schema(d) + fks = schema.tables['child'].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.target_table == 'parent' + assert fk.target_columns == ['a', 'b'] + assert fk.source_columns == ['c', 'b'] + + +# --------------------------------------------------------------------------- +# More than one PK column missing (bridge search skipped) +# --------------------------------------------------------------------------- + +class TestMoreThanOneMissingPK: + """len(missing_pk_cols) > 1 — falls into the final elif arm (no bridge attempted).""" + + @pytest.fixture + def schema(self): + cats = [ + _cat('parent', 'parent', 'Loop', ['_parent.a', '_parent.b', '_parent.c']), + _cat('child', 'child', 'Loop', ['_child.id']), + ] + items = [ + _item('_parent.a', 'parent', 'a', type_purpose='Key', type_contents='Text'), + _item('_parent.b', 'parent', 'b', type_purpose='Key', type_contents='Text'), + _item('_parent.c', 'parent', 'c', type_purpose='Key', type_contents='Text'), + _item('_child.id', 'child', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.a', 'child', 'a', type_purpose='Link', + linked_item_id='_parent.a', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + def test_no_fk_produced(self, schema): + assert schema.tables['child'].foreign_keys == [] + + def test_warning_emitted(self, schema): + assert any('_child.a' in w and 'skipping' in w for w in schema.warnings) + + def test_partial_link_recorded(self, schema): + assert any( + pl.source_table == 'child' and pl.source_column == 'a' + for pl in schema.partial_links + ) + + +# --------------------------------------------------------------------------- +# partial_links field +# --------------------------------------------------------------------------- + +class TestPartialLinks: + """PartialLinkDef entries are recorded for unresolvable Link items.""" + + @pytest.fixture + def schema(self): + # src.ref → tgt.extra where 'extra' is not a PK → partial link + cats = [ + _cat('src', 'src', 'Loop', ['_src.id']), + _cat('tgt', 'tgt', 'Loop', ['_tgt.id']), + ] + items = [ + _item('_src.id', 'src', 'id', type_purpose='Key', type_contents='Text'), + _item('_tgt.id', 'tgt', 'id', type_purpose='Key', type_contents='Text'), + _item('_tgt.extra', 'tgt', 'extra', type_contents='Text'), + _item('_src.ref', 'src', 'ref', type_purpose='Link', + linked_item_id='_tgt.extra', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + def test_partial_link_in_list(self, schema): + assert any( + pl.source_table == 'src' and pl.source_column == 'ref' + for pl in schema.partial_links + ) + + def test_partial_link_target_fields(self, schema): + pl = next(p for p in schema.partial_links if p.source_column == 'ref') + assert pl.target_table == 'tgt' + assert pl.target_column == 'extra' + + def test_partial_link_missing_pks(self, schema): + pl = next(p for p in schema.partial_links if p.source_column == 'ref') + assert 'id' in pl.missing_pk_cols + + +# --------------------------------------------------------------------------- +# Non-PK Link items in propagation_links +# --------------------------------------------------------------------------- + +class TestPropagationLinksNonPK: + """Non-PK Link items enter propagation_links only when enumeration_default is set.""" + + def test_non_pk_link_with_default_in_propagation_links(self): + cats = [ + _cat('parent', 'parent', 'Set', ['_parent.id']), + _cat('child', 'child', 'Loop', ['_child.id']), + ] + items = [ + _item('_parent.id', 'parent', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.id', 'child', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.ref', 'child', 'ref', type_purpose='Link', + linked_item_id='_parent.id', type_contents='Text', + enumeration_default='DEFAULT_VAL'), + ] + d = _make_dict(cats, items) + schema = generate_schema(d) + assert 'child' in schema.propagation_links + assert any(col == 'ref' for col, _, _ in schema.propagation_links['child']) + + def test_non_pk_link_default_value_stored(self): + cats = [ + _cat('parent', 'parent', 'Set', ['_parent.id']), + _cat('child', 'child', 'Loop', ['_child.id']), + ] + items = [ + _item('_parent.id', 'parent', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.id', 'child', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.ref', 'child', 'ref', type_purpose='Link', + linked_item_id='_parent.id', type_contents='Text', + enumeration_default='MY_DEFAULT'), + ] + d = _make_dict(cats, items) + schema = generate_schema(d) + entry = next(e for e in schema.propagation_links['child'] if e[0] == 'ref') + assert entry[2] == 'MY_DEFAULT' + + def test_non_pk_link_without_default_excluded(self): + cats = [ + _cat('parent', 'parent', 'Set', ['_parent.id']), + _cat('child', 'child', 'Loop', ['_child.id']), + ] + items = [ + _item('_parent.id', 'parent', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.id', 'child', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.ref', 'child', 'ref', type_purpose='Link', + linked_item_id='_parent.id', type_contents='Text'), + ] + d = _make_dict(cats, items) + schema = generate_schema(d) + entries = schema.propagation_links.get('child', []) + assert not any(col == 'ref' for col, _, _ in entries) + + def test_non_pk_link_column_remains_nullable(self): + # Non-PK column should already be nullable — propagation_links does not change it + cats = [ + _cat('parent', 'parent', 'Set', ['_parent.id']), + _cat('child', 'child', 'Loop', ['_child.id']), + ] + items = [ + _item('_parent.id', 'parent', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.id', 'child', 'id', type_purpose='Key', type_contents='Text'), + _item('_child.ref', 'child', 'ref', type_purpose='Link', + linked_item_id='_parent.id', type_contents='Text', + enumeration_default='X'), + ] + d = _make_dict(cats, items) + schema = generate_schema(d) + ref_col = next(c for c in schema.tables['child'].columns if c.name == 'ref') + assert ref_col.nullable is True + + +# --------------------------------------------------------------------------- +# Category parent map edge cases +# --------------------------------------------------------------------------- + +class TestCategoryParentMapEdgeCases: + """Self-reference and parent-not-in-schema produce None in category_parent.""" + + def test_self_referential_category_has_none_parent(self): + # _cat('x', 'x', 'Set') → category_id == definition_id == 'x' → parent_tbl == tbl_name → None + cats = [_cat('x', 'x', 'Set')] + d = _make_dict(cats, []) + schema = generate_schema(d) + assert schema.category_parent.get('x') is None + + def test_parent_not_in_schema_gives_none(self): + # child's category_id points to a Head category (not in tables) → None + head_cat = DdlmItem( + definition_id='headcat', scope='Category', definition_class='Head', + category_id='headcat', object_id=None, type_purpose=None, type_source=None, + type_container='Single', type_contents=None, linked_item_id=None, + units_code=None, description=None, category_keys=[], + ) + child_cat = DdlmItem( + definition_id='child', scope='Category', definition_class='Loop', + category_id='headcat', # parent is the Head category + object_id=None, type_purpose=None, type_source=None, + type_container='Single', type_contents=None, linked_item_id=None, + units_code=None, description=None, category_keys=[], + ) + cats_map = {'headcat': head_cat, 'child': child_cat} + d = DdlmDictionary( + name='T', title=None, version=None, + categories=cats_map, items={}, tag_to_item=cats_map, + alias_to_definition_id={}, deprecated_ids=set(), + ) + schema = generate_schema(d) + assert schema.category_parent.get('child') is None + + +# --------------------------------------------------------------------------- +# tag_to_category_class +# --------------------------------------------------------------------------- + +class TestTagToCategoryClass: + def test_set_item_tagged_as_set(self): + cats = [_cat('cfg', 'cfg', 'Set', ['_cfg.id'])] + items = [_item('_cfg.id', 'cfg', 'id', type_contents='Text')] + d = _make_dict(cats, items) + schema = generate_schema(d) + assert schema.tag_to_category_class.get('_cfg.id') == 'Set' + + def test_loop_item_tagged_as_loop(self): + cats = [_cat('meas', 'meas', 'Loop', ['_meas.id'])] + items = [_item('_meas.id', 'meas', 'id', type_contents='Text')] + d = _make_dict(cats, items) + schema = generate_schema(d) + assert schema.tag_to_category_class.get('_meas.id') == 'Loop' + + def test_item_with_head_category_excluded(self): + # item whose category is Head → not in tag_to_category_class + head_cat = DdlmItem( + definition_id='hd', scope='Category', definition_class='Head', + category_id='hd', object_id=None, type_purpose=None, type_source=None, + type_container='Single', type_contents=None, linked_item_id=None, + units_code=None, description=None, category_keys=[], + ) + head_item = DdlmItem( + definition_id='_hd.val', scope='Item', definition_class='Datum', + category_id='hd', object_id='val', type_purpose=None, type_source=None, + type_container='Single', type_contents='Text', linked_item_id=None, + units_code=None, description=None, + ) + cats_map = {'hd': head_cat} + item_map = {'_hd.val': head_item} + tag_to_item = {**cats_map, **item_map} + d = DdlmDictionary( + name='T', title=None, version=None, + categories=cats_map, items=item_map, tag_to_item=tag_to_item, + alias_to_definition_id={}, deprecated_ids=set(), + ) + schema = generate_schema(d) + assert '_hd.val' not in schema.tag_to_category_class + + +# --------------------------------------------------------------------------- +# deprecated_replacements +# --------------------------------------------------------------------------- + +class TestDeprecatedReplacements: + def test_deprecated_item_in_deprecated_replacements(self): + cats = [_cat('cfg', 'cfg', 'Set', ['_cfg.id'])] + old_item = DdlmItem( + definition_id='_cfg.old_name', scope='Item', definition_class='Datum', + category_id='cfg', object_id='old_name', type_purpose=None, type_source=None, + type_container='Single', type_contents='Text', linked_item_id=None, + units_code=None, description=None, + is_deprecated=True, replaced_by=['_cfg.new_name'], + ) + cats_d = {c.definition_id: c for c in cats} + item_map = {'_cfg.old_name': old_item} + tag_to_item = {**cats_d, **item_map} + d = DdlmDictionary( + name='T', title=None, version=None, + categories=cats_d, items=item_map, tag_to_item=tag_to_item, + alias_to_definition_id={}, deprecated_ids={'_cfg.old_name'}, + ) + schema = generate_schema(d) + assert '_cfg.old_name' in schema.deprecated_replacements + assert schema.deprecated_replacements['_cfg.old_name'] == ['_cfg.new_name'] + + def test_non_deprecated_item_absent_from_deprecated_replacements(self): + cats = [_cat('cfg', 'cfg', 'Set', ['_cfg.id'])] + items = [_item('_cfg.id', 'cfg', 'id', type_contents='Text')] + d = _make_dict(cats, items) + schema = generate_schema(d) + assert '_cfg.id' not in schema.deprecated_replacements + + +# --------------------------------------------------------------------------- +# SchemaSpec passthrough fields +# --------------------------------------------------------------------------- + +class TestSchemaSpecPassthrough: + def test_alias_to_definition_id_passthrough(self): + cats = [_cat('cfg', 'cfg', 'Set')] + cats_d = {c.definition_id: c for c in cats} + d = DdlmDictionary( + name='T', title=None, version=None, + categories=cats_d, items={}, tag_to_item=cats_d, + alias_to_definition_id={'_cfg.old': '_cfg.new'}, + deprecated_ids=set(), + ) + schema = generate_schema(d) + assert schema.alias_to_definition_id == {'_cfg.old': '_cfg.new'} + + def test_deprecated_ids_passthrough(self): + cats = [_cat('cfg', 'cfg', 'Set')] + cats_d = {c.definition_id: c for c in cats} + d = DdlmDictionary( + name='T', title=None, version=None, + categories=cats_d, items={}, tag_to_item=cats_d, + alias_to_definition_id={}, + deprecated_ids={'_cfg.old_tag'}, + ) + schema = generate_schema(d) + assert '_cfg.old_tag' in schema.deprecated_ids + + def test_dictionary_metadata_passthrough(self): + cats = [_cat('cfg', 'cfg', 'Set')] + cats_d = {c.definition_id: c for c in cats} + d = DdlmDictionary( + name='MY_DICT', title='My Dictionary', version='2.0', + categories=cats_d, items={}, tag_to_item=cats_d, + alias_to_definition_id={}, deprecated_ids=set(), + uri='https://example.com/mydict.dic', + ) + schema = generate_schema(d) + assert schema.dictionary_name == 'MY_DICT' + assert schema.dictionary_title == 'My Dictionary' + assert schema.dictionary_version == '2.0' + assert schema.dictionary_uri == 'https://example.com/mydict.dic' diff --git a/tests/dictionary/test_visualise.py b/tests/dictionary/test_visualise.py index b9ff38a..b2274a5 100644 --- a/tests/dictionary/test_visualise.py +++ b/tests/dictionary/test_visualise.py @@ -788,3 +788,137 @@ def test_no_external_urls(self): assert 'cdn.jsdelivr.net' not in html assert 'unpkg.com' not in html assert 'cdnjs' not in html + + +# --------------------------------------------------------------------------- +# visualise_schema — branch coverage for Stage 1 extraction targets +# --------------------------------------------------------------------------- + +class TestVisualiseSchemaContextSetup: + """_build_vis_context branches: deprecated + real_tables + ghost filtering.""" + + def test_hide_deprecated_table_removed_from_ghost_candidates(self): + # A deprecated table that is also referenced by another table's FK must not + # become a [MISSING] ghost node when hide_deprecated=True removes it from + # real_tables AND from ghost_tables. + fk = ForeignKeyDef('child', ['dep_id'], 'dep', ['id']) + child_cols = [_col('id', definition_id='_child.id', is_primary_key=True), + _col('dep_id', definition_id='_child.dep_id')] + dep_cols = [_col('id', definition_id='_dep.id', is_primary_key=True)] + child = _table('child', columns=child_cols, primary_keys=['id'], foreign_keys=[fk]) + dep = _table('dep', columns=dep_cols, primary_keys=['id']) + s = SchemaSpec( + tables={'child': child, 'dep': dep}, column_to_tag={}, + bridge_columns=[], category_parent={}, + deprecated_ids={'_dep.id'}, + ) + dot = visualise_schema(s, hide_deprecated=True) + assert '[MISSING]' not in dot + assert '"child" -> "dep"' not in dot + + +class TestVisualiseSchemaClusteredNodes: + """_emit_clustered_nodes branches.""" + + def test_partial_component_visible_members_rendered_in_subgraph(self): + # Component {active, dep} where dep is hidden by hide_deprecated=True. + # partial_structural_components picks up {active, dep} because + # not c.issubset(real_tables) but any(t in real_tables for t in c). + # 'active' should still appear in a cluster subgraph. + fk = ForeignKeyDef('active', ['dep_id'], 'dep', ['id']) + active_cols = [_col('id', definition_id='_active.id', is_primary_key=True), + _col('dep_id', definition_id='_active.dep_id')] + dep_cols = [_col('id', definition_id='_dep.id', is_primary_key=True)] + active = _table('active', columns=active_cols, primary_keys=['id'], foreign_keys=[fk]) + dep = _table('dep', columns=dep_cols, primary_keys=['id']) + s = SchemaSpec( + tables={'active': active, 'dep': dep}, column_to_tag={}, + bridge_columns=[], category_parent={}, + deprecated_ids={'_dep.id'}, + ) + dot = visualise_schema(s, highlight_components=True, hide_deprecated=True) + assert 'subgraph cluster_' in dot + assert 'active' in dot + assert '"dep"' not in dot # deprecated table must not appear as a DOT node + + +class TestVisualiseSchemaFkEdges: + """_emit_fk_edges branches.""" + + def test_fk_edge_skipped_when_target_not_in_real_or_ghost(self): + # FK target is in real_tables normally, but with show_orphans=False the + # target (an orphan-connected table) disappears. Actually the easiest + # path: target is deprecated + hide_deprecated=True → not in real_tables, + # not a ghost → edge must be suppressed. + fk = ForeignKeyDef('child', ['dep_id'], 'dep', ['id']) + child_cols = [_col('id', definition_id='_child.id', is_primary_key=True), + _col('dep_id', definition_id='_child.dep_id')] + dep_cols = [_col('id', definition_id='_dep.id', is_primary_key=True)] + child = _table('child', columns=child_cols, primary_keys=['id'], foreign_keys=[fk]) + dep = _table('dep', columns=dep_cols, primary_keys=['id']) + s = SchemaSpec( + tables={'child': child, 'dep': dep}, column_to_tag={}, + bridge_columns=[], category_parent={}, + deprecated_ids={'_dep.id'}, + ) + dot = visualise_schema(s, hide_deprecated=True) + assert '"child" -> "dep"' not in dot + + +class TestVisualiseSchemaEdgeGuards: + """_emit_bridge_edges and _emit_parent_edges guard branches.""" + + def test_bridge_edge_skipped_when_target_hidden_by_deprecated(self): + # Bridge target table is fully deprecated → removed from real_tables, not a ghost + # → neither target_in_real nor target_is_ghost → edge must be suppressed. + bc = BridgeColumnDef('src', 'dep_id', [('via_col', 'dep', 'id')], 'dep_id') + dep_cols = [_col('id', definition_id='_dep.id', is_primary_key=True)] + src = _table('src') + dep = _table('dep', columns=dep_cols, primary_keys=['id']) + s = SchemaSpec( + tables={'src': src, 'dep': dep}, column_to_tag={}, + bridge_columns=[bc], category_parent={}, + deprecated_ids={'_dep.id'}, + ) + dot = visualise_schema(s, show_bridge=True, hide_deprecated=True) + assert '"src" -> "dep"' not in dot + + def test_parent_none_value_produces_no_edge(self): + # category_parent entry with None value → `if not parent: continue` fires. + s = _schema([_table('t')], category_parent={'t': None}) + dot = visualise_schema(s, show_parent_edges=True) + # No parent edge for 't' since parent is None + assert '"t" ->' not in dot + + def test_parent_edge_skipped_when_child_not_in_real_tables(self): + # hide_deprecated=True removes the child table from real_tables. + # child_in_real → False → `if not child_in_real: continue` fires, no edge. + child_cols = [_col('id', definition_id='_child.id', is_primary_key=True)] + par_cols = [_col('id', definition_id='_par.id', is_primary_key=True)] + child = _table('child', columns=child_cols, primary_keys=['id']) + par = _table('par', columns=par_cols, primary_keys=['id']) + s = SchemaSpec( + tables={'child': child, 'par': par}, column_to_tag={}, + bridge_columns=[], category_parent={'child': 'par'}, + deprecated_ids={'_child.id'}, + ) + dot = visualise_schema(s, show_parent_edges=True, hide_deprecated=True) + assert '"child" -> "par"' not in dot + + def test_parent_edge_skipped_when_parent_not_ghost_and_not_in_real(self): + # Parent table is fully deprecated → not in real_tables, not a ghost + # (it's still in schema.tables) → `not parent_is_ghost and not parent_in_real` + # fires → edge must be suppressed even with show_parent_edges=True. + par_cols = [_col('id', definition_id='_par.id', is_primary_key=True)] + par = _table('par', columns=par_cols, primary_keys=['id']) + fk = ForeignKeyDef('child', ['par_id'], 'par', ['id']) + child_cols = [_col('id', definition_id='_child.id', is_primary_key=True), + _col('par_id', definition_id='_child.par_id')] + child = _table('child', columns=child_cols, primary_keys=['id'], foreign_keys=[fk]) + s = SchemaSpec( + tables={'child': child, 'par': par}, column_to_tag={}, + bridge_columns=[], category_parent={'child': 'par'}, + deprecated_ids={'_par.id'}, + ) + dot = visualise_schema(s, show_parent_edges=True, hide_deprecated=True) + assert '"child" -> "par"' not in dot diff --git a/tests/ingestion/test_fk_fill_pass.py b/tests/ingestion/test_fk_fill_pass.py new file mode 100644 index 0000000..49ada2b --- /dev/null +++ b/tests/ingestion/test_fk_fill_pass.py @@ -0,0 +1,299 @@ +"""Branch-coverage tests for _run_fk_fill_pass helper functions. + +Tests target _fill_single_fk, _fill_composite_fk, and _fill_propagation_links +in isolation before the refactoring. +""" +import duckdb +import pytest + +from cifflow.dictionary.ddlm_item import DdlmItem +from cifflow.dictionary.ddlm_parser import DdlmDictionary +from cifflow.dictionary.schema import generate_schema +from cifflow.ingestion.duckdb_ingest import ( + _fill_composite_fk, + _fill_propagation_links, + _fill_single_fk, + setup_duckdb, +) +from cifflow.ingestion.ingest import build_tag_to_column + + +# --------------------------------------------------------------------------- +# Schema helpers +# --------------------------------------------------------------------------- + +def _item(definition_id, category_id, object_id, *, type_purpose=None, + type_contents=None, linked_item_id=None, enumeration_default=None): + return DdlmItem( + definition_id=definition_id, scope='Item', definition_class='Datum', + category_id=category_id, object_id=object_id, + type_purpose=type_purpose, type_source=None, type_container='Single', + type_contents=type_contents, linked_item_id=linked_item_id, + units_code=None, description=None, + enumeration_default=enumeration_default, + ) + + +def _cat(definition_id, cat_class, category_keys=None): + return DdlmItem( + definition_id=definition_id, scope='Category', + definition_class=cat_class, category_id=None, object_id=None, + type_purpose=None, type_source=None, type_container='Single', + type_contents=None, linked_item_id=None, units_code=None, + description=None, category_keys=category_keys or [], + ) + + +def _make_dict(cats, items): + categories = {c.definition_id: c for c in cats} + item_map = {i.definition_id: i for i in items} + return DdlmDictionary( + name='TEST', title=None, version=None, + categories=categories, items=item_map, + tag_to_item={**categories, **item_map}, + alias_to_definition_id={}, deprecated_ids=set(), + ) + + +_SCALARS = '__scalars__' + + +def _insert(db, table, rows): + for i, row in enumerate(rows): + sys_cols = ['_cifflow_block_id', '_cifflow_block_idx', '_loop_id', '_iter_idx', '_cifflow_row_id'] + sys_vals = [ + f"'{row.get('_cifflow_block_id', 'blk1')}'", + str(row.get('_cifflow_block_idx', 0)), + f"'{row.get('_loop_id', _SCALARS)}'", + str(row.get('_iter_idx', 0)), + str(row.get('_cifflow_row_id', i + 1)), + ] + data_cols = [f'"{k}"' for k in row + if k not in ('_cifflow_block_id', '_cifflow_block_idx', '_loop_id', '_iter_idx', '_cifflow_row_id')] + data_vals = [ + f"'{v}'" if v is not None else 'NULL' + for k, v in row.items() + if k not in ('_cifflow_block_id', '_cifflow_block_idx', '_loop_id', '_iter_idx', '_cifflow_row_id') + ] + all_cols = ', '.join(sys_cols + data_cols) + all_vals = ', '.join(sys_vals + data_vals) + db.execute(f'INSERT INTO "_raw_{table}" ({all_cols}) VALUES ({all_vals})') + + +def _fetch(db, table, col): + return [r[0] for r in db.execute(f'SELECT "{col}" FROM "_raw_{table}"').fetchall()] + + +# --------------------------------------------------------------------------- +# Schemas +# --------------------------------------------------------------------------- + +def _schema_parent_child(): + """structure (Set PK=id) → cell (Set PK=structure_id key-FK→structure.id).""" + cats = [ + _cat('_structure', 'Set', ['_structure.id']), + _cat('_cell', 'Set', ['_cell.structure_id']), + ] + items = [ + _item('_structure.id', '_structure', 'id', type_purpose='Key', type_contents='Text'), + _item('_cell.structure_id', '_cell', 'structure_id', + type_purpose='Link', linked_item_id='_structure.id', type_contents='Text'), + _item('_cell.length_a', '_cell', 'length_a', + type_purpose='Measurand', type_contents='Real'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_non_key_fk(): + """structure (Set PK=id) and atom_site (Loop PK=label non-key FK structure_id→structure.id).""" + cats = [ + _cat('_structure', 'Set', ['_structure.id']), + _cat('_atom_site', 'Loop', ['_atom_site.label']), + ] + items = [ + _item('_structure.id', '_structure', 'id', type_purpose='Key', type_contents='Text'), + _item('_atom_site.label', '_atom_site', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom_site.structure_id', '_atom_site', 'structure_id', + type_purpose='Link', linked_item_id='_structure.id', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_composite_fk(): + """phase (Set PK=(id1,id2)), atom (Loop PK=label composite FK→phase).""" + cats = [ + _cat('_phase', 'Set', ['_phase.id1', '_phase.id2']), + _cat('_atom', 'Loop', ['_atom.label']), + ] + items = [ + _item('_phase.id1', '_phase', 'id1', type_purpose='Key', type_contents='Text'), + _item('_phase.id2', '_phase', 'id2', type_purpose='Key', type_contents='Text'), + _item('_atom.label', '_atom', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom.phase_id1', '_atom', 'phase_id1', + type_purpose='Link', linked_item_id='_phase.id1', type_contents='Text'), + _item('_atom.phase_id2', '_atom', 'phase_id2', + type_purpose='Link', linked_item_id='_phase.id2', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_with_default(): + """structure (Set PK=id) and atom_site (Loop, PK=label, non-PK Link col with default).""" + cats = [ + _cat('_structure', 'Set', ['_structure.id']), + _cat('_atom_site', 'Loop', ['_atom_site.label']), + ] + items = [ + _item('_structure.id', '_structure', 'id', type_purpose='Key', type_contents='Text'), + _item('_atom_site.label', '_atom_site', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom_site.structure_id', '_atom_site', 'structure_id', + type_purpose='Link', linked_item_id='_structure.id', + type_contents='Text', enumeration_default='default_struct'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _col_by_name(schema, tbl_name): + table = schema.tables[tbl_name] + return {c.name: c for c in table.columns if not c.is_synthetic} + + +# --------------------------------------------------------------------------- +# Tests: _fill_single_fk +# --------------------------------------------------------------------------- + +class TestFillSingleFk: + def test_key_fk_null_filled_from_loop_match(self): + """Key-FK NULL col is filled from parent row with same block+loop+iter.""" + schema = _schema_parent_child() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_loop_id': _SCALARS, '_iter_idx': 0, '_cifflow_row_id': 1}]) + _insert(db, 'cell', [{'structure_id': None, 'length_a': '5.0', '_loop_id': _SCALARS, '_iter_idx': 0, '_cifflow_row_id': 2}]) + table = schema.tables['cell'] + col_by_name = _col_by_name(schema, 'cell') + _fill_single_fk(db, 'cell', table, col_by_name, schema, False, lambda *a: None) + assert _fetch(db, 'cell', 'structure_id') == ['s1'] + + def test_key_fk_already_set_unchanged(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_cifflow_row_id': 1}]) + _insert(db, 'cell', [{'structure_id': 's1', 'length_a': '5.0', '_cifflow_row_id': 2}]) + table = schema.tables['cell'] + col_by_name = _col_by_name(schema, 'cell') + _fill_single_fk(db, 'cell', table, col_by_name, schema, False, lambda *a: None) + assert _fetch(db, 'cell', 'structure_id') == ['s1'] + + def test_non_key_fk_propagate_false_stays_null(self): + schema = _schema_non_key_fk() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_cifflow_row_id': 1}]) + _insert(db, 'atom_site', [{'label': 'C1', 'structure_id': None, '_cifflow_row_id': 2}]) + table = schema.tables['atom_site'] + col_by_name = _col_by_name(schema, 'atom_site') + _fill_single_fk(db, 'atom_site', table, col_by_name, schema, False, lambda *a: None) + assert _fetch(db, 'atom_site', 'structure_id') == [None] + + def test_non_key_fk_propagate_true_filled(self): + schema = _schema_non_key_fk() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_cifflow_row_id': 1}]) + _insert(db, 'atom_site', [{'label': 'C1', 'structure_id': None, '_cifflow_row_id': 2}]) + table = schema.tables['atom_site'] + col_by_name = _col_by_name(schema, 'atom_site') + _fill_single_fk(db, 'atom_site', table, col_by_name, schema, True, lambda *a: None) + assert _fetch(db, 'atom_site', 'structure_id') == ['s1'] + + def test_scalars_fallback_used_when_no_loop_match(self): + """Parent row is in scalars loop; child is in a named loop.""" + schema = _schema_non_key_fk() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_loop_id': _SCALARS, '_cifflow_row_id': 1}]) + _insert(db, 'atom_site', [{'label': 'C1', 'structure_id': None, '_loop_id': 'lp1', '_cifflow_row_id': 2}]) + table = schema.tables['atom_site'] + col_by_name = _col_by_name(schema, 'atom_site') + _fill_single_fk(db, 'atom_site', table, col_by_name, schema, True, lambda *a: None) + assert _fetch(db, 'atom_site', 'structure_id') == ['s1'] + + +# --------------------------------------------------------------------------- +# Tests: _fill_composite_fk +# --------------------------------------------------------------------------- + +class TestFillCompositeFk: + def test_composite_fk_nulls_filled_from_parent_when_propagate_true(self): + # phase_id1/phase_id2 are non-PK in atom → only filled when propagate_fk=True + schema = _schema_composite_fk() + db = setup_duckdb(schema) + _insert(db, 'phase', [{'id1': 'p1', 'id2': 'q1', '_loop_id': _SCALARS, '_cifflow_row_id': 1}]) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': None, 'phase_id2': None, + '_loop_id': _SCALARS, '_cifflow_row_id': 2}]) + table = schema.tables['atom'] + col_by_name = _col_by_name(schema, 'atom') + _fill_composite_fk(db, 'atom', table, col_by_name, schema, True) + assert _fetch(db, 'atom', 'phase_id1') == ['p1'] + assert _fetch(db, 'atom', 'phase_id2') == ['q1'] + + def test_composite_non_key_propagate_false_stays_null(self): + # phase_id1/phase_id2 are non-PK in atom (PK is label), propagate_fk=False + schema = _schema_composite_fk() + db = setup_duckdb(schema) + _insert(db, 'phase', [{'id1': 'p1', 'id2': 'q1', '_cifflow_row_id': 1}]) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': None, 'phase_id2': None, '_cifflow_row_id': 2}]) + table = schema.tables['atom'] + col_by_name = _col_by_name(schema, 'atom') + _fill_composite_fk(db, 'atom', table, col_by_name, schema, False) + assert _fetch(db, 'atom', 'phase_id1') == [None] + + def test_composite_non_key_propagate_true_filled(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + _insert(db, 'phase', [{'id1': 'p1', 'id2': 'q1', '_loop_id': _SCALARS, '_cifflow_row_id': 1}]) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': None, 'phase_id2': None, + '_loop_id': _SCALARS, '_cifflow_row_id': 2}]) + table = schema.tables['atom'] + col_by_name = _col_by_name(schema, 'atom') + _fill_composite_fk(db, 'atom', table, col_by_name, schema, True) + assert _fetch(db, 'atom', 'phase_id1') == ['p1'] + assert _fetch(db, 'atom', 'phase_id2') == ['q1'] + + +# --------------------------------------------------------------------------- +# Tests: _fill_propagation_links +# --------------------------------------------------------------------------- + +class TestFillPropagationLinks: + def test_propagation_link_fills_from_linked_table(self): + """Key-FK with propagation link: cell.structure_id filled from structure.id.""" + schema = _schema_parent_child() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1', '_loop_id': _SCALARS, '_cifflow_row_id': 1}]) + _insert(db, 'cell', [{'structure_id': None, 'length_a': '5.0', + '_loop_id': _SCALARS, '_cifflow_row_id': 2}]) + table = schema.tables['cell'] + col_by_name = _col_by_name(schema, 'cell') + tag_to_column = build_tag_to_column(schema) + _fill_propagation_links(db, 'cell', table, col_by_name, schema, True, tag_to_column) + assert _fetch(db, 'cell', 'structure_id') == ['s1'] + + def test_default_value_applied_when_no_match(self): + """Non-PK Link col with enumeration_default gets default when still NULL.""" + schema = _schema_with_default() + db = setup_duckdb(schema) + # No structure rows → propagation chain finds nothing; default should apply + _insert(db, 'atom_site', [{'label': 'C1', 'structure_id': None, '_cifflow_row_id': 1}]) + table = schema.tables['atom_site'] + col_by_name = _col_by_name(schema, 'atom_site') + tag_to_column = build_tag_to_column(schema) + _fill_propagation_links(db, 'atom_site', table, col_by_name, schema, False, tag_to_column) + assert _fetch(db, 'atom_site', 'structure_id') == ['default_struct'] + + def test_already_set_col_not_overwritten_by_default(self): + schema = _schema_with_default() + db = setup_duckdb(schema) + _insert(db, 'atom_site', [{'label': 'C1', 'structure_id': 'existing', '_cifflow_row_id': 1}]) + table = schema.tables['atom_site'] + col_by_name = _col_by_name(schema, 'atom_site') + tag_to_column = build_tag_to_column(schema) + _fill_propagation_links(db, 'atom_site', table, col_by_name, schema, False, tag_to_column) + assert _fetch(db, 'atom_site', 'structure_id') == ['existing'] diff --git a/tests/ingestion/test_propagate_fk_sql.py b/tests/ingestion/test_propagate_fk_sql.py new file mode 100644 index 0000000..0682209 --- /dev/null +++ b/tests/ingestion/test_propagate_fk_sql.py @@ -0,0 +1,339 @@ +"""Branch-coverage tests for propagate_fk_sql helpers. + +Tests are written against the extracted helper functions before refactoring +so that the refactoring can be verified to preserve behaviour. +""" +import duckdb +import pytest + +from cifflow.dictionary.ddlm_item import DdlmItem +from cifflow.dictionary.ddlm_parser import DdlmDictionary +from cifflow.dictionary.schema import generate_schema +from cifflow.ingestion.duckdb_ingest import ( + _create_composite_fk_stub_parents, + _create_single_fk_stub_parents, + _generate_uuid_pks, + _topo_order, + setup_duckdb, +) + + +# --------------------------------------------------------------------------- +# Schema helpers +# --------------------------------------------------------------------------- + +def _item(definition_id, category_id, object_id, *, type_purpose=None, + type_contents=None, linked_item_id=None): + return DdlmItem( + definition_id=definition_id, scope='Item', definition_class='Datum', + category_id=category_id, object_id=object_id, + type_purpose=type_purpose, type_source=None, type_container='Single', + type_contents=type_contents, linked_item_id=linked_item_id, + units_code=None, description=None, + ) + + +def _cat(definition_id, cat_class, category_keys=None): + return DdlmItem( + definition_id=definition_id, scope='Category', + definition_class=cat_class, category_id=None, object_id=None, + type_purpose=None, type_source=None, type_container='Single', + type_contents=None, linked_item_id=None, units_code=None, + description=None, category_keys=category_keys or [], + ) + + +def _make_dict(cats, items): + categories = {c.definition_id: c for c in cats} + item_map = {i.definition_id: i for i in items} + return DdlmDictionary( + name='TEST', title=None, version=None, + categories=categories, items=item_map, + tag_to_item={**categories, **item_map}, + alias_to_definition_id={}, deprecated_ids=set(), + ) + + +# --------------------------------------------------------------------------- +# Data helpers +# --------------------------------------------------------------------------- + +_SCALARS = '__scalars__' + + +def _insert(db, table, rows): + """Insert rows into _raw_{table}. Each row dict may include system cols or just data cols.""" + for i, row in enumerate(rows): + sys_defaults = { + '_cifflow_block_id': 'blk1', + '_cifflow_block_idx': 0, + '_loop_id': _SCALARS, + '_iter_idx': 0, + '_cifflow_row_id': i + 1, + } + sys_cols = ['_cifflow_block_id', '_cifflow_block_idx', '_loop_id', '_iter_idx', '_cifflow_row_id'] + sys_vals = [ + f"'{row.get('_cifflow_block_id', sys_defaults['_cifflow_block_id'])}'", + str(row.get('_cifflow_block_idx', sys_defaults['_cifflow_block_idx'])), + f"'{row.get('_loop_id', sys_defaults['_loop_id'])}'", + str(row.get('_iter_idx', sys_defaults['_iter_idx'])), + str(row.get('_cifflow_row_id', sys_defaults['_cifflow_row_id'])), + ] + data_cols = [f'"{k}"' for k in row if not k.startswith('_cifflow_') and k != '_loop_id' and k != '_iter_idx'] + data_vals = [ + f"'{v}'" if v is not None else 'NULL' + for k, v in row.items() + if not k.startswith('_cifflow_') and k != '_loop_id' and k != '_iter_idx' + ] + all_cols = ', '.join(sys_cols + data_cols) + all_vals = ', '.join(sys_vals + data_vals) + db.execute(f'INSERT INTO "_raw_{table}" ({all_cols}) VALUES ({all_vals})') + + +def _fetch(db, table, col): + return [r[0] for r in db.execute(f'SELECT "{col}" FROM "_raw_{table}"').fetchall()] + + +def _count(db, table): + return db.execute(f'SELECT COUNT(*) FROM "_raw_{table}"').fetchone()[0] + + +# --------------------------------------------------------------------------- +# Schemas +# --------------------------------------------------------------------------- + +def _schema_simple(): + """structure (Set, PK=id) — no FK on id.""" + cats = [_cat('_structure', 'Set', ['_structure.id'])] + items = [ + _item('_structure.id', '_structure', 'id', type_purpose='Key', type_contents='Text'), + _item('_structure.name', '_structure', 'name', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_parent_child(): + """structure (Set, PK=id) → cell (Set, PK=structure_id key-FK→structure.id).""" + cats = [ + _cat('_structure', 'Set', ['_structure.id']), + _cat('_cell', 'Set', ['_cell.structure_id']), + ] + items = [ + _item('_structure.id', '_structure', 'id', type_purpose='Key', type_contents='Text'), + _item('_cell.structure_id', '_cell', 'structure_id', + type_purpose='Link', linked_item_id='_structure.id', type_contents='Text'), + _item('_cell.length_a', '_cell', 'length_a', type_contents='Real'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_composite_fk(): + """phase (Set, PK=(id1,id2)), atom (Loop, PK=label, composite FK→phase).""" + cats = [ + _cat('_phase', 'Set', ['_phase.id1', '_phase.id2']), + _cat('_atom', 'Loop', ['_atom.label']), + ] + items = [ + _item('_phase.id1', '_phase', 'id1', type_purpose='Key', type_contents='Text'), + _item('_phase.id2', '_phase', 'id2', type_purpose='Key', type_contents='Text'), + _item('_atom.label', '_atom', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom.phase_id1', '_atom', 'phase_id1', + type_purpose='Link', linked_item_id='_phase.id1', type_contents='Text'), + _item('_atom.phase_id2', '_atom', 'phase_id2', + type_purpose='Link', linked_item_id='_phase.id2', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_siblings(): + """Two Loop tables (cat_a, cat_b) sharing the same PK column name 'label', no FK between them.""" + cats = [ + _cat('_cat_a', 'Loop', ['_cat_a.label']), + _cat('_cat_b', 'Loop', ['_cat_b.label']), + ] + items = [ + _item('_cat_a.label', '_cat_a', 'label', type_purpose='Key', type_contents='Text'), + _item('_cat_a.value', '_cat_a', 'value', type_contents='Text'), + _item('_cat_b.label', '_cat_b', 'label', type_purpose='Key', type_contents='Text'), + _item('_cat_b.value', '_cat_b', 'value', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +# --------------------------------------------------------------------------- +# Tests: _generate_uuid_pks +# --------------------------------------------------------------------------- + +class TestGenerateUuidPks: + def test_null_pk_gets_uuid(self): + schema = _schema_simple() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': None, 'name': 'test'}]) + _generate_uuid_pks(db, schema, None) + ids = _fetch(db, 'structure', 'id') + assert len(ids) == 1 + assert ids[0] is not None + assert len(ids[0]) > 0 + + def test_existing_pk_unchanged(self): + schema = _schema_simple() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 'my_id', 'name': 'test'}]) + _generate_uuid_pks(db, schema, None) + assert _fetch(db, 'structure', 'id') == ['my_id'] + + def test_pk_with_single_fk_skipped(self): + # cell.structure_id has a single FK to structure.id — should NOT get UUID assigned + schema = _schema_parent_child() + db = setup_duckdb(schema) + _insert(db, 'structure', [{'id': 's1'}]) + _insert(db, 'cell', [{'structure_id': None, 'length_a': '5.0'}]) + _generate_uuid_pks(db, schema, None) + # structure_id has has_single_fk=True → skipped; stays NULL + assert _fetch(db, 'cell', 'structure_id') == [None] + + def test_null_pk_multiple_rows(self): + schema = _schema_simple() + db = setup_duckdb(schema) + _insert(db, 'structure', [ + {'id': None, 'name': 'a', '_cifflow_row_id': 1}, + {'id': None, 'name': 'b', '_cifflow_row_id': 2}, + ]) + _generate_uuid_pks(db, schema, None) + ids = _fetch(db, 'structure', 'id') + assert all(v is not None for v in ids) + assert ids[0] != ids[1] # each row gets a distinct UUID + + def test_sibling_copies_from_canonical(self): + # cat_a and cat_b share PK 'label'; cat_a is canonical (sorted first) + schema = _schema_siblings() + db = setup_duckdb(schema) + _insert(db, 'cat_a', [{'label': 'L1', 'value': 'x', '_loop_id': 'lp1', '_iter_idx': 0, '_cifflow_row_id': 1}]) + _insert(db, 'cat_b', [{'label': None, 'value': 'y', '_loop_id': 'lp1', '_iter_idx': 0, '_cifflow_row_id': 2}]) + _generate_uuid_pks(db, schema, None) + # cat_b should have copied 'L1' from cat_a (same block, loop, iter) + assert _fetch(db, 'cat_b', 'label') == ['L1'] + + def test_sibling_gets_own_uuid_when_canonical_null(self): + schema = _schema_siblings() + db = setup_duckdb(schema) + # Both NULL at different iter_idx — no match to copy from + _insert(db, 'cat_a', [{'label': None, '_loop_id': 'lp1', '_iter_idx': 0, '_cifflow_row_id': 1}]) + _insert(db, 'cat_b', [{'label': None, '_loop_id': 'lp1', '_iter_idx': 1, '_cifflow_row_id': 2}]) + _generate_uuid_pks(db, schema, None) + vals_a = _fetch(db, 'cat_a', 'label') + vals_b = _fetch(db, 'cat_b', 'label') + assert vals_a[0] is not None + assert vals_b[0] is not None + + def test_populated_filter_restricts_tables(self): + schema = _schema_siblings() + db = setup_duckdb(schema) + _insert(db, 'cat_a', [{'label': None, '_cifflow_row_id': 1}]) + _insert(db, 'cat_b', [{'label': None, '_cifflow_row_id': 2}]) + _generate_uuid_pks(db, schema, {'cat_a'}) + # Only cat_a is in populated — cat_a gets UUID, cat_b stays NULL + assert _fetch(db, 'cat_a', 'label')[0] is not None + assert _fetch(db, 'cat_b', 'label')[0] is None + + +# --------------------------------------------------------------------------- +# Tests: _create_composite_fk_stub_parents +# --------------------------------------------------------------------------- + +class TestCreateCompositeFkStubParents: + def test_missing_composite_parent_created(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': 'p1', 'phase_id2': 'q1'}]) + _create_composite_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'phase') == 1 + assert _fetch(db, 'phase', 'id1') == ['p1'] + assert _fetch(db, 'phase', 'id2') == ['q1'] + + def test_existing_composite_parent_not_duplicated(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'phase', [{'id1': 'p1', 'id2': 'q1', '_cifflow_row_id': 1}]) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': 'p1', 'phase_id2': 'q1', '_cifflow_row_id': 2}]) + _create_composite_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'phase') == 1 # no duplicate inserted + + def test_null_composite_fk_not_inserted(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': None, 'phase_id2': None}]) + _create_composite_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'phase') == 0 + + def test_populated_filter_composite(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': 'p1', 'phase_id2': 'q1'}]) + # atom not in populated → no stub created + _create_composite_fk_stub_parents(db, schema, topo, {'phase'}) + assert _count(db, 'phase') == 0 + + def test_populated_set_updated(self): + schema = _schema_composite_fk() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'atom', [{'label': 'A1', 'phase_id1': 'p1', 'phase_id2': 'q1'}]) + populated = {'atom', 'phase'} + _create_composite_fk_stub_parents(db, schema, topo, populated) + # phase should remain in populated (was added or already present) + assert 'phase' in populated + + +# --------------------------------------------------------------------------- +# Tests: _create_single_fk_stub_parents +# --------------------------------------------------------------------------- + +class TestCreateSingleFkStubParents: + def test_missing_single_parent_created(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'cell', [{'structure_id': 's1', 'length_a': '5.0'}]) + _create_single_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'structure') == 1 + assert _fetch(db, 'structure', 'id') == ['s1'] + + def test_existing_single_parent_not_duplicated(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'structure', [{'id': 's1', '_cifflow_row_id': 1}]) + _insert(db, 'cell', [{'structure_id': 's1', 'length_a': '5.0', '_cifflow_row_id': 2}]) + _create_single_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'structure') == 1 + + def test_null_single_fk_not_inserted(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'cell', [{'structure_id': None, 'length_a': '5.0'}]) + _create_single_fk_stub_parents(db, schema, topo, None) + assert _count(db, 'structure') == 0 + + def test_populated_filter_single(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'cell', [{'structure_id': 's1', 'length_a': '5.0'}]) + # cell not in populated → no stub created for structure + _create_single_fk_stub_parents(db, schema, topo, {'structure'}) + assert _count(db, 'structure') == 0 + + def test_populated_set_updated_on_insert(self): + schema = _schema_parent_child() + db = setup_duckdb(schema) + topo = _topo_order(schema) + _insert(db, 'cell', [{'structure_id': 's1', 'length_a': '5.0'}]) + populated = {'cell', 'structure'} + _create_single_fk_stub_parents(db, schema, topo, populated) + assert 'structure' in populated diff --git a/tests/output/test_collect_grouped.py b/tests/output/test_collect_grouped.py new file mode 100644 index 0000000..2261cf0 --- /dev/null +++ b/tests/output/test_collect_grouped.py @@ -0,0 +1,962 @@ +""" +Branch-coverage tests for _collect_grouped in cifflow.output.emit. + +Tests call _collect_grouped directly and assert on _BlockData structure. +Branch groups (Stage 1 analysis): + 1. Empty / no-data + 2. Pure-loop blocks (keyless Set, fallback-only) + 3. Set-only source blocks (no loop tables) + 4. Basic merge / separate (keyed Set + PK-FK Loop) + 5. No-pkreach Loop with incidental Set + 6. Primary-anchor skip for incidental block + 7. Child-Set BFS in incidental block + 8. Orphan Loop table (rows span multiple fps) + 9. Single-fp Loop table absorbed into fingerprint block + 10. Multi-anchor bridge block (PK-stripping) +""" +from __future__ import annotations + +import duckdb +import pytest + +from cifflow import build, ingest, generate_schema +from cifflow.dictionary import DictionaryLoader +from cifflow.dictionary.schema import SchemaSpec +from cifflow.output.emit import _collect_grouped, _BlockData +from cifflow.types import CifVersion + +_V = CifVersion.CIF_2_0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _load_schema(ddl: str) -> SchemaSpec: + return generate_schema(DictionaryLoader().load(ddl)) + + +def _conn(cif: str, schema: SchemaSpec | None = None) -> duckdb.DuckDBPyConnection: + cf, errs = build(cif) + assert not errs, errs + conn, _ = ingest(cf, None, schema) + return conn + + +def _cg(conn: duckdb.DuckDBPyConnection, schema: SchemaSpec) -> list[_BlockData]: + return _collect_grouped(conn, schema, _V) + + +def _empty_schema() -> SchemaSpec: + return SchemaSpec(tables={}, column_to_tag={}) + + +def _find_block(blocks: list[_BlockData], **criteria) -> _BlockData | None: + """Return first block matching ALL criteria (anchor_frozenset, name prefix, has_table).""" + for b in blocks: + if 'anchor' in criteria and b.anchor_frozenset != criteria['anchor']: + continue + if 'name_startswith' in criteria and not b.name.startswith(criteria['name_startswith']): + continue + if 'has_table' in criteria and criteria['has_table'] not in b.table_rows: + continue + if 'no_table' in criteria and criteria['no_table'] in b.table_rows: + continue + return b + return None + + +# --------------------------------------------------------------------------- +# DDL strings +# --------------------------------------------------------------------------- + +_KEYLESS_SET_DDL = """\ +#\\#CIF_2.0 +data_keyless_dic + +save_CELL + _definition.id CELL + _definition.scope Category + _definition.class Set + _name.category_id cell +save_ + +save_cell.length_a + _definition.id '_cell.length_a' + _definition.class Attribute + _name.category_id cell + _name.object_id length_a + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ +""" + +_EXPT_PEAK_DDL = """\ +#\\#CIF_2.0 +data_ep_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt + _category_key.name '_expt.id' +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_expt.title + _definition.id '_expt.title' + _definition.class Attribute + _name.category_id expt + _name.object_id title + _type.purpose Describe + _type.source Recorded + _type.container Single + _type.contents Text +save_ + +save_PEAK + _definition.id PEAK + _definition.scope Category + _definition.class Loop + _name.category_id peak + loop_ + _category_key.name + '_peak.expt_id' + '_peak.id' +save_ + +save_peak.id + _definition.id '_peak.id' + _definition.class Attribute + _name.category_id peak + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_peak.expt_id + _definition.id '_peak.expt_id' + _definition.class Attribute + _name.category_id peak + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_peak.intensity + _definition.id '_peak.intensity' + _definition.class Attribute + _name.category_id peak + _name.object_id intensity + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ +""" + +# CRYSTAL (Set keyed) + ATOM (Loop, non-PK FK atom.crystal_id → crystal.id) +_INCIDENTAL_DDL = """\ +#\\#CIF_2.0 +data_incidental_dic + +save_CRYSTAL + _definition.id CRYSTAL + _definition.scope Category + _definition.class Set + _name.category_id crystal + _category_key.name '_crystal.id' +save_ + +save_crystal.id + _definition.id '_crystal.id' + _definition.class Attribute + _name.category_id crystal + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_ATOM + _definition.id ATOM + _definition.scope Category + _definition.class Loop + _name.category_id atom + _category_key.name '_atom.id' +save_ + +save_atom.id + _definition.id '_atom.id' + _definition.class Attribute + _name.category_id atom + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_atom.crystal_id + _definition.id '_atom.crystal_id' + _definition.class Attribute + _name.category_id atom + _name.object_id crystal_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_crystal.id' +save_ +""" + +# CRYSTAL (Set) + SYMMETRY (child-Set: sole domain PK crystal_id FKs to CRYSTAL) +# + ATOM (Loop, non-PK FK crystal_id → CRYSTAL) +_CHILD_SET_DDL = """\ +#\\#CIF_2.0 +data_child_set_dic + +save_CRYSTAL + _definition.id CRYSTAL + _definition.scope Category + _definition.class Set + _name.category_id crystal + _category_key.name '_crystal.id' +save_ + +save_crystal.id + _definition.id '_crystal.id' + _definition.class Attribute + _name.category_id crystal + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_SYMMETRY + _definition.id SYMMETRY + _definition.scope Category + _definition.class Set + _name.category_id symmetry + _category_key.name '_symmetry.crystal_id' +save_ + +save_symmetry.crystal_id + _definition.id '_symmetry.crystal_id' + _definition.class Attribute + _name.category_id symmetry + _name.object_id crystal_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_crystal.id' +save_ + +save_symmetry.space_group + _definition.id '_symmetry.space_group' + _definition.class Attribute + _name.category_id symmetry + _name.object_id space_group + _type.purpose Describe + _type.source Recorded + _type.container Single + _type.contents Text +save_ + +save_ATOM + _definition.id ATOM + _definition.scope Category + _definition.class Loop + _name.category_id atom + _category_key.name '_atom.id' +save_ + +save_atom.id + _definition.id '_atom.id' + _definition.class Attribute + _name.category_id atom + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_atom.crystal_id + _definition.id '_atom.crystal_id' + _definition.class Attribute + _name.category_id atom + _name.object_id crystal_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_crystal.id' +save_ +""" + +# EXPT (Set keyed) + PEAK (Loop PK-FK to EXPT) + ELEMENT (Loop, no FK to any Set) +_ORPHAN_DDL = """\ +#\\#CIF_2.0 +data_orphan_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt + _category_key.name '_expt.id' +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_PEAK + _definition.id PEAK + _definition.scope Category + _definition.class Loop + _name.category_id peak + loop_ + _category_key.name + '_peak.expt_id' + '_peak.id' +save_ + +save_peak.id + _definition.id '_peak.id' + _definition.class Attribute + _name.category_id peak + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_peak.expt_id + _definition.id '_peak.expt_id' + _definition.class Attribute + _name.category_id peak + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_ELEMENT + _definition.id ELEMENT + _definition.scope Category + _definition.class Loop + _name.category_id element + _category_key.name '_element.symbol' +save_ + +save_element.symbol + _definition.id '_element.symbol' + _definition.class Attribute + _name.category_id element + _name.object_id symbol + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ +""" + +# EXPT (Set) + SAMPLE (Set) + PATTERN (Loop, composite PK-FK to both EXPT and SAMPLE) +_BRIDGE_DDL = """\ +#\\#CIF_2.0 +data_bridge_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt + _category_key.name '_expt.id' +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_expt.title + _definition.id '_expt.title' + _definition.class Attribute + _name.category_id expt + _name.object_id title + _type.purpose Describe + _type.source Recorded + _type.container Single + _type.contents Text +save_ + +save_SAMPLE + _definition.id SAMPLE + _definition.scope Category + _definition.class Set + _name.category_id sample + _category_key.name '_sample.id' +save_ + +save_sample.id + _definition.id '_sample.id' + _definition.class Attribute + _name.category_id sample + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_sample.name + _definition.id '_sample.name' + _definition.class Attribute + _name.category_id sample + _name.object_id name + _type.purpose Describe + _type.source Recorded + _type.container Single + _type.contents Text +save_ + +save_PATTERN + _definition.id PATTERN + _definition.scope Category + _definition.class Loop + _name.category_id pattern + loop_ + _category_key.name + '_pattern.expt_id' + '_pattern.sample_id' +save_ + +save_pattern.expt_id + _definition.id '_pattern.expt_id' + _definition.class Attribute + _name.category_id pattern + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_pattern.sample_id + _definition.id '_pattern.sample_id' + _definition.class Attribute + _name.category_id pattern + _name.object_id sample_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_sample.id' +save_ + +save_pattern.counts + _definition.id '_pattern.counts' + _definition.class Attribute + _name.category_id pattern + _name.object_id counts + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ +""" + + +# =========================================================================== +# 1. Empty / no-data +# =========================================================================== + +class TestEmpty: + + def test_bare_connection_no_tables_returns_empty_list(self): + schema = _load_schema(_EXPT_PEAK_DDL) + conn = duckdb.connect() + blocks = _cg(conn, schema) + assert blocks == [] + + def test_no_schema_fallback_only_returns_pure_loop_block(self): + # Tags not in schema go to _cif_fallback → pure_loop block with no anchor key. + conn = _conn('#\\#CIF_2.0\ndata_test\n_foo.bar 1\n') + blocks = _cg(conn, _empty_schema()) + assert len(blocks) == 1 + assert blocks[0].anchor_key_dict == {} + + +# =========================================================================== +# 2. Keyless Set → pure_loop block +# =========================================================================== + +class TestKeylessSetPureLoop: + + @pytest.fixture + def schema(self): + return _load_schema(_KEYLESS_SET_DDL) + + def test_keyless_set_routes_to_pure_loop(self, schema): + conn = _conn('#\\#CIF_2.0\ndata_b1\n_cell.length_a 5.4\n', schema) + blocks = _cg(conn, schema) + assert len(blocks) == 1 + assert 'cell' in blocks[0].table_rows + + def test_keyless_set_pure_loop_block_has_empty_anchor_key_dict(self, schema): + conn = _conn('#\\#CIF_2.0\ndata_b1\n_cell.length_a 5.4\n', schema) + blocks = _cg(conn, schema) + assert blocks[0].anchor_key_dict == {} + + +# =========================================================================== +# 3. Set-only source block (keyed Set, no Loop tables) +# =========================================================================== + +class TestSetOnlyBlock: + + @pytest.fixture + def schema(self): + return _load_schema(_EXPT_PEAK_DDL) + + def test_set_only_block_creates_fingerprint_block(self, schema): + conn = _conn('#\\#CIF_2.0\ndata_e1\n_expt.id E1\n', schema) + blocks = _cg(conn, schema) + fp_blocks = [b for b in blocks if b.anchor_frozenset == frozenset({'expt'})] + assert len(fp_blocks) == 1 + + def test_set_only_block_anchor_frozenset_contains_set_table(self, schema): + conn = _conn('#\\#CIF_2.0\ndata_e1\n_expt.id E1\n', schema) + blocks = _cg(conn, schema) + assert any(b.anchor_frozenset == frozenset({'expt'}) for b in blocks) + + def test_set_only_block_anchor_key_dict_has_pk_value(self, schema): + conn = _conn('#\\#CIF_2.0\ndata_e1\n_expt.id E1\n', schema) + blocks = _cg(conn, schema) + fp = _find_block(blocks, anchor=frozenset({'expt'})) + assert fp is not None + assert fp.anchor_key_dict.get('expt.id') == ['E1'] + + +# =========================================================================== +# 4. Basic merge / separate (keyed Set + PK-FK Loop) +# =========================================================================== + +_MERGE_CIF = ( + '#\\#CIF_2.0\n' + 'data_run1\n' + '_expt.id X1\n' + 'loop_\n _peak.id\n _peak.expt_id\n _peak.intensity\n' + ' p1 X1 100.0\n' + '\n\n' + 'data_run2\n' + '_expt.id X1\n' + 'loop_\n _peak.id\n _peak.expt_id\n _peak.intensity\n' + ' p2 X1 200.0\n' +) + +_SEPARATE_CIF = ( + '#\\#CIF_2.0\n' + 'data_run1\n' + '_expt.id X1\n' + 'loop_\n _peak.id\n _peak.expt_id\n _peak.intensity\n' + ' p1 X1 100.0\n' + '\n\n' + 'data_run2\n' + '_expt.id X2\n' + 'loop_\n _peak.id\n _peak.expt_id\n _peak.intensity\n' + ' p2 X2 200.0\n' +) + + +class TestMergeAndSeparate: + + @pytest.fixture + def schema(self): + return _load_schema(_EXPT_PEAK_DDL) + + def test_same_set_key_produces_one_output_block(self, schema): + conn = _conn(_MERGE_CIF, schema) + blocks = _cg(conn, schema) + fp_blocks = [b for b in blocks if 'expt' in b.anchor_frozenset] + assert len(fp_blocks) == 1 + + def test_merged_block_contains_all_loop_rows(self, schema): + conn = _conn(_MERGE_CIF, schema) + blocks = _cg(conn, schema) + fp = _find_block(blocks, anchor=frozenset({'expt'})) + assert fp is not None + assert len(fp.table_rows.get('peak', [])) == 2 + + def test_different_set_keys_produce_two_output_blocks(self, schema): + conn = _conn(_SEPARATE_CIF, schema) + blocks = _cg(conn, schema) + fp_blocks = [b for b in blocks if 'expt' in b.anchor_frozenset] + assert len(fp_blocks) == 2 + + def test_suppress_fk_pk_is_true(self, schema): + conn = _conn(_MERGE_CIF, schema) + blocks = _cg(conn, schema) + assert all(b.suppress_fk_pk for b in blocks) + + def test_suppress_all_fk_to_set_is_true(self, schema): + conn = _conn(_MERGE_CIF, schema) + blocks = _cg(conn, schema) + assert all(b.suppress_all_fk_to_set for b in blocks) + + +# =========================================================================== +# 5. No-pkreach Loop with incidental Set +# =========================================================================== + +# Block with both CRYSTAL (Set) and ATOM (Loop, non-PK FK to CRYSTAL). +# ATOM has no PK-FK path to any Set → no-pkreach arm in _block_fingerprint. +_INCIDENTAL_CIF = ( + '#\\#CIF_2.0\n' + 'data_b1\n' + '_crystal.id C1\n' + 'loop_\n _atom.id\n _atom.crystal_id\n' + ' A1 C1\n' + ' A2 C1\n' +) + + +class TestNoPKReachIncidental: + + @pytest.fixture + def schema(self): + return _load_schema(_INCIDENTAL_DDL) + + def test_no_pkreach_loop_source_block_goes_to_pure_loop(self, schema): + # The source block has no PK-FK path from atom to crystal, so main_fps=[]. + # → source block routes to pure_loop_block_ids. + conn = _conn(_INCIDENTAL_CIF, schema) + blocks = _cg(conn, schema) + # At least one block with empty anchor_key_dict (pure_loop or incidental). + # Pure_loop block has 'atom' data (and 'crystal' from cache.rows_for_block). + pure_loop = _find_block(blocks, anchor=frozenset({'crystal'}), no_table='atom') + # The pure_loop block from b1 contains atom rows (all schema tables fetched). + assert any('atom' in b.table_rows for b in blocks) + + def test_no_pkreach_incidental_set_creates_dedicated_block(self, schema): + conn = _conn(_INCIDENTAL_CIF, schema) + blocks = _cg(conn, schema) + # An incidental block for crystal is produced with anchor_frozenset={'crystal'}. + inc = _find_block(blocks, anchor=frozenset({'crystal'}), has_table='crystal') + assert inc is not None + + def test_incidental_block_includes_loop_rows_via_non_pk_fk(self, schema): + conn = _conn(_INCIDENTAL_CIF, schema) + blocks = _cg(conn, schema) + inc = _find_block(blocks, anchor=frozenset({'crystal'}), has_table='crystal') + assert inc is not None + assert 'atom' in inc.table_rows + assert len(inc.table_rows['atom']) == 2 + + def test_incidental_block_anchor_frozenset_is_set_table(self, schema): + conn = _conn(_INCIDENTAL_CIF, schema) + blocks = _cg(conn, schema) + inc = _find_block(blocks, anchor=frozenset({'crystal'}), has_table='crystal') + assert inc is not None + assert inc.anchor_frozenset == frozenset({'crystal'}) + + +# =========================================================================== +# 6. Primary-anchor skip for incidental block +# =========================================================================== + +# Two source blocks: b1 is Set-only (crystal.id=C1) → creates a main fingerprint. +# b2 is Loop+Set with same crystal.id via tag_presence. +# Because (crystal, C1) is already a primary fp anchor AND crystal has no child-Sets, +# the incidental block for (crystal, C1) from b2 is skipped. +_PRIMARY_SKIP_CIF = ( + '#\\#CIF_2.0\n' + 'data_b1\n' + '_crystal.id C1\n' + '\n\n' + 'data_b2\n' + '_crystal.id C1\n' + 'loop_\n _atom.id\n _atom.crystal_id\n' + ' A1 C1\n' +) + +_CHILD_SET_SKIP_CIF = ( + '#\\#CIF_2.0\n' + 'data_b1\n' + '_crystal.id C1\n' + '\n\n' + 'data_b2\n' + '_crystal.id C1\n' + '_symmetry.crystal_id C1\n' + '_symmetry.space_group Fm3m\n' + 'loop_\n _atom.id\n _atom.crystal_id\n' + ' A1 C1\n' +) + + +class TestPrimaryAnchorSkip: + + def test_incidental_skipped_when_already_primary_anchor_no_child_sets(self): + schema = _load_schema(_INCIDENTAL_DDL) + conn = _conn(_PRIMARY_SKIP_CIF, schema) + blocks = _cg(conn, schema) + # Should have: 1 fingerprint block (crystal C1 from b1) + 1 pure_loop block (atom from b2). + # No extra incidental block for crystal. + crystal_blocks = [b for b in blocks if 'crystal' in b.anchor_frozenset] + assert len(crystal_blocks) == 1 # only the fingerprint block + + def test_incidental_not_skipped_when_has_child_sets(self): + # b1: crystal C1 Set-only → primary fp anchor for (crystal, C1). + # b2: crystal C1 (tag_presence) + symmetry C1 (child-Set of crystal) + atom. + # Crystal has child-Set SYMMETRY → _expand_with_child_sets != {crystal} → NOT skipped. + schema = _load_schema(_CHILD_SET_DDL) + conn = _conn(_CHILD_SET_SKIP_CIF, schema) + blocks = _cg(conn, schema) + crystal_blocks = [b for b in blocks if 'crystal' in b.anchor_frozenset] + # Fingerprint block (b1) + incidental block (b2, not skipped because of child-Set) + assert len(crystal_blocks) >= 2 + + +# =========================================================================== +# 7. Child-Set BFS in incidental block +# =========================================================================== + +_CHILD_BFS_CIF = ( + '#\\#CIF_2.0\n' + 'data_b1\n' + '_crystal.id C1\n' + '_symmetry.crystal_id C1\n' + '_symmetry.space_group Fm3m\n' + 'loop_\n _atom.id\n _atom.crystal_id\n' + ' A1 C1\n' + ' A2 C1\n' +) + + +class TestChildSetBFS: + + @pytest.fixture + def schema(self): + return _load_schema(_CHILD_SET_DDL) + + def test_child_set_collected_in_incidental_block(self, schema): + # CRYSTAL is incidental (no PK-FK path from ATOM). + # SYMMETRY is a child-Set of CRYSTAL → collected via BFS into the crystal incidental block. + conn = _conn(_CHILD_BFS_CIF, schema) + blocks = _cg(conn, schema) + inc = _find_block(blocks, anchor=frozenset({'crystal'}), has_table='crystal') + assert inc is not None + assert 'symmetry' in inc.table_rows + + def test_child_set_bfs_applies_fk_filter(self, schema): + # symmetry row's crystal_id must match the crystal anchor pk_val. + conn = _conn(_CHILD_BFS_CIF, schema) + blocks = _cg(conn, schema) + inc = _find_block(blocks, anchor=frozenset({'crystal'}), has_table='crystal') + assert inc is not None + sym_rows = inc.table_rows.get('symmetry', []) + assert all(r.get('crystal_id') == 'C1' for r in sym_rows) + + +# =========================================================================== +# 8. Orphan Loop table (rows span multiple fingerprint groups) +# =========================================================================== + +# Two experiments; ELEMENT appears in both source blocks → orphan block. +# Symbols must be distinct per block so each block owns its rows (ingest +# deduplicates Loop rows by PK: e2's Fe row would belong to e1 if shared). +_ORPHAN_CIF = ( + '#\\#CIF_2.0\n' + 'data_e1\n' + '_expt.id E1\n' + 'loop_\n _peak.id\n _peak.expt_id\n p1 E1\n' + 'loop_\n _element.symbol\n Fe\n Cu\n' + '\n\n' + 'data_e2\n' + '_expt.id E2\n' + 'loop_\n _peak.id\n _peak.expt_id\n p2 E2\n' + 'loop_\n _element.symbol\n Ag\n' +) + +# Only block e1 has ELEMENT rows → single-fp routing (not orphan). +_SINGLE_FP_CIF = ( + '#\\#CIF_2.0\n' + 'data_e1\n' + '_expt.id E1\n' + 'loop_\n _peak.id\n _peak.expt_id\n p1 E1\n' + 'loop_\n _element.symbol\n Fe\n' + '\n\n' + 'data_e2\n' + '_expt.id E2\n' + 'loop_\n _peak.id\n _peak.expt_id\n p2 E2\n' +) + + +class TestOrphanTable: + + @pytest.fixture + def schema(self): + return _load_schema(_ORPHAN_DDL) + + def test_element_spanning_two_fps_produces_orphan_block(self, schema): + conn = _conn(_ORPHAN_CIF, schema) + blocks = _cg(conn, schema) + orphan = _find_block(blocks, anchor=frozenset(), has_table='element') + assert orphan is not None + + def test_orphan_block_contains_elements_from_all_source_blocks(self, schema): + # Orphan block aggregates element rows owned by e1 (Fe, Cu) and e2 (Ag). + conn = _conn(_ORPHAN_CIF, schema) + blocks = _cg(conn, schema) + orphan = _find_block(blocks, anchor=frozenset(), has_table='element') + assert orphan is not None + symbols = {r['symbol'] for r in orphan.table_rows['element']} + assert 'Fe' in symbols + assert 'Cu' in symbols + assert 'Ag' in symbols + + def test_fingerprint_blocks_do_not_contain_orphan_table(self, schema): + conn = _conn(_ORPHAN_CIF, schema) + blocks = _cg(conn, schema) + for b in blocks: + if 'expt' in b.anchor_frozenset: + assert 'element' not in b.table_rows + + +# =========================================================================== +# 9. Single-fp Loop table absorbed into fingerprint block +# =========================================================================== + +class TestSingleFpTable: + + @pytest.fixture + def schema(self): + return _load_schema(_ORPHAN_DDL) + + def test_single_fp_element_absorbed_into_its_fp_block(self, schema): + # ELEMENT rows are only in block e1 (fp for E1) → single_fp routing → included in E1 block. + conn = _conn(_SINGLE_FP_CIF, schema) + blocks = _cg(conn, schema) + e1_block = next( + b for b in blocks + if b.anchor_frozenset == frozenset({'expt'}) + and b.anchor_key_dict.get('expt.id') == ['E1'] + ) + assert 'element' in e1_block.table_rows + + def test_single_fp_element_not_in_other_fp_block(self, schema): + # E2 block should NOT contain ELEMENT rows (not in E2's source block). + conn = _conn(_SINGLE_FP_CIF, schema) + blocks = _cg(conn, schema) + e2_block = next( + b for b in blocks + if b.anchor_frozenset == frozenset({'expt'}) + and b.anchor_key_dict.get('expt.id') == ['E2'] + ) + assert 'element' not in e2_block.table_rows + + def test_single_fp_no_orphan_block_produced(self, schema): + # ELEMENT is in exactly one fp → single_fp_tables, not orphan_tables → no orphan block. + conn = _conn(_SINGLE_FP_CIF, schema) + blocks = _cg(conn, schema) + orphan = _find_block(blocks, anchor=frozenset(), has_table='element') + assert orphan is None + + +# =========================================================================== +# 10. Multi-anchor bridge block (PK-stripping for sets_with_own_block) +# =========================================================================== + +# Block A: expt E1 + sample S1 + pattern(E1, S1) → bridge fp (two Set anchors). +# Block B: expt E2 only → single-anchor fp; adds 'expt' to sets_with_own_block. +# In the bridge block: expt rows are stripped to PK only (title removed); +# sample rows are kept full (sample not in sets_with_own_block). +_BRIDGE_CIF = ( + '#\\#CIF_2.0\n' + 'data_a\n' + '_expt.id E1\n' + '_expt.title "Experiment 1"\n' + '_sample.id S1\n' + '_sample.name "Sample 1"\n' + 'loop_\n _pattern.expt_id\n _pattern.sample_id\n _pattern.counts\n' + ' E1 S1 100.0\n' + '\n\n' + 'data_b\n' + '_expt.id E2\n' + '_expt.title "Experiment 2"\n' +) + + +class TestMultiAnchorBridge: + + @pytest.fixture + def schema(self): + return _load_schema(_BRIDGE_DDL) + + def test_bridge_block_has_two_set_anchors(self, schema): + conn = _conn(_BRIDGE_CIF, schema) + blocks = _cg(conn, schema) + bridge = next( + (b for b in blocks if len(b.anchor_frozenset) == 2), + None, + ) + assert bridge is not None + assert bridge.anchor_frozenset == frozenset({'expt', 'sample'}) + + def test_bridge_strips_set_with_own_block_to_pk_only(self, schema): + # expt has a single-anchor fp (block B, expt E2) → sets_with_own_block. + # In the bridge block for A, expt rows must be stripped to PK columns only. + conn = _conn(_BRIDGE_CIF, schema) + blocks = _cg(conn, schema) + bridge = next(b for b in blocks if len(b.anchor_frozenset) == 2) + expt_row = bridge.table_rows['expt'][0] + assert 'title' not in expt_row + + def test_bridge_keeps_full_data_for_set_not_in_sets_with_own_block(self, schema): + # sample only appears in the bridge block → NOT in sets_with_own_block → full data kept. + conn = _conn(_BRIDGE_CIF, schema) + blocks = _cg(conn, schema) + bridge = next(b for b in blocks if len(b.anchor_frozenset) == 2) + sample_row = bridge.table_rows['sample'][0] + assert 'name' in sample_row and sample_row['name'] is not None diff --git a/tests/output/test_collect_structure.py b/tests/output/test_collect_structure.py new file mode 100644 index 0000000..afe18ef --- /dev/null +++ b/tests/output/test_collect_structure.py @@ -0,0 +1,180 @@ +"""Branch-coverage tests for _collect_structure helper functions. + +Tests target _segregate_structure_blocks and _collect_satellite_merges +in isolation using minimal _BlockData instances (no DuckDB required). +""" +import pytest + +from cifflow.output.emit import ( + _BlockData, + _collect_satellite_merges, + _segregate_structure_blocks, +) + + +# --------------------------------------------------------------------------- +# _BlockData factory +# --------------------------------------------------------------------------- + +def _bd(anchor_frozenset, anchor_key_dict=None, table_rows=None, name='blk'): + return _BlockData( + name=name, + table_rows=table_rows or {}, + fallback_rows=[], + anchor_frozenset=anchor_frozenset, + anchor_key_dict=anchor_key_dict or {}, + suppress_fk_pk=True, + suppress_all_fk_to_set=True, + ) + + +# --------------------------------------------------------------------------- +# Tests: _segregate_structure_blocks +# --------------------------------------------------------------------------- + +class TestSegregateStructureBlocks: + def test_structure_block_goes_to_structure_list(self): + b = _bd(frozenset({'structure'})) + structs, pd_phases, sgs, models, others = _segregate_structure_blocks([b]) + assert structs == [b] + assert not pd_phases and not sgs and not models and not others + + def test_pd_phase_block_keyed_by_phase_id(self): + b = _bd(frozenset({'pd_phase'}), anchor_key_dict={'pd_phase.id': ['p1']}) + _, pd_phases, _, _, others = _segregate_structure_blocks([b]) + assert pd_phases == {'p1': b} + assert not others + + def test_pd_phase_block_multiple_ids(self): + b = _bd(frozenset({'pd_phase'}), anchor_key_dict={'pd_phase.id': ['p1', 'p2']}) + _, pd_phases, _, _, _ = _segregate_structure_blocks([b]) + assert pd_phases['p1'] is b and pd_phases['p2'] is b + + def test_space_group_block_keyed_by_sg_id(self): + b = _bd(frozenset({'space_group'}), anchor_key_dict={'space_group.id': ['sg1']}) + _, _, sgs, _, others = _segregate_structure_blocks([b]) + assert sgs == {'sg1': b} + assert not others + + def test_model_block_with_structure_id_routed_to_model_map(self): + b = _bd(frozenset({'model'}), table_rows={'model': [{'structure_id': 's1'}]}) + _, _, _, models, others = _segregate_structure_blocks([b]) + assert models == {'s1': [b]} + assert not others + + def test_model_block_without_structure_id_goes_to_other(self): + b = _bd(frozenset({'model'}), table_rows={'model': [{'label': 'M1'}]}) + _, _, _, models, others = _segregate_structure_blocks([b]) + assert not models + assert others == [b] + + def test_model_block_empty_rows_goes_to_other(self): + b = _bd(frozenset({'model'}), table_rows={}) + _, _, _, models, others = _segregate_structure_blocks([b]) + assert not models + assert others == [b] + + def test_unrecognised_anchor_goes_to_other(self): + b = _bd(frozenset({'atom_site'})) + structs, pd_phases, sgs, models, others = _segregate_structure_blocks([b]) + assert others == [b] + assert not structs and not pd_phases and not sgs and not models + + def test_empty_input_returns_all_empty(self): + structs, pd_phases, sgs, models, others = _segregate_structure_blocks([]) + assert structs == [] and not pd_phases and not sgs and not models and others == [] + + def test_multiple_block_types_mixed(self): + s = _bd(frozenset({'structure'}), name='s') + p = _bd(frozenset({'pd_phase'}), anchor_key_dict={'pd_phase.id': ['p1']}, name='p') + o = _bd(frozenset({'diffractogram'}), name='o') + structs, pd_phases, _, _, others = _segregate_structure_blocks([s, p, o]) + assert structs == [s] + assert 'p1' in pd_phases + assert others == [o] + + def test_pd_phase_block_with_no_phase_id_goes_nowhere(self): + # anchor_key_dict has no 'pd_phase.id' key → no entry in pd_phases, not in others either + b = _bd(frozenset({'pd_phase'}), anchor_key_dict={}) + _, pd_phases, _, _, others = _segregate_structure_blocks([b]) + assert not pd_phases + assert not others + + +# --------------------------------------------------------------------------- +# Tests: _collect_satellite_merges +# --------------------------------------------------------------------------- + +class TestCollectSatelliteMerges: + def _structure_block(self, rows): + return _bd(frozenset({'structure'}), table_rows={'structure': rows}) + + def test_matching_phase_id_absorbed(self): + phase_blk = _bd(frozenset({'pd_phase'})) + struct = self._structure_block([{'phase_id': 'p1', 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {'p1': phase_blk}, {}, {}, consumed) + assert phase_blk in to_merge + assert id(phase_blk) in consumed + + def test_placeholder_phase_id_not_absorbed(self): + phase_blk = _bd(frozenset({'pd_phase'})) + struct = self._structure_block([{'phase_id': '.', 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {'.': phase_blk}, {}, {}, consumed) + assert not to_merge + + def test_unknown_phase_id_not_absorbed(self): + phase_blk = _bd(frozenset({'pd_phase'})) + struct = self._structure_block([{'phase_id': 'p1', 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {'p2': phase_blk}, {}, {}, consumed) + assert not to_merge + + def test_matching_space_group_id_absorbed(self): + sg_blk = _bd(frozenset({'space_group'})) + struct = self._structure_block([{'phase_id': None, 'space_group_id': 'sg1', 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {}, {'sg1': sg_blk}, {}, consumed) + assert sg_blk in to_merge + assert id(sg_blk) in consumed + + def test_single_model_referent_absorbed(self): + model_blk = _bd(frozenset({'model'})) + struct = self._structure_block([{'phase_id': None, 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {}, {}, {'s1': [model_blk]}, consumed) + assert model_blk in to_merge + assert id(model_blk) in consumed + + def test_multiple_model_referents_not_absorbed(self): + m1 = _bd(frozenset({'model'}), name='m1') + m2 = _bd(frozenset({'model'}), name='m2') + struct = self._structure_block([{'phase_id': None, 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {}, {}, {'s1': [m1, m2]}, consumed) + assert not to_merge + + def test_no_matching_satellites_returns_empty(self): + struct = self._structure_block([{'phase_id': None, 'space_group_id': None, 'id': 's1'}]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {}, {}, {}, consumed) + assert to_merge == [] + assert not consumed + + def test_same_satellite_not_added_twice(self): + # Two structure rows referencing same phase block + phase_blk = _bd(frozenset({'pd_phase'})) + struct = self._structure_block([ + {'phase_id': 'p1', 'space_group_id': None, 'id': 's1'}, + {'phase_id': 'p1', 'space_group_id': None, 'id': 's1'}, + ]) + consumed = set() + to_merge = _collect_satellite_merges(struct, {'p1': phase_blk}, {}, {}, consumed) + assert to_merge.count(phase_blk) == 1 + + def test_no_structure_rows_returns_empty(self): + struct = _bd(frozenset({'structure'}), table_rows={'structure': []}) + consumed = set() + to_merge = _collect_satellite_merges(struct, {}, {}, {}, consumed) + assert to_merge == [] diff --git a/tests/output/test_emit.py b/tests/output/test_emit.py index 1768148..b709d20 100644 --- a/tests/output/test_emit.py +++ b/tests/output/test_emit.py @@ -749,6 +749,12 @@ def test_all_blocks_shared_dataset_id(self, grouped_schema): assert len(dataset_ids) == len(cif2.blocks) assert len(set(dataset_ids)) == 1 + def test_keyless_set_zero_rows_no_raise(self, mini_schema): + """Keyless Set table with 0 rows should not raise — guard only fires on non-empty tables.""" + conn = _ingest_src('#\\#CIF_2.0\ndata_empty\n', mini_schema) + result = emit(conn, schema=mini_schema, mode=EmitMode.ALL_BLOCKS) + assert result is not None + # Schema with Loop table whose PK includes a Set FK — exercises header scalars. _SET_KEY_IN_PK_DIC = """\ @@ -909,6 +915,221 @@ def test_output_is_valid_cif(self, conn, schema): assert not errors, errors +# --------------------------------------------------------------------------- +# ALL_BLOCKS — pure Loop (no Set FK in PK) +# --------------------------------------------------------------------------- + +class TestAllBlocksPureLoop: + """ALL_BLOCKS: Loop table whose PK contains only Loop-category keys. + + _collect_all_blocks must produce exactly one block for all rows + (``set_key_cols`` is empty → the pure-Loop branch). + """ + + # _COMPOSITE_DIC contains SCAN (Loop, key=scan.id, no FK to any Set) + # alongside EXPT (Set) and RESULT (Loop with Set FK in PK). + # Ingesting only scan data exercises the pure-Loop path. + + @pytest.fixture + def schema(self): + return _make_schema(_COMPOSITE_DIC) + + _SCAN_ONLY_CIF = ( + '#\\#CIF_2.0\n' + 'data_scans\n' + 'loop_\n _scan.id\n' + ' sc1\n sc2\n sc3\n' + ) + + def test_pure_loop_one_block_for_all_rows(self, schema): + """Pure Loop emits exactly one block containing all rows.""" + conn = _ingest_src(self._SCAN_ONLY_CIF, schema) + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + cif2, errors = build(result) + assert not errors + scan_blocks = [n for n in cif2.blocks if '_scan.id' in cif2[n]] + assert len(scan_blocks) == 1 + + def test_pure_loop_block_contains_all_rows(self, schema): + """The single scan block contains all 3 scan rows.""" + conn = _ingest_src(self._SCAN_ONLY_CIF, schema) + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + cif2, errors = build(result) + assert not errors + scan_block = next(n for n in cif2.blocks if '_scan.id' in cif2[n]) + ids = sorted(str(v) for v in cif2[scan_block]['_scan.id']) + assert ids == ['sc1', 'sc2', 'sc3'] + + def test_pure_loop_block_named_after_table(self, schema): + """Block name for a pure Loop is derived from the table name.""" + conn = _ingest_src(self._SCAN_ONLY_CIF, schema) + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + headers = [l[len('data_'):] for l in result.splitlines() if l.startswith('data_')] + assert any(h.startswith('scan') for h in headers) + + def test_pure_loop_has_loop_header(self, schema): + """The single scan block uses a loop_ structure (not scalar tags).""" + conn = _ingest_src(self._SCAN_ONLY_CIF, schema) + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + assert 'loop_' in result + + def test_pure_loop_output_is_valid_cif(self, schema): + conn = _ingest_src(self._SCAN_ONLY_CIF, schema) + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + _, errors = build(result) + assert not errors, errors + + +# Schema with a nested Set (child Set whose PK FK-links to a parent Set). +# EXPT (Set, key=expt.id) + SAMPLE (Set, composite key: expt_id→expt + id). +# Exercises the Set path in _collect_all_blocks where set_key_cols is non-empty. +_NESTED_SET_DIC = """\ +#\\#CIF_2.0 +data_nested_set_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt + _category_key.name '_expt.id' +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_SAMPLE + _definition.id SAMPLE + _definition.scope Category + _definition.class Set + _name.category_id sample + loop_ + _category_key.name + '_sample.expt_id' + '_sample.id' +save_ + +save_sample.expt_id + _definition.id '_sample.expt_id' + _definition.class Attribute + _name.category_id sample + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_sample.id + _definition.id '_sample.id' + _definition.class Attribute + _name.category_id sample + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_sample.name + _definition.id '_sample.name' + _definition.class Attribute + _name.category_id sample + _name.object_id name + _type.purpose Describe + _type.source Assigned + _type.container Single + _type.contents Text +save_ +""" + +# Two sample entities in the same experiment, across two CIF blocks. +_NESTED_SET_CIF = ( + '#\\#CIF_2.0\n' + 'data_e1s1\n' + '_expt.id E1\n' + '_sample.expt_id E1\n' + '_sample.id S1\n' + '_sample.name Alpha\n' + '\n\n' + 'data_e1s2\n' + '_expt.id E1\n' + '_sample.expt_id E1\n' + '_sample.id S2\n' + '_sample.name Beta\n' +) + + +# --------------------------------------------------------------------------- +# ALL_BLOCKS — Set category with FK in its own PK (nested Set) +# --------------------------------------------------------------------------- + +class TestAllBlocksNestedSet: + """ALL_BLOCKS: child Set whose composite PK includes a FK to a parent Set. + + _collect_all_blocks must inject a synthetic parent row (expt) so that + _suppressed_fk_pk_cols suppresses the FK column and emits _expt.id as a + scalar above the sample tags. + """ + + @pytest.fixture + def schema(self): + return _make_schema(_NESTED_SET_DIC) + + @pytest.fixture + def conn(self, schema): + return _ingest_src(_NESTED_SET_CIF, schema) + + def test_sample_blocks_one_per_row(self, conn, schema): + """Each sample entity gets its own block.""" + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + cif2, errors = build(result) + assert not errors + sample_blocks = [n for n in cif2.blocks if '_sample.id' in cif2[n]] + assert len(sample_blocks) == 2 + + def test_parent_set_scalar_injected(self, conn, schema): + """_expt.id appears as a scalar tag-value pair in each sample block.""" + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + cif2, errors = build(result) + assert not errors + sample_blocks = [n for n in cif2.blocks if '_sample.id' in cif2[n]] + for bname in sample_blocks: + assert '_expt.id' in cif2[bname], f"_expt.id missing in block {bname}" + assert str(cif2[bname]['_expt.id'][0]) == 'E1' + + def test_fk_column_suppressed(self, conn, schema): + """_sample.expt_id is suppressed (the scalar _expt.id takes its place).""" + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + assert '_sample.expt_id' not in result + + def test_sample_values_correct(self, conn, schema): + """Each sample block contains the correct name for its id.""" + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + cif2, errors = build(result) + assert not errors + sample_blocks = [n for n in cif2.blocks if '_sample.id' in cif2[n]] + id_to_name = { + str(cif2[b]['_sample.id'][0]): str(cif2[b]['_sample.name'][0]) + for b in sample_blocks + } + assert id_to_name == {'S1': 'Alpha', 'S2': 'Beta'} + + def test_output_is_valid_cif(self, conn, schema): + result = emit(conn, schema=schema, mode=EmitMode.ALL_BLOCKS) + _, errors = build(result) + assert not errors, errors + + # --------------------------------------------------------------------------- # ORIGINAL mode — multiple source blocks grouped by _cifflow_block_id # --------------------------------------------------------------------------- @@ -2390,6 +2611,254 @@ def test_incompatible_merge_group_emits_plain_loops(self, schema, conn): assert '_meas.intensity' in result assert '_expt.id' in result + def test_empty_group_emits_nothing(self, schema): + """When no category in the group has rows, merge group returns [] (guard branch).""" + conn_empty = _ingest_src('#\\#CIF_2.0\ndata_b\n_expt.id E1\n', schema) + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn_empty, schema, mode=EmitMode.ONE_BLOCK, plan=plan) + assert 'loop_' not in result + + def test_pretty_false_compact_output(self, conn, schema): + """pretty=False disables column alignment; all merge-group data still present.""" + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan, pretty=False) + assert 'loop_' in result + assert '_meas.intensity' in result + assert '_calc.intensity' in result + + +# Schema: EXPT (Set) + MEAS and CALC (both Loop with composite FK-in-PK pointing to EXPT). +# Used to test _render_merge_group with suppress_all_fk_to_set=True (GROUPED mode). +# After suppressing the expt_id FK, both tables share effective PK {point_id} → compatible. +_MERGE_GROUPED_FK_DIC = """\ +#\\#CIF_2.0 +data_merge_grouped_fk_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt + _category_key.name '_expt.id' +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Code +save_ + +save_MEAS + _definition.id MEAS + _definition.scope Category + _definition.class Loop + _name.category_id meas + loop_ + _category_key.name + '_meas.expt_id' + '_meas.point_id' +save_ + +save_meas.expt_id + _definition.id '_meas.expt_id' + _definition.class Attribute + _name.category_id meas + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_meas.point_id + _definition.id '_meas.point_id' + _definition.class Attribute + _name.category_id meas + _name.object_id point_id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_meas.intensity + _definition.id '_meas.intensity' + _definition.class Attribute + _name.category_id meas + _name.object_id intensity + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ + +save_CALC + _definition.id CALC + _definition.scope Category + _definition.class Loop + _name.category_id calc + loop_ + _category_key.name + '_calc.expt_id' + '_calc.point_id' +save_ + +save_calc.expt_id + _definition.id '_calc.expt_id' + _definition.class Attribute + _name.category_id calc + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Code + _name.linked_item_id '_expt.id' +save_ + +save_calc.point_id + _definition.id '_calc.point_id' + _definition.class Attribute + _name.category_id calc + _name.object_id point_id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_calc.intensity + _definition.id '_calc.intensity' + _definition.class Attribute + _name.category_id calc + _name.object_id intensity + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ +""" + +_MERGE_GROUPED_FK_CIF = ( + '#\\#CIF_2.0\n' + 'data_run1\n' + '_expt.id E1\n' + 'loop_\n _meas.expt_id\n _meas.point_id\n _meas.intensity\n' + ' E1 1 10.0\n E1 2 20.0\n' + 'loop_\n _calc.expt_id\n _calc.point_id\n _calc.intensity\n' + ' E1 1 11.0\n E1 2 21.0\n' +) + + +class TestMergeGroupSuppressedFK: + """_render_merge_group with suppress_all_fk_to_set=True (GROUPED mode). + + MEAS and CALC both have composite PK [expt_id→expt, point_id]. After + removing the FK-to-Set column from the effective PK, both share {point_id} + and are compatible for merging. + """ + + @pytest.fixture + def schema(self): + return _make_schema(_MERGE_GROUPED_FK_DIC) + + @pytest.fixture + def conn(self, schema): + return _ingest_src(_MERGE_GROUPED_FK_CIF, schema) + + def test_compatible_after_fk_suppression(self, conn, schema): + """With expt_id suppressed, meas and calc share effective PK {point_id} → one loop_.""" + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + assert result.count('loop_') == 1 + + def test_merged_loop_contains_both_intensities(self, conn, schema): + """Merged loop contains columns from both meas and calc.""" + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + assert '_meas.intensity' in result + assert '_calc.intensity' in result + + def test_fk_col_suppressed_from_loop_header(self, conn, schema): + """_meas.expt_id and _calc.expt_id must not appear in the loop_ header.""" + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + assert '_meas.expt_id' not in result + assert '_calc.expt_id' not in result + + def test_set_scalar_present_in_block(self, conn, schema): + """_expt.id is emitted as a scalar tag-value pair in the block.""" + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + cif2, errors = build(result) + assert not errors + block = cif2[cif2.blocks[0]] + assert str(block['_expt.id'][0]) == 'E1' + + def test_output_is_valid_cif(self, conn, schema): + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + _, errors = build(result) + assert not errors, errors + + +class TestMergeGroupReconstructSU: + """_render_merge_group with reconstruct_su=True. + + _SU_FK_PK_DIC has pd_data (Loop, PK=point_id) and meas (Loop, PK=point_id) + with an SU pair (intensity / intensity_su). A merge group [pd_data, meas] + exercises the token-matrix SU-reconstruction branch. + """ + + @pytest.fixture + def schema(self): + return _make_schema(_SU_FK_PK_DIC) + + @pytest.fixture + def conn(self, schema): + return _ingest_src(_SU_FK_PK_CIF, schema) + + def test_su_merged_into_parenthetical_form(self, conn, schema): + """reconstruct_su=True merges intensity_su into intensity(su) in the merged loop.""" + spec = BlockSpec(category_order=[['pd_data', 'meas']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan, reconstruct_su=True) + # _merge_su('10.0', '0.1') → '10.0(1)' + assert '10.0(1)' in result + + def test_su_col_absent_from_merged_header(self, conn, schema): + """_meas.intensity_su must not appear as a standalone column header.""" + spec = BlockSpec(category_order=[['pd_data', 'meas']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan, reconstruct_su=True) + assert '_meas.intensity_su' not in result + + def test_one_merged_loop_emitted(self, conn, schema): + """pd_data and meas share PK point_id → compatible → one loop_.""" + spec = BlockSpec(category_order=[['pd_data', 'meas']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan, reconstruct_su=True) + assert result.count('loop_') == 1 + + def test_output_is_valid_cif(self, conn, schema): + spec = BlockSpec(category_order=[['pd_data', 'meas']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan, reconstruct_su=True) + _, errors = build(result) + assert not errors, errors + class TestSingleBlock: """BlockSpec.single_block=True collapses all matching blocks into one.""" diff --git a/tests/output/test_emit_render_block.py b/tests/output/test_emit_render_block.py new file mode 100644 index 0000000..82790f2 --- /dev/null +++ b/tests/output/test_emit_render_block.py @@ -0,0 +1,719 @@ +""" +Branch-coverage tests for _render_block in cifflow.output.emit. + +Branches correspond to the five logical sections identified in Stage 1 of the +complexity-reduction refactor: + + 1. Guard — CIF 1.1 block-name length + 2. organise_fallback — scalar / pure-loop / mixed-fallback / audit-id stripping + 3. inject_header — conformance tags and _audit_dataset.id injection + 4. merge_group_item — ORIGINAL vs GROUPED merge-group dispatch + 5. single_table_item — column filtering, FK-PK suppression, Set/Loop dispatch + 6. collect_remnant — unrendered mixed-fallback falls to plain remnant +""" + +from __future__ import annotations + +import pytest + +from cifflow import build, ingest, emit, EmitMode, generate_schema +from cifflow.dictionary import DictionaryLoader +from cifflow.dictionary.schema import ColumnDef, ForeignKeyDef, SchemaSpec, TableDef +from cifflow.output import BlockSpec, OutputPlan +from cifflow.output.emit import _BlockData, _render_block +from cifflow.types import CifVersion + + +# --------------------------------------------------------------------------- +# Helpers — construct _BlockData and SchemaSpec directly +# --------------------------------------------------------------------------- + +def _col( + name: str, + *, + pk: bool = False, + synthetic: bool = False, + linked: str | None = None, +) -> ColumnDef: + return ColumnDef( + name=name, + definition_id=f'_test.{name}', + type_contents='Text', + nullable=not pk, + is_primary_key=pk, + is_synthetic=synthetic, + linked_item_id=linked, + ) + + +def _table( + name: str, + cols: list[ColumnDef], + *, + cls: str = 'Loop', + fks: list[ForeignKeyDef] | None = None, +) -> TableDef: + pks = [c.name for c in cols if c.is_primary_key] + return TableDef( + name=name, + definition_id=f'_{name}', + category_class=cls, + columns=cols, + primary_keys=pks, + foreign_keys=fks or [], + ) + + +def _schema(*tables: TableDef, extra_c2t: dict | None = None) -> SchemaSpec: + c2t: dict[tuple[str, str], str] = { + (t.name, c.name): f'_{t.name}.{c.name}' + for t in tables + for c in t.columns + if not c.is_synthetic + } + if extra_c2t: + c2t.update(extra_c2t) + return SchemaSpec(tables={t.name: t for t in tables}, column_to_tag=c2t) + + +def _bd(**kwargs) -> _BlockData: + defaults: dict = dict( + name='test', + table_rows={}, + fallback_rows=[], + anchor_frozenset=frozenset(), + anchor_key_dict={}, + suppress_fk_pk=False, + ) + defaults.update(kwargs) + return _BlockData(**defaults) + + +CIF20 = CifVersion.CIF_2_0 +CIF11 = CifVersion.CIF_1_1 + + +def _render(block_name: str, data: _BlockData, schema: SchemaSpec, + version: CifVersion = CIF20) -> str: + lines = _render_block(block_name, data, schema, version, None, False, False) + return '\n'.join(lines) + + +# --------------------------------------------------------------------------- +# Helpers — emit() round-trip +# --------------------------------------------------------------------------- + +def _make_schema(dic_src: str) -> SchemaSpec: + d = DictionaryLoader().load(dic_src) + return generate_schema(d) + + +def _ingest_src(cif_src: str, schema: SchemaSpec | None = None): + cif, errors = build(cif_src) + assert not errors, errors + conn, _ = ingest(cif, None, schema) + return conn + + +# --------------------------------------------------------------------------- +# Mini dictionaries +# --------------------------------------------------------------------------- + +_EXPT_DIC = """\ +#\\#CIF_2.0 +data_expt_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Text +save_ +""" + +_ATOM_DIC = """\ +#\\#CIF_2.0 +data_atom_dic + +save_ATOM + _definition.id ATOM + _definition.scope Category + _definition.class Loop + _name.category_id atom + _category_key.name '_atom.id' +save_ + +save_atom.id + _definition.id '_atom.id' + _definition.class Attribute + _name.category_id atom + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_atom.label + _definition.id '_atom.label' + _definition.class Attribute + _name.category_id atom + _name.object_id label + _type.purpose Descriptor + _type.source Assigned + _type.container Single + _type.contents Text +save_ +""" + +_EXPT_ATOM_DIC = """\ +#\\#CIF_2.0 +data_expt_atom_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Text +save_ + +save_ATOM + _definition.id ATOM + _definition.scope Category + _definition.class Loop + _name.category_id atom + _category_key.name '_atom.id' +save_ + +save_atom.id + _definition.id '_atom.id' + _definition.class Attribute + _name.category_id atom + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_atom.label + _definition.id '_atom.label' + _definition.class Attribute + _name.category_id atom + _name.object_id label + _type.purpose Descriptor + _type.source Assigned + _type.container Single + _type.contents Text +save_ + +save_atom.expt_id + _definition.id '_atom.expt_id' + _definition.class Attribute + _name.category_id atom + _name.object_id expt_id + _type.purpose Link + _type.source Related + _type.container Single + _type.contents Text + _name.linked_item_id '_expt.id' +save_ +""" + +_MEAS_CALC_DIC = """\ +#\\#CIF_2.0 +data_meas_calc_dic + +save_EXPT + _definition.id EXPT + _definition.scope Category + _definition.class Set + _name.category_id expt +save_ + +save_expt.id + _definition.id '_expt.id' + _definition.class Attribute + _name.category_id expt + _name.object_id id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Text +save_ + +save_MEAS + _definition.id MEAS + _definition.scope Category + _definition.class Loop + _name.category_id meas + _category_key.name '_meas.point_id' +save_ + +save_meas.point_id + _definition.id '_meas.point_id' + _definition.class Attribute + _name.category_id meas + _name.object_id point_id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_meas.intensity + _definition.id '_meas.intensity' + _definition.class Attribute + _name.category_id meas + _name.object_id intensity + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ + +save_CALC + _definition.id CALC + _definition.scope Category + _definition.class Loop + _name.category_id calc + _category_key.name '_calc.point_id' +save_ + +save_calc.point_id + _definition.id '_calc.point_id' + _definition.class Attribute + _name.category_id calc + _name.object_id point_id + _type.purpose Key + _type.source Assigned + _type.container Single + _type.contents Integer +save_ + +save_calc.intensity + _definition.id '_calc.intensity' + _definition.class Attribute + _name.category_id calc + _name.object_id intensity + _type.purpose Number + _type.source Measured + _type.container Single + _type.contents Real +save_ +""" + + +# --------------------------------------------------------------------------- +# Branch group 1: Guard +# --------------------------------------------------------------------------- + +class TestRenderBlockGuard: + """CIF 1.1 block-name length check (line 1728).""" + + def test_cif11_long_block_name_raises(self): + data = _bd() + schema = SchemaSpec(tables={}, column_to_tag={}) + with pytest.raises(ValueError, match='75-character'): + _render_block('x' * 76, data, schema, CIF11, None, False, False) + + def test_cif20_long_block_name_allowed(self): + name = 'x' * 100 + data = _bd() + schema = SchemaSpec(tables={}, column_to_tag={}) + lines = _render_block(name, data, schema, CIF20, None, False, False) + assert lines[0] == f'data_{name}' + + +# --------------------------------------------------------------------------- +# Branch group 2: organise_fallback_rows +# --------------------------------------------------------------------------- + +class TestOrganiseFallbackRows: + """Fallback-row partitioning: scalar, pure-loop, mixed-fallback, audit-id stripping.""" + + def test_scalar_fallback_rendered(self): + # loop_id is None → remnant_rows path → rendered as scalar by _render_fallback + conn = _ingest_src('#\\#CIF_2.0\ndata_t\n_unknown.tag hello\n') + result = emit(conn, SchemaSpec(tables={}, column_to_tag={})) + assert '_unknown.tag' in result + assert 'hello' in result + + def test_pure_loop_fallback_rendered_as_loop(self): + # loop_id set + no ref_table → pure_loop_rows path → loop_ output + conn = _ingest_src( + '#\\#CIF_2.0\ndata_t\nloop_\n _x.a\n _x.b\n 1 2\n 3 4\n' + ) + result = emit(conn, SchemaSpec(tables={}, column_to_tag={})) + assert 'loop_' in result + assert '_x.a' in result + assert '_x.b' in result + + def test_mixed_fallback_extra_col_in_known_loop(self): + # loop_id set + ref_table set → mixed_fallback → extra col appears in structured loop + schema = _make_schema(_ATOM_DIC) + cif_src = ( + '#\\#CIF_2.0\ndata_t\n' + 'loop_\n _atom.id\n _atom.label\n _atom.extra\n' + ' 1 H val1\n 2 C val2\n' + ) + conn = _ingest_src(cif_src, schema) + result = emit(conn, schema) + assert '_atom.extra' in result + assert 'val1' in result + assert 'val2' in result + + def test_audit_id_stripped_from_pure_loop_when_dataset_id_set(self): + # _audit_dataset.id in a pure fallback loop is stripped when dataset_id is also set, + # so the scalar injection controls the output instead of emitting a spurious loop_. + schema = SchemaSpec( + tables={}, + column_to_tag={('audit_dataset', 'id'): '_audit_dataset.id'}, + ) + data = _bd( + fallback_rows=[ + { + 'loop_id': 1, + 'ref_table': None, + 'tag': '_audit_dataset.id', + 'value': 'DS1', + 'value_type': 'string', + }, + ], + dataset_id='DS1', + ) + result = _render('test', data, schema) + assert 'loop_' not in result + assert '_audit_dataset.id' in result + assert 'DS1' in result + + +# --------------------------------------------------------------------------- +# Branch group 3: inject_header_items +# --------------------------------------------------------------------------- + +class TestInjectHeaderItems: + """Conformance tags and _audit_dataset.id injection.""" + + def test_no_header_items_produces_only_data_line(self): + # No conformance_tags, no dataset_id → output is exactly 'data_blk' + data = _bd() + schema = SchemaSpec(tables={}, column_to_tag={}) + result = _render('blk', data, schema) + assert result == 'data_blk' + + def test_conformance_tags_emitted_immediately_after_data_line(self): + # conformance_tags → lines added before structured content + tbl = _table('cell', [_col('len', pk=True)], cls='Set') + schema = _schema(tbl) + data = _bd( + table_rows={'cell': [{'len': '5.4', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}]}, + anchor_frozenset=frozenset({'cell'}), + suppress_fk_pk=True, + conformance_tags=[('_conform.dict_name', 'mydict')], + ) + result = _render('blk', data, schema) + lines = result.splitlines() + assert lines[0] == 'data_blk' + conform_idx = next(i for i, ln in enumerate(lines) if '_conform.dict_name' in ln) + cell_idx = next(i for i, ln in enumerate(lines) if '_cell.len' in ln) + assert conform_idx < cell_idx + + def test_dataset_id_string_emitted_as_scalar(self): + # dataset_id is a plain string → scalar tag-value pair, no loop_ + schema = SchemaSpec( + tables={}, + column_to_tag={('audit_dataset', 'id'): '_audit_dataset.id'}, + ) + data = _bd(dataset_id='MY_DS') + result = _render('blk', data, schema) + assert '_audit_dataset.id MY_DS' in result + assert 'loop_' not in result + + def test_dataset_id_list_emitted_as_loop(self): + # dataset_id is a list → loop_ with one value per row + schema = SchemaSpec( + tables={}, + column_to_tag={('audit_dataset', 'id'): '_audit_dataset.id'}, + ) + data = _bd(dataset_id=['DS1', 'DS2']) + result = _render('blk', data, schema) + assert 'loop_' in result + assert 'DS1' in result + assert 'DS2' in result + + def test_dataset_id_skipped_when_audit_dataset_is_set_in_table_rows(self): + # audit_dataset (Set class) already in table_rows → injection skipped to avoid duplicate + ad_col = _col('id', pk=True) + ad_tbl = _table('audit_dataset', [ad_col], cls='Set') + schema = _schema(ad_tbl, extra_c2t={('audit_dataset', 'id'): '_audit_dataset.id'}) + data = _bd( + table_rows={ + 'audit_dataset': [{'id': 'DS1', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}] + }, + anchor_frozenset=frozenset({'audit_dataset'}), + suppress_fk_pk=True, + dataset_id='DS1', + ) + result = _render('blk', data, schema) + # _audit_dataset.id appears exactly once (rendered from table_rows, not injected) + assert result.count('_audit_dataset.id') == 1 + + def test_dataset_id_skipped_when_present_in_fallback_remnant(self): + # _audit_dataset.id already in scalar fallback → injection skipped + schema = SchemaSpec( + tables={}, + column_to_tag={('audit_dataset', 'id'): '_audit_dataset.id'}, + ) + data = _bd( + fallback_rows=[ + { + 'loop_id': None, + 'ref_table': None, + 'tag': '_audit_dataset.id', + 'value': 'DS1', + 'value_type': 'string', + }, + ], + dataset_id='DS1', + ) + result = _render('blk', data, schema) + assert result.count('_audit_dataset.id') == 1 + + def test_audit_dataset_loop_table_popped_and_scalar_injected(self): + # audit_dataset is Loop class in table_rows → table is popped, scalar is injected instead + ad_col = _col('id', pk=True) + ad_tbl = _table('audit_dataset', [ad_col], cls='Loop') # Loop, not Set + schema = _schema(ad_tbl, extra_c2t={('audit_dataset', 'id'): '_audit_dataset.id'}) + data = _bd( + table_rows={ + 'audit_dataset': [{'id': 'DS1', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}] + }, + suppress_fk_pk=False, + dataset_id='DS1', + ) + result = _render('blk', data, schema) + # Table popped → scalar emission, not loop_ + assert '_audit_dataset.id DS1' in result + assert 'loop_' not in result + + +# --------------------------------------------------------------------------- +# Branch group 4: render_merge_group_item +# --------------------------------------------------------------------------- + +class TestRenderMergeGroupItem: + """Merge-group dispatch: ORIGINAL (suppress_loop_fk_pk=True) vs GROUPED path.""" + + @pytest.fixture + def schema(self): + return _make_schema(_MEAS_CALC_DIC) + + @pytest.fixture + def conn(self, schema): + return _ingest_src( + '#\\#CIF_2.0\ndata_b\n_expt.id E1\n' + 'loop_\n _meas.point_id\n _meas.intensity\n 1 10.0\n 2 20.0\n' + 'loop_\n _calc.point_id\n _calc.intensity\n 1 11.0\n 2 21.0\n', + schema, + ) + + def test_original_path_renders_all_columns(self, conn, schema): + # suppress_loop_fk_pk=True (ORIGINAL mode) → _render_original_loop_group path + result = emit(conn, schema, mode=EmitMode.ORIGINAL) + assert '_meas.intensity' in result + assert '_calc.intensity' in result + + def test_grouped_path_merges_compatible_tables_to_single_loop(self, conn, schema): + # suppress_loop_fk_pk=False (ONE_BLOCK + merge spec) → _render_merge_group path + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.ONE_BLOCK, plan=plan) + assert result.count('loop_') == 1 + assert '_meas.intensity' in result + assert '_calc.intensity' in result + + def test_suppress_pkg_computed_for_merge_group_in_grouped_mode(self, conn, schema): + # GROUPED mode (suppress_fk_pk=True + suppress_all_fk_to_set=True): + # suppress_pkg is computed for the merge group before calling _render_merge_group. + # Even when no FK-PK columns exist, the code path is exercised and the merge + # group still produces a single loop_ with columns from both tables. + spec = BlockSpec(category_order=[['meas', 'calc']]) + plan = OutputPlan(specs=[spec]) + result = emit(conn, schema, mode=EmitMode.GROUPED, plan=plan) + assert result.count('loop_') == 1 + assert '_meas.intensity' in result + assert '_calc.intensity' in result + + +# --------------------------------------------------------------------------- +# Branch group 5: render_single_table_item +# --------------------------------------------------------------------------- + +class TestRenderSingleTableItem: + """Column filtering, FK-PK suppression, and Set/Loop dispatch.""" + + def test_table_with_no_rows_not_rendered(self): + # data.table_rows.get(table_name) is falsy → table skipped + schema = _make_schema(_ATOM_DIC) + conn = _ingest_src('#\\#CIF_2.0\ndata_t\n', schema) + result = emit(conn, schema) + assert '_atom.id' not in result + assert 'loop_' not in result + + def test_set_category_rendered_as_scalar_pairs(self): + # Set category, single row → tag-value pairs, no loop_ + schema = _make_schema(_EXPT_DIC) + conn = _ingest_src('#\\#CIF_2.0\ndata_t\n_expt.id E1\n', schema) + result = emit(conn, schema) + assert '_expt.id' in result + assert 'loop_' not in result + + def test_loop_category_rendered_as_loop(self): + # Loop category, multiple rows → loop_ with column headers and data values + schema = _make_schema(_ATOM_DIC) + conn = _ingest_src( + '#\\#CIF_2.0\ndata_t\nloop_\n _atom.id\n _atom.label\n 1 H\n 2 C\n', + schema, + ) + result = emit(conn, schema) + assert 'loop_' in result + assert '_atom.id' in result + assert '_atom.label' in result + + def test_fk_pk_col_suppressed_in_grouped_mode(self): + # suppress_fk_pk=True + suppress_all_fk_to_set=True: a column that is BOTH an FK + # and a PK (composite key) pointing to a co-emitted Set row is suppressed. + expt_tbl = _table('expt', [_col('id', pk=True)], cls='Set') + fk = ForeignKeyDef( + source_table='atom', + source_columns=['expt_id'], + target_table='expt', + target_columns=['id'], + ) + atom_tbl = _table( + 'atom', + [_col('expt_id', pk=True), _col('id', pk=True), _col('label')], + cls='Loop', + fks=[fk], + ) + schema = _schema(expt_tbl, atom_tbl) + data = _bd( + table_rows={ + 'expt': [{'id': 'E1', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}], + 'atom': [ + {'expt_id': 'E1', 'id': '1', 'label': 'H', + '_cifflow_block_id': 'b', '_cifflow_row_id': 2}, + {'expt_id': 'E1', 'id': '2', 'label': 'C', + '_cifflow_block_id': 'b', '_cifflow_row_id': 3}, + ], + }, + anchor_frozenset=frozenset({'expt'}), + suppress_fk_pk=True, + suppress_all_fk_to_set=True, + ) + result = _render('blk', data, schema) + assert '_atom.expt_id' not in result + assert '_atom.label' in result + + def test_inapplicable_col_filtered_in_grouped_mode(self): + # suppress_all_fk_to_set=True: column where every value is '.' is removed; + # other columns still render + col_id = _col('id', pk=True) + col_val = _col('val') + col_inapplicable = _col('dead') + tbl = _table('t', [col_id, col_val, col_inapplicable], cls='Loop') + schema = _schema(tbl) + data = _bd( + table_rows={'t': [ + {'id': '1', 'val': 'v1', 'dead': '.', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}, + {'id': '2', 'val': 'v2', 'dead': '.', '_cifflow_block_id': 'b', '_cifflow_row_id': 2}, + ]}, + suppress_fk_pk=True, + suppress_all_fk_to_set=True, + ) + result = _render('blk', data, schema) + assert '_t.val' in result + assert '_t.dead' not in result + assert 'loop_' in result + + def test_all_inapplicable_cols_causes_table_skip(self): + # suppress_all_fk_to_set=True + every active column has all-'.' values → table skipped + col_id = _col('id', pk=True) + col_val = _col('val') + tbl = _table('t', [col_id, col_val], cls='Loop') + schema = _schema(tbl) + data = _bd( + table_rows={'t': [ + {'id': '.', 'val': '.', '_cifflow_block_id': 'b', '_cifflow_row_id': 1}, + ]}, + suppress_fk_pk=True, + suppress_all_fk_to_set=True, + ) + result = _render('blk', data, schema) + assert 'loop_' not in result + assert '_t.' not in result + + +# --------------------------------------------------------------------------- +# Branch group 6: collect_remnant_rows +# --------------------------------------------------------------------------- + +class TestCollectRemnantRows: + """Remnant row collection: scalar fallback and unrendered mixed-fallback.""" + + def test_scalar_fallback_rendered_after_structured_tables(self): + # Scalar (loop_id=None) fallback rows appear after structured table content + schema = _make_schema(_EXPT_DIC) + conn = _ingest_src( + '#\\#CIF_2.0\ndata_t\n_expt.id E1\n_extra.tag hello\n', + schema, + ) + result = emit(conn, schema) + assert '_expt.id' in result + assert '_extra.tag' in result + assert result.index('_expt.id') < result.index('_extra.tag') + + def test_mixed_fallback_with_absent_ref_table_falls_to_remnant(self): + # A mixed-fallback row whose ref_table is not in table_rows is treated as plain remnant + tbl = _table('atom', [_col('id', pk=True), _col('label')], cls='Loop') + schema = _schema(tbl) + data = _bd( + table_rows={}, # 'atom' not rendered in this block + fallback_rows=[ + { + 'loop_id': 1, + 'ref_table': 'atom', + 'col_index': 1, + '_cifflow_row_id': 1, + 'tag': '_atom.extra', + 'value': 'v1', + 'value_type': 'string', + }, + ], + ) + result = _render('blk', data, schema) + assert '_atom.extra' in result + assert 'v1' in result diff --git a/tests/output/test_render_original_loop_group.py b/tests/output/test_render_original_loop_group.py new file mode 100644 index 0000000..a128dab --- /dev/null +++ b/tests/output/test_render_original_loop_group.py @@ -0,0 +1,207 @@ +"""Branch-coverage tests for _render_positional_join (extracted from _render_original_loop_group). + +Tests cover the positional-join rendering path in isolation, without DuckDB. +""" +import pytest + +from cifflow.dictionary.ddlm_item import DdlmItem +from cifflow.dictionary.ddlm_parser import DdlmDictionary +from cifflow.dictionary.schema import generate_schema +from cifflow.output.emit import _render_positional_join +from cifflow.types import CifVersion + + +# --------------------------------------------------------------------------- +# Schema helpers +# --------------------------------------------------------------------------- + +def _item(definition_id, category_id, object_id, *, type_purpose=None, + type_contents=None, linked_item_id=None): + return DdlmItem( + definition_id=definition_id, scope='Item', definition_class='Datum', + category_id=category_id, object_id=object_id, + type_purpose=type_purpose, type_source=None, type_container='Single', + type_contents=type_contents, linked_item_id=linked_item_id, + units_code=None, description=None, + ) + + +def _cat(definition_id, cat_class, category_keys=None): + return DdlmItem( + definition_id=definition_id, scope='Category', + definition_class=cat_class, category_id=None, object_id=None, + type_purpose=None, type_source=None, type_container='Single', + type_contents=None, linked_item_id=None, units_code=None, + description=None, category_keys=category_keys or [], + ) + + +def _make_dict(cats, items): + categories = {c.definition_id: c for c in cats} + item_map = {i.definition_id: i for i in items} + return DdlmDictionary( + name='TEST', title=None, version=None, + categories=categories, items=item_map, + tag_to_item={**categories, **item_map}, + alias_to_definition_id={}, deprecated_ids=set(), + ) + + +def _schema_two_tables(): + """atom_site (Loop, PK=label, col=x_fract) and atom_site_b (Loop, PK=label, col=y_fract).""" + cats = [ + _cat('_atom_site', 'Loop', ['_atom_site.label']), + _cat('_atom_site_b', 'Loop', ['_atom_site_b.label']), + ] + items = [ + _item('_atom_site.label', '_atom_site', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom_site.x_fract', '_atom_site', 'x_fract', type_contents='Real'), + _item('_atom_site_b.label', '_atom_site_b', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom_site_b.y_fract', '_atom_site_b', 'y_fract', type_contents='Real'), + ] + return generate_schema(_make_dict(cats, items)) + + +def _schema_with_su(): + """atom_site (Loop, PK=label, col=x_fract with SU).""" + cats = [_cat('_atom_site', 'Loop', ['_atom_site.label'])] + items = [ + _item('_atom_site.label', '_atom_site', 'label', type_purpose='Key', type_contents='Text'), + _item('_atom_site.x_fract', '_atom_site', 'x_fract', type_contents='Real'), + _item('_atom_site.x_fract_su', '_atom_site', 'x_fract_su', + type_purpose='SU', linked_item_id='_atom_site.x_fract', type_contents='Real'), + ] + return generate_schema(_make_dict(cats, items)) + + +VER = CifVersion.CIF_2_0 + + +def _row(rid, **kwargs): + return {'_cifflow_row_id': rid, **kwargs} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestRenderPositionalJoin: + def test_no_rows_returns_empty(self): + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], []), + ('atom_site_b', ['label', 'y_fract'], []), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=False, line_limit=None) + assert result == [] + + def test_basic_two_table_join(self): + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], [_row(1, label='C1', x_fract='0.5')]), + ('atom_site_b', ['label', 'y_fract'], [_row(1, label='C1', y_fract='0.3')]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=False, line_limit=None) + assert result[0] == 'loop_' + # Tags for both tables should appear + joined = '\n'.join(result) + assert '_atom_site.label' in joined + assert '_atom_site_b.y_fract' in joined + # Data row: 4 values on one line + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + assert len(data_lines) == 1 + assert 'C1' in data_lines[0] + assert '0.3' in data_lines[0] + + def test_sparse_rows_padded_with_placeholder(self): + """Table B has 1 row but table A has 2 — second B row should be '.'""" + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], [ + _row(1, label='C1', x_fract='0.1'), + _row(2, label='C2', x_fract='0.2'), + ]), + ('atom_site_b', ['label', 'y_fract'], [_row(1, label='C1', y_fract='0.5')]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=False, line_limit=None) + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + assert len(data_lines) == 2 + assert '.' in data_lines[1] # padded for missing B row + + def test_rows_sorted_by_row_id(self): + """Rows out of insertion order should be sorted by _cifflow_row_id.""" + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], [ + _row(2, label='C2', x_fract='0.2'), + _row(1, label='C1', x_fract='0.1'), + ]), + ('atom_site_b', ['label', 'y_fract'], [ + _row(2, label='C2', y_fract='0.5'), + _row(1, label='C1', y_fract='0.3'), + ]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=False, line_limit=None) + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + # First row should be C1 (row_id=1) + assert data_lines[0].split()[0] == 'C1' + assert data_lines[1].split()[0] == 'C2' + + def test_reconstruct_su_merges_value(self): + """With reconstruct_su=True, value(su) should be merged into the output token.""" + schema = _schema_with_su() + per_table = [ + ('atom_site', ['label', 'x_fract'], + [_row(1, label='C1', x_fract='0.123', x_fract_su='0.002')]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=True, pretty=False, line_limit=None) + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + assert '0.123(2)' in data_lines[0] + + def test_reconstruct_su_null_su_leaves_value_unchanged(self): + schema = _schema_with_su() + per_table = [ + ('atom_site', ['label', 'x_fract'], + [_row(1, label='C1', x_fract='0.123', x_fract_su=None)]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=True, pretty=False, line_limit=None) + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + assert '0.123' in data_lines[0] + assert '(' not in data_lines[0] + + def test_pretty_mode_runs_without_error(self): + """pretty=True should trigger decimal-align path without crashing.""" + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], [ + _row(1, label='C1', x_fract='1.0'), + _row(2, label='C2', x_fract='10.5'), + ]), + ('atom_site_b', ['label', 'y_fract'], [ + _row(1, label='C1', y_fract='0.3'), + _row(2, label='C2', y_fract='1.20'), + ]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=True, line_limit=None) + assert result[0] == 'loop_' + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + assert len(data_lines) == 2 + + def test_null_value_emitted_as_placeholder(self): + schema = _schema_two_tables() + per_table = [ + ('atom_site', ['label', 'x_fract'], [_row(1, label='C1', x_fract=None)]), + ('atom_site_b', ['label', 'y_fract'], [_row(1, label='C1', y_fract='0.3')]), + ] + result = _render_positional_join(per_table, schema, VER, + reconstruct_su=False, pretty=False, line_limit=None) + data_lines = [l for l in result if not l.startswith('loop_') and not l.startswith(' _')] + tokens = data_lines[0].split() + assert '.' in tokens # NULL emitted as '.' diff --git a/tests/output/test_render_set_category.py b/tests/output/test_render_set_category.py new file mode 100644 index 0000000..3e01d3b --- /dev/null +++ b/tests/output/test_render_set_category.py @@ -0,0 +1,236 @@ +"""Branch-coverage tests for _render_set_category helper functions. + +Tests cover _build_set_quads, _requote_set_quads, and _decimal_align_set_quads +extracted from _render_set_category, plus the orchestrator itself. +""" +import pytest + +from cifflow.dictionary.ddlm_item import DdlmItem +from cifflow.dictionary.ddlm_parser import DdlmDictionary +from cifflow.dictionary.schema import generate_schema +from cifflow.output.emit import ( + _build_set_quads, + _decimal_align_set_quads, + _requote_set_quads, + _render_set_category, +) +from cifflow.types import CifVersion + +VER = CifVersion.CIF_2_0 + + +# --------------------------------------------------------------------------- +# Schema helpers +# --------------------------------------------------------------------------- + +def _item(definition_id, category_id, object_id, *, type_purpose=None, + type_contents=None, linked_item_id=None): + return DdlmItem( + definition_id=definition_id, scope='Item', definition_class='Datum', + category_id=category_id, object_id=object_id, + type_purpose=type_purpose, type_source=None, type_container='Single', + type_contents=type_contents, linked_item_id=linked_item_id, + units_code=None, description=None, + ) + + +def _cat(definition_id, cat_class, category_keys=None): + return DdlmItem( + definition_id=definition_id, scope='Category', + definition_class=cat_class, category_id=None, object_id=None, + type_purpose=None, type_source=None, type_container='Single', + type_contents=None, linked_item_id=None, units_code=None, + description=None, category_keys=category_keys or [], + ) + + +def _make_dict(cats, items): + categories = {c.definition_id: c for c in cats} + item_map = {i.definition_id: i for i in items} + return DdlmDictionary( + name='TEST', title=None, version=None, + categories=categories, items=item_map, + tag_to_item={**categories, **item_map}, + alias_to_definition_id={}, deprecated_ids=set(), + ) + + +def _schema_set(): + """cell (Set PK=id, col=length_a Real, col=name Text, col=length_a_su SU for length_a).""" + cats = [_cat('_cell', 'Set', ['_cell.id'])] + items = [ + _item('_cell.id', '_cell', 'id', type_purpose='Key', type_contents='Text'), + _item('_cell.length_a', '_cell', 'length_a', type_contents='Real'), + _item('_cell.length_a_su', '_cell', 'length_a_su', + type_purpose='SU', linked_item_id='_cell.length_a', type_contents='Real'), + _item('_cell.name', '_cell', 'name', type_contents='Text'), + ] + return generate_schema(_make_dict(cats, items)) + + +_SCHEMA = _schema_set() +_TABLE_DEF = _SCHEMA.tables['cell'] +_SU_MAP = {'length_a': 'length_a_su'} + + +# --------------------------------------------------------------------------- +# Tests: _build_set_quads +# --------------------------------------------------------------------------- + +class TestBuildSetQuads: + def test_basic_quads_built(self): + row = {'id': 'S1', 'length_a': '5.0', 'name': 'test'} + quads = _build_set_quads(['id', 'length_a', 'name'], row, 'cell', + _SCHEMA, VER, False, {}, None) + assert len(quads) == 3 + tags = [q[0] for q in quads] + assert '_cell.id' in tags + + def test_null_value_skipped(self): + row = {'id': 'S1', 'length_a': None, 'name': 'test'} + quads = _build_set_quads(['id', 'length_a', 'name'], row, 'cell', + _SCHEMA, VER, False, {}, None) + cols = [q[1] for q in quads] + assert 'length_a' not in cols + + def test_su_merged_when_present(self): + row = {'length_a': '5.123', 'length_a_su': '0.003'} + quads = _build_set_quads(['length_a'], row, 'cell', + _SCHEMA, VER, True, _SU_MAP, None) + assert len(quads) == 1 + assert quads[0][2] == '5.123(3)' # value with SU merged + + def test_su_null_leaves_value_unchanged(self): + row = {'length_a': '5.123', 'length_a_su': None} + quads = _build_set_quads(['length_a'], row, 'cell', + _SCHEMA, VER, True, _SU_MAP, None) + assert quads[0][2] == '5.123' + + def test_reconstruct_su_false_ignores_su_map(self): + row = {'length_a': '5.123', 'length_a_su': '0.003'} + quads = _build_set_quads(['length_a'], row, 'cell', + _SCHEMA, VER, False, _SU_MAP, None) + assert quads[0][2] == '5.123' # no SU merging + + def test_multiline_token_initially_folded_with_line_limit(self): + # A very long value that quote() would render as multiline + long_val = 'A' * 200 + row = {'name': long_val} + quads = _build_set_quads(['name'], row, 'cell', + _SCHEMA, VER, False, {}, 80) + # The initial quote produces a semicolon text field (\n at start) + # line_limit path should apply make_text_field + assert len(quads) == 1 + + +# --------------------------------------------------------------------------- +# Tests: _requote_set_quads +# --------------------------------------------------------------------------- + +class TestRequoteSetQuads: + def _quad(self, tag, col, value, token): + return (tag, col, value, token) + + def test_line_too_long_requoted_as_multiline(self): + q = self._quad('_cell.name', 'name', 'hello', 'hello') + quads, new_width = _requote_set_quads([q], tag_width=0, line_limit=5, pretty=False) + # '_cell.name hello' is 17 chars > 5; should be requoted as multiline + assert quads[0][3].startswith('\n') + + def test_line_fits_unchanged(self): + q = self._quad('_cell.id', 'id', 'S1', 'S1') + quads, new_width = _requote_set_quads([q], tag_width=0, line_limit=200, pretty=False) + assert quads[0][3] == 'S1' + + def test_already_multiline_not_requoted(self): + token = '\n; text field\n;' + q = self._quad('_cell.name', 'name', 'text field', token) + quads, _ = _requote_set_quads([q], tag_width=0, line_limit=5, pretty=False) + assert quads[0][3] == token # unchanged + + def test_pretty_tag_width_recomputed(self): + q1 = self._quad('_cell.length_a', 'length_a', '5.0', '5.0') + q2 = self._quad('_cell.id', 'id', 'X', 'X') + quads, new_width = _requote_set_quads([q1, q2], tag_width=5, line_limit=200, pretty=True) + # No requoting happened; tag_width should be max of inline tag lengths + assert new_width == max(len('_cell.length_a'), len('_cell.id')) + + def test_pretty_tag_width_excludes_multiline_tokens(self): + q1 = self._quad('_cell.length_a', 'length_a', '5.0', '5.0') + q2 = self._quad('_cell.name', 'name', 'X', '\n; X\n;') + quads, new_width = _requote_set_quads([q1, q2], tag_width=5, line_limit=200, pretty=True) + # Only q1 is inline; width = len('_cell.length_a') + assert new_width == len('_cell.length_a') + + +# --------------------------------------------------------------------------- +# Tests: _decimal_align_set_quads +# --------------------------------------------------------------------------- + +class TestDecimalAlignSetQuads: + def test_real_tokens_aligned(self): + quads = [ + ('_cell.length_a', 'length_a', '5.0', '5.0'), + ('_cell.length_a', 'length_a', '10.50', '10.50'), + ] + result = _decimal_align_set_quads(quads, _TABLE_DEF) + tokens = [q[3] for q in result] + # Decimal-aligned: both have same width + assert len(tokens[0]) == len(tokens[1]) + + def test_non_real_tokens_unchanged(self): + quads = [('_cell.name', 'name', 'hello', 'hello')] + result = _decimal_align_set_quads(quads, _TABLE_DEF) + assert result[0][3] == 'hello' + + def test_multiline_real_token_excluded_from_alignment(self): + quads = [ + ('_cell.length_a', 'length_a', '5.0', '5.0'), + ('_cell.length_a', 'length_a', '10.5', '\n; 10.5\n;'), + ] + result = _decimal_align_set_quads(quads, _TABLE_DEF) + # The multiline token should not affect alignment and remain unchanged + assert result[1][3] == '\n; 10.5\n;' + + def test_no_real_tokens_unchanged(self): + quads = [('_cell.name', 'name', 'hello', 'hello')] + result = _decimal_align_set_quads(quads, _TABLE_DEF) + assert result == quads + + +# --------------------------------------------------------------------------- +# Integration tests: _render_set_category +# --------------------------------------------------------------------------- + +class TestRenderSetCategory: + def test_basic_rendering(self): + row = {'id': 'S1', 'length_a': '5.0', 'name': 'test'} + result = _render_set_category(row, ['id', 'length_a', 'name'], + 'cell', _SCHEMA, VER, _TABLE_DEF, + False, False, None) + assert any('_cell.id' in l for l in result) + assert any('S1' in l for l in result) + + def test_null_cols_omitted(self): + row = {'id': 'S1', 'length_a': None} + result = _render_set_category(row, ['id', 'length_a'], + 'cell', _SCHEMA, VER, _TABLE_DEF, + False, False, None) + assert not any('length_a' in l for l in result) + + def test_pretty_mode(self): + row = {'id': 'S1', 'length_a': '5.0', 'name': 'test'} + result = _render_set_category(row, ['id', 'length_a', 'name'], + 'cell', _SCHEMA, VER, _TABLE_DEF, + False, True, None) + assert len(result) == 3 + assert any('S1' in l for l in result) + assert any('5.0' in l for l in result) + assert any('test' in l for l in result) + + def test_reconstruct_su(self): + row = {'length_a': '5.123', 'length_a_su': '0.003'} + result = _render_set_category(row, ['length_a'], + 'cell', _SCHEMA, VER, _TABLE_DEF, + True, False, None) + assert any('5.123(3)' in l for l in result) diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 06920cb..62288e1 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -491,6 +491,287 @@ def test_schema_warnings_shown(self): assert 'warning' in out.lower() or '!' in out +# --------------------------------------------------------------------------- +# inspect_schema — branch coverage +# --------------------------------------------------------------------------- + +# A DIC with a Set category (tests Set class colour path and non-floating loops). +_SET_AND_LOOP_DIC = """\ +#\\#CIF_2.0 +data_TEST + +save_BASE + _definition.id BASE + _definition.scope Category + _definition.class Set + _name.category_id base + _category_key.name '_base.id' +save_ + +save_base.id + _definition.id '_base.id' + _definition.class Attribute + _name.category_id base + _name.object_id id + _type.purpose Key + _type.contents Text +save_ + +save_CHILD + _definition.id CHILD + _definition.scope Category + _definition.class Loop + _name.category_id child + _category_key.name '_child.id' +save_ + +save_child.id + _definition.id '_child.id' + _definition.class Attribute + _name.category_id child + _name.object_id id + _type.purpose Key + _name.linked_item_id '_base.id' + _type.contents Text +save_ +""" + +# A DIC for transitive _resolves_to_set: A.id -> B.code (Loop) -> S.id (Set). +_TRANSITIVE_DIC = """\ +#\\#CIF_2.0 +data_TEST + +save_S + _definition.id S + _definition.scope Category + _definition.class Set + _name.category_id s + _category_key.name '_s.id' +save_ + +save_s.id + _definition.id '_s.id' + _definition.class Attribute + _name.category_id s + _name.object_id id + _type.purpose Key + _type.contents Text +save_ + +save_B + _definition.id B + _definition.scope Category + _definition.class Loop + _name.category_id b + _category_key.name '_b.code' +save_ + +save_b.code + _definition.id '_b.code' + _definition.class Attribute + _name.category_id b + _name.object_id code + _type.purpose Key + _name.linked_item_id '_s.id' + _type.contents Text +save_ + +save_A + _definition.id A + _definition.scope Category + _definition.class Loop + _name.category_id a + _category_key.name '_a.id' +save_ + +save_a.id + _definition.id '_a.id' + _definition.class Attribute + _name.category_id a + _name.object_id id + _type.purpose Key + _name.linked_item_id '_b.code' + _type.contents Text +save_ +""" + + +class TestInspectSchemaBranchCoverage: + """Branch-coverage tests written before the CC-61 refactor of inspect_schema.""" + + # --- _depr_suffix branches --- + + def test_deprecated_table_with_replacement_shown(self): + """_depr_suffix: deprecated_ids contains definition_id AND replacements exist.""" + from cifflow.dictionary.schema import SchemaSpec, TableDef, ColumnDef + col = ColumnDef(name='_cifflow_block_id', definition_id='', type_contents=None, + nullable=False, is_primary_key=True, is_synthetic=True, + linked_item_id=None) + tdef = TableDef(name='old_tbl', definition_id='OLD_TBL', + category_class='Loop', columns=[col], + primary_keys=['_cifflow_block_id']) + schema = SchemaSpec( + tables={'old_tbl': tdef}, + column_to_tag={}, + deprecated_ids={'OLD_TBL'}, + deprecated_replacements={'OLD_TBL': ['NEW_TBL']}, + ) + out = _capture(inspect_schema, schema) + assert 'DEPRECATED' in out + assert 'NEW_TBL' in out + + def test_deprecated_table_no_replacement_shown(self): + """_depr_suffix: deprecated_ids contains definition_id but replacements empty.""" + from cifflow.dictionary.schema import SchemaSpec, TableDef, ColumnDef + col = ColumnDef(name='_cifflow_block_id', definition_id='', type_contents=None, + nullable=False, is_primary_key=True, is_synthetic=True, + linked_item_id=None) + tdef = TableDef(name='old_tbl', definition_id='OLD_TBL', + category_class='Loop', columns=[col], + primary_keys=['_cifflow_block_id']) + schema = SchemaSpec( + tables={'old_tbl': tdef}, + column_to_tag={}, + deprecated_ids={'OLD_TBL'}, + deprecated_replacements={'OLD_TBL': []}, + ) + out = _capture(inspect_schema, schema) + assert 'DEPRECATED' in out + assert '->' not in out + + def test_deprecated_column_with_replacement_shown(self): + """_depr_suffix on a column definition_id with a replacement.""" + from cifflow.dictionary.schema import SchemaSpec, TableDef, ColumnDef + col = ColumnDef(name='old_col', definition_id='_t.old', + type_contents='Text', nullable=True, + is_primary_key=False, is_synthetic=False, + linked_item_id=None) + tdef = TableDef(name='t', definition_id='T', + category_class='Loop', + columns=[col], primary_keys=[]) + schema = SchemaSpec( + tables={'t': tdef}, + column_to_tag={('t', 'old_col'): '_t.old'}, + deprecated_ids={'_t.old'}, + deprecated_replacements={'_t.old': ['_t.new']}, + ) + out = _capture(inspect_schema, schema) + assert 'DEPRECATED' in out + assert '_t.new' in out + + # --- Set category class path --- + + def test_set_category_class_shown(self): + """A Set-class table displays '[Set]' in the header.""" + out = _capture(inspect_schema, _SET_AND_LOOP_DIC) + assert 'Set' in out + + def test_set_category_in_summary_count(self): + """Summary line reports correct Set/Loop split.""" + out = _capture(inspect_schema, _SET_AND_LOOP_DIC) + assert '1 Set' in out + assert '1 Loop' in out + + # --- Summary line content --- + + def test_summary_table_count(self): + """Summary line contains the correct number of tables.""" + out = _capture(inspect_schema, _SCHEMA_DIC) + assert '1 table' in out + + def test_summary_fk_count(self): + """Summary line contains FK count.""" + out = _capture(inspect_schema, _SET_AND_LOOP_DIC) + assert 'FK' in out + + # --- Column flag: UNIQUE on _cifflow_row_id --- + + def test_unique_flag_on_cifflow_row_id(self): + """_cifflow_row_id is marked UNIQUE when it is not part of the PK.""" + out = _capture(inspect_schema, _SCHEMA_DIC) + # _cifflow_row_id is NOT the PK for a keyed Loop → UNIQUE flag shown + assert 'UNIQUE' in out + + # --- show_ddl=False path --- + + def test_show_ddl_false_no_create_table(self): + """When show_ddl=False (default) no CREATE TABLE appears.""" + out = _capture(inspect_schema, _SCHEMA_DIC) + assert 'CREATE TABLE' not in out + + # --- Floating loop section --- + + def test_floating_loop_section_shown_when_no_set_link(self): + """A Loop with no Set-derived PK appears in the floating-loop section.""" + out = _capture(inspect_schema, _SCHEMA_DIC) + # widget.id has no linked_item_id → widget is floating + assert 'loop tables without Set-derived' in out + assert 'widget' in out + + def test_non_floating_loop_absent_from_floating_section(self): + """A Loop whose PK links to a Set category does NOT appear in floating section.""" + out = _capture(inspect_schema, _SET_AND_LOOP_DIC) + # child.id links to base.id (Set) → child is not floating + assert 'child' not in out.split('loop tables without Set-derived')[-1] \ + if 'loop tables without Set-derived' in out else True + + def test_no_floating_section_when_all_loops_anchored(self): + """No floating-loop section header when every Loop has a Set-derived key.""" + out = _capture(inspect_schema, _SET_AND_LOOP_DIC) + assert 'loop tables without Set-derived' not in out + + # --- _resolves_to_set transitive chain --- + + def test_transitive_loop_to_set_not_floating(self): + """A.id -> B.code (Loop) -> S.id (Set): A is not floating.""" + out = _capture(inspect_schema, _TRANSITIVE_DIC) + # a is not floating because it transitively reaches Set S + floating_section = '' + if 'loop tables without Set-derived' in out: + floating_section = out.split('loop tables without Set-derived')[1] + assert 'a' not in floating_section + + # --- Warnings section --- + + def test_no_warnings_section_when_schema_clean(self): + """When schema has no warnings, the warnings header does not appear.""" + from cifflow.dictionary.loader import DictionaryLoader + from cifflow.dictionary.schema import generate_schema + loader = DictionaryLoader() + schema = generate_schema(loader.load(_SCHEMA_DIC)) + if not schema.warnings: + out = _capture(inspect_schema, schema) + assert 'schema warnings' not in out + + def test_warnings_section_shown_when_warnings_present(self): + """When schema.warnings is non-empty, the warnings block is printed.""" + from cifflow.dictionary.schema import SchemaSpec, TableDef, ColumnDef + col = ColumnDef(name='_cifflow_block_id', definition_id='', type_contents=None, + nullable=False, is_primary_key=True, is_synthetic=True, + linked_item_id=None) + tdef = TableDef(name='t', definition_id='T', category_class='Loop', + columns=[col], primary_keys=['_cifflow_block_id']) + schema = SchemaSpec( + tables={'t': tdef}, + column_to_tag={}, + warnings=['something went wrong'], + ) + out = _capture(inspect_schema, schema) + assert 'schema warnings' in out + assert 'something went wrong' in out + + # --- Source dispatch: DdlmDictionary import fallback --- + + def test_source_as_schema_spec_bypasses_loading(self): + """Passing a SchemaSpec directly skips all loading paths.""" + from cifflow.dictionary.loader import DictionaryLoader + from cifflow.dictionary.schema import generate_schema + loader = DictionaryLoader() + schema = generate_schema(loader.load(_SCHEMA_DIC)) + out = _capture(inspect_schema, schema) + assert 'widget' in out + + # --------------------------------------------------------------------------- # inspect_ingest + TraceEvent # ---------------------------------------------------------------------------