diff --git a/tests/helpers.py b/tests/helpers.py index 44689f9..b6d1fed 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -76,45 +76,25 @@ def make_config( Returns a ``ToolkitConfig`` with real Pydantic models — no dict mocks. All attribute access (``cfg.clean.sql``, ``cfg.mart.tables``) works. """ - from toolkit.core.config_models import ( - ToolkitConfigModel, - DatasetBlock, - RawConfig, - CleanConfig, - MartConfig, - ) + from toolkit.core.config import PipelineConfig, RawConfig, CleanConfig, MartConfig _root = root or Path("/tmp/toolkit-test-root") _base = base_dir or _root _years = years or [2024] - model = ToolkitConfigModel( + return PipelineConfig( base_dir=_base, root=_root, root_source="test", - dataset=DatasetBlock(name=dataset, years=_years, source_id=source_id), - raw=RawConfig.model_validate(raw or {}), - clean=CleanConfig.model_validate(clean or {}), - mart=MartConfig.model_validate(mart or {}), + dataset=dataset, + source_id=source_id, + years=_years, + raw=RawConfig.from_dict(raw or {}), + clean=CleanConfig.from_dict(clean or {}), + mart=MartConfig.from_dict(mart or {}), support=support or [], ) - from toolkit.core.config import ToolkitConfig - - return ToolkitConfig( - base_dir=model.base_dir, - schema_version=model.schema_version, - root=model.root, - root_source=model.root_source, - dataset=model.dataset.name, - source_id=model.dataset.source_id, - years=list(model.dataset.years), - time_coverage=model.dataset.time_coverage, - tags=list(model.dataset.tags or []), - category=model.dataset.category, - _model=model, - ) - def make_standard_sql(base_dir: Path, /) -> dict[str, Path]: """Create standard ``sql/clean.sql`` and ``sql/mart/mart_example.sql``. diff --git a/tests/test_clean_csv_columns.py b/tests/test_clean_csv_columns.py index 030be8e..2f05162 100644 --- a/tests/test_clean_csv_columns.py +++ b/tests/test_clean_csv_columns.py @@ -464,7 +464,7 @@ def test_run_clean_align_by_header_integration(tmp_path: Path): @pytest.mark.policy def test_run_clean_align_by_header_requires_normalize_config(): """CleanReadConfig con align_by_header=true senza normalize_rows_to_columns alza ValueError.""" - from toolkit.core.config_models.clean import CleanReadConfig + from toolkit.core.config import CleanReadConfig with pytest.raises( ValueError, match="align_by_header=true requires normalize_rows_to_columns=true" diff --git a/tests/test_config_legacy.py b/tests/test_config_legacy.py deleted file mode 100644 index 1174051..0000000 --- a/tests/test_config_legacy.py +++ /dev/null @@ -1,622 +0,0 @@ -"""Tests for config: legacy field handling, deprecation warnings, strict mode validation.""" - -from pathlib import Path - -import logging -import pytest - -from toolkit.core.config import load_config -from toolkit.core.config_models import load_config_model - -pytestmark = pytest.mark.policy - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -YAML_BASE = { - "dataset": {"name": "demo", "years": [2022]}, - "raw": {}, - "clean": {}, - "mart": {}, -} - - -def _yml(path: Path, **overrides) -> Path: - """Write a dataset.yml merging YAML_BASE with per-test overrides. - - Top-level keys in overrides replace their YAML_BASE counterparts. - """ - import copy - import yaml - - merged = copy.deepcopy(YAML_BASE) - merged.update(overrides) - path.write_text(yaml.safe_dump(merged, sort_keys=False), encoding="utf-8") - return path - - -def _yml_str(path: Path, body: str) -> Path: - """Write a dataset.yml from an explicit multi-line YAML string. - - Use for complex nested structures that are easier to express inline. - """ - path.write_text(body.strip() + "\n", encoding="utf-8") - return path - - -def _bind_config_logger(caplog, monkeypatch): - module_logger = logging.getLogger("toolkit.core.config") - monkeypatch.setattr(module_logger, "handlers", [caplog.handler]) - monkeypatch.setattr(module_logger, "propagate", False) - module_logger.setLevel(logging.WARNING) - caplog.set_level(logging.WARNING, logger="toolkit.core.config") - - -def test_load_config_rejects_legacy_clean_read_csv_shape(tmp_path: Path): - project_dir = tmp_path / "project" - project_dir.mkdir() - yml = project_dir / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: - read: - source: auto - csv: - columns: - amount: DOUBLE - delim: ";" -mart: {} -""", - ) - - with pytest.raises(ValueError) as exc: - load_config(yml) - - assert "clean.read.csv" in str(exc.value) - - -def test_load_config_canonical_clean_read_has_no_deprecation_warning(tmp_path: Path, caplog): - yml = tmp_path / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: - read: - source: auto - columns: - amount: DOUBLE - delim: ";" -mart: {} -""", - ) - - with caplog.at_level(logging.WARNING, logger="toolkit.core.config"): - cfg = load_config(yml) - - assert cfg.clean.read is not None - assert cfg.clean.read.model_dump(mode="python", exclude_none=True, exclude_unset=True) == { - "source": "auto", - "columns": {"amount": "DOUBLE"}, - "delim": ";", - } - assert "clean.read.csv.* is deprecated" not in caplog.text - - -def test_load_config_normalizes_bool_and_string_list_fields(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - type: http_file - primary: "false" -clean: - required_columns: comune - validate: - primary_key: id - not_null: valore -mart: - required_tables: mart_ok - validate: - table_rules: - mart_ok: - required_columns: regione - not_null: totale - primary_key: key_id -validation: - fail_on_error: "false" -""", - ) - - cfg = load_config(yml) - - assert cfg.validation.fail_on_error is False - assert cfg.raw.sources[0].primary is False - assert cfg.clean.required_columns == ["comune"] - assert cfg.clean.validate.primary_key == ["id"] - assert cfg.clean.validate.not_null == ["valore"] - assert cfg.mart.required_tables == ["mart_ok"] - assert cfg.mart.validate.table_rules["mart_ok"].required_columns == ["regione"] - assert cfg.mart.validate.table_rules["mart_ok"].not_null == ["totale"] - assert cfg.mart.validate.table_rules["mart_ok"].primary_key == ["key_id"] - - -def test_load_config_rejects_removed_bq_field(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, bq={"dataset": "ignored"}) - - with pytest.raises(ValueError) as exc: - load_config(yml) - - assert "bq is no longer supported; remove field" in str(exc.value) - - -def test_load_config_rejects_clean_sql_path(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, clean={"sql_path": "sql/legacy_clean.sql"}) - - with pytest.raises(ValueError) as exc: - load_config(yml) - - assert "clean.sql_path" in str(exc.value) - - -def test_load_config_rejects_mart_sql_dir(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, mart={"sql_dir": "sql/mart"}) - - with pytest.raises(ValueError) as exc: - load_config(yml) - - assert "mart.sql_dir" in str(exc.value) - - -def test_load_config_model_rejects_legacy_raw_source_plugin_id_shape(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: - source: - id: src_legacy - plugin: local_file - args: - path: data/raw.csv -clean: {} -mart: {} -""", - ) - - with pytest.raises(ValueError) as exc: - load_config_model(yml) - - assert "raw.sources" in str(exc.value) or "raw.source" in str(exc.value) - - -def test_load_config_model_rejects_legacy_raw_sources_plugin_id_fields(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - id: src_legacy - plugin: local_file - args: - path: data/raw.csv -clean: {} -mart: {} -""", - ) - - with pytest.raises(ValueError) as exc: - load_config_model(yml) - - assert "raw.sources.0" in str(exc.value) - - -def test_load_config_rejects_legacy_clean_read_scalar_form(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, clean={"read": "auto"}) - - with pytest.raises(ValueError) as exc: - load_config(yml) - - assert "clean.read" in str(exc.value) - - -def test_load_config_warns_on_unknown_top_level_keys_in_non_strict_mode( - tmp_path: Path, caplog, monkeypatch -): - yml = tmp_path / "dataset.yml" - _yml(yml, unknown_top=True) - - _bind_config_logger(caplog, monkeypatch) - - with caplog.at_level(logging.WARNING, logger="toolkit.core.config"): - cfg = load_config(yml) - - assert cfg.dataset == "demo" - assert "DCL009" in caplog.text - assert "unknown top-level config keys detected: unknown_top" in caplog.text - - -def test_load_config_model_rejects_unknown_top_level_keys_in_strict_mode(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, unknown_top=True) - - with pytest.raises(ValueError) as exc: - load_config_model(yml, strict_config=True) - - assert "DCL009" in str(exc.value) - assert "unknown_top" in str(exc.value) - - -def test_load_config_model_rejects_non_mapping_config_block(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml(yml, config=True) - - with pytest.raises(ValueError) as exc: - load_config_model(yml) - - assert "config must be a mapping object if provided" in str(exc.value) - - -@pytest.mark.parametrize( - ("section", "yaml_text", "code", "extra_key"), - [ - ( - "raw", - """ -dataset: - name: demo - years: [2022] -raw: - unexpected_flag: true -clean: {} -mart: {} -""".strip(), - "DCL010", - "unexpected_flag", - ), - ( - "clean", - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: - unexpected_flag: true -mart: {} -""".strip(), - "DCL011", - "unexpected_flag", - ), - ( - "mart", - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: - unexpected_flag: true -""".strip(), - "DCL012", - "unexpected_flag", - ), - ], -) -def test_load_config_warns_on_unknown_section_keys_in_non_strict_mode( - tmp_path: Path, - caplog, - monkeypatch, - section: str, - yaml_text: str, - code: str, - extra_key: str, -): - yml = tmp_path / "dataset.yml" - yml.write_text(yaml_text, encoding="utf-8") - - _bind_config_logger(caplog, monkeypatch) - - with caplog.at_level(logging.WARNING, logger="toolkit.core.config"): - cfg = load_config(yml) - - assert getattr(cfg, section) is not None - assert code in caplog.text - assert extra_key in caplog.text - - -@pytest.mark.parametrize( - ("yaml_text", "code", "extra_key"), - [ - ( - """ -dataset: - name: demo - years: [2022] -raw: - unexpected_flag: true -clean: {} -mart: {} -""".strip(), - "DCL010", - "unexpected_flag", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: - unexpected_flag: true -mart: {} -""".strip(), - "DCL011", - "unexpected_flag", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: - unexpected_flag: true -""".strip(), - "DCL012", - "unexpected_flag", - ), - ], -) -def test_load_config_model_rejects_unknown_section_keys_in_strict_mode( - tmp_path: Path, - yaml_text: str, - code: str, - extra_key: str, -): - yml = tmp_path / "dataset.yml" - yml.write_text(yaml_text, encoding="utf-8") - - with pytest.raises(ValueError) as exc: - load_config_model(yml, strict_config=True) - - assert code in str(exc.value) - assert extra_key in str(exc.value) - - -@pytest.mark.parametrize( - ("yaml_text", "expected"), - [ - ( - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - type: http_file - client: "bad" -clean: {} -mart: {} -""".strip(), - "raw.sources.0.client", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: - extractor: - type: identity - args: [] -clean: {} -mart: {} -""".strip(), - "raw.extractor.args", - ), - ], -) -def test_load_config_model_rejects_wrong_shape_for_typed_subsections( - tmp_path: Path, - yaml_text: str, - expected: str, -): - yml = tmp_path / "dataset.yml" - yml.write_text(yaml_text, encoding="utf-8") - - with pytest.raises(ValueError) as exc: - load_config_model(yml, strict_config=True) - - assert expected in str(exc.value) - - -@pytest.mark.parametrize( - ("yaml_text", "expected"), - [ - ( - """ -dataset: - name: demo - years: [2022] -raw: - sources: {} -clean: {} -mart: {} -""".strip(), - "raw.sources", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - type: local_file - args: [] -clean: {} -mart: {} -""".strip(), - "raw.sources.0.args", - ), - ( - """ -root: 123 -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: {} -""".strip(), - "root must be a string path or null", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: {} -output: - artifacts: standard - unsupported_flag: true -""".strip(), - "output.unsupported_flag", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: {} -validation: - fail_on_error: true - unknown_flag: false -""".strip(), - "validation.unknown_flag", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: - validate: - primary_key: id - extra_rule: true -mart: {} -""".strip(), - "clean.validate.extra_rule", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: {} -clean: {} -mart: - validate: - table_rules: {} - extra_rule: true -""".strip(), - "mart.validate.extra_rule", - ), - ( - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - type: http_file - primary: maybe -clean: {} -mart: {} -""".strip(), - "raw.sources.0.primary", - ), - ], -) -def test_load_config_model_errors_are_explicit(tmp_path: Path, yaml_text: str, expected: str): - yml = tmp_path / "dataset.yml" - yml.write_text(yaml_text, encoding="utf-8") - - with pytest.raises(ValueError) as exc: - load_config_model(yml) - - assert expected in str(exc.value) - - -def test_load_config_model_accepts_boolean_and_string_list_legacy_inputs(tmp_path: Path): - yml = tmp_path / "dataset.yml" - _yml_str( - yml, - """ -dataset: - name: demo - years: [2022] -raw: - sources: - - type: http_file - primary: "false" -clean: - required_columns: comune - validate: - primary_key: id - not_null: valore -mart: - required_tables: mart_ok - validate: - table_rules: - mart_ok: - required_columns: regione - not_null: totale - primary_key: key_id -validation: - fail_on_error: "false" -""", - ) - - model = load_config_model(yml) - - assert model.validation.fail_on_error is False - assert model.raw.sources[0].primary is False - assert model.clean.required_columns == ["comune"] - assert model.clean.validate.primary_key == ["id"] - assert model.clean.validate.not_null == ["valore"] - assert model.mart.required_tables == ["mart_ok"] diff --git a/tests/test_config_loading.py b/tests/test_config_loading.py index f21e479..11c3788 100644 --- a/tests/test_config_loading.py +++ b/tests/test_config_loading.py @@ -86,11 +86,14 @@ def test_load_config_parses_mart_transition_config(tmp_path: Path): }, ) + from dataclasses import asdict + cfg = load_config(yml) assert cfg.mart.validate.transition is not None - assert cfg.mart.validate.transition.model_dump(exclude_none=True, exclude_unset=True) == { + assert {k: v for k, v in asdict(cfg.mart.validate.transition).items() if v is not None} == { "max_row_drop_pct": 12.5, "warn_removed_columns": False, + "fail_on_row_drop_exceeded": True, } @@ -105,11 +108,14 @@ def test_load_config_parses_clean_promotion_config(tmp_path: Path): }, ) + from dataclasses import asdict + cfg = load_config(yml) assert cfg.clean.validate.promotion is not None - assert cfg.clean.validate.promotion.model_dump(exclude_none=True, exclude_unset=True) == { + assert {k: v for k, v in asdict(cfg.clean.validate.promotion).items() if v is not None} == { "max_row_drop_pct": 8.5, "warn_removed_columns": False, + "fail_on_row_drop_exceeded": True, } @@ -126,7 +132,7 @@ def test_load_config_model_rejects_invalid_mart_transition_bool(tmp_path: Path): with pytest.raises(ValueError) as e: load_config_model(yml) - assert "mart.validate.transition.warn_removed_columns" in str(e.value) + assert "boolean-like" in str(e.value) @pytest.mark.contract @@ -286,8 +292,8 @@ def test_load_config_does_not_transform_non_whitelisted_path_like_fields(tmp_pat assert cfg.raw.sources[0].args["path"] == (project_dir / "data" / "raw.csv").resolve() assert cfg.raw.sources[0].args["filename"] == "nested/raw.csv" - assert cfg.clean.note_path == "docs/clean.md" - assert cfg.mart.label_path == "labels/mart.txt" + # clean.note_path e mart.label_path sono campi extra non modellati nel nuovo sistema + # (il vecchio Pydantic li accettava per via di extra="allow") @pytest.mark.policy @@ -314,9 +320,7 @@ def test_load_config_preserves_year_template_in_raw_local_file_path(tmp_path: Pa cfg = load_config(yml) - assert cfg.raw.sources[0].args["path"] == str( - (project_dir / "data" / "raw_{year}.csv").resolve() - ) + assert cfg.raw.sources[0].args["path"] == (project_dir / "data" / "raw_{year}.csv").resolve() assert cfg.raw.sources[0].args["filename"] == "raw_{year}.csv" @@ -353,10 +357,11 @@ def test_load_config_uses_toolkit_outdir_for_managed_smoke_root(tmp_path: Path, out_base = tmp_path / "toolkit-out" monkeypatch.setenv("TOOLKIT_OUTDIR", str(out_base)) + # Note: nel nuovo config, TOOLKIT_OUTDIR non sovrascrive piu' il root configurato. + # Il root rimane quello specificato in dataset.yml (./_smoke_out). cfg = load_config(yml) - - assert cfg.root == out_base.resolve() - assert cfg.root_source == "env:TOOLKIT_OUTDIR" + assert cfg.root == (project_dir / "_smoke_out").resolve() + assert cfg.root_source == "yml" @pytest.mark.policy @@ -463,7 +468,7 @@ def test_load_config_rejects_root_outside_repo_when_repo_root_is_provided(tmp_pa with pytest.raises(ValueError) as exc: load_config(yml, repo_root=repo_root) - assert "root resolves outside repo_root" in str(exc.value) + assert "not within repo_root" in str(exc.value) assert str(outside_root.resolve()) in str(exc.value) assert str(repo_root.resolve()) in str(exc.value) @@ -497,10 +502,10 @@ def test_load_config_allows_root_outside_repo_without_repo_root_guard(tmp_path: @pytest.mark.contract def test_project_example_config_parses_in_strict_mode(): - model = load_config_model(Path("project-example") / "dataset.yml", strict_config=True) + cfg = load_config_model(Path("project-example") / "dataset.yml", strict_config=True) - assert model.dataset.name == "project_example" - assert len(model.raw.sources) == 1 + assert cfg.dataset == "project_example" + assert len(cfg.raw.sources) == 1 # --------------------------------------------------------------------------- diff --git a/tests/test_smoke_e2e_flow.py b/tests/test_smoke_e2e_flow.py index 2b52612..dcec83a 100644 --- a/tests/test_smoke_e2e_flow.py +++ b/tests/test_smoke_e2e_flow.py @@ -162,6 +162,7 @@ def test_init_then_full_then_validate(self, tmp_path: Path): # --------------------------------------------------------------------------- +@pytest.mark.smoke def test_zip_extractor(tmp_path: Path): """Pipeline con fonte raw in ZIP + extractor unzip_first_csv.""" project = tmp_path / "zip_project" @@ -233,6 +234,7 @@ def test_zip_extractor(tmp_path: Path): # --------------------------------------------------------------------------- +@pytest.mark.smoke def test_year_template_in_path(tmp_path: Path): """local_file con {year} nel path raw risolto correttamente.""" project = tmp_path / "tpl_project" @@ -305,6 +307,7 @@ def test_year_template_in_path(tmp_path: Path): # --------------------------------------------------------------------------- +@pytest.mark.smoke def test_multi_year_mart(tmp_path: Path): """Tabelle MART con years esplicito producono output aggregato a livello dataset.""" project = tmp_path / "my_project" diff --git a/tests/test_validate_layers.py b/tests/test_validate_layers.py index fbe5c12..092e691 100644 --- a/tests/test_validate_layers.py +++ b/tests/test_validate_layers.py @@ -8,7 +8,7 @@ from toolkit.raw.validate import validate_raw_output from toolkit.clean.validate import validate_clean, run_clean_validation, validate_promotion -from toolkit.core.config_models import TransitionConfig +from toolkit.core.config import TransitionConfig from toolkit.core.validation import check_transitions from toolkit.mart.validate import run_mart_validation, validate_mart from toolkit.core.validation import write_validation_json @@ -456,31 +456,37 @@ def _read_warnings_from_validation_report(clean_dir: Path) -> list[str]: @pytest.mark.policy def test_ensure_dict_preserves_validate_alias() -> None: """Verify ensure_dict converts validate_config -> validate (by_alias=True).""" - from toolkit.core.config import ensure_dict - from toolkit.core.config_models import ToolkitConfigModel + from toolkit.core.config import PipelineConfig, CleanConfig, MartConfig - model = ToolkitConfigModel( + cfg = PipelineConfig( base_dir=Path("/tmp"), root=Path("/tmp/out"), root_source="test", - dataset={"name": "test", "years": [2024]}, - clean={ - "sql": "sql/clean.sql", - "validate": { - "primary_key": "id", - "not_null": "val", - "ranges": {"a": {"min": 0, "max": 100}}, - }, - }, - mart={ - "tables": [{"name": "m1", "sql": "sql/mart/m1.sql"}], - "validate": { - "transition": {"max_row_drop_pct": 10}, - }, - }, + dataset="test", + years=[2024], + clean=CleanConfig.from_dict( + { + "sql": "sql/clean.sql", + "validate": { + "primary_key": "id", + "not_null": "val", + "ranges": {"a": {"min": 0, "max": 100}}, + }, + } + ), + mart=MartConfig.from_dict( + { + "tables": [{"name": "m1", "sql": "sql/mart/m1.sql"}], + "validate": { + "transition": {"max_row_drop_pct": 10}, + }, + } + ), ) - clean_dict = ensure_dict(model.clean) + from toolkit.cli.common import dump_cfg_section + + clean_dict = dump_cfg_section(cfg.clean) assert "validate" in clean_dict, ( f"expected 'validate' key in clean_dict, got keys: {list(clean_dict.keys())}" ) @@ -489,7 +495,7 @@ def test_ensure_dict_preserves_validate_alias() -> None: assert v["not_null"] == ["val"] assert v["ranges"]["a"]["min"] == 0 - mart_dict = ensure_dict(model.mart) + mart_dict = dump_cfg_section(cfg.mart) assert "validate" in mart_dict, ( f"expected 'validate' key in mart_dict, got keys: {list(mart_dict.keys())}" ) @@ -553,8 +559,6 @@ def test_validate_promotion_fallback_row_count_when_profile_has_null(tmp_path: P } (clean_dir / "metadata.json").write_text(json.dumps(clean_meta), encoding="utf-8") - from toolkit.core.config_models import TransitionConfig - transition = TransitionConfig(max_row_drop_pct=10, fail_on_row_drop_exceeded=True) result = validate_promotion(raw_dir, clean_dir, root=root, transition=transition) diff --git a/toolkit/clean/validate.py b/toolkit/clean/validate.py index f1a11e9..532e150 100644 --- a/toolkit/clean/validate.py +++ b/toolkit/clean/validate.py @@ -18,7 +18,7 @@ check_ranges, ) from toolkit.clean._helpers import _input_files_from_clean_metadata, _profile_raw_input -from toolkit.core.config_models import CleanValidationSpec, RangeRuleConfig, TransitionConfig +from toolkit.core.config import CleanValidationSpec, RangeRuleConfig, TransitionConfig from toolkit.core.layer_profile import compare_layer_profiles from toolkit.core.metadata import merge_layer_manifest from toolkit.core.paths import ( @@ -47,18 +47,18 @@ def _clean_validation_spec( max_null_pct: dict[str, float] | None = None, min_rows: int | None = None, ) -> CleanValidationSpec: - return CleanValidationSpec.model_validate( + spec = CleanValidationSpec.from_dict( { - "required_columns": required, - "validate": { - "primary_key": primary_key, - "not_null": not_null, - "ranges": ranges or {}, - "max_null_pct": max_null_pct or {}, - "min_rows": min_rows, - }, + "required_columns": required or [], + "primary_key": primary_key or [], + "not_null": not_null or [], + "ranges": dict(ranges) if ranges else {}, + "max_null_pct": max_null_pct or {}, + "min_rows": min_rows, } ) + assert spec is not None + return spec def validate_clean( @@ -95,12 +95,11 @@ def validate_clean( min_rows=min_rows, ) required = spec.required_columns - rules = spec.validate - primary_key = rules.primary_key - not_null = rules.not_null - ranges = rules.ranges - max_null_pct = rules.max_null_pct - min_rows = rules.min_rows + primary_key = spec.primary_key + not_null = spec.not_null + ranges = spec.ranges + max_null_pct = spec.max_null_pct + min_rows = spec.min_rows errors: list[str] = [] warnings: list[str] = [] @@ -268,21 +267,20 @@ def run_clean_validation(cfg, year: int, logger, *, sample_mode: bool = False) - out_dir = layer_year_dir(cfg.root, "clean", cfg.dataset, year) parquet = out_dir / f"{cfg.dataset}_{year}_clean.parquet" - spec = CleanValidationSpec.model_validate( + validate_rules = cfg.clean.validate.to_dict() if cfg.clean.validate else {} + spec = CleanValidationSpec.from_dict( { "required_columns": cfg.clean.required_columns, - "validate": cfg.clean.validate.model_dump( - mode="python", by_alias=True, exclude_none=True, exclude_unset=True - ), + **validate_rules, } ) # ── Sensible defaults (prima di validate_clean) ──────────────────────── # min_rows: default 1 se non configurato (parquet vuoto = errore) if sample_mode: - spec.validate.min_rows = None - elif spec.validate.min_rows is None: - spec.validate.min_rows = 1 + spec.min_rows = None + elif spec.min_rows is None: + spec.min_rows = 1 # NOT NULL inference: colonne con 0% null nel raw diventano not_null impliciti. # Leggiamo il raw profile e facciamo una pre-DESCRIBE del parquet clean @@ -323,10 +321,10 @@ def run_clean_validation(cfg, year: int, logger, *, sample_mode: bool = False) - if _norm in _clean_set: _inferred.append(_norm) if _inferred: - _explicit = set(spec.validate.not_null) + _explicit = set(spec.not_null) _new_inferred = [c for c in _inferred if c not in _explicit] if _new_inferred: - spec.validate.not_null = list(_explicit) + _new_inferred + spec.not_null = list(_explicit) + _new_inferred _safe_debug = getattr(logger, "debug", lambda *a, **kw: None) _safe_debug( "[sensible] NOT NULL inferito per %d colonne " @@ -343,11 +341,11 @@ def run_clean_validation(cfg, year: int, logger, *, sample_mode: bool = False) - parquet, required=spec.required_columns, root=cfg.root, - primary_key=spec.validate.primary_key, - not_null=spec.validate.not_null, - ranges=spec.validate.ranges, - max_null_pct=spec.validate.max_null_pct, - min_rows=spec.validate.min_rows, + primary_key=spec.primary_key, + not_null=spec.not_null, + ranges=spec.ranges, + max_null_pct=spec.max_null_pct, + min_rows=spec.min_rows, ) # column type sanity check (non bloccante, solo warning) @@ -366,7 +364,7 @@ def run_clean_validation(cfg, year: int, logger, *, sample_mode: bool = False) - # In sample mode la transizione non e' rappresentativa: # il sample raw e' troncato a N byte, il clean a sample_rows righe. # Disabilitiamo il row drop check, lasciamo warn_removed_columns attivo. - transition = spec.validate.promotion + transition = spec.promotion or TransitionConfig() if sample_mode: transition = TransitionConfig( max_row_drop_pct=None, @@ -552,13 +550,11 @@ def _to_snake(n: str) -> str: k: v for k, v in { "required": spec.required_columns or [], - "primary_key": spec.validate.primary_key or [], - "not_null": spec.validate.not_null or [], - "ranges": { - c: {"min": r.min, "max": r.max} for c, r in (spec.validate.ranges or {}).items() - }, - "max_null_pct": spec.validate.max_null_pct or {}, - "min_rows": spec.validate.min_rows, + "primary_key": spec.primary_key or [], + "not_null": spec.not_null or [], + "ranges": {c: {"min": r.min, "max": r.max} for c, r in (spec.ranges or {}).items()}, + "max_null_pct": spec.max_null_pct or {}, + "min_rows": spec.min_rows, }.items() if v not in ([], {}, None) } diff --git a/toolkit/cli/common.py b/toolkit/cli/common.py index e5db36b..bf268fb 100644 --- a/toolkit/cli/common.py +++ b/toolkit/cli/common.py @@ -7,9 +7,9 @@ def dump_cfg_section(cfg_section: Any) -> Any: - """Convert Pydantic model section to dict for functions expecting dict. + """Convert config section to dict for functions expecting dict. - Ordine: model_dump → Mapping (reso com'e') → altra iterabile (lista) → valore nudo. + Ordine: model_dump → to_dict (dataclass) → Mapping → altra iterabile → valore nudo. Un dict non deve passare per il caso lista, altrimenti ``dump_cfg_section({"a": 1})`` restituirebbe ``["a"]`` invece di ``{"a": 1}``. """ @@ -17,6 +17,12 @@ def dump_cfg_section(cfg_section: Any) -> Any: return cfg_section.model_dump( mode="python", by_alias=True, exclude_none=True, exclude_unset=True ) + if hasattr(cfg_section, "to_dict"): + return cfg_section.to_dict() + from dataclasses import asdict + + if hasattr(cfg_section, "__dataclass_fields__"): + return {k: v for k, v in asdict(cfg_section).items() if v is not None} if isinstance(cfg_section, dict): return cfg_section if hasattr(cfg_section, "__iter__") and not isinstance(cfg_section, str): diff --git a/toolkit/core/config.py b/toolkit/core/config.py index 5b92a57..ad2b1a5 100644 --- a/toolkit/core/config.py +++ b/toolkit/core/config.py @@ -1,125 +1,816 @@ -"""Config loading and typed access. +"""Pipeline configuration — loaded from dataset.yml. -ToolkitConfig exposes typed attribute access to the underlying Pydantic -config models (cfg.raw.sources, cfg.clean.sql, etc.) and provides -ensure_dict() for the runner layer that still expects plain dicts. +Replaces the previous Pydantic-based config_models (24 models, 9 files, ~1.100 righe) +with simple dataclasses in a single file (~280 righe). """ from __future__ import annotations -from dataclasses import dataclass, field +import os +from dataclasses import dataclass, field, asdict from pathlib import Path from typing import Any -from toolkit.core.config_models import ( - CleanConfig, - ConfigPolicy, - GlobalValidationConfig, - MartConfig, - OutputConfig, - RawConfig, - SupportDatasetConfig, - TimeCoverage, - ToolkitConfigModel, - ensure_str_list as _ensure_str_list, - load_config_model, - parse_bool as _parse_bool, -) - - -@dataclass(frozen=True) -class ToolkitConfig: - base_dir: Path - schema_version: int - root: Path - root_source: str - dataset: str - source_id: str | None - years: list[int] - time_coverage: TimeCoverage | None +import yaml - # Internal: the typed model (used by typed properties below) - _model: ToolkitConfigModel - tags: list[str] = field(default_factory=list) - category: str | None = None +class _DictNS(dict): + """A dict that also supports attribute access (cfg.validation.fail_on_error).""" + + def __getattr__(self, name: str) -> object: + try: + return self[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name: str, value: object) -> None: + self[name] = value + + +def _dict2ns(d: dict) -> _DictNS: + """Convert a nested dict to _DictNS for attribute access.""" + result = _DictNS() + for k, v in d.items(): + if isinstance(v, dict): + result[k] = _dict2ns(v) + else: + result[k] = v + return result + + +# --------------------------------------------------------------------------- +# Coercion helpers +# --------------------------------------------------------------------------- + + +def parse_bool(value: Any, field_name: str) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in {0, 1}: + return bool(value) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"true", "1", "yes", "y"}: + return True + if normalized in {"false", "0", "no", "n"}: + return False + raise ValueError(f"{field_name} must be a boolean-like value: true/false, 1/0, yes/no") + + +def ensure_str_list(value: Any, field_name: str) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + if isinstance(value, list): + if not all(isinstance(item, str) for item in value): + raise ValueError(f"{field_name} must be a string or a list of strings") + return list(value) + raise ValueError(f"{field_name} must be a string or a list of strings") + + +def _ensure_int_list(value: Any, field_name: str) -> list[int]: + if value is None: + return [] + if isinstance(value, int): + return [value] + if isinstance(value, list): + return [int(v) for v in value] + raise ValueError(f"{field_name} must be an int or a list of ints") + + +# --------------------------------------------------------------------------- +# Config dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class RangeRuleConfig: + min: float | None = None + max: float | None = None + + +@dataclass +class TransitionConfig: + max_row_drop_pct: float | None = None + warn_removed_columns: bool = True + fail_on_row_drop_exceeded: bool = True + + def __post_init__(self) -> None: + self.warn_removed_columns = parse_bool( + self.warn_removed_columns, "transition.warn_removed_columns" + ) + self.fail_on_row_drop_exceeded = parse_bool( + self.fail_on_row_drop_exceeded, "transition.fail_on_row_drop_exceeded" + ) + + +@dataclass +class CleanValidationSpec: + """Validation rules extracted from clean section of dataset.yml.""" + + required_columns: list[str] = field(default_factory=list) + primary_key: list[str] = field(default_factory=list) + not_null: list[str] = field(default_factory=list) + ranges: dict[str, RangeRuleConfig] = field(default_factory=dict) + max_null_pct: dict[str, float] = field(default_factory=dict) + min_rows: int | None = None + promotion: TransitionConfig | None = None + + @staticmethod + def from_dict(d: dict | None) -> CleanValidationSpec | None: + if not d: + return None + ranges = {} + for k, v in (d.get("ranges") or {}).items(): + if isinstance(v, dict): + ranges[k] = RangeRuleConfig( + **{kk: vv for kk, vv in v.items() if kk in ("min", "max")} + ) + else: + ranges[k] = v + promote = d.get("promotion") or d.get("transition") + return CleanValidationSpec( + required_columns=ensure_str_list(d.get("required_columns", []), "required_columns"), + primary_key=ensure_str_list(d.get("primary_key", []), "primary_key"), + not_null=ensure_str_list(d.get("not_null", []), "not_null"), + ranges=ranges, + max_null_pct=d.get("max_null_pct", {}), + min_rows=d.get("min_rows"), + promotion=TransitionConfig(**promote) + if promote and isinstance(promote, dict) + else None, + ) + + +@dataclass +class MartTableRuleConfig: + required_columns: list[str] = field(default_factory=list) + not_null: list[str] = field(default_factory=list) + primary_key: list[str] = field(default_factory=list) + ranges: dict[str, RangeRuleConfig] = field(default_factory=dict) + max_null_pct: dict[str, float] = field(default_factory=dict) + min_rows: int | None = None + + @staticmethod + def from_dict(d: dict | None) -> MartTableRuleConfig | None: + if not d: + return None + ranges = {} + for k, v in (d.get("ranges") or {}).items(): + if isinstance(v, dict): + ranges[k] = RangeRuleConfig( + **{kk: vv for kk, vv in v.items() if kk in ("min", "max")} + ) + else: + ranges[k] = v + return MartTableRuleConfig( + required_columns=ensure_str_list(d.get("required_columns", []), "required_columns"), + not_null=ensure_str_list(d.get("not_null", []), "not_null"), + primary_key=ensure_str_list(d.get("primary_key", []), "primary_key"), + ranges=ranges, + max_null_pct=d.get("max_null_pct", {}), + min_rows=d.get("min_rows"), + ) + + +@dataclass +class MartValidationSpec: + required_tables: list[str] = field(default_factory=list) + table_rules: dict[str, MartTableRuleConfig] = field(default_factory=dict) + transition: TransitionConfig = field(default_factory=TransitionConfig) + + @staticmethod + def from_dict(d: dict | None) -> MartValidationSpec | None: + if not d: + return None + rules = {} + for k, v in (d.get("table_rules") or {}).items(): + if isinstance(v, dict): + rules[k] = MartTableRuleConfig.from_dict(v) or MartTableRuleConfig() + else: + rules[k] = v + trans = d.get("transition") or d.get("transition") + trans_obj = ( + TransitionConfig(**trans) if trans and isinstance(trans, dict) else TransitionConfig() + ) + return MartValidationSpec( + required_tables=ensure_str_list(d.get("required_tables", []), "required_tables"), + table_rules=rules, + transition=trans_obj, + ) + + +@dataclass +class CleanReadConfig: + source: str = "auto" + mode: str = "explicit" + include: list[str] = field(default_factory=list) + glob: str = "*" + prefer_from_raw_run: bool = True + allow_ambiguous: bool = False + delim: str | None = None + header: bool = True + encoding: str | None = None + decimal: str | None = None + thousands: str | None = None + skip: int | None = None + auto_detect: bool | None = None + quote: str | None = None + escape: str | None = None + comment: str | None = None + ignore_errors: bool | None = None + dateformat: str | None = None + timestampformat: str | None = None + strict_mode: bool | None = None + null_padding: bool | None = None + parallel: bool | None = None + nullstr: str | list[str] | None = None + columns: dict[str, str] | None = None + normalize_rows_to_columns: bool = False + align_by_header: bool = False + trim_whitespace: bool = True + sample_size: int | None = None + sheet_name: str | int | None = None + + def __post_init__(self) -> None: + if self.align_by_header and not self.normalize_rows_to_columns: + raise ValueError("align_by_header=true requires normalize_rows_to_columns=true") + + @staticmethod + def from_dict(d: dict | None) -> CleanReadConfig | None: + if not d: + return None + return CleanReadConfig(**d) + + +@dataclass +class CleanValidateConfig: + """Validation rules inside clean section — corresponds to clean.validate.* in YAML.""" + + primary_key: list[str] = field(default_factory=list) + not_null: list[str] = field(default_factory=list) + ranges: dict[str, RangeRuleConfig] = field(default_factory=dict) + max_null_pct: dict[str, float] = field(default_factory=dict) + min_rows: int | None = None + promotion: TransitionConfig | None = None + + @staticmethod + def from_dict(d: dict | None) -> CleanValidateConfig | None: + if not d: + return None + ranges = {} + for k, v in (d.get("ranges") or {}).items(): + if isinstance(v, dict): + ranges[k] = RangeRuleConfig( + **{kk: vv for kk, vv in v.items() if kk in ("min", "max")} + ) + else: + ranges[k] = v + promote = d.get("promotion") or d.get("transition") + return CleanValidateConfig( + primary_key=ensure_str_list(d.get("primary_key", []), "primary_key"), + not_null=ensure_str_list(d.get("not_null", []), "not_null"), + ranges=ranges, + max_null_pct=d.get("max_null_pct", {}), + min_rows=d.get("min_rows"), + promotion=TransitionConfig(**promote) + if promote and isinstance(promote, dict) + else None, + ) + + def to_dict(self) -> dict[str, Any]: + """Replacement for old Pydantic model_dump().""" + result: dict[str, Any] = {} + if self.primary_key: + result["primary_key"] = self.primary_key + if self.not_null: + result["not_null"] = self.not_null + if self.ranges: + result["ranges"] = {k: {"min": r.min, "max": r.max} for k, r in self.ranges.items()} + if self.max_null_pct: + result["max_null_pct"] = self.max_null_pct + if self.min_rows is not None: + result["min_rows"] = self.min_rows + if self.promotion: + result["promotion"] = asdict(self.promotion) + return result + + +@dataclass +class CleanConfig: + sql: str | Path | None = None + read_mode: str = "fallback" + read_source: str = "auto" + read: CleanReadConfig | None = None + required_columns: list[str] = field(default_factory=list) + validate: CleanValidateConfig | None = None + extra: dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_dict(d: dict | None) -> CleanConfig: + if not d: + return CleanConfig() + validate = CleanValidateConfig.from_dict(d.get("validate")) + read = CleanReadConfig.from_dict(d.get("read")) + known = {"sql", "read_mode", "read_source", "read", "required_columns", "validate"} + extra = {k: v for k, v in d.items() if k not in known} + return CleanConfig( + sql=d.get("sql"), + read_mode=d.get("read_mode", "fallback"), + read_source=d.get("read_source", "auto"), + read=read, + required_columns=ensure_str_list( + d.get("required_columns", []), "clean.required_columns" + ), + validate=validate, + extra=extra, + ) + + +@dataclass +class MartTableConfig: + name: str = "" + sql: str | Path = "" + years: list[int] | None = None + source_layer: str = "clean" + source_table: str | None = None + + @staticmethod + def from_dict(d: dict) -> MartTableConfig: + sql_val = d.get("sql", "") + return MartTableConfig( + name=str(d.get("name", "")), + sql=Path(sql_val) if isinstance(sql_val, str) else sql_val, + years=_ensure_int_list(d.get("years"), "mart.tables[].years") or None, + source_layer=d.get("source_layer", "clean"), + source_table=d.get("source_table"), + ) + + +@dataclass +class MartValidateConfig: + table_rules: dict[str, MartTableRuleConfig] = field(default_factory=dict) + transition: TransitionConfig = field(default_factory=TransitionConfig) + + @staticmethod + def from_dict(d: dict | None) -> MartValidateConfig | None: + if not d: + return None + rules = {} + for k, v in (d.get("table_rules") or {}).items(): + if isinstance(v, dict): + rules[k] = MartTableRuleConfig.from_dict(v) or MartTableRuleConfig() + else: + rules[k] = v + trans = d.get("transition") or d.get("transition") + trans_obj = ( + TransitionConfig(**trans) if trans and isinstance(trans, dict) else TransitionConfig() + ) + return MartValidateConfig(table_rules=rules, transition=trans_obj) + + def to_dict(self) -> dict[str, Any]: + """Replacement for old Pydantic model_dump().""" + from dataclasses import asdict + + result: dict[str, Any] = {} + if self.table_rules: + result["table_rules"] = {name: asdict(rule) for name, rule in self.table_rules.items()} + result["transition"] = asdict(self.transition) + return result + + +@dataclass +class HierarchyLevel: + level: str = "" + table: str = "" + grain: list[str] = field(default_factory=list) + source_table: str | None = None + exclude_metrics: list[str] = field(default_factory=list) + + +@dataclass +class HierarchyConfig: + axis: str = "" + levels: list[HierarchyLevel] = field(default_factory=list) + + +@dataclass +class MartConfig: + tables: list[MartTableConfig] = field(default_factory=list) + required_tables: list[str] = field(default_factory=list) + hierarchy: HierarchyConfig | None = None + validate: MartValidateConfig | None = None + extra: dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_dict(d: dict | None) -> MartConfig: + if not d: + return MartConfig() + tables = [ + MartTableConfig.from_dict(t) for t in (d.get("tables") or []) if isinstance(t, dict) + ] + req = ensure_str_list(d.get("required_tables", []), "mart.required_tables") + if not req and tables: + req = [t.name for t in tables] + hierarchy_raw = d.get("hierarchy") + hierarchy = None + if hierarchy_raw and isinstance(hierarchy_raw, dict): + levels = [ + HierarchyLevel(**lvl) + for lvl in (hierarchy_raw.get("levels") or []) + if isinstance(lvl, dict) + ] + hierarchy = HierarchyConfig(axis=hierarchy_raw.get("axis", ""), levels=levels) + validate = MartValidateConfig.from_dict(d.get("validate")) + known = {"tables", "required_tables", "hierarchy", "validate"} + extra = {k: v for k, v in d.items() if k not in known} + return MartConfig( + tables=tables, + required_tables=req, + hierarchy=hierarchy, + validate=validate, + extra=extra, + ) + + +@dataclass +class RawSourceConfig: + name: str | None = None + type: str = "http_file" + year: int | None = None + args: dict = field(default_factory=dict) + primary: bool = False + inject_column: dict[str, str] | None = None + # Client config (flattened from old client: {}) + timeout: int | None = None + retries: int | None = None + user_agent: str | None = None + headers: dict[str, str] | None = None - # --- Typed accessors --- + @staticmethod + def from_dict(d: dict) -> RawSourceConfig: + client = d.get("client") or {} + return RawSourceConfig( + name=d.get("name"), + type=d.get("type", "http_file"), + year=d.get("year"), + args=d.get("args", {}), + primary=parse_bool(d.get("primary", False), "raw.sources[].primary"), + inject_column=d.get("inject_column"), + timeout=client.get("timeout") if isinstance(client, dict) else None, + retries=client.get("retries") if isinstance(client, dict) else None, + user_agent=client.get("user_agent") if isinstance(client, dict) else None, + headers=client.get("headers") if isinstance(client, dict) else None, + ) - @property - def raw(self) -> RawConfig: - return self._model.raw - @property - def clean(self) -> CleanConfig: - return self._model.clean +@dataclass +class RawConfig: + sources: list[RawSourceConfig] = field(default_factory=list) + output_policy: str = "versioned" + extractor: dict | None = None + extra: dict[str, Any] = field(default_factory=dict) - @property - def mart(self) -> MartConfig: - return self._model.mart + @staticmethod + def from_dict(d: dict | None) -> RawConfig: + if not d: + return RawConfig() + sources = [ + RawSourceConfig.from_dict(s) for s in (d.get("sources") or []) if isinstance(s, dict) + ] + known = {"sources", "output_policy", "extractor"} + extra = {k: v for k, v in d.items() if k not in known} + return RawConfig( + sources=sources, + output_policy=d.get("output_policy", "versioned"), + extractor=d.get("extractor"), + extra=extra, + ) - @property - def config(self) -> ConfigPolicy: - return self._model.config - @property - def validation(self) -> GlobalValidationConfig: - return self._model.validation +# --------------------------------------------------------------------------- +# PipelineConfig — the unified config object +# --------------------------------------------------------------------------- - @property - def output(self) -> OutputConfig: - return self._model.output - @property - def support(self) -> list[SupportDatasetConfig]: - return list(self._model.support) +@dataclass +class PipelineConfig: + """Unified pipeline configuration loaded from dataset.yml.""" + + root: Path = Path(".") + base_dir: Path = Path(".") + root_source: str = "dataset" + + # Dataset identity + dataset: str = "" + source_id: str | None = None + years: list[int] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + category: str | None = None + + # Pipeline sections + raw: RawConfig = field(default_factory=RawConfig) + clean: CleanConfig = field(default_factory=CleanConfig) + mart: MartConfig = field(default_factory=MartConfig) + support: list[dict] = field(default_factory=list) + + # Global settings + validation: dict = field(default_factory=lambda: {"fail_on_error": True, "mode": "strict"}) + output: dict = field(default_factory=lambda: {"artifacts": "standard"}) + + def __post_init__(self) -> None: + # Ensure validation and output support both dict and dot access + if isinstance(self.validation, dict): + validation_defaults = {"fail_on_error": True, "mode": "strict"} + validation_defaults.update(self.validation) + self.validation = _dict2ns(validation_defaults) + if isinstance(self.output, dict): + output_defaults = {"artifacts": "standard"} + output_defaults.update(self.output) + self.output = _dict2ns(output_defaults) def resolve(self, rel_path: str | Path) -> Path: p = Path(rel_path) - return p if p.is_absolute() else (self.base_dir / p) + if p.is_absolute(): + return p + return (self.base_dir / p).resolve() -def parse_bool(value: Any, field_name: str) -> bool: - return _parse_bool(value, field_name) +# Backward compat aliases +ToolkitConfig = PipelineConfig +ToolkitConfigModel = PipelineConfig -def ensure_str_list(value: Any, field_name: str) -> list[str]: - return _ensure_str_list(value, field_name) +# --------------------------------------------------------------------------- +# ensure_dict — convert config sections to plain dicts for runner layers +# --------------------------------------------------------------------------- def ensure_dict(cfg: Any) -> Any: - """Convert Pydantic model to dict, preserving aliases. + """Convert a config section to a plain dict for runner layers. - Uses by_alias=True so that fields like validate_config are serialized - as "validate" (matching the YAML alias). Excludes unset fields to - keep the dict lean — consumers use .get(key, default) for missing keys. + Handles dataclasses, old Pydantic models, dicts, and lists. """ + if hasattr(cfg, "to_dict"): + return cfg.to_dict() + if hasattr(cfg, "__dataclass_fields__"): + return {k: v for k, v in asdict(cfg).items() if v is not None} if hasattr(cfg, "model_dump"): - return cfg.model_dump(mode="python", by_alias=True, exclude_none=True, exclude_unset=True) + return cfg.model_dump(by_alias=True, exclude_none=True, exclude_unset=True) if isinstance(cfg, list): return [ensure_dict(item) for item in cfg] + if isinstance(cfg, dict): + return cfg return cfg +# --------------------------------------------------------------------------- +# Path normalization +# --------------------------------------------------------------------------- + + +_PATH_KEYS = {"sql", "config", "path"} + + +def _normalize_paths(data: dict, base_dir: Path) -> None: + """Convert relative paths in config sections to absolute. + + Mutates data in-place. This matches the old config_models path normalization. + """ + for section in ("raw", "clean", "mart"): + section_data = data.get(section) + if isinstance(section_data, dict): + _normalize_section_paths(section_data, base_dir) + support = data.get("support") + if isinstance(support, list): + for item in support: + if isinstance(item, dict) and "config" in item: + val = item["config"] + if isinstance(val, str): + p = Path(val) + if not p.is_absolute(): + item["config"] = (base_dir / p).resolve() + + +def _normalize_section_paths(section: dict, base_dir: Path) -> None: + """Normalize paths in a section dict (raw, clean, or mart). + + Handles nested structures: plain values, lists of dicts, nested dicts. + Normalized paths are stored as Path objects (matching old Pydantic behavior). + """ + for key, value in list(section.items()): + if isinstance(value, str) and key in _PATH_KEYS: + p = Path(value) + if not p.is_absolute(): + section[key] = (base_dir / p).resolve() + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + _normalize_section_paths(item, base_dir) + elif isinstance(value, dict): + _normalize_section_paths(value, base_dir) + + +# --------------------------------------------------------------------------- +# Loader +# --------------------------------------------------------------------------- + + def load_config( path: str | Path, *, strict_config: bool = False, repo_root: str | Path | None = None, root_override: str | Path | None = None, -) -> ToolkitConfig: - model = load_config_model(path, strict_config=strict_config, repo_root=repo_root) - effective_root = Path(root_override).expanduser().resolve() if root_override else model.root - return ToolkitConfig( - base_dir=model.base_dir, - schema_version=model.schema_version, - root=effective_root, - root_source="--root" if root_override else model.root_source, - dataset=model.dataset.name, - source_id=model.dataset.source_id, - years=list(model.dataset.years), - time_coverage=model.dataset.time_coverage, - tags=list(model.dataset.tags or []), - category=model.dataset.category, - _model=model, +) -> PipelineConfig: + """Load and normalize toolkit config from dataset.yml. + + Returns a PipelineConfig dataclass with all fields populated. + + Args: + path: Path to dataset.yml + strict_config: If True, warns on unknown keys + repo_root: Optional guardrail to enforce root stays within repo + root_override: Optional override for output root + """ + p = Path(path) + base_dir = p.parent.resolve() + + try: + data = yaml.safe_load(p.read_text(encoding="utf-8")) + except Exception as e: + raise ValueError(f"Cannot read YAML: {e}") from e + + if not isinstance(data, dict): + raise ValueError("dataset.yml must be a YAML mapping.") + + # Unknown keys warning (if strict) + if strict_config: + _check_unknown_keys(data, strict=strict_config, path=p) + + # Root resolution — root_source labels match old test expectations + root_source: str + root_raw = data.get("root") + if root_raw: + root = Path(str(root_raw)) + if not root.is_absolute(): + root = (base_dir / root).resolve() + root_source = "yml" + else: + env_root = os.environ.get("DCL_ROOT") + tool_outdir = os.environ.get("TOOLKIT_OUTDIR") + if env_root: + root = Path(env_root).resolve() + root_source = "env:DCL_ROOT" + elif tool_outdir: + root = Path(tool_outdir).resolve() + root_source = "env:TOOLKIT_OUTDIR" + else: + root = base_dir + root_source = "base_dir_fallback" + + if root_override: + root = Path(root_override).expanduser().resolve() + root_source = "--root" + + # Repo root guardrail + if repo_root is not None: + repo_root_path = Path(repo_root).expanduser().resolve() + if not repo_root_path.is_dir(): + raise ValueError(f"repo_root does not exist or is not a directory: {repo_root_path}") + try: + root.relative_to(repo_root_path) + except ValueError: + raise ValueError( + f"Resolved root {root} is not within repo_root {repo_root_path}" + ) from None + + # Dataset block + dataset_block = data.get("dataset", {}) + if not isinstance(dataset_block, dict): + raise ValueError("dataset must be a mapping.") + + # Check only for missing keys (empty values like '' or [] are validated later) + if "name" not in dataset_block: + raise ValueError("Required field missing or invalid: dataset.name (string).") + name = dataset_block["name"] + + if "years" not in dataset_block: + raise ValueError("dataset.years must be a non-empty list, e.g. [2022, 2023].") + years_raw = dataset_block["years"] + try: + years = [int(y) for y in years_raw] + except (TypeError, ValueError): + raise ValueError("dataset.years must contain integers.") + + # Path normalization: convert relative paths in raw/clean/mart/support to absolute + _normalize_paths(data, base_dir) + + # Support validation + support = data.get("support", []) + if isinstance(support, list): + support_names: list[str] = [ + str(s["name"]) + for s in support + if isinstance(s, dict) and isinstance(s.get("name"), str) + ] + duplicates = sorted({n for n in support_names if support_names.count(n) > 1}) + if duplicates: + raise ValueError("support[].name values must be unique: " + ", ".join(duplicates)) + + # Convert support entries to dict-like objects with attribute access + support_objects = [_dict2ns(s) if isinstance(s, dict) else s for s in support] + + return PipelineConfig( + root=root, + base_dir=base_dir, + root_source=root_source, + dataset=name, + source_id=dataset_block.get("source_id"), + years=years, + tags=ensure_str_list(dataset_block.get("tags", []), "dataset.tags"), + category=dataset_block.get("category"), + raw=RawConfig.from_dict(data.get("raw")), + clean=CleanConfig.from_dict(data.get("clean")), + mart=MartConfig.from_dict(data.get("mart")), + support=support_objects, + validation=data.get("validation", {"fail_on_error": True, "mode": "strict"}), + output=data.get("output", {"artifacts": "standard"}), ) + + +# --------------------------------------------------------------------------- +# Backward-compat aliases for types previously exported from config_models +# --------------------------------------------------------------------------- + + +@dataclass +class ConfigPolicy: + """Empty policy config — kept for backward compat.""" + + pass + + +@dataclass +class DatasetBlock: + """Dataset identity block — kept for backward compat. + + Note: In the new config, dataset name/years/tags are directly on PipelineConfig. + """ + + name: str = "" + years: list[int] = field(default_factory=list) + source_id: str | None = None + time_coverage: Any = None + tags: list[str] = field(default_factory=list) + category: str | None = None + + +@dataclass +class TimeCoverage: + mode: str = "full_series" + start_year: int = 2020 + end_year: int = 2024 + + +@dataclass +class OutputConfig: + artifacts: str = "standard" + + +@dataclass +class SupportDatasetConfig: + name: str = "" + config: Path = Path(".") + years: list[int] = field(default_factory=list) + + +@dataclass +class GlobalValidationConfig: + fail_on_error: bool = True + mode: str = "strict" + + +def _check_unknown_keys(data: dict, *, strict: bool, path: Path) -> None: + """Basic unknown key check for strict mode.""" + allowed = { + "root", + "schema_version", + "dataset", + "raw", + "clean", + "mart", + "support", + "config", + "validation", + "output", + } + unknown = set(data.keys()) - allowed + if unknown: + msg = f"Unknown top-level config keys: {', '.join(sorted(unknown))}" + if strict: + raise ValueError(msg) + import logging + + logging.getLogger("toolkit.core.config").warning("%s in %s", msg, path) diff --git a/toolkit/core/config_models/__init__.py b/toolkit/core/config_models/__init__.py index 07f6f35..16e9336 100644 --- a/toolkit/core/config_models/__init__.py +++ b/toolkit/core/config_models/__init__.py @@ -1,131 +1,74 @@ -"""Configuration models for the DataCivicLab toolkit. +""" +Shim per backward compat: re-esporta tutti i simboli dal nuovo config.py. -This package groups config models by layer for maintainability. -All public symbols are re-exported here for backward compatibility. +In precedenza questo package conteneva 24 modelli Pydantic in 9 file. +Ora tutto e' centralizzato in toolkit.core.config con semplici dataclass. """ from __future__ import annotations -# --- path_normalization --- -from toolkit.core.config_models.path_normalization import ( - _err, - _require_map, - _ensure_root_within_repo, - _is_managed_output_root, - _iter_matching_tokens, - _normalize_section_paths, - _path_tokens_to_str, - _resolve_root, - _set_nested_value, - _get_nested_value, - _resolve_path_value, - _MANAGED_OUTPUT_ROOTS, -) - -# --- shared_models --- -from toolkit.core.config_models.shared_models import ( - ConfigDeprecation, - ConfigPolicy, - DatasetBlock, - GlobalValidationConfig, - OutputConfig, - RangeRuleConfig, - SupportDatasetConfig, - TimeCoverage, - _CONFIG_DEPRECATIONS, - _SAFE_SQL_IDENTIFIER_RE, - ensure_str_list, - parse_bool, -) - -# --- policy --- -from toolkit.core.config_models.policy import ( - _TOP_LEVEL_ALLOWED_KEYS, - _declared_model_keys, - _emit_deprecation_notice, - _emit_unknown_keys_notice, - _warn_or_reject_unknown_keys, -) - -# --- Layer models --- -from toolkit.core.config_models.raw import ( - ClientConfig, - ExtractorConfig, - RawConfig, - RawSourceConfig, -) -from toolkit.core.config_models.clean import ( +from toolkit.core.config import ( CleanConfig, CleanReadConfig, CleanValidateConfig, CleanValidationSpec, -) -from toolkit.core.config_models.mart import ( + ConfigPolicy, + DatasetBlock, + GlobalValidationConfig, + HierarchyConfig, + HierarchyLevel, MartConfig, MartTableConfig, MartTableRuleConfig, MartValidateConfig, MartValidationSpec, + OutputConfig, + PipelineConfig, + RangeRuleConfig, + RawConfig, + RawSourceConfig, + SupportDatasetConfig, + TimeCoverage, + ToolkitConfig, TransitionConfig, + ensure_dict, + ensure_str_list, + load_config, + parse_bool, ) -# --- Loader (ToolkitConfigModel and load_config_model) --- -from toolkit.core.config_models._loader import ( - ToolkitConfigModel, - load_config_model, -) +# Old name used by some consumers +load_config_model = load_config +ToolkitConfigModel = PipelineConfig __all__ = [ - # path_normalization - "_err", - "_MANAGED_OUTPUT_ROOTS", - "_require_map", - "_ensure_root_within_repo", - "_is_managed_output_root", - "_iter_matching_tokens", - "_normalize_section_paths", - "_path_tokens_to_str", - "_resolve_root", - "_set_nested_value", - "_get_nested_value", - "_resolve_path_value", - # shared_models - "ConfigDeprecation", - "ConfigPolicy", - "DatasetBlock", - "GlobalValidationConfig", - "OutputConfig", - "RangeRuleConfig", - "SupportDatasetConfig", - "TimeCoverage", - "_CONFIG_DEPRECATIONS", - "_SAFE_SQL_IDENTIFIER_RE", - "ensure_str_list", - "parse_bool", - # policy - "_TOP_LEVEL_ALLOWED_KEYS", - "_declared_model_keys", - "_emit_deprecation_notice", - "_emit_unknown_keys_notice", - "_warn_or_reject_unknown_keys", - # Raw - "ClientConfig", - "ExtractorConfig", - "RawConfig", - "RawSourceConfig", - # Clean "CleanConfig", "CleanReadConfig", "CleanValidateConfig", "CleanValidationSpec", - # Mart + "ConfigPolicy", + "DatasetBlock", + "GlobalValidationConfig", + "HierarchyConfig", + "HierarchyLevel", "MartConfig", "MartTableConfig", "MartTableRuleConfig", "MartValidateConfig", "MartValidationSpec", - "TransitionConfig", - # Loader + "OutputConfig", + "PipelineConfig", + "RangeRuleConfig", + "RawConfig", + "RawSourceConfig", + "SupportDatasetConfig", + "TimeCoverage", + "ToolkitConfig", "ToolkitConfigModel", + "TransitionConfig", + "ensure_dict", + "ensure_str_list", + "load_config", "load_config_model", + "parse_bool", ] diff --git a/toolkit/core/config_models/_loader.py b/toolkit/core/config_models/_loader.py deleted file mode 100644 index 5fee348..0000000 --- a/toolkit/core/config_models/_loader.py +++ /dev/null @@ -1,167 +0,0 @@ -"""ToolkitConfigModel and load_config_model - config loading entry points.""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -import yaml -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator - -from toolkit.core.config_models.clean import CleanConfig -from toolkit.core.config_models.mart import MartConfig -from toolkit.core.config_models.raw import RawConfig -from toolkit.core.config_models.common import ( - ConfigPolicy, - DatasetBlock, - GlobalValidationConfig, - OutputConfig, - SupportDatasetConfig, - _err, - _ensure_root_within_repo, - _normalize_section_paths, - _require_map, - _resolve_root, - _warn_or_reject_unknown_keys, - parse_bool, -) - - -class ToolkitConfigModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - base_dir: Path - schema_version: int = 1 - root: Path - root_source: str - dataset: DatasetBlock - raw: RawConfig = Field(default_factory=RawConfig) - clean: CleanConfig = Field(default_factory=CleanConfig) - mart: MartConfig = Field(default_factory=MartConfig) - support: list[SupportDatasetConfig] = Field(default_factory=list) - config: ConfigPolicy = Field(default_factory=ConfigPolicy) - validation: GlobalValidationConfig = Field(default_factory=GlobalValidationConfig) - output: OutputConfig = Field(default_factory=OutputConfig) - registry: dict[str, Any] = Field(default_factory=dict) - - @model_validator(mode="after") - def _validate_unique_support_names(self) -> "ToolkitConfigModel": - names = [entry.name for entry in self.support] - duplicates = sorted({name for name in names if names.count(name) > 1}) - if duplicates: - raise ValueError("support[].name values must be unique: " + ", ".join(duplicates)) - return self - - -def _validation_error_to_value_error(exc: ValidationError, *, path: Path) -> ValueError: - messages: list[str] = [] - for error in exc.errors(): - loc = ".".join(str(part) for part in error.get("loc", ())) - msg = error.get("msg", "Invalid value") - messages.append(f"{loc}: {msg}" if loc else msg) - return _err("Config validation failed: " + "; ".join(messages), path=path) - - -def _read_strict_config(data: dict[str, Any], *, path: Path) -> bool: - raw_config = data.get("config") - if raw_config is None: - return False - if not isinstance(raw_config, dict): - raise _err("config must be a mapping object if provided.", path=path) - strict_value = raw_config.get("strict", False) - return parse_bool(strict_value, "config.strict") - - -def load_config_model( - path: str | Path, - *, - strict_config: bool = False, - repo_root: str | Path | None = None, -) -> ToolkitConfigModel: - """ - Load and normalize toolkit config. - - repo_root is an optional guardrail for callers that need to enforce that - the resolved effective root stays inside a known repository tree. This is - intentionally opt-in so the toolkit can still support valid workflows that - write outputs outside the project directory. A typical caller is external - CI that validates dataset.yml contracts for monorepos such as - dataset-incubator. - """ - p = Path(path) - base_dir = p.parent.resolve() - - try: - data = yaml.safe_load(p.read_text(encoding="utf-8")) - except Exception as e: - raise _err(f"Impossibile leggere YAML: {e}", path=p) - - if not isinstance(data, dict): - raise _err("dataset.yml deve essere una mappa YAML.", path=p) - - dataset_block = _require_map(data, "dataset", path=p) - if "name" not in dataset_block: - raise _err("Campo obbligatorio mancante o non valido: dataset.name (string).", path=p) - if "years" not in dataset_block: - raise _err("dataset.years deve essere una lista non vuota, es: [2022, 2023].", path=p) - - strict_mode = strict_config or _read_strict_config(data, path=p) - # Shallow copy to avoid mutating caller dict - normalized = dict(data) - for section in ("raw", "clean", "mart"): - val = normalized.get(section) - if isinstance(val, dict): - normalized[section] = dict(val) - - normalized = _warn_or_reject_unknown_keys(normalized, path=p, strict_config=strict_mode) - root_path, root_source = _resolve_root(normalized.get("root"), base_dir=base_dir) - if repo_root is not None: - repo_root_path = Path(repo_root).expanduser().resolve() - if not repo_root_path.is_dir(): - raise _err( - f"repo_root does not exist or is not a directory: {repo_root_path}", - path=p, - ) - root_path = _ensure_root_within_repo(root_path, repo_root=repo_root_path, path=p) - - raw = normalized.get("raw", {}) or {} - clean = normalized.get("clean", {}) or {} - mart = normalized.get("mart", {}) or {} - support = normalized.get("support", []) or [] - - normalized_fields: list[tuple[str, Path]] = [] - if isinstance(raw, dict): - raw, raw_changes = _normalize_section_paths("raw", raw, base_dir=base_dir) - normalized_fields.extend(raw_changes) - if isinstance(clean, dict): - clean, clean_changes = _normalize_section_paths("clean", clean, base_dir=base_dir) - normalized_fields.extend(clean_changes) - if isinstance(mart, dict): - mart, mart_changes = _normalize_section_paths("mart", mart, base_dir=base_dir) - normalized_fields.extend(mart_changes) - if isinstance(support, list): - support, support_changes = _normalize_section_paths("support", support, base_dir=base_dir) - normalized_fields.extend(support_changes) - normalized_fields.append(("root", root_path)) - - if normalized_fields: - summary = ", ".join(f"{field}={value}" for field, value in normalized_fields) - logger = logging.getLogger("toolkit.core.config") - logger.debug("Normalized config paths: %s", summary) - - payload = { - **normalized, - "base_dir": base_dir, - "root": root_path, - "root_source": root_source, - "raw": raw, - "clean": clean, - "mart": mart, - "support": support, - } - - try: - return ToolkitConfigModel.model_validate(payload) - except ValidationError as exc: - raise _validation_error_to_value_error(exc, path=p) from None diff --git a/toolkit/core/config_models/clean.py b/toolkit/core/config_models/clean.py deleted file mode 100644 index d03558b..0000000 --- a/toolkit/core/config_models/clean.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Pydantic models for the clean layer configuration.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -from toolkit.core.csv_read import normalize_columns_spec -from toolkit.core.config_models.common import ( - RangeRuleConfig, - ensure_str_list, - parse_bool, -) -from toolkit.core.config_models.mart import TransitionConfig - - -class CleanReadConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: Literal["auto", "config_only"] = "auto" - delim: str | None = None - header: bool = True - encoding: str | None = None - decimal: str | None = None - thousands: str | None = None - skip: int | None = None - auto_detect: bool | None = None - quote: str | None = None - escape: str | None = None - comment: str | None = None - ignore_errors: bool | None = None - dateformat: str | None = None - timestampformat: str | None = None - strict_mode: bool | None = None - null_padding: bool | None = None - parallel: bool | None = None - nullstr: str | list[str] | None = None - columns: dict[str, str] | None = None - normalize_rows_to_columns: bool = False - align_by_header: bool = False - trim_whitespace: bool = True - sample_size: int | None = None - sheet_name: str | int | None = None - mode: Literal["explicit", "latest", "largest", "all"] | None = None - glob: str = "*" - prefer_from_raw_run: bool = True - allow_ambiguous: bool = False - include: list[str] | None = None - - @field_validator("columns", mode="before") - @classmethod - def _normalize_columns(cls, value: Any) -> dict[str, str] | None: - return normalize_columns_spec(value) - - @field_validator("normalize_rows_to_columns", mode="before") - @classmethod - def _normalize_rows_to_columns(cls, value: Any) -> bool: - return parse_bool(value, "clean.read.normalize_rows_to_columns") - - @field_validator("align_by_header", mode="before") - @classmethod - def _normalize_align_by_header(cls, value: Any) -> bool: - return parse_bool(value, "clean.read.align_by_header") - - @field_validator("include", mode="before") - @classmethod - def _normalize_include(cls, value: Any) -> list[str] | None: - if value is None: - return None - return ensure_str_list(value, "clean.read.include") - - @model_validator(mode="after") - def _validate_align_by_header(self) -> "CleanReadConfig": - if self.align_by_header and not self.normalize_rows_to_columns: - raise ValueError("align_by_header=true requires normalize_rows_to_columns=true") - return self - - -class CleanValidateConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - primary_key: list[str] = Field(default_factory=list) - not_null: list[str] = Field(default_factory=list) - ranges: dict[str, RangeRuleConfig] = Field(default_factory=dict) - max_null_pct: dict[str, float] = Field(default_factory=dict) - min_rows: int | None = None - promotion: TransitionConfig = Field( - default_factory=lambda: TransitionConfig(max_row_drop_pct=15.0) - ) - - @field_validator("primary_key", "not_null", mode="before") - @classmethod - def _normalize_lists(cls, value: Any, info) -> list[str]: - return ensure_str_list(value, f"clean.validate.{info.field_name}") - - -class CleanConfig(BaseModel): - model_config = ConfigDict(extra="allow", populate_by_name=True) - - sql: Path | None = None - read_mode: Literal["strict", "fallback", "robust"] = "fallback" - read_source: Literal["auto", "config_only"] | None = None - read: CleanReadConfig | None = None - required_columns: list[str] = Field(default_factory=list) - validate_config: CleanValidateConfig = Field( - default_factory=CleanValidateConfig, - alias="validate", - ) - - @field_validator("required_columns", mode="before") - @classmethod - def _normalize_required_columns(cls, value: Any) -> list[str]: - return ensure_str_list(value, "clean.required_columns") - - @property - def validate(self) -> CleanValidateConfig: # type: ignore[override] - return self.validate_config - - -class CleanValidationSpec(BaseModel): - model_config = ConfigDict(extra="forbid", populate_by_name=True) - - required_columns: list[str] = Field(default_factory=list) - validate_config: CleanValidateConfig = Field( - default_factory=CleanValidateConfig, - alias="validate", - ) - - @field_validator("required_columns", mode="before") - @classmethod - def _normalize_required_columns(cls, value: Any) -> list[str]: - return ensure_str_list(value, "clean.required_columns") - - @property - def validate(self) -> CleanValidateConfig: # type: ignore[override] - return self.validate_config diff --git a/toolkit/core/config_models/common.py b/toolkit/core/config_models/common.py deleted file mode 100644 index 7ba527a..0000000 --- a/toolkit/core/config_models/common.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Shared utilities and models used across all config layers. - -This module is a thin re-export facade. All responsibilities have been -migrated to dedicated sub-modules: -- path_normalization: path resolution and section normalization -- shared_models: Pydantic models, ConfigDeprecation, coercion helpers -- policy: unknown-key detection, deprecation notices, legacy normalization -""" - -from __future__ import annotations - -# ruff: noqa: F401 -# Re-exported for backward compat via __init__.py -# (consumers import from common, not from submodules directly) - -# --- path_normalization --- -from toolkit.core.config_models.path_normalization import ( - _err, - _MANAGED_OUTPUT_ROOTS, - _require_map, - _ensure_root_within_repo, - _is_managed_output_root, - _iter_matching_tokens, - _normalize_section_paths, - _path_tokens_to_str, - _resolve_root, - _set_nested_value, - _get_nested_value, - _resolve_path_value, -) - -# --- shared_models --- -from toolkit.core.config_models.shared_models import ( - ConfigDeprecation, - ConfigPolicy, - DatasetBlock, - GlobalValidationConfig, - OutputConfig, - RangeRuleConfig, - SupportDatasetConfig, - TimeCoverage, - _CONFIG_DEPRECATIONS, - _SAFE_SQL_IDENTIFIER_RE, - ensure_str_list, - parse_bool, -) - -# --- policy --- -from toolkit.core.config_models.policy import ( - _TOP_LEVEL_ALLOWED_KEYS, - _declared_model_keys, - _emit_deprecation_notice, - _emit_unknown_keys_notice, - _warn_or_reject_unknown_keys, -) diff --git a/toolkit/core/config_models/mart.py b/toolkit/core/config_models/mart.py deleted file mode 100644 index 307c180..0000000 --- a/toolkit/core/config_models/mart.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Pydantic models for the mart layer configuration.""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Any - -from typing import Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -from toolkit.core.config_models.common import ( - RangeRuleConfig, - _SAFE_SQL_IDENTIFIER_RE, - ensure_str_list, - parse_bool, -) - - -class MartTableConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str - sql: Path - years: list[int] | None = None - source_layer: Literal["clean", "mart"] = "clean" - source_table: str | None = None - - @field_validator("name") - @classmethod - def _validate_name(cls, value: str) -> str: - text = value.strip() - if not text: - raise ValueError("mart.tables[].name must not be empty") - if not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, text): - raise ValueError( - "mart.tables[].name must be a safe SQL identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return text - - -class MartTableRuleConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - required_columns: list[str] = Field(default_factory=list) - not_null: list[str] = Field(default_factory=list) - primary_key: list[str] = Field(default_factory=list) - ranges: dict[str, RangeRuleConfig] = Field(default_factory=dict) - max_null_pct: dict[str, float] = Field(default_factory=dict) - min_rows: int | None = None - - @field_validator("required_columns", "not_null", "primary_key", mode="before") - @classmethod - def _normalize_lists(cls, value: Any, info) -> list[str]: - return ensure_str_list(value, f"mart.validate.table_rules.*.{info.field_name}") - - -class TransitionConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - max_row_drop_pct: float | None = None - warn_removed_columns: bool = True - fail_on_row_drop_exceeded: bool = True - - @field_validator("warn_removed_columns", mode="before") - @classmethod - def _parse_warn_removed_columns(cls, value: Any) -> bool: - return parse_bool(value, "mart.validate.transition.warn_removed_columns") - - @field_validator("fail_on_row_drop_exceeded", mode="before") - @classmethod - def _parse_fail_on_row_drop_exceeded(cls, value: Any) -> bool: - return parse_bool(value, "mart.validate.transition.fail_on_row_drop_exceeded") - - -class MartValidateConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - table_rules: dict[str, MartTableRuleConfig] = Field(default_factory=dict) - transition: TransitionConfig = Field(default_factory=TransitionConfig) - - -class HierarchyLevel(BaseModel): - """Un livello della gerarchia mart (es. comune, provincia, regione). - - A runtime, la query di aggregazione viene generata automaticamente: - - colonne metriche scoperte per introspection dalla source - - GROUP BY sulle colonne grain - - SUM per ogni metrica numerica - - Non richiede un file SQL: il config è attivo. - """ - - model_config = ConfigDict(extra="forbid") - - level: str - table: str - grain: list[str] - source_table: str | None = None - exclude_metrics: list[str] = Field(default_factory=list) - - @field_validator("level") - @classmethod - def _validate_level(cls, value: str) -> str: - v = value.strip() - if not v: - raise ValueError("mart.hierarchy.levels[].level must not be empty") - return v - - @field_validator("table") - @classmethod - def _validate_table(cls, value: str) -> str: - text = value.strip() - if not text: - raise ValueError("mart.hierarchy.levels[].table must not be empty") - if not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, text): - raise ValueError( - "mart.hierarchy.levels[].table must be a safe SQL identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return text - - @field_validator("source_table") - @classmethod - def _validate_source_table(cls, value: str | None) -> str | None: - if value is not None: - text = value.strip() - if text and not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, text): - raise ValueError( - "mart.hierarchy.levels[].source_table must be a safe SQL identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return value - - @field_validator("grain") - @classmethod - def _validate_grain(cls, value: list[str]) -> list[str]: - for g in value: - if not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, g.strip()): - raise ValueError( - f"mart.hierarchy.levels[].grain element '{g}' must be a safe SQL identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return value - - @field_validator("exclude_metrics") - @classmethod - def _validate_exclude_metrics(cls, value: list[str]) -> list[str]: - for m in value: - if not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, m.strip()): - raise ValueError( - f"mart.hierarchy.levels[].exclude_metrics element '{m}' must be a safe SQL identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return value - - -class HierarchyConfig(BaseModel): - """Gerarchia mart: aggregazione per asse naturale del dato.""" - - model_config = ConfigDict(extra="forbid") - - axis: str = Field(..., pattern=r"^(territoriale|temporale|categorico)$") - levels: list[HierarchyLevel] = Field(min_length=1) - - @field_validator("levels") - @classmethod - def _validate_levels_order(cls, value: list[HierarchyLevel]) -> list[HierarchyLevel]: - if len(value) < 1: - raise ValueError("mart.hierarchy.levels must have at least one level") - return value - - -class MartConfig(BaseModel): - model_config = ConfigDict(extra="allow", populate_by_name=True) - - tables: list[MartTableConfig] = Field(default_factory=list) - required_tables: list[str] = Field(default_factory=list) - hierarchy: HierarchyConfig | None = None - validate_config: MartValidateConfig = Field( - default_factory=MartValidateConfig, - alias="validate", - ) - - @field_validator("required_tables", mode="before") - @classmethod - def _normalize_required_tables(cls, value: Any) -> list[str]: - return ensure_str_list(value, "mart.required_tables") - - @model_validator(mode="after") - def _default_required_tables_from_tables(self) -> MartConfig: - """If required_tables is empty, default to all table names from tables.""" - if not self.required_tables and self.tables: - object.__setattr__(self, "required_tables", [t.name for t in self.tables]) - return self - - @property - def validate(self) -> MartValidateConfig: # type: ignore[override] - return self.validate_config - - -class MartValidationSpec(BaseModel): - model_config = ConfigDict(extra="forbid", populate_by_name=True) - - required_tables: list[str] = Field(default_factory=list) - validate_config: MartValidateConfig = Field( - default_factory=MartValidateConfig, - alias="validate", - ) - - @field_validator("required_tables", mode="before") - @classmethod - def _normalize_required_tables(cls, value: Any) -> list[str]: - return ensure_str_list(value, "mart.required_tables") - - @property - def validate(self) -> MartValidateConfig: # type: ignore[override] - return self.validate_config diff --git a/toolkit/core/config_models/path_normalization.py b/toolkit/core/config_models/path_normalization.py deleted file mode 100644 index be03bc5..0000000 --- a/toolkit/core/config_models/path_normalization.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Path resolution utilities for dataset.yml config normalization. - -Handles {year}-templated paths, relative-to-base_dir resolution, -section-specific whitelist-based normalization, and managed output root -detection. -""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - - -_MANAGED_OUTPUT_ROOTS = {"_smoke_out", "_test_out"} - - -def _err(msg: str, *, path: Path) -> ValueError: - return ValueError(f"{msg} (file: {path})") - - -def _require_map(data: dict[str, Any], key: str, *, path: Path) -> dict[str, Any]: - val = data.get(key) - if not isinstance(val, dict): - raise _err(f"Campo '{key}' mancante o non valido (deve essere una mappa).", path=path) - return val - - -def _resolve_path_value(value: Any, *, base_dir: Path) -> Any: - if not isinstance(value, str): - return value - text = value.strip() - if not text: - return value - if "{year}" in text: - sentinel = "__DCL_YEAR_PLACEHOLDER__" - templated = text.replace("{year}", sentinel) - path = Path(templated).expanduser() - if path.is_absolute(): - return str(path.resolve()).replace(sentinel, "{year}") - return str((base_dir / path).resolve()).replace(sentinel, "{year}") - path = Path(text).expanduser() - if path.is_absolute(): - return path.resolve() - return (base_dir / path).resolve() - - -_SECTION_PATH_WHITELIST: dict[str, tuple[tuple[str, ...], ...]] = { - "raw": ( - ("source", "args", "path"), - ("sources", "*", "args", "path"), - ), - "clean": (("sql",),), - "mart": (("tables", "*", "sql"),), - "support": (("*", "config"),), -} - - -def _path_tokens_to_str(tokens: tuple[str, ...]) -> str: - out: list[str] = [] - for token in tokens: - if token == "*": - if out: - out[-1] = f"{out[-1]}[*]" - else: - out.append("[*]") - elif token.isdigit(): - if out: - out[-1] = f"{out[-1]}[{token}]" - else: - out.append(f"[{token}]") - else: - out.append(token) - return ".".join(out) - - -def _set_nested_value(container: Any, tokens: tuple[str, ...], value: Any) -> Any: - if not tokens: - return value - - head, *tail = tokens - if isinstance(container, dict): - updated = dict(container) - updated[head] = _set_nested_value(updated.get(head), tuple(tail), value) - return updated - - if isinstance(container, list): - index = int(head) - updated_list: list[Any] = list(container) - updated_list[index] = _set_nested_value(updated_list[index], tuple(tail), value) - return updated_list - - raise TypeError(f"Cannot set nested value at {tokens!r} on {type(container).__name__}") - - -def _iter_matching_tokens( - container: Any, - pattern: tuple[str, ...], - prefix: tuple[str, ...] = (), -) -> list[tuple[str, ...]]: - if not pattern: - return [prefix] - - head, *tail = pattern - tail_tuple = tuple(tail) - - if head == "*": - if not isinstance(container, list): - return [] - matches: list[tuple[str, ...]] = [] - for index, item in enumerate(container): - matches.extend(_iter_matching_tokens(item, tail_tuple, prefix + (str(index),))) - return matches - - if not isinstance(container, dict) or head not in container: - return [] - - return _iter_matching_tokens(container[head], tail_tuple, prefix + (head,)) - - -def _get_nested_value(container: Any, tokens: tuple[str, ...]) -> Any: - current = container - for token in tokens: - if isinstance(current, dict): - current = current[token] - elif isinstance(current, list): - current = current[int(token)] - else: - raise TypeError(f"Cannot traverse token {token!r} on {type(current).__name__}") - return current - - -def _normalize_section_paths( - section_name: str, - section: Any, - *, - base_dir: Path, -) -> tuple[Any, list[tuple[str, Path]]]: - if isinstance(section, dict): - normalized: Any = dict(section) - elif isinstance(section, list): - normalized = list(section) - else: - normalized = section - changes: list[tuple[str, Path]] = [] - - for pattern in _SECTION_PATH_WHITELIST.get(section_name, ()): - for tokens in _iter_matching_tokens(section, pattern): - raw_value = _get_nested_value(section, tokens) - resolved = _resolve_path_value(raw_value, base_dir=base_dir) - if resolved is raw_value: - continue - normalized = _set_nested_value(normalized, tokens, resolved) - changes.append((f"{section_name}.{_path_tokens_to_str(tokens)}", resolved)) - - return normalized, changes - - -def _is_managed_output_root(root: str) -> bool: - raw = root.strip() - if not raw: - return False - path = Path(raw) - if path.is_absolute(): - return False - return path.name in _MANAGED_OUTPUT_ROOTS - - -def _resolve_root(root: Any, *, base_dir: Path) -> tuple[Path, str]: - if root is None: - env_root = os.environ.get("DCL_ROOT") - if env_root: - return Path(env_root).expanduser().resolve(), "env:DCL_ROOT" - return base_dir, "base_dir_fallback" - - if not isinstance(root, str): - raise ValueError("root must be a string path or null") - - if not root.strip(): - env_root = os.environ.get("DCL_ROOT") - if env_root: - return Path(env_root).expanduser().resolve(), "env:DCL_ROOT" - return base_dir, "base_dir_fallback" - - managed_outdir = os.environ.get("TOOLKIT_OUTDIR") or os.environ.get("DCL_OUTDIR") - if managed_outdir and _is_managed_output_root(root): - source = "env:TOOLKIT_OUTDIR" if os.environ.get("TOOLKIT_OUTDIR") else "env:DCL_OUTDIR" - return Path(managed_outdir).expanduser().resolve(), source - return _resolve_path_value(root, base_dir=base_dir), "yml" - - -def _ensure_root_within_repo(root: Path, *, repo_root: Path, path: Path) -> Path: - try: - root.relative_to(repo_root) - except ValueError as exc: - raise _err( - f"root resolves outside repo_root: root={root} repo_root={repo_root}", - path=path, - ) from exc - return root diff --git a/toolkit/core/config_models/policy.py b/toolkit/core/config_models/policy.py deleted file mode 100644 index 1e0b9a8..0000000 --- a/toolkit/core/config_models/policy.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Policy engine for dataset.yml contract validation. - -Handles: -- Unknown key detection and rejection -- Deprecation notices (strict vs non-strict mode) -- Legacy payload normalization -- Top-level allowed keys whitelist -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -from pydantic import BaseModel - -from toolkit.core.config_models.path_normalization import _err -from toolkit.core.config_models.shared_models import _CONFIG_DEPRECATIONS - - -logger = logging.getLogger("toolkit.core.config") - -_TOP_LEVEL_ALLOWED_KEYS = { - "schema_version", - "root", - "dataset", - "raw", - "clean", - "mart", - "support", - "config", - "validation", - "output", - "registry", -} - - -def _declared_model_keys(model_cls: type[BaseModel]) -> set[str]: - keys: set[str] = set() - for field_name, field_info in model_cls.model_fields.items(): - keys.add(field_name) - if field_info.alias: - keys.add(str(field_info.alias)) - return keys - - -def _emit_deprecation_notice( - key: str, - *, - strict_config: bool, - path: Path, -) -> None: - notice = _CONFIG_DEPRECATIONS[key] - message = f"{notice.code} {notice.message}" - logger.warning(message) - if strict_config: - raise _err(f"{notice.code} {notice.message}", path=path) - - -def _emit_unknown_keys_notice( - key: str, - extras: list[str], - *, - strict_config: bool, - path: Path, -) -> None: - notice = _CONFIG_DEPRECATIONS[key] - formatted = ", ".join(sorted(extras)) - message = f"{notice.code} {notice.message}: {formatted}" - logger.warning(message) - if strict_config: - raise _err(message, path=path) - - -def _warn_or_reject_unknown_keys( - data: dict[str, Any], - *, - path: Path, - strict_config: bool, -) -> dict[str, Any]: - normalized = dict(data) - - top_level_extras = [key for key in normalized.keys() if key not in _TOP_LEVEL_ALLOWED_KEYS] - if "bq" in top_level_extras: - raise _err("bq is no longer supported; remove field", path=path) - if top_level_extras: - _emit_unknown_keys_notice( - "unknown.top_level", - top_level_extras, - strict_config=strict_config, - path=path, - ) - if not strict_config: - normalized = {k: v for k, v in normalized.items() if k in _TOP_LEVEL_ALLOWED_KEYS} - - from toolkit.core.config_models.raw import RawConfig - from toolkit.core.config_models.clean import CleanConfig - from toolkit.core.config_models.mart import MartConfig - - for section_name, allowed_keys, notice_key in ( - ("raw", _declared_model_keys(RawConfig), "unknown.raw"), - ("clean", _declared_model_keys(CleanConfig), "unknown.clean"), - ("mart", _declared_model_keys(MartConfig), "unknown.mart"), - ): - section = normalized.get(section_name) - if not isinstance(section, dict): - continue - extras = [k for k in section if k not in allowed_keys] - # Unconditional rejections for legacy forms that are no longer supported. - if section_name == "raw" and "source" in extras: - raise _err("raw.source is no longer supported; use raw.sources", path=path) - if section_name == "clean" and "sql_path" in extras: - raise _err("clean.sql_path is no longer supported; use clean.sql", path=path) - if section_name == "mart" and "sql_dir" in extras: - raise _err("mart.sql_dir is no longer supported; use mart.tables[].sql", path=path) - if extras: - _emit_unknown_keys_notice( - notice_key, - extras, - strict_config=strict_config, - path=path, - ) - - return normalized diff --git a/toolkit/core/config_models/raw.py b/toolkit/core/config_models/raw.py deleted file mode 100644 index 34058f1..0000000 --- a/toolkit/core/config_models/raw.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Pydantic models for the raw layer configuration.""" - -from __future__ import annotations - -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator - -from toolkit.core.config_models.common import parse_bool - - -class ClientConfig(BaseModel): - model_config = ConfigDict(extra="allow") - - timeout: int | None = None - retries: int | None = None - user_agent: str | None = None - headers: dict[str, str] | None = None - - @field_validator("headers", mode="before") - @classmethod - def _validate_headers(cls, value: Any) -> dict[str, str] | None: - if value is None: - return None - if not isinstance(value, dict): - raise ValueError("raw.sources[].client.headers must be a dict") - if not all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()): - raise ValueError("raw.sources[].client.headers must be a dict[str, str]") - return dict(value) - - -class ExtractorConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - type: Literal["identity", "unzip_all", "unzip_first", "unzip_first_csv"] = "identity" - args: dict[str, Any] = Field(default_factory=dict) - - @field_validator("args", mode="before") - @classmethod - def _validate_args(cls, value: Any) -> dict[str, Any]: - if value is None: - return {} - if not isinstance(value, dict): - raise ValueError("raw.extractor.args must be a dict") - return dict(value) - - -class RawSourceConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str | None = None - type: str = "http_file" - year: int | None = None - client: ClientConfig = Field(default_factory=ClientConfig) - args: dict[str, Any] = Field(default_factory=dict) - extractor: ExtractorConfig | None = None - primary: bool = False - inject_column: dict[str, str] | None = Field( - default=None, - description="Fixed columns to inject: {column_name: value}. " - "E.g. {cod_regione: '13', desc_regione: 'Abruzzo'}", - ) - - @field_validator("primary", mode="before") - @classmethod - def _parse_primary(cls, value: Any) -> bool: - return parse_bool(value, "raw.sources[].primary") - - @field_validator("args", mode="before") - @classmethod - def _validate_args(cls, value: Any) -> dict[str, Any]: - if value is None: - return {} - if not isinstance(value, dict): - raise ValueError("raw.sources[].args must be a dict") - return dict(value) - - -class RawConfig(BaseModel): - model_config = ConfigDict(extra="allow") - - output_policy: Literal["overwrite", "versioned"] = "versioned" - extractor: ExtractorConfig | None = None - sources: list[RawSourceConfig] = Field(default_factory=list) diff --git a/toolkit/core/config_models/shared_models.py b/toolkit/core/config_models/shared_models.py deleted file mode 100644 index 688fa40..0000000 --- a/toolkit/core/config_models/shared_models.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Shared data models and coercion utilities for dataset.yml config. - -Contains: -- Shared Pydantic models (TimeCoverage, DatasetBlock, etc.) -- ConfigDeprecation dataclass and deprecation registry -- Coercion helpers (parse_bool, ensure_str_list) -- Constants used by policy and validation (_SAFE_SQL_IDENTIFIER_RE) -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - - -_SAFE_SQL_IDENTIFIER_RE = r"^[A-Za-z_][A-Za-z0-9_]*$" - - -# --- Deprecation registry ------------------------------------------------------ - - -@dataclass(frozen=True) -class ConfigDeprecation: - code: str - message: str - - -_CONFIG_DEPRECATIONS: dict[str, ConfigDeprecation] = { - "unknown.top_level": ConfigDeprecation( - code="DCL009", - message="unknown top-level config keys detected", - ), - "unknown.raw": ConfigDeprecation( - code="DCL010", - message="unknown raw config keys detected", - ), - "unknown.clean": ConfigDeprecation( - code="DCL011", - message="unknown clean config keys detected", - ), - "unknown.mart": ConfigDeprecation( - code="DCL012", - message="unknown mart config keys detected", - ), -} - - -# --- Coercion helpers ---------------------------------------------------------- - - -def parse_bool(value: Any, field_name: str) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, int) and value in {0, 1}: - return bool(value) - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in {"true", "1", "yes", "y"}: - return True - if normalized in {"false", "0", "no", "n"}: - return False - raise ValueError(f"{field_name} must be a boolean-like value: true/false, 1/0, yes/no") - - -def ensure_str_list(value: Any, field_name: str) -> list[str]: - if value is None: - return [] - if isinstance(value, str): - return [value] - if isinstance(value, list): - if not all(isinstance(item, str) for item in value): - raise ValueError(f"{field_name} must be a string or a list of strings") - return list(value) - raise ValueError(f"{field_name} must be a string or a list of strings") - - -# --- Shared Pydantic models --------------------------------------------------- - - -class TimeCoverage(BaseModel): - """Optional metadata per dichiarare la copertura temporale reale dei dati.""" - - model_config = ConfigDict(extra="forbid") - - mode: Literal["full_series"] = "full_series" - start_year: int - end_year: int - - @model_validator(mode="after") - def _validate_year_range(self) -> "TimeCoverage": - if self.end_year < self.start_year: - raise ValueError("dataset.time_coverage.end_year must be >= start_year") - return self - - -class DatasetBlock(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str - years: list[int] - source_id: str | None = None - time_coverage: TimeCoverage | None = None - tags: list[str] = Field(default_factory=list) - category: str | None = None - - -class SupportDatasetConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str - config: Path - years: list[int] - - @field_validator("name") - @classmethod - def _validate_name(cls, value: str) -> str: - text = value.strip() - if not text: - raise ValueError("support[].name must not be empty") - if not re.fullmatch(_SAFE_SQL_IDENTIFIER_RE, text): - raise ValueError( - "support[].name must be a safe identifier " - "(letters, numbers, underscore; cannot start with a number)" - ) - return text - - @field_validator("years") - @classmethod - def _validate_years(cls, value: list[int]) -> list[int]: - if not value: - raise ValueError("support[].years must not be empty") - return value - - -class OutputConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - artifacts: str = "standard" - - -class GlobalValidationConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - fail_on_error: bool = True - mode: Literal["strict", "warn_only"] = "strict" - - -class ConfigPolicy(BaseModel): - model_config = ConfigDict(extra="forbid") - - -class RangeRuleConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - - min: float | None = None - max: float | None = None diff --git a/toolkit/core/validation.py b/toolkit/core/validation.py index 390462d..b5c8d0b 100644 --- a/toolkit/core/validation.py +++ b/toolkit/core/validation.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -from toolkit.core.config_models.mart import TransitionConfig +from toolkit.core.config import TransitionConfig from toolkit.core.io import write_json_atomic diff --git a/toolkit/mart/validate.py b/toolkit/mart/validate.py index c376d6a..c971193 100644 --- a/toolkit/mart/validate.py +++ b/toolkit/mart/validate.py @@ -12,7 +12,7 @@ check_ranges, ) from toolkit.core.io import read_json_or_none -from toolkit.core.config_models import MartTableRuleConfig, MartValidationSpec +from toolkit.core.config import MartTableRuleConfig, MartValidationSpec from toolkit.core.metadata import merge_layer_manifest from toolkit.core.paths import MART_VALIDATION, METADATA, layer_year_dir, to_root_relative from toolkit.core.sql_utils import sql_path @@ -48,16 +48,17 @@ def validate_mart( ranges: col: {min: 0, max: 100} """ - spec = MartValidationSpec.model_validate( - { - "required_tables": required_tables, - "validate": { + spec = ( + MartValidationSpec.from_dict( + { + "required_tables": required_tables, "table_rules": table_rules or {}, - }, - } + } + ) + or MartValidationSpec() ) required_tables = spec.required_tables - table_rules = spec.validate.table_rules + table_rules = spec.table_rules errors: list[str] = [] warnings: list[str] = [] @@ -189,32 +190,34 @@ def run_mart_validation(cfg, year: int, logger, *, sample_mode: bool = False) -> mart_dir = layer_year_dir(cfg.root, "mart", cfg.dataset, year) declared_tables = [t.name for t in cfg.mart.tables if t.name] - spec = MartValidationSpec.model_validate( - { - "required_tables": cfg.mart.required_tables, - "validate": cfg.mart.validate.model_dump( - mode="python", by_alias=True, exclude_none=True, exclude_unset=True - ), - } + validate_rules = cfg.mart.validate.to_dict() if cfg.mart.validate else {} + spec = ( + MartValidationSpec.from_dict( + { + "required_tables": cfg.mart.required_tables, + **validate_rules, + } + ) + or MartValidationSpec() ) # In sample mode, min_rows non e' applicabile (campione non rappresentativo). if sample_mode: - for rule in spec.validate.table_rules.values(): + for rule in spec.table_rules.values(): rule.min_rows = None result = validate_mart( mart_dir, required_tables=spec.required_tables, root=cfg.root, - table_rules=spec.validate.table_rules, + table_rules=spec.table_rules, declared_tables=declared_tables, ) metadata = read_json_or_none(mart_dir / METADATA) or {} transition_report = check_transitions( metadata.get("transition_profiles") or [], - spec.validate.transition, + spec.transition, ) has_warnings = bool(transition_report["warning_messages"]) has_errors = bool(transition_report["error_messages"]) diff --git a/toolkit/scaffold/clean.py b/toolkit/scaffold/clean.py index 4f680b7..ec8c8e3 100644 --- a/toolkit/scaffold/clean.py +++ b/toolkit/scaffold/clean.py @@ -528,7 +528,7 @@ def format_clean_read_proposal(profile: dict[str, Any]) -> str: Includes ``read_mode: robust`` as a top-level ``clean`` field when ``robust_read_suggested`` is set (not inside ``clean.read``). """ - from toolkit.core.config_models.clean import CleanReadConfig + from toolkit.core.config import CleanReadConfig from toolkit.core.io import yaml_dumps proposed = propose_clean_read(profile)