From 16fd1d1b09bc0c6219f45e058f59bdbb4f1f8d26 Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Fri, 10 Jul 2026 00:56:54 +0100 Subject: [PATCH 1/2] fix: update error logging message for csv mapping in fetch_pipeline_csvs --- request-processor/src/application/core/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor/src/application/core/workflow.py b/request-processor/src/application/core/workflow.py index b4d20b2..4f62736 100644 --- a/request-processor/src/application/core/workflow.py +++ b/request-processor/src/application/core/workflow.py @@ -359,7 +359,7 @@ def fetch_pipeline_csvs( dataset, pipeline_dir, geom_type, resource, pipeline_csv ) except Exception as e: - logger.error(f"Error saving new mapping: {e}") + logger.error(f"Error saving new csv mapping: {e}") return {} From 93d783775e944326b85c378bba1ff9c01b8b3e65 Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Fri, 10 Jul 2026 01:25:35 +0100 Subject: [PATCH 2/2] fix: enhance column mapping for tree polygons and add test coverage --- .../src/application/core/workflow.py | 30 +++++------ request-processor/tests/conftest.py | 7 +++ .../src/application/core/test_workflow.py | 2 +- .../src/application/core/test_workflow.py | 52 +++++++++++++++++++ 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/request-processor/src/application/core/workflow.py b/request-processor/src/application/core/workflow.py index 4f62736..d0d1588 100644 --- a/request-processor/src/application/core/workflow.py +++ b/request-processor/src/application/core/workflow.py @@ -314,8 +314,9 @@ def fetch_pipeline_csvs( ): os.makedirs(pipeline_dir, exist_ok=True) pipeline_csvs = ["column.csv", "transform.csv"] - downloaded = False + not_mapped_columns = [] for pipeline_csv in pipeline_csvs: + downloaded = False try: csv_path = os.path.join(pipeline_dir, pipeline_csv) print( @@ -345,6 +346,10 @@ def fetch_pipeline_csvs( if downloaded: try: if pipeline_csv == "column.csv": + if geom_type: + add_geom_mapping( + dataset, pipeline_dir, geom_type, resource, pipeline_csv + ) if column_mapping: not_mapped_columns = add_extra_column_mappings( csv_path, @@ -353,14 +358,9 @@ def fetch_pipeline_csvs( resource, specification, ) - return not_mapped_columns - if geom_type: - add_geom_mapping( - dataset, pipeline_dir, geom_type, resource, pipeline_csv - ) except Exception as e: logger.error(f"Error saving new csv mapping: {e}") - return {} + return not_mapped_columns def add_geom_mapping(dataset, pipeline_dir, geom_type, resource, pipeline_csv): @@ -740,6 +740,10 @@ def fetch_add_data_pipeline_csvs( else: column_csv_path = os.path.join(pipeline_dir, "column.csv") try: + if geom_type and resource and dataset: + add_geom_mapping( + dataset, pipeline_dir, geom_type, resource, "column.csv" + ) if column_mapping and resource and dataset and specification: add_extra_column_mappings( column_csv_path, @@ -749,10 +753,6 @@ def fetch_add_data_pipeline_csvs( specification, endpoint_hash=endpoint_hash, ) - elif geom_type and resource and dataset: - add_geom_mapping( - dataset, pipeline_dir, geom_type, resource, "column.csv" - ) except Exception as e: logger.error(f"Error saving column mappings to column.csv: {e}") return True @@ -769,6 +769,10 @@ def fetch_add_data_pipeline_csvs( if csv_name == "column.csv": try: + if geom_type and resource and dataset: + add_geom_mapping( + dataset, pipeline_dir, geom_type, resource, csv_name + ) if column_mapping and resource and dataset and specification: add_extra_column_mappings( csv_path, @@ -778,10 +782,6 @@ def fetch_add_data_pipeline_csvs( specification, endpoint_hash=endpoint_hash, ) - elif geom_type and resource and dataset: - add_geom_mapping( - dataset, pipeline_dir, geom_type, resource, csv_name - ) except Exception as e: logger.error(f"Error saving column mappings to column.csv: {e}") diff --git a/request-processor/tests/conftest.py b/request-processor/tests/conftest.py index 1c5b2dc..1d192d8 100644 --- a/request-processor/tests/conftest.py +++ b/request-processor/tests/conftest.py @@ -167,6 +167,13 @@ def _mock_extract_dataset_field_rows(dataset_name): "guidance": "", "hint": "", }, + { + "dataset": dataset_name, + "field": "start-date", + "field-dataset": "", + "guidance": "", + "hint": "", + }, ] fieldnames = rows[0].keys() with open(mock_field_csv, "w") as f: diff --git a/request-processor/tests/integration/src/application/core/test_workflow.py b/request-processor/tests/integration/src/application/core/test_workflow.py index f4c5a89..9983b1f 100644 --- a/request-processor/tests/integration/src/application/core/test_workflow.py +++ b/request-processor/tests/integration/src/application/core/test_workflow.py @@ -292,7 +292,7 @@ def test_run_workflow_brownfield_land( dataset = "brownfield-land" organisation = "local-authority:CTY" geom_type = "" - column_mapping = {"ref": "reference"} + column_mapping = {"ref": "reference", "start_date": "start-date"} fileName = uploaded_csv_brownfield_land source_organisation_csv = f"{test_data_dir}/csvs/organisation.csv" destination_organisation_csv = os.path.join( diff --git a/request-processor/tests/unit/src/application/core/test_workflow.py b/request-processor/tests/unit/src/application/core/test_workflow.py index 5886a6e..bec294c 100644 --- a/request-processor/tests/unit/src/application/core/test_workflow.py +++ b/request-processor/tests/unit/src/application/core/test_workflow.py @@ -588,6 +588,58 @@ def raise_http_error(url, path): assert os.path.exists(pipeline_dir_str) +def test_fetch_add_data_pipeline_csvs_applies_geom_and_column_mapping( + monkeypatch, tmp_path +): + collection = "tree-collection" + pipeline_dir = tmp_path / "pipeline" + monkeypatch.setattr( + "src.application.core.workflow.CONFIG_URL", "http://example.com/config/" + ) + + def fake_download_file(url, path): + if path.endswith("column.csv"): + _write_column_csv( + path, + [ + { + "dataset": "tree", + "resource": "resource-hash", + "column": "id", + "field": "reference", + } + ], + ) + else: + with open(path, "w") as f: + f.write("dummy data") + + monkeypatch.setattr( + "src.application.core.workflow.download_file", fake_download_file + ) + mock_spec = MagicMock(dataset_field={"tree": ["geometry", "reference"]}) + + fetch_add_data_pipeline_csvs( + collection, + str(pipeline_dir), + column_mapping={"na": "IGNORE"}, + geom_type="polygon", + resource="resource-hash", + dataset="tree", + specification=mock_spec, + ) + + with open(pipeline_dir / "column.csv", newline="") as f: + rows = list(csv.DictReader(f)) + + assert { + "dataset": "tree", + "resource": "resource-hash", + "column": "WKT", + "field": "geometry", + } in rows + + def test_download_file_does_not_add_auth_without_github_env(monkeypatch, tmp_path): destination = tmp_path / "column.csv" requests = []