Fix column mapping for tree polygons and enhance tests for geometry h…

request-processor/src/application/core/workflow.py

@@ -314,8 +314,9 @@ def fetch_pipeline_csvs(
 ):
     os.makedirs(pipeline_dir, exist_ok=True)
     pipeline_csvs = ["column.csv", "transform.csv"]
-    downloaded = False
+    not_mapped_columns = []
     for pipeline_csv in pipeline_csvs:
+        downloaded = False
         try:
             csv_path = os.path.join(pipeline_dir, pipeline_csv)
             print(
@@ -345,6 +346,10 @@ def fetch_pipeline_csvs(
         if downloaded:
             try:
                 if pipeline_csv == "column.csv":
+                    if geom_type:
+                        add_geom_mapping(
+                            dataset, pipeline_dir, geom_type, resource, pipeline_csv
+                        )
                     if column_mapping:
                         not_mapped_columns = add_extra_column_mappings(
                             csv_path,
@@ -353,14 +358,9 @@ def fetch_pipeline_csvs(
                             resource,
                             specification,
                         )
-                        return not_mapped_columns
-                    if geom_type:
-                        add_geom_mapping(
-                            dataset, pipeline_dir, geom_type, resource, pipeline_csv
-                        )
             except Exception as e:
-                logger.error(f"Error saving new mapping: {e}")
-    return {}
+                logger.error(f"Error saving new csv mapping: {e}")
+    return not_mapped_columns
 
 
 def add_geom_mapping(dataset, pipeline_dir, geom_type, resource, pipeline_csv):
@@ -740,6 +740,10 @@ def fetch_add_data_pipeline_csvs(
         else:
             column_csv_path = os.path.join(pipeline_dir, "column.csv")
             try:
+                if geom_type and resource and dataset:
+                    add_geom_mapping(
+                        dataset, pipeline_dir, geom_type, resource, "column.csv"
+                    )
                 if column_mapping and resource and dataset and specification:
                     add_extra_column_mappings(
                         column_csv_path,
@@ -749,10 +753,6 @@ def fetch_add_data_pipeline_csvs(
                         specification,
                         endpoint_hash=endpoint_hash,
                     )
-                elif geom_type and resource and dataset:
-                    add_geom_mapping(
-                        dataset, pipeline_dir, geom_type, resource, "column.csv"
-                    )
             except Exception as e:
                 logger.error(f"Error saving column mappings to column.csv: {e}")
             return True
@@ -769,6 +769,10 @@ def fetch_add_data_pipeline_csvs(
 
         if csv_name == "column.csv":
             try:
+                if geom_type and resource and dataset:
+                    add_geom_mapping(
+                        dataset, pipeline_dir, geom_type, resource, csv_name
+                    )
                 if column_mapping and resource and dataset and specification:
                     add_extra_column_mappings(
                         csv_path,
@@ -778,10 +782,6 @@ def fetch_add_data_pipeline_csvs(
                         specification,
                         endpoint_hash=endpoint_hash,
                     )
-                elif geom_type and resource and dataset:
-                    add_geom_mapping(
-                        dataset, pipeline_dir, geom_type, resource, csv_name
-                    )
             except Exception as e:
                 logger.error(f"Error saving column mappings to column.csv: {e}")
 

request-processor/tests/conftest.py

@@ -167,6 +167,13 @@ def _mock_extract_dataset_field_rows(dataset_name):
                 "guidance": "",
                 "hint": "",
             },
+            {
+                "dataset": dataset_name,
+                "field": "start-date",
+                "field-dataset": "",
+                "guidance": "",
+                "hint": "",
+            },
         ]
         fieldnames = rows[0].keys()
         with open(mock_field_csv, "w") as f:

request-processor/tests/integration/src/application/core/test_workflow.py

@@ -292,7 +292,7 @@ def test_run_workflow_brownfield_land(
     dataset = "brownfield-land"
     organisation = "local-authority:CTY"
     geom_type = ""
-    column_mapping = {"ref": "reference"}
+    column_mapping = {"ref": "reference", "start_date": "start-date"}
     fileName = uploaded_csv_brownfield_land
     source_organisation_csv = f"{test_data_dir}/csvs/organisation.csv"
     destination_organisation_csv = os.path.join(

request-processor/tests/unit/src/application/core/test_workflow.py

@@ -588,6 +588,58 @@ def raise_http_error(url, path):
     assert os.path.exists(pipeline_dir_str)
 
 
+def test_fetch_add_data_pipeline_csvs_applies_geom_and_column_mapping(
+    monkeypatch, tmp_path
+):
+    collection = "tree-collection"
+    pipeline_dir = tmp_path / "pipeline"
+    monkeypatch.setattr(
+        "src.application.core.workflow.CONFIG_URL", "http://example.com/config/"
+    )
+
+    def fake_download_file(url, path):
+        if path.endswith("column.csv"):
+            _write_column_csv(
+                path,
+                [
+                    {
+                        "dataset": "tree",
+                        "resource": "resource-hash",
+                        "column": "id",
+                        "field": "reference",
+                    }
+                ],
+            )
+        else:
+            with open(path, "w") as f:
+                f.write("dummy data")
+
+    monkeypatch.setattr(
+        "src.application.core.workflow.download_file", fake_download_file
+    )
+    mock_spec = MagicMock(dataset_field={"tree": ["geometry", "reference"]})
+
+    fetch_add_data_pipeline_csvs(
+        collection,
+        str(pipeline_dir),
+        column_mapping={"na": "IGNORE"},
+        geom_type="polygon",
+        resource="resource-hash",
+        dataset="tree",
+        specification=mock_spec,
+    )
+
+    with open(pipeline_dir / "column.csv", newline="") as f:
+        rows = list(csv.DictReader(f))
+
+    assert {
+        "dataset": "tree",
+        "resource": "resource-hash",
+        "column": "WKT",
+        "field": "geometry",
+    } in rows
+
+
 def test_download_file_does_not_add_auth_without_github_env(monkeypatch, tmp_path):
     destination = tmp_path / "column.csv"
     requests = []