From 1d75928397c76d1364cd467806a5a86c0d414ef3 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 7 Jul 2026 15:58:57 +0100 Subject: [PATCH 1/4] preview page flags if overlap detected in entiy organiisation csv --- .../datamanager/controllers/preview.py | 87 ++++++++++++--- .../datamanager/entities_preview.html | 20 +++- .../datamanager/controllers/test_preview.py | 100 ++++++++++++++++++ 3 files changed, 190 insertions(+), 17 deletions(-) create mode 100644 tests/unit/blueprints/datamanager/controllers/test_preview.py diff --git a/application/blueprints/datamanager/controllers/preview.py b/application/blueprints/datamanager/controllers/preview.py index 846733e..ac8d189 100644 --- a/application/blueprints/datamanager/controllers/preview.py +++ b/application/blueprints/datamanager/controllers/preview.py @@ -28,6 +28,67 @@ logger = logging.getLogger(__name__) +def _build_entity_organisation_summary(new_entities, authoritative, pipeline_summary): + """ + Build entity-organisation CSV preview context - only relevant when new + entities were actually created; otherwise there is nothing to map. + + Returns (entity_org_table_params, has_entity_org, entity_org_warning, + entity_org_overlap_warning, entity_org_error_warning) + """ + entity_org_table_params = None + has_entity_org = False + entity_org_warning = None + entity_org_overlap_warning = None + entity_org_error_warning = None + + if not new_entities: + return ( + entity_org_table_params, + has_entity_org, + entity_org_warning, + entity_org_overlap_warning, + entity_org_error_warning, + ) + + if not authoritative: + entity_org_warning = "Non-authoritative data being submitted" + return ( + entity_org_table_params, + has_entity_org, + entity_org_warning, + entity_org_overlap_warning, + entity_org_error_warning, + ) + + entity_organisation_data = pipeline_summary.get("entity-organisation") or [] + if entity_organisation_data: + ( + entity_org_table_params, + has_entity_org, + ) = build_entity_organisation_csv(entity_organisation_data) + + entry = entity_organisation_data[0] + if entry.get("overlap"): + entity_org_overlap_warning = ( + "Entity org range mapping already assigned for these new " + "entities - likely a single source dataset" + ) + if entry.get("error"): + entity_org_error_warning = ( + "An error occurred creating the entity-organisation csv, " + "please re-run if you believe this is required" + ) + + return ( + entity_org_table_params, + has_entity_org, + entity_org_warning, + entity_org_overlap_warning, + entity_org_error_warning, + ) + + def handle_entities_preview(request_id, req): # Check State status = req.get("status") @@ -138,23 +199,15 @@ def handle_entities_preview(request_id, req): } ) - # Build entity-organisation CSV preview (only for authoritative data) + # Build entity-organisation CSV preview authoritative = params.get("authoritative", False) - entity_org_table_params = None - has_entity_org = False - entity_org_warning = None - - if authoritative: - entity_organisation_data = pipeline_summary.get("entity-organisation") or [] - if entity_organisation_data: - ( - entity_org_table_params, - has_entity_org, - ) = build_entity_organisation_csv(entity_organisation_data) - else: - entity_org_warning = ( - "This must be manually created currently for non-authoritative data" - ) + ( + entity_org_table_params, + has_entity_org, + entity_org_warning, + entity_org_overlap_warning, + entity_org_error_warning, + ) = _build_entity_organisation_summary(new_entities, authoritative, pipeline_summary) return render_template( "datamanager/entities_preview.html", @@ -177,6 +230,8 @@ def handle_entities_preview(request_id, req): entity_org_table_params=entity_org_table_params, has_entity_org=has_entity_org, entity_org_warning=entity_org_warning, + entity_org_overlap_warning=entity_org_overlap_warning, + entity_org_error_warning=entity_org_error_warning, ) diff --git a/application/templates/datamanager/entities_preview.html b/application/templates/datamanager/entities_preview.html index f1e6acb..c1bfde6 100644 --- a/application/templates/datamanager/entities_preview.html +++ b/application/templates/datamanager/entities_preview.html @@ -135,7 +135,7 @@

column.csv (new mappings)

{% endif %} - {% if entity_org_warning or (has_entity_org and entity_org_table_params) %} + {% if entity_org_warning or entity_org_overlap_warning or entity_org_error_warning or (has_entity_org and entity_org_table_params) %}

Entity Org Summary

@@ -149,10 +149,28 @@

entity-organisation.csv

{% else %} + {% if has_entity_org and entity_org_table_params %}
{{ table(entity_org_table_params) }}
{% endif %} + {% if entity_org_overlap_warning %} +
+ + + Warning + {{ entity_org_overlap_warning }} + +
+ {% endif %} + {% if entity_org_error_warning %} + + {% endif %} + {% endif %}
{% endif %} diff --git a/tests/unit/blueprints/datamanager/controllers/test_preview.py b/tests/unit/blueprints/datamanager/controllers/test_preview.py new file mode 100644 index 0000000..c3c5b56 --- /dev/null +++ b/tests/unit/blueprints/datamanager/controllers/test_preview.py @@ -0,0 +1,100 @@ +from application.blueprints.datamanager.controllers.preview import ( + _build_entity_organisation_summary, +) + +NEW_ENTITIES = [{"entity": "10100002", "reference": "REF001"}] + + +def test_no_new_entities_hides_section(): + result = _build_entity_organisation_summary([], True, {"entity-organisation": []}) + + assert result == (None, False, None, None, None) + + +def test_non_authoritative_is_informational_only(): + """Non-authoritative just flags the data as such - nothing needs to be created.""" + ( + table_params, + has_entity_org, + warning, + overlap_warning, + error_warning, + ) = _build_entity_organisation_summary(NEW_ENTITIES, False, {"entity-organisation": []}) + + assert table_params is None + assert has_entity_org is False + assert warning == "Non-authoritative data being submitted" + assert overlap_warning is None + assert error_warning is None + + +def test_authoritative_overlap_shows_table_and_overlap_warning(): + pipeline_summary = { + "entity-organisation": [ + { + "dataset": "nature-improvement-area", + "entity-minimum": 10100002, + "entity-maximum": 10100002, + "organisation": "government-organisation:PB202", + "overlap": True, + "error": False, + } + ] + } + + ( + table_params, + has_entity_org, + warning, + overlap_warning, + error_warning, + ) = _build_entity_organisation_summary(NEW_ENTITIES, True, pipeline_summary) + + assert has_entity_org is True + assert table_params is not None + assert warning is None + assert overlap_warning == ( + "Entity org range mapping already assigned for these new " + "entities - likely a single source dataset" + ) + assert error_warning is None + + +def test_authoritative_error_shows_table_and_error_warning(): + pipeline_summary = { + "entity-organisation": [ + { + "dataset": "nature-improvement-area", + "entity-minimum": 10100002, + "entity-maximum": 10100002, + "organisation": "government-organisation:PB202", + "overlap": False, + "error": True, + } + ] + } + + ( + table_params, + has_entity_org, + warning, + overlap_warning, + error_warning, + ) = _build_entity_organisation_summary(NEW_ENTITIES, True, pipeline_summary) + + assert has_entity_org is True + assert table_params is not None + assert warning is None + assert overlap_warning is None + assert error_warning == ( + "An error occurred creating the entity-organisation csv, " + "please re-run if you believe this is required" + ) + + +def test_authoritative_no_entity_organisation_data_hides_section(): + result = _build_entity_organisation_summary( + NEW_ENTITIES, True, {"entity-organisation": []} + ) + + assert result == (None, False, None, None, None) From 4f24b724ae3e1268bae39acfc227d31c45b9c20d Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 7 Jul 2026 16:20:06 +0100 Subject: [PATCH 2/4] Improved message of overlap detected --- .../datamanager/controllers/preview.py | 31 +++++----- .../datamanager/entities_preview.html | 26 +++----- .../datamanager/controllers/test_preview.py | 62 +++++++++++++------ 3 files changed, 66 insertions(+), 53 deletions(-) diff --git a/application/blueprints/datamanager/controllers/preview.py b/application/blueprints/datamanager/controllers/preview.py index ac8d189..238b58d 100644 --- a/application/blueprints/datamanager/controllers/preview.py +++ b/application/blueprints/datamanager/controllers/preview.py @@ -34,12 +34,12 @@ def _build_entity_organisation_summary(new_entities, authoritative, pipeline_sum entities were actually created; otherwise there is nothing to map. Returns (entity_org_table_params, has_entity_org, entity_org_warning, - entity_org_overlap_warning, entity_org_error_warning) + entity_org_overlap_info, entity_org_error_warning) """ entity_org_table_params = None has_entity_org = False entity_org_warning = None - entity_org_overlap_warning = None + entity_org_overlap_info = None entity_org_error_warning = None if not new_entities: @@ -47,7 +47,7 @@ def _build_entity_organisation_summary(new_entities, authoritative, pipeline_sum entity_org_table_params, has_entity_org, entity_org_warning, - entity_org_overlap_warning, + entity_org_overlap_info, entity_org_error_warning, ) @@ -57,34 +57,31 @@ def _build_entity_organisation_summary(new_entities, authoritative, pipeline_sum entity_org_table_params, has_entity_org, entity_org_warning, - entity_org_overlap_warning, + entity_org_overlap_info, entity_org_error_warning, ) entity_organisation_data = pipeline_summary.get("entity-organisation") or [] if entity_organisation_data: - ( - entity_org_table_params, - has_entity_org, - ) = build_entity_organisation_csv(entity_organisation_data) - entry = entity_organisation_data[0] if entry.get("overlap"): - entity_org_overlap_warning = ( - "Entity org range mapping already assigned for these new " - "entities - likely a single source dataset" - ) - if entry.get("error"): + entity_org_overlap_info = "Entity org already exists - no action needed" + elif entry.get("error"): entity_org_error_warning = ( "An error occurred creating the entity-organisation csv, " "please re-run if you believe this is required" ) + else: + ( + entity_org_table_params, + has_entity_org, + ) = build_entity_organisation_csv(entity_organisation_data) return ( entity_org_table_params, has_entity_org, entity_org_warning, - entity_org_overlap_warning, + entity_org_overlap_info, entity_org_error_warning, ) @@ -205,7 +202,7 @@ def handle_entities_preview(request_id, req): entity_org_table_params, has_entity_org, entity_org_warning, - entity_org_overlap_warning, + entity_org_overlap_info, entity_org_error_warning, ) = _build_entity_organisation_summary(new_entities, authoritative, pipeline_summary) @@ -230,7 +227,7 @@ def handle_entities_preview(request_id, req): entity_org_table_params=entity_org_table_params, has_entity_org=has_entity_org, entity_org_warning=entity_org_warning, - entity_org_overlap_warning=entity_org_overlap_warning, + entity_org_overlap_info=entity_org_overlap_info, entity_org_error_warning=entity_org_error_warning, ) diff --git a/application/templates/datamanager/entities_preview.html b/application/templates/datamanager/entities_preview.html index c1bfde6..6462c84 100644 --- a/application/templates/datamanager/entities_preview.html +++ b/application/templates/datamanager/entities_preview.html @@ -135,7 +135,7 @@

column.csv (new mappings)

{% endif %} - {% if entity_org_warning or entity_org_overlap_warning or entity_org_error_warning or (has_entity_org and entity_org_table_params) %} + {% if entity_org_warning or entity_org_overlap_info or entity_org_error_warning or (has_entity_org and entity_org_table_params) %}

Entity Org Summary

@@ -148,28 +148,20 @@

entity-organisation.csv

{{ entity_org_warning }}
- {% else %} - {% if has_entity_org and entity_org_table_params %} -
- {{ table(entity_org_table_params) }} -
- {% endif %} - {% if entity_org_overlap_warning %} -
- - - Warning - {{ entity_org_overlap_warning }} - + {% elif entity_org_overlap_info %} +
+ {{ entity_org_overlap_info }}
- {% endif %} - {% if entity_org_error_warning %} + {% elif entity_org_error_warning %} - {% endif %} + {% elif has_entity_org and entity_org_table_params %} +
+ {{ table(entity_org_table_params) }} +
{% endif %}
diff --git a/tests/unit/blueprints/datamanager/controllers/test_preview.py b/tests/unit/blueprints/datamanager/controllers/test_preview.py index c3c5b56..29dae95 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_preview.py +++ b/tests/unit/blueprints/datamanager/controllers/test_preview.py @@ -17,24 +17,23 @@ def test_non_authoritative_is_informational_only(): table_params, has_entity_org, warning, - overlap_warning, + overlap_info, error_warning, ) = _build_entity_organisation_summary(NEW_ENTITIES, False, {"entity-organisation": []}) assert table_params is None assert has_entity_org is False assert warning == "Non-authoritative data being submitted" - assert overlap_warning is None + assert overlap_info is None assert error_warning is None -def test_authoritative_overlap_shows_table_and_overlap_warning(): +def test_authoritative_overlap_shows_info_message_only(): + """Overlap is informational, not a warning, and the table is skipped.""" pipeline_summary = { "entity-organisation": [ { "dataset": "nature-improvement-area", - "entity-minimum": 10100002, - "entity-maximum": 10100002, "organisation": "government-organisation:PB202", "overlap": True, "error": False, @@ -46,27 +45,23 @@ def test_authoritative_overlap_shows_table_and_overlap_warning(): table_params, has_entity_org, warning, - overlap_warning, + overlap_info, error_warning, ) = _build_entity_organisation_summary(NEW_ENTITIES, True, pipeline_summary) - assert has_entity_org is True - assert table_params is not None + assert has_entity_org is False + assert table_params is None assert warning is None - assert overlap_warning == ( - "Entity org range mapping already assigned for these new " - "entities - likely a single source dataset" - ) + assert overlap_info == "Entity org already exists - no action needed" assert error_warning is None -def test_authoritative_error_shows_table_and_error_warning(): +def test_authoritative_error_shows_error_message_only(): + """Error skips the table too, since there's no trustworthy range to show.""" pipeline_summary = { "entity-organisation": [ { "dataset": "nature-improvement-area", - "entity-minimum": 10100002, - "entity-maximum": 10100002, "organisation": "government-organisation:PB202", "overlap": False, "error": True, @@ -78,20 +73,49 @@ def test_authoritative_error_shows_table_and_error_warning(): table_params, has_entity_org, warning, - overlap_warning, + overlap_info, error_warning, ) = _build_entity_organisation_summary(NEW_ENTITIES, True, pipeline_summary) - assert has_entity_org is True - assert table_params is not None + assert has_entity_org is False + assert table_params is None assert warning is None - assert overlap_warning is None + assert overlap_info is None assert error_warning == ( "An error occurred creating the entity-organisation csv, " "please re-run if you believe this is required" ) +def test_authoritative_no_overlap_or_error_shows_table(): + pipeline_summary = { + "entity-organisation": [ + { + "dataset": "nature-improvement-area", + "entity-minimum": 10100002, + "entity-maximum": 10100002, + "organisation": "government-organisation:PB202", + "overlap": False, + "error": False, + } + ] + } + + ( + table_params, + has_entity_org, + warning, + overlap_info, + error_warning, + ) = _build_entity_organisation_summary(NEW_ENTITIES, True, pipeline_summary) + + assert has_entity_org is True + assert table_params is not None + assert warning is None + assert overlap_info is None + assert error_warning is None + + def test_authoritative_no_entity_organisation_data_hides_section(): result = _build_entity_organisation_summary( NEW_ENTITIES, True, {"entity-organisation": []} From d93fffbfa8d0dc387853cc56e04febedab26a40a Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Thu, 9 Jul 2026 15:33:03 +0100 Subject: [PATCH 3/4] lint --- application/blueprints/datamanager/controllers/preview.py | 4 +++- tests/unit/blueprints/datamanager/controllers/test_preview.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/application/blueprints/datamanager/controllers/preview.py b/application/blueprints/datamanager/controllers/preview.py index 238b58d..988fe9e 100644 --- a/application/blueprints/datamanager/controllers/preview.py +++ b/application/blueprints/datamanager/controllers/preview.py @@ -204,7 +204,9 @@ def handle_entities_preview(request_id, req): entity_org_warning, entity_org_overlap_info, entity_org_error_warning, - ) = _build_entity_organisation_summary(new_entities, authoritative, pipeline_summary) + ) = _build_entity_organisation_summary( + new_entities, authoritative, pipeline_summary + ) return render_template( "datamanager/entities_preview.html", diff --git a/tests/unit/blueprints/datamanager/controllers/test_preview.py b/tests/unit/blueprints/datamanager/controllers/test_preview.py index 29dae95..83efff4 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_preview.py +++ b/tests/unit/blueprints/datamanager/controllers/test_preview.py @@ -19,7 +19,9 @@ def test_non_authoritative_is_informational_only(): warning, overlap_info, error_warning, - ) = _build_entity_organisation_summary(NEW_ENTITIES, False, {"entity-organisation": []}) + ) = _build_entity_organisation_summary( + NEW_ENTITIES, False, {"entity-organisation": []} + ) assert table_params is None assert has_entity_org is False From ff5123c8d29620c8cf50a44c0992d082972cb505 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Mon, 13 Jul 2026 14:42:06 +0100 Subject: [PATCH 4/4] change async prod url --- application/blueprints/datamanager/services/github-add.md | 2 +- config/config.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/blueprints/datamanager/services/github-add.md b/application/blueprints/datamanager/services/github-add.md index 2d22a65..f393dcc 100644 --- a/application/blueprints/datamanager/services/github-add.md +++ b/application/blueprints/datamanager/services/github-add.md @@ -88,7 +88,7 @@ The workflow fetches request data from: {ASYNC_API_BASE_URL}/requests/{request_id} ``` -**Default base URL:** `http://development-pub-async-api-lb-69142969.eu-west-2.elb.amazonaws.com` +**Default base URL:** `https://pub-async.development.planning.data.gov.uk` To override, set the `ASYNC_API_BASE_URL` repository variable in GitHub Settings > Secrets and variables > Actions > Variables. diff --git a/config/config.py b/config/config.py index 996feff..7648c3e 100644 --- a/config/config.py +++ b/config/config.py @@ -100,9 +100,9 @@ def get_request_api_endpoint(): mapping = { "local": "http://localhost:8000", - "development": "http://development-pub-async-api-lb-69142969.eu-west-2.elb.amazonaws.com", - "staging": "http://staging-pub-async-api-lb-12493311.eu-west-2.elb.amazonaws.com", - "production": "http://production-pub-async-api-lb-636110663.eu-west-2.elb.amazonaws.com", + "development": "https://pub-async.development.planning.data.gov.uk", + "staging": "https://pub-async.staging.planning.data.gov.uk", + "production": "https://pub-async.planning.data.gov.uk", } return mapping.get(env, mapping["local"])