diff --git a/application/blueprints/datamanager/controllers/preview.py b/application/blueprints/datamanager/controllers/preview.py index 726b9e6..8dd8ad2 100644 --- a/application/blueprints/datamanager/controllers/preview.py +++ b/application/blueprints/datamanager/controllers/preview.py @@ -99,6 +99,29 @@ def _load_json_list(value: str | None) -> list: return loaded if isinstance(loaded, list) else [] +def _build_endpoint_summary( + selected_hashes, existing_endpoints, dataset_id, organisation_code +): + summary = [] + if not selected_hashes: + return summary + dataset_display = get_dataset_name(dataset_id, default=dataset_id) + org_display = get_organisation_name(organisation_code) + for ep in existing_endpoints: + ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep + ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep + if ep_hash in selected_hashes: + summary.append( + { + "endpoint": ep_hash, + "endpoint-url": ep_url, + "dataset": dataset_display, + "organisation": org_display, + } + ) + return summary + + def _count_excluded_references(params: dict) -> int: references = params.get("excluded_references") or [] if not isinstance(references, list): @@ -237,6 +260,9 @@ def handle_entities_preview(request_id, req): endpoints_to_retire = ( _load_json_list(request_meta.endpoints_to_retire) if request_meta else [] ) + endpoints_to_unretire = ( + _load_json_list(request_meta.endpoints_to_unretire) if request_meta else [] + ) old_entity_rows = pipeline_summary.get("old-entity") or [] old_entity_redirect_table_params = build_old_entity_redirect_table(old_entity_rows) old_entity_redirect_count = ( @@ -250,22 +276,13 @@ def handle_entities_preview(request_id, req): if isinstance(existing_endpoints, str): existing_endpoints = [existing_endpoints] if existing_endpoints else [] organisation_code = params.get("organisationName") or params.get("organisation", "") - retire_summary = [] - if endpoints_to_retire: - dataset_display = get_dataset_name(dataset_id, default=dataset_id) - org_display = get_organisation_name(organisation_code) - for ep in existing_endpoints: - ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep - ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep - if ep_hash in endpoints_to_retire: - retire_summary.append( - { - "endpoint": ep_hash, - "endpoint-url": ep_url, - "dataset": dataset_display, - "organisation": org_display, - } - ) + + retire_summary = _build_endpoint_summary( + endpoints_to_retire, existing_endpoints, dataset_id, organisation_code + ) + unretire_summary = _build_endpoint_summary( + endpoints_to_unretire, existing_endpoints, dataset_id, organisation_code + ) # Build entity-organisation CSV preview authoritative = params.get("authoritative", False) @@ -286,6 +303,7 @@ def handle_entities_preview(request_id, req): source_flow=source_flow, return_url=return_url, retire_summary=retire_summary, + unretire_summary=unretire_summary, old_entity_redirect_table_params=old_entity_redirect_table_params, old_entity_redirect_count=old_entity_redirect_count, new_count=len(new_entities), @@ -318,6 +336,9 @@ def handle_add_data_confirm( endpoints_to_retire = ( _load_json_list(request_meta.endpoints_to_retire) if request_meta else [] ) + endpoints_to_unretire = ( + _load_json_list(request_meta.endpoints_to_unretire) if request_meta else [] + ) # Stale-assessment guard: if the config branch has advanced for this collection # since the assessment was taken, the assigned entity numbers may now collide. @@ -364,6 +385,7 @@ def handle_add_data_confirm( triggered_by=f"{session.get('user', {}).get('login', 'unknown')}", github_branch=github_branch, endpoints_to_retire=endpoints_to_retire, + endpoints_to_unretire=endpoints_to_unretire, ) except GitHubWorkflowError as e: logger.exception(f"GitHub async workflow error: {e}") diff --git a/application/blueprints/datamanager/controllers/transform.py b/application/blueprints/datamanager/controllers/transform.py index f2645d5..3f4b67d 100644 --- a/application/blueprints/datamanager/controllers/transform.py +++ b/application/blueprints/datamanager/controllers/transform.py @@ -8,11 +8,15 @@ from shapely.geometry import mapping from . import ControllerError +from application.utils import compute_hash from ..services.async_api import fetch_response_details from ..services.dataset import get_dataset_name, get_dataset_typology from ..services.organisation import get_org_entity, get_organisation_name from ..services.doc_crawler import check_endpoint_in_doc, is_gov_uk_url -from ..services.endpoint import get_endpoint_urls_for_hashes +from ..services.endpoint import ( + get_endpoint_log_summary_for_hashes, + get_endpoint_info_for_hashes, +) from ..services.planning_data import ( get_entities_for_organisation_and_dataset, get_entity_count_for_organisation_and_dataset, @@ -407,22 +411,48 @@ def _count_categories(rows: list) -> dict: return counts -def _resolve_existing_endpoints(source_summary: dict) -> list: +def _date_only(date_str: str) -> str: + """Truncate an ISO datetime (``YYYY-MM-DDThh:mm:ssZ``) to ``YYYY-MM-DD``.""" + return date_str[:10] if date_str else "" + + +def _resolve_existing_endpoints( + source_summary: dict, current_endpoint_url: str = "" +) -> list: existing_endpoints = ( source_summary.get("existing_endpoint_for_organisation_dataset") or [] ) if isinstance(existing_endpoints, str): existing_endpoints = [existing_endpoints] if existing_endpoints else [] - if existing_endpoints: - endpoint_data = get_endpoint_urls_for_hashes(existing_endpoints) - existing_endpoints = [ - { - "endpoint": h, - "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""), - "end-date": endpoint_data.get(h, {}).get("end_date", ""), - } - for h in existing_endpoints - ] + # The same endpoint hash can appear on more than one source.csv row for an + # org/dataset; retiring matches on the hash so it actions every row at once. + # Collapse duplicates so the retire table shows one row per endpoint. + existing_endpoints = list(dict.fromkeys(existing_endpoints)) + if not existing_endpoints: + return existing_endpoints + + endpoint_data = get_endpoint_info_for_hashes(existing_endpoints) + log_data = get_endpoint_log_summary_for_hashes(existing_endpoints) + # The endpoint hash is sha256(endpoint_url), so this matches the endpoint + # being added when it is already present in the endpoint CSV. + current_hash = compute_hash(current_endpoint_url) if current_endpoint_url else None + + existing_endpoints = [ + { + "endpoint": h, + "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""), + "entry-date": _date_only(endpoint_data.get(h, {}).get("entry_date", "")), + "end-date": _date_only(endpoint_data.get(h, {}).get("end_date", "")), + "latest-status": log_data.get(h, {}).get("latest_status", ""), + "latest-log-entry-date": _date_only( + log_data.get(h, {}).get("latest_log_entry_date", "") + ), + "is_retired": bool(endpoint_data.get(h, {}).get("end_date", "")), + "is_current": h == current_hash, + } + for h in existing_endpoints + ] + existing_endpoints.sort(key=lambda e: e["entry-date"] or "", reverse=True) return existing_endpoints @@ -779,7 +809,7 @@ def handle_check_transform( response_payload = req.get("response") or {} response_data = response_payload.get("data") or {} source_summary = response_data.get("source-summary") or {} - existing_endpoints = _resolve_existing_endpoints(source_summary) + existing_endpoints = _resolve_existing_endpoints(source_summary, endpoint_url) pipelines_append_required = source_summary.get("pipelines_append_required") pipeline_summary = response_data.get("pipeline-summary") or {} show_dedup_tab = is_assign_entities and dataset_id == "conservation-area" diff --git a/application/blueprints/datamanager/router.py b/application/blueprints/datamanager/router.py index a7445ad..5a60db1 100644 --- a/application/blueprints/datamanager/router.py +++ b/application/blueprints/datamanager/router.py @@ -16,6 +16,7 @@ from application.blueprints.base.views import ADD_DATA_LOCK, ASSIGN_ENTITIES_LOCK from application.db.models import RequestMeta, ServiceLock from application.extensions import db +from application.utils import compute_hash from .controllers.form import ( handle_dashboard_get, @@ -274,20 +275,45 @@ def check_transform(request_id): def check_transform_post(request_id): - """Store selected endpoints to retire from transform page and continue to preview.""" - hashes = request.form.getlist("retire_endpoints") + """Store selected endpoints to retire/unretire from the transform page.""" + checked = request.form.getlist("retire_endpoints") + presented = request.form.getlist("presented_endpoints") + currently_retired = request.form.getlist("currently_retired") + + # Never allow the endpoint being added (already in the CSV) to be changed, + current_hash = _current_endpoint_hash(request_id) + presented = [h for h in presented if h != current_hash] + + # to_retire = presented and checked and not currently retired → newly retiring + # to_unretire = currently retired and presented and now unchecked → unretiring + to_retire = [h for h in presented if h in checked and h not in currently_retired] + to_unretire = [h for h in currently_retired if h in presented and h not in checked] + meta = db.session.get(RequestMeta, request_id) if meta is None: meta = RequestMeta( - request_id=request_id, endpoints_to_retire=json.dumps(hashes) + request_id=request_id, + endpoints_to_retire=json.dumps(to_retire), + endpoints_to_unretire=json.dumps(to_unretire), ) db.session.add(meta) else: - meta.endpoints_to_retire = json.dumps(hashes) + meta.endpoints_to_retire = json.dumps(to_retire) + meta.endpoints_to_unretire = json.dumps(to_unretire) db.session.commit() return redirect(url_for("datamanager.entities_preview", request_id=request_id)) +def _current_endpoint_hash(request_id): + """sha256 of the endpoint URL being added, or None if it can't be resolved.""" + try: + req = fetch_request(request_id) + except AsyncAPIError: + return None + url = (req.get("params") or {}).get("url", "") + return compute_hash(url) if url else None + + def add_data_confirm_async(request_id): logger.info(f"Triggering async GitHub workflow for request_id: {request_id}") github_branch = request.form.get("github_branch") or None diff --git a/application/blueprints/datamanager/services/endpoint.py b/application/blueprints/datamanager/services/endpoint.py index cedeb1e..ee295a3 100644 --- a/application/blueprints/datamanager/services/endpoint.py +++ b/application/blueprints/datamanager/services/endpoint.py @@ -8,9 +8,10 @@ logger = logging.getLogger(__name__) -def get_endpoint_urls_for_hashes(hashes: list) -> dict: +def get_endpoint_info_for_hashes(hashes: list) -> dict: """ - Given a list of endpoint hashes, returns a dict mapping {hash: endpoint_url} + Given a list of endpoint hashes, returns a dict mapping + {hash: {"endpoint_url": ..., "entry_date": ..., "end_date": ...}} by querying the datasette endpoint table. """ if not hashes: @@ -34,9 +35,53 @@ def get_endpoint_urls_for_hashes(hashes: list) -> dict: if h: result[h] = { "endpoint_url": row.get("endpoint_url", ""), + "entry_date": row.get("entry_date") or "", "end_date": row.get("end_date") or "", } except Exception as e: logger.error(f"Failed to fetch endpoint URLs for hashes: {e}", exc_info=True) return result + + +def get_endpoint_log_summary_for_hashes(hashes: list) -> dict: + """ + Given a list of endpoint hashes, returns a dict mapping + {hash: {"latest_status": ..., "latest_log_entry_date": ...}}. + """ + if not hashes: + return {} + + datasette_url = current_app.config.get("DATASETTE_BASE_URL") + datasette_root = datasette_url.rsplit("/", 1)[0] + url = ( + f"{datasette_root}/performance/reporting_historic_endpoints.json" + f"?endpoint__in={','.join(hashes)}" + f"&_shape=objects" + f"&_size=max" + ) + + result = {} + try: + response = requests.get(url, timeout=REQUESTS_TIMEOUT) + response.raise_for_status() + data = response.json() + for row in data.get("rows", []): + h = row.get("endpoint") + if not h: + continue + log_date = row.get("latest_log_entry_date") or "" + existing = result.get(h) + # Keep the most recent record per endpoint (dates are ISO, so string + # comparison orders them correctly). + if existing is None or log_date > existing["latest_log_entry_date"]: + result[h] = { + "latest_status": row.get("latest_status") or "", + "latest_log_entry_date": log_date, + } + except Exception as e: + logger.error( + f"Failed to fetch endpoint log summary for hashes: {e}", exc_info=True + ) + + return result diff --git a/application/blueprints/datamanager/services/github.py b/application/blueprints/datamanager/services/github.py index fff8eb1..751b9fc 100644 --- a/application/blueprints/datamanager/services/github.py +++ b/application/blueprints/datamanager/services/github.py @@ -277,7 +277,8 @@ def trigger_add_data_async_workflow( request_id: str, triggered_by: str = "config-manager", github_branch: str = None, - endpoints_to_retire: list = None, + endpoints_to_retire: list[str] | None = None, + endpoints_to_unretire: list[str] | None = None, ) -> dict: """ Trigger the 'add-data-async-script' workflow in the digital-land/config repository. @@ -297,6 +298,11 @@ def trigger_add_data_async_workflow( "retire_endpoints": ( ",".join(endpoints_to_retire or []) if endpoints_to_retire else "" ), + "unretire_endpoints": ( + ",".join(endpoints_to_unretire or []) + if endpoints_to_unretire + else "" + ), "environment": current_app.config.get("ENVIRONMENT"), }, } diff --git a/application/db/models.py b/application/db/models.py index 5adf74a..14024ce 100644 --- a/application/db/models.py +++ b/application/db/models.py @@ -18,6 +18,9 @@ class RequestMeta(db.Model): endpoints_to_retire = db.Column( db.Text, nullable=True ) # JSON list of endpoint hashes + endpoints_to_unretire = db.Column( + db.Text, nullable=True + ) # JSON list of endpoint hashes to clear the end-date for branch_sha = db.Column( db.Text, nullable=True ) # config-manager-update HEAD SHA when the assessment was submitted diff --git a/application/templates/datamanager/check-transform.html b/application/templates/datamanager/check-transform.html index e9bf044..0f36c67 100644 --- a/application/templates/datamanager/check-transform.html +++ b/application/templates/datamanager/check-transform.html @@ -40,24 +40,49 @@ ▼
-

Select any existing endpoints to retire when adding this data:

-
- {% for ep in existing_endpoints %} - {% set ep_hash = ep.endpoint if ep is mapping else ep %} - {% set ep_url = ep['endpoint-url'] if ep is mapping else ep %} - {% set ep_end_date = ep['end-date'] if ep is mapping else "" %} -
- - -
- {% endfor %} -
+

Tick an endpoint to retire it when adding this data, or untick an already-retired endpoint to unretire it:

+ + + + + + + + + + + + {% for ep in existing_endpoints %} + {% set disabled = ep['is_current'] %} + + + + + + + + {% endfor %} + +
RetireEndpoint URLEntry dateLatest statusLatest log entry date
+
+
+ + +
+
+ {% if not disabled %} + + {% if ep['is_retired'] %} + + Currently retired {{ ep['end-date'] }} + {% endif %} + {% else %} + Endpoint being added + {% endif %} + 		+ {{ ep['endpoint-url'] }} + {{ ep.endpoint }} + {{ ep['entry-date'] or "—" }}{{ ep['latest-status'] or "—" }}{{ ep['latest-log-entry-date'] or "—" }}
diff --git a/application/templates/datamanager/entities_preview.html b/application/templates/datamanager/entities_preview.html index ae86b66..01e6aed 100644 --- a/application/templates/datamanager/entities_preview.html +++ b/application/templates/datamanager/entities_preview.html @@ -205,6 +205,33 @@

Retire Summary

{% endif %} + + {% if unretire_summary %} +
+

Unretire Summary

+
+
+
+
+
Dataset
+
{{ unretire_summary[0].dataset }}
+
+
+
Organisation
+
{{ unretire_summary[0].organisation }}
+
+ {% for ep in unretire_summary %} +
+
Endpoint to unretire
+
{{ ep['endpoint-url'] or ep.endpoint }}
+
+ {% endfor %} +
+
+
+
+ {% endif %} + {% if old_entity_redirect_table_params %}
diff --git a/docs/datamanager/add-data.md b/docs/datamanager/add-data.md index 2ca2b7f..f36806e 100644 --- a/docs/datamanager/add-data.md +++ b/docs/datamanager/add-data.md @@ -61,10 +61,14 @@ redirects to check-transform. ### 4. Check transform (`datamanager.check_transform`, `controllers/transform.py`) `GET /check-transform/` renders `check-transform-loading.html` then -`check-transform.html`: the transformed facts, issue logs, and an entity-growth view, plus the -option to select **endpoints to retire**. `POST /check-transform/` (`check_transform_post`, -`router.py`) stores the chosen `retire_endpoints` on the `RequestMeta` row and redirects to the -entities preview. +`check-transform.html`: the transformed facts, issue logs, and an entity-growth view, plus a +**retire/unretire** table of the org/dataset's existing endpoints (endpoint URL, entry date, +latest status and latest log entry date — enriched from the `endpoint` table and +`performance/reporting_historic_endpoints`). Each checkbox reflects the desired *retired* state: +ticking an active endpoint retires it, unticking a currently-retired one unretires it; the +endpoint being added is shown disabled and cannot be changed. `POST /check-transform/` +(`check_transform_post`, `router.py`) diffs the submission and stores `retire_endpoints` / +`endpoints_to_unretire` on the `RequestMeta` row, then redirects to the entities preview. > **Scope has grown.** This page started as a home for optional pre-commit *actions* (notably > selecting endpoints to retire), but has since become a fully-fledged **comparison of the entities @@ -119,9 +123,9 @@ landing page). The `datamanager` before-request guard redirects to the landing p left behind. - **`authoritative` must be `yes`/`no`.** It is validated on the form and decides whether `entity-organisation.csv` rows are written by the commit workflow. -- **`retire_endpoints` is stored, not applied.** The check-transform POST saves the selected hashes - on the `RequestMeta` row; the actual end-dating happens later in the commit workflow, not in - config-manager. +- **`retire_endpoints` / `endpoints_to_unretire` are stored, not applied.** The check-transform POST + saves the selected hashes on the `RequestMeta` row; the actual end-dating (retire) and end-date + clearing (unretire) happen later in the commit workflow, not in config-manager. - **The stale guard can silently no-op.** It only runs when submitting onto the shared branch *and* a baseline was captured at submission — see [github-add.md](github-add.md#stale-assessment-guard). diff --git a/docs/datamanager/architecture.md b/docs/datamanager/architecture.md index f2a40de..7a18e35 100644 --- a/docs/datamanager/architecture.md +++ b/docs/datamanager/architecture.md @@ -69,7 +69,7 @@ Controllers receive a request context and orchestrate the workflow: validate inp #### `RequestMeta` and `source_flow` -The entities preview and confirm routes live under `/datamanager` but are reused by the assign-entities flow (which redirects into them rather than having its own copies). Because those two endpoints are shared, the URL prefix alone can't tell which process lock applies. So at submission time each flow records `source_flow` (`"add_data"` / `"assign_entities"`) on `RequestMeta` via `record_source_flow`; the router's lock guard and the preview render then read it back — the single source of truth for which flow a request belongs to. +The entities preview and confirm routes live under `/datamanager` but are reused by the assign-entities flow (which redirects into them rather than having its own copies). Because those two endpoints are shared, the URL prefix alone can't tell which process lock applies. So at submission time each flow records `source_flow` (`"add_data"` / `"assign_entities"`) on `RequestMeta` via `record_source_flow`; the router's lock guard and the preview render then read it back — the single source of truth for which flow a request belongs to. `RequestMeta` also carries the check-transform retire/unretire selections (`endpoints_to_retire` / `endpoints_to_unretire`) that the async request's params can't — stored on POST and forwarded to the commit workflow. #### `ControllerError` diff --git a/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py new file mode 100644 index 0000000..7dc3071 --- /dev/null +++ b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py @@ -0,0 +1,26 @@ +"""add request_meta endpoints_to_unretire column + +Revision ID: b8c9d0e1f2a3 +Revises: b9c0d1e2f3a4 +Create Date: 2026-07-27 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +revision = "b8c9d0e1f2a3" +down_revision = "b9c0d1e2f3a4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "request_meta", + sa.Column("endpoints_to_unretire", sa.Text(), nullable=True), + ) + + +def downgrade(): + op.drop_column("request_meta", "endpoints_to_unretire") diff --git a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py index e941ff1..14f5d89 100644 --- a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py +++ b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py @@ -542,13 +542,17 @@ def test_assign_entities_check_results_does_not_show_retire_endpoints(client): transform_controller = "application.blueprints.datamanager.controllers.transform" with patch( - f"{transform_controller}.get_endpoint_urls_for_hashes", + f"{transform_controller}.get_endpoint_info_for_hashes", return_value={ "endpoint-a": { "endpoint_url": "https://example.com/data.csv", + "entry_date": "2026-01-01", "end_date": "", } }, + ), patch( + f"{transform_controller}.get_endpoint_log_summary_for_hashes", + return_value={}, ): with patch(f"{transform_controller}.get_org_entity", return_value=90): with patch(f"{transform_controller}.get_organisation_name"): diff --git a/tests/integration/blueprints/datamanager/test_datamanager_integration.py b/tests/integration/blueprints/datamanager/test_datamanager_integration.py index 7ff47f6..b745d98 100644 --- a/tests/integration/blueprints/datamanager/test_datamanager_integration.py +++ b/tests/integration/blueprints/datamanager/test_datamanager_integration.py @@ -1,3 +1,4 @@ +import json from unittest.mock import patch import responses as rsps @@ -1097,3 +1098,49 @@ def test_entity_showing_text_uses_entity_ids(self, client): ): response = client.get("/datamanager/check-transform/test-id") assert b"Showing entities 7010000100 to 7010000109" in response.data + + +class TestCheckTransformPostRetireUnretire: + @rsps.activate + def test_diffs_retire_and_unretire_and_protects_current(self, client): + from application.db.models import RequestMeta + from application.extensions import db + from application.utils import compute_hash + + current_url = "https://example.com/current.csv" + current_hash = compute_hash(current_url) + rsps.add( + rsps.GET, + f"{ASYNC_BASE}/req-diff", + json={ + **COMPLETED_TRANSFORM_REQUEST, + "id": "req-diff", + "params": {"url": current_url}, + }, + status=200, + ) + + # active endpoint "hash-active" ticked -> retire + # retired endpoint "hash-retired" left unticked -> unretire + # current endpoint slipped into the checked list -> must be ignored + from werkzeug.datastructures import MultiDict + + response = client.post( + "/datamanager/check-transform/req-diff", + data=MultiDict( + [ + ("presented_endpoints", "hash-active"), + ("presented_endpoints", "hash-retired"), + ("presented_endpoints", current_hash), + ("currently_retired", "hash-retired"), + ("retire_endpoints", "hash-active"), + ("retire_endpoints", current_hash), + ] + ), + ) + assert response.status_code == 302 + + with client.application.app_context(): + meta = db.session.get(RequestMeta, "req-diff") + assert json.loads(meta.endpoints_to_retire) == ["hash-active"] + assert json.loads(meta.endpoints_to_unretire) == ["hash-retired"] diff --git a/tests/unit/blueprints/datamanager/controllers/test_transform.py b/tests/unit/blueprints/datamanager/controllers/test_transform.py index 8b090ec..65a8f05 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_transform.py +++ b/tests/unit/blueprints/datamanager/controllers/test_transform.py @@ -1,10 +1,15 @@ import json +from unittest.mock import patch +from application.utils import compute_hash from application.blueprints.datamanager.controllers.transform import ( _dedup_candidate_form_value, _prepare_duplicate_candidates, + _resolve_existing_endpoints, ) +TRANSFORM_MODULE = "application.blueprints.datamanager.controllers.transform" + def test_dedup_candidate_form_value_builds_redirect_payload(): form_value = _dedup_candidate_form_value( @@ -152,3 +157,55 @@ def test_prepare_duplicate_candidates_disables_redirects_for_excluded_references assert candidates[0]["redirect_can_select"] is False assert candidates[1]["redirect_can_select"] is True + + +def test_resolve_existing_endpoints_enriches_sorts_and_flags(): + current_url = "https://example.com/current.csv" + current_hash = compute_hash(current_url) + source_summary = { + "existing_endpoint_for_organisation_dataset": [ + "hash-old", + current_hash, + "hash-new", + ] + } + endpoint_data = { + "hash-old": { + "endpoint_url": "https://example.com/old.csv", + "entry_date": "2026-01-01", + "end_date": "2026-06-01", + }, + current_hash: { + "endpoint_url": current_url, + "entry_date": "2026-03-01", + "end_date": "", + }, + "hash-new": { + "endpoint_url": "https://example.com/new.csv", + "entry_date": "2026-05-01", + "end_date": "", + }, + } + log_data = { + "hash-new": { + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + } + } + + with patch( + f"{TRANSFORM_MODULE}.get_endpoint_info_for_hashes", return_value=endpoint_data + ), patch( + f"{TRANSFORM_MODULE}.get_endpoint_log_summary_for_hashes", return_value=log_data + ): + result = _resolve_existing_endpoints(source_summary, current_url) + + # Sorted by entry-date desc: hash-new (05-01), current (03-01), hash-old (01-01) + assert [r["endpoint"] for r in result] == ["hash-new", current_hash, "hash-old"] + + by_hash = {r["endpoint"]: r for r in result} + assert by_hash["hash-old"]["is_retired"] is True + assert by_hash["hash-old"]["is_current"] is False + assert by_hash[current_hash]["is_current"] is True + assert by_hash["hash-new"]["latest-status"] == "200" + assert by_hash["hash-new"]["latest-log-entry-date"] == "2026-07-20" diff --git a/tests/unit/blueprints/datamanager/services/test_endpoint.py b/tests/unit/blueprints/datamanager/services/test_endpoint.py new file mode 100644 index 0000000..973f0a2 --- /dev/null +++ b/tests/unit/blueprints/datamanager/services/test_endpoint.py @@ -0,0 +1,95 @@ +from unittest.mock import MagicMock, patch + +from application.blueprints.datamanager.services.endpoint import ( + get_endpoint_log_summary_for_hashes, + get_endpoint_info_for_hashes, +) + +ENDPOINT_MODULE = "application.blueprints.datamanager.services.endpoint" + + +def _objects_response(rows): + resp = MagicMock() + resp.raise_for_status = MagicMock() + resp.json.return_value = {"rows": rows} + return resp + + +class TestGetEndpointUrlsForHashes: + def test_empty_input_returns_empty_dict(self): + assert get_endpoint_info_for_hashes([]) == {} + + def test_maps_url_entry_and_end_date(self, app): + rows = [ + { + "endpoint": "hash-a", + "endpoint_url": "https://example.com/a.csv", + "entry_date": "2026-01-01", + "end_date": "", + } + ] + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get") as mock_get: + mock_get.return_value = _objects_response(rows) + result = get_endpoint_info_for_hashes(["hash-a"]) + + assert result == { + "hash-a": { + "endpoint_url": "https://example.com/a.csv", + "entry_date": "2026-01-01", + "end_date": "", + } + } + called_url = mock_get.call_args[0][0] + assert "endpoint.json" in called_url + assert "endpoint__in=hash-a" in called_url + + def test_exception_returns_empty_dict(self, app): + with app.app_context(): + with patch( + f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("boom") + ): + assert get_endpoint_info_for_hashes(["hash-a"]) == {} + + +class TestGetEndpointLogSummaryForHashes: + def test_empty_input_returns_empty_dict(self): + assert get_endpoint_log_summary_for_hashes([]) == {} + + def test_maps_latest_row_per_endpoint(self, app): + # Two rows for the same endpoint; the most recent log date wins. + rows = [ + { + "endpoint": "hash-a", + "latest_status": "404", + "latest_log_entry_date": "2026-07-10", + }, + { + "endpoint": "hash-a", + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + }, + ] + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get") as mock_get: + mock_get.return_value = _objects_response(rows) + result = get_endpoint_log_summary_for_hashes(["hash-a", "hash-b"]) + + assert result == { + "hash-a": { + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + } + } + # A single precomputed-table lookup is issued for all hashes. + assert mock_get.call_count == 1 + called_url = mock_get.call_args[0][0] + assert "performance/reporting_historic_endpoints.json" in called_url + assert "endpoint__in=hash-a,hash-b" in called_url + + def test_exception_returns_empty_dict(self, app): + with app.app_context(): + with patch( + f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("slow") + ): + assert get_endpoint_log_summary_for_hashes(["hash-a"]) == {}