From 54205a43448f85e2bd777a869a6ee1d6169397c7 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 28 Jul 2026 13:52:31 +0100 Subject: [PATCH 1/7] better context in retire endpoints and unretire option --- .../datamanager/controllers/preview.py | 54 +++++++---- .../datamanager/controllers/transform.py | 69 +++++++++++--- application/blueprints/datamanager/router.py | 39 +++++++- .../datamanager/services/endpoint.py | 54 ++++++++++- .../blueprints/datamanager/services/github.py | 6 ++ application/db/models.py | 3 + .../datamanager/check-transform.html | 63 +++++++++---- .../datamanager/entities_preview.html | 27 ++++++ ..._add_request_meta_endpoints_to_unretire.py | 26 +++++ .../datamanager/test_flagged_resources.py | 4 + .../test_datamanager_integration.py | 46 +++++++++ .../datamanager/controllers/test_transform.py | 73 ++++++++++++++ .../datamanager/services/test_endpoint.py | 94 +++++++++++++++++++ 13 files changed, 505 insertions(+), 53 deletions(-) create mode 100644 migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py create mode 100644 tests/unit/blueprints/datamanager/services/test_endpoint.py diff --git a/application/blueprints/datamanager/controllers/preview.py b/application/blueprints/datamanager/controllers/preview.py index 7d722bc..1ca4d9f 100644 --- a/application/blueprints/datamanager/controllers/preview.py +++ b/application/blueprints/datamanager/controllers/preview.py @@ -101,6 +101,29 @@ def _load_json_list(value: str | None) -> list: return loaded if isinstance(loaded, list) else [] +def _build_endpoint_summary( + selected_hashes, existing_endpoints, dataset_id, organisation_code +): + summary = [] + if not selected_hashes: + return summary + dataset_display = get_dataset_name(dataset_id, default=dataset_id) + org_display = get_organisation_name(organisation_code) + for ep in existing_endpoints: + ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep + ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep + if ep_hash in selected_hashes: + summary.append( + { + "endpoint": ep_hash, + "endpoint-url": ep_url, + "dataset": dataset_display, + "organisation": org_display, + } + ) + return summary + + def _count_excluded_references(params: dict) -> int: references = params.get("excluded_references") or [] if not isinstance(references, list): @@ -276,6 +299,9 @@ def handle_entities_preview(request_id, req): endpoints_to_retire = ( _load_json_list(request_meta.endpoints_to_retire) if request_meta else [] ) + endpoints_to_unretire = ( + _load_json_list(request_meta.endpoints_to_unretire) if request_meta else [] + ) old_entity_rows = pipeline_summary.get("old-entity") or [] old_entity_redirect_table_params = build_old_entity_redirect_table(old_entity_rows) old_entity_redirect_count = ( @@ -289,22 +315,13 @@ def handle_entities_preview(request_id, req): if isinstance(existing_endpoints, str): existing_endpoints = [existing_endpoints] if existing_endpoints else [] organisation_code = params.get("organisationName") or params.get("organisation", "") - retire_summary = [] - if endpoints_to_retire: - dataset_display = get_dataset_name(dataset_id, default=dataset_id) - org_display = get_organisation_name(organisation_code) - for ep in existing_endpoints: - ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep - ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep - if ep_hash in endpoints_to_retire: - retire_summary.append( - { - "endpoint": ep_hash, - "endpoint-url": ep_url, - "dataset": dataset_display, - "organisation": org_display, - } - ) + + retire_summary = _build_endpoint_summary( + endpoints_to_retire, existing_endpoints, dataset_id, organisation_code + ) + unretire_summary = _build_endpoint_summary( + endpoints_to_unretire, existing_endpoints, dataset_id, organisation_code + ) # Build entity-organisation CSV preview authoritative = params.get("authoritative", False) @@ -325,6 +342,7 @@ def handle_entities_preview(request_id, req): source_flow=source_flow, return_url=return_url, retire_summary=retire_summary, + unretire_summary=unretire_summary, old_entity_redirect_table_params=old_entity_redirect_table_params, old_entity_redirect_count=old_entity_redirect_count, new_count=len(new_entities), @@ -357,6 +375,9 @@ def handle_add_data_confirm( endpoints_to_retire = ( _load_json_list(request_meta.endpoints_to_retire) if request_meta else [] ) + endpoints_to_unretire = ( + _load_json_list(request_meta.endpoints_to_unretire) if request_meta else [] + ) # Stale-assessment guard: if the config branch has advanced for this collection # since the assessment was taken, the assigned entity numbers may now collide. @@ -403,6 +424,7 @@ def handle_add_data_confirm( triggered_by=f"{session.get('user', {}).get('login', 'unknown')}", github_branch=github_branch, endpoints_to_retire=endpoints_to_retire, + endpoints_to_unretire=endpoints_to_unretire, ) except GitHubWorkflowError as e: logger.exception(f"GitHub async workflow error: {e}") diff --git a/application/blueprints/datamanager/controllers/transform.py b/application/blueprints/datamanager/controllers/transform.py index f2645d5..95960c1 100644 --- a/application/blueprints/datamanager/controllers/transform.py +++ b/application/blueprints/datamanager/controllers/transform.py @@ -1,6 +1,7 @@ import json import logging import re +from datetime import date, datetime import requests from flask import current_app, render_template, request as flask_request @@ -8,11 +9,15 @@ from shapely.geometry import mapping from . import ControllerError +from application.utils import compute_hash from ..services.async_api import fetch_response_details from ..services.dataset import get_dataset_name, get_dataset_typology from ..services.organisation import get_org_entity, get_organisation_name from ..services.doc_crawler import check_endpoint_in_doc, is_gov_uk_url -from ..services.endpoint import get_endpoint_urls_for_hashes +from ..services.endpoint import ( + get_endpoint_log_summary_for_hashes, + get_endpoint_info_for_hashes, +) from ..services.planning_data import ( get_entities_for_organisation_and_dataset, get_entity_count_for_organisation_and_dataset, @@ -407,22 +412,60 @@ def _count_categories(rows: list) -> dict: return counts -def _resolve_existing_endpoints(source_summary: dict) -> list: +def _date_only(date_str: str) -> str: + """Truncate an ISO datetime (``YYYY-MM-DDThh:mm:ssZ``) to ``YYYY-MM-DD``.""" + return date_str[:10] if date_str else "" + + +def _days_since(date_str: str): + """Whole days between ``date_str`` (ISO ``YYYY-MM-DD...``) and today, or None.""" + if not date_str: + return None + try: + d = datetime.fromisoformat(date_str[:10]).date() + except ValueError: + return None + return (date.today() - d).days + + +def _resolve_existing_endpoints( + source_summary: dict, current_endpoint_url: str = "" +) -> list: existing_endpoints = ( source_summary.get("existing_endpoint_for_organisation_dataset") or [] ) if isinstance(existing_endpoints, str): existing_endpoints = [existing_endpoints] if existing_endpoints else [] - if existing_endpoints: - endpoint_data = get_endpoint_urls_for_hashes(existing_endpoints) - existing_endpoints = [ - { - "endpoint": h, - "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""), - "end-date": endpoint_data.get(h, {}).get("end_date", ""), - } - for h in existing_endpoints - ] + # The same endpoint hash can appear on more than one source.csv row for an + # org/dataset; retiring matches on the hash so it actions every row at once. + # Collapse duplicates so the retire table shows one row per endpoint. + existing_endpoints = list(dict.fromkeys(existing_endpoints)) + if not existing_endpoints: + return existing_endpoints + + endpoint_data = get_endpoint_info_for_hashes(existing_endpoints) + log_data = get_endpoint_log_summary_for_hashes(existing_endpoints) + # The endpoint hash is sha256(endpoint_url), so this matches the endpoint + # being added when it is already present in the endpoint CSV. + current_hash = compute_hash(current_endpoint_url) if current_endpoint_url else None + + existing_endpoints = [ + { + "endpoint": h, + "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""), + "entry-date": _date_only(endpoint_data.get(h, {}).get("entry_date", "")), + "end-date": _date_only(endpoint_data.get(h, {}).get("end_date", "")), + "latest-status": log_data.get(h, {}).get("latest_status", ""), + "latest-log-entry-date": _date_only( + log_data.get(h, {}).get("latest_log_entry_date", "") + ), + "days-since-200": _days_since(log_data.get(h, {}).get("latest_200_date")), + "is_retired": bool(endpoint_data.get(h, {}).get("end_date", "")), + "is_current": h == current_hash, + } + for h in existing_endpoints + ] + existing_endpoints.sort(key=lambda e: e["entry-date"] or "", reverse=True) return existing_endpoints @@ -779,7 +822,7 @@ def handle_check_transform( response_payload = req.get("response") or {} response_data = response_payload.get("data") or {} source_summary = response_data.get("source-summary") or {} - existing_endpoints = _resolve_existing_endpoints(source_summary) + existing_endpoints = _resolve_existing_endpoints(source_summary, endpoint_url) pipelines_append_required = source_summary.get("pipelines_append_required") pipeline_summary = response_data.get("pipeline-summary") or {} show_dedup_tab = is_assign_entities and dataset_id == "conservation-area" diff --git a/application/blueprints/datamanager/router.py b/application/blueprints/datamanager/router.py index 69b83fe..7725ce4 100644 --- a/application/blueprints/datamanager/router.py +++ b/application/blueprints/datamanager/router.py @@ -16,6 +16,7 @@ from application.blueprints.base.views import ADD_DATA_LOCK, ASSIGN_ENTITIES_LOCK from application.db.models import RequestMeta, ServiceLock from application.extensions import db +from application.utils import compute_hash from .controllers.form import ( handle_dashboard_get, @@ -247,20 +248,50 @@ def check_transform(request_id): def check_transform_post(request_id): - """Store selected endpoints to retire from transform page and continue to preview.""" - hashes = request.form.getlist("retire_endpoints") + """Store selected endpoints to retire/unretire from the transform page. + """ + checked = request.form.getlist("retire_endpoints") + presented = request.form.getlist("presented_endpoints") + currently_retired = request.form.getlist("currently_retired") + + # Never allow the endpoint being added (already in the CSV) to be changed, + current_hash = _current_endpoint_hash(request_id) + presented = [h for h in presented if h != current_hash] + + # to_retire = presented and checked and not currently retired → newly retiring + # to_unretire = currently retired and presented and now unchecked → unretiring + to_retire = [ + h for h in presented if h in checked and h not in currently_retired + ] + to_unretire = [ + h for h in currently_retired if h in presented and h not in checked + ] + meta = db.session.get(RequestMeta, request_id) if meta is None: meta = RequestMeta( - request_id=request_id, endpoints_to_retire=json.dumps(hashes) + request_id=request_id, + endpoints_to_retire=json.dumps(to_retire), + endpoints_to_unretire=json.dumps(to_unretire), ) db.session.add(meta) else: - meta.endpoints_to_retire = json.dumps(hashes) + meta.endpoints_to_retire = json.dumps(to_retire) + meta.endpoints_to_unretire = json.dumps(to_unretire) db.session.commit() return redirect(url_for("datamanager.entities_preview", request_id=request_id)) +def _current_endpoint_hash(request_id): + """sha256 of the endpoint URL being added, or None if it can't be resolved.""" + try: + req = fetch_request(request_id) + except AsyncAPIError: + return None + url = (req.get("params") or {}).get("url", "") + return compute_hash(url) if url else None + + def add_data_confirm_async(request_id): logger.info(f"Triggering async GitHub workflow for request_id: {request_id}") github_branch = request.form.get("github_branch") or None diff --git a/application/blueprints/datamanager/services/endpoint.py b/application/blueprints/datamanager/services/endpoint.py index cedeb1e..7cb837e 100644 --- a/application/blueprints/datamanager/services/endpoint.py +++ b/application/blueprints/datamanager/services/endpoint.py @@ -1,4 +1,5 @@ import logging +from urllib.parse import quote import requests from flask import current_app @@ -8,9 +9,10 @@ logger = logging.getLogger(__name__) -def get_endpoint_urls_for_hashes(hashes: list) -> dict: +def get_endpoint_info_for_hashes(hashes: list) -> dict: """ - Given a list of endpoint hashes, returns a dict mapping {hash: endpoint_url} + Given a list of endpoint hashes, returns a dict mapping + {hash: {"endpoint_url": ..., "entry_date": ..., "end_date": ...}} by querying the datasette endpoint table. """ if not hashes: @@ -34,9 +36,57 @@ def get_endpoint_urls_for_hashes(hashes: list) -> dict: if h: result[h] = { "endpoint_url": row.get("endpoint_url", ""), + "entry_date": row.get("entry_date") or "", "end_date": row.get("end_date") or "", } except Exception as e: logger.error(f"Failed to fetch endpoint URLs for hashes: {e}", exc_info=True) return result + + +def get_endpoint_log_summary_for_hashes(hashes: list) -> dict: + """ + Given a list of endpoint hashes, returns a dict mapping + {hash: {"latest_status": ..., "latest_log_entry_date": ..., + "latest_200_date": ...}} by querying the datasette ``log`` table. + + Warning the ``log`` table is large and slow + """ + if not hashes: + return {} + + datasette_url = current_app.config.get("DATASETTE_BASE_URL") + hash_list = ",".join(f"'{h}'" for h in hashes) + sql = ( + "SELECT l.endpoint AS endpoint, " + "l.status AS latest_status, " + "l.entry_date AS latest_log_entry_date, " + "(SELECT max(l2.entry_date) FROM log l2 " + "WHERE l2.endpoint = l.endpoint AND l2.status = '200') AS latest_200_date " + "FROM log l " + "WHERE l.rowid IN (" + f"SELECT max(rowid) FROM log WHERE endpoint IN ({hash_list}) GROUP BY endpoint)" + ) + url = f"{datasette_url}.json?sql={quote(sql)}&_shape=array&_size=max" + + result = {} + try: + response = requests.get(url, timeout=REQUESTS_TIMEOUT) + response.raise_for_status() + rows = response.json() + # _shape=array returns a list of row dicts. + for row in rows or []: + h = row.get("endpoint") + if h: + result[h] = { + "latest_status": row.get("latest_status") or "", + "latest_log_entry_date": row.get("latest_log_entry_date") or "", + "latest_200_date": row.get("latest_200_date") or "", + } + except Exception as e: + logger.error( + f"Failed to fetch endpoint log summary for hashes: {e}", exc_info=True + ) + + return result diff --git a/application/blueprints/datamanager/services/github.py b/application/blueprints/datamanager/services/github.py index fff8eb1..237aa3f 100644 --- a/application/blueprints/datamanager/services/github.py +++ b/application/blueprints/datamanager/services/github.py @@ -278,6 +278,7 @@ def trigger_add_data_async_workflow( triggered_by: str = "config-manager", github_branch: str = None, endpoints_to_retire: list = None, + endpoints_to_unretire: list = None, ) -> dict: """ Trigger the 'add-data-async-script' workflow in the digital-land/config repository. @@ -297,6 +298,11 @@ def trigger_add_data_async_workflow( "retire_endpoints": ( ",".join(endpoints_to_retire or []) if endpoints_to_retire else "" ), + "unretire_endpoints": ( + ",".join(endpoints_to_unretire or []) + if endpoints_to_unretire + else "" + ), "environment": current_app.config.get("ENVIRONMENT"), }, } diff --git a/application/db/models.py b/application/db/models.py index ea7edb9..a52b590 100644 --- a/application/db/models.py +++ b/application/db/models.py @@ -18,6 +18,9 @@ class RequestMeta(db.Model): endpoints_to_retire = db.Column( db.Text, nullable=True ) # JSON list of endpoint hashes + endpoints_to_unretire = db.Column( + db.Text, nullable=True + ) # JSON list of endpoint hashes to clear the end-date for branch_sha = db.Column( db.Text, nullable=True ) # config-manager-update HEAD SHA when the assessment was submitted diff --git a/application/templates/datamanager/check-transform.html b/application/templates/datamanager/check-transform.html index e9bf044..1c0d98e 100644 --- a/application/templates/datamanager/check-transform.html +++ b/application/templates/datamanager/check-transform.html @@ -40,24 +40,51 @@ ▼
-

Select any existing endpoints to retire when adding this data:

-
- {% for ep in existing_endpoints %} - {% set ep_hash = ep.endpoint if ep is mapping else ep %} - {% set ep_url = ep['endpoint-url'] if ep is mapping else ep %} - {% set ep_end_date = ep['end-date'] if ep is mapping else "" %} -
- - -
- {% endfor %} -
+

Tick an endpoint to retire it when adding this data, or untick an already-retired endpoint to unretire it:

+ + + + + + + + + + + + + {% for ep in existing_endpoints %} + {% set disabled = ep['is_current'] %} + + + + + + + + + {% endfor %} + +
Retire?Endpoint URLEntry dateLatest statusLatest log entry dateDays since 200
+
+
+ + +
+
+ {% if not disabled %} + + {% if ep['is_retired'] %} + + Currently retired {{ ep['end-date'] }} + {% endif %} + {% else %} + Endpoint being added + {% endif %} + 		+ {{ ep['endpoint-url'] }} + {{ ep.endpoint }} + {{ ep['entry-date'] or "—" }}{{ ep['latest-status'] or "—" }}{{ ep['latest-log-entry-date'] or "—" }}{{ ep['days-since-200'] if ep['days-since-200'] is not none else "N/A" }}
diff --git a/application/templates/datamanager/entities_preview.html b/application/templates/datamanager/entities_preview.html index ae86b66..01e6aed 100644 --- a/application/templates/datamanager/entities_preview.html +++ b/application/templates/datamanager/entities_preview.html @@ -205,6 +205,33 @@

Retire Summary

{% endif %} + + {% if unretire_summary %} +
+

Unretire Summary

+
+
+
+
+
Dataset
+
{{ unretire_summary[0].dataset }}
+
+
+
Organisation
+
{{ unretire_summary[0].organisation }}
+
+ {% for ep in unretire_summary %} +
+
Endpoint to unretire
+
{{ ep['endpoint-url'] or ep.endpoint }}
+
+ {% endfor %} +
+
+
+
+ {% endif %} + {% if old_entity_redirect_table_params %}
diff --git a/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py new file mode 100644 index 0000000..0897e2f --- /dev/null +++ b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py @@ -0,0 +1,26 @@ +"""add request_meta endpoints_to_unretire column + +Revision ID: b8c9d0e1f2a3 +Revises: a7b8c9d0e1f2 +Create Date: 2026-07-27 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +revision = "b8c9d0e1f2a3" +down_revision = "a7b8c9d0e1f2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "request_meta", + sa.Column("endpoints_to_unretire", sa.Text(), nullable=True), + ) + + +def downgrade(): + op.drop_column("request_meta", "endpoints_to_unretire") diff --git a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py index 287d790..6c99133 100644 --- a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py +++ b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py @@ -442,9 +442,13 @@ def test_assign_entities_check_results_does_not_show_retire_endpoints(client): return_value={ "endpoint-a": { "endpoint_url": "https://example.com/data.csv", + "entry_date": "2026-01-01", "end_date": "", } }, + ), patch( + f"{transform_controller}.get_endpoint_log_summary_for_hashes", + return_value={}, ): with patch(f"{transform_controller}.get_org_entity", return_value=90): with patch(f"{transform_controller}.get_organisation_name"): diff --git a/tests/integration/blueprints/datamanager/test_datamanager_integration.py b/tests/integration/blueprints/datamanager/test_datamanager_integration.py index 7ff47f6..d0d7cd3 100644 --- a/tests/integration/blueprints/datamanager/test_datamanager_integration.py +++ b/tests/integration/blueprints/datamanager/test_datamanager_integration.py @@ -1,3 +1,4 @@ +import json from unittest.mock import patch import responses as rsps @@ -1097,3 +1098,48 @@ def test_entity_showing_text_uses_entity_ids(self, client): ): response = client.get("/datamanager/check-transform/test-id") assert b"Showing entities 7010000100 to 7010000109" in response.data + + +class TestCheckTransformPostRetireUnretire: + @rsps.activate + def test_diffs_retire_and_unretire_and_protects_current(self, client): + from application.db.models import RequestMeta + from application.extensions import db + from application.utils import compute_hash + + current_url = "https://example.com/current.csv" + current_hash = compute_hash(current_url) + rsps.add( + rsps.GET, + f"{ASYNC_BASE}/req-diff", + json={ + **COMPLETED_TRANSFORM_REQUEST, + "id": "req-diff", + "params": {"url": current_url}, + }, + status=200, + ) + + # active endpoint "hash-active" ticked -> retire + # retired endpoint "hash-retired" left unticked -> unretire + # current endpoint slipped into the checked list -> must be ignored + from werkzeug.datastructures import MultiDict + + response = client.post( + "/datamanager/check-transform/req-diff", + data=MultiDict( + [ + ("presented_endpoints", "hash-active"), + ("presented_endpoints", "hash-retired"), + ("currently_retired", "hash-retired"), + ("retire_endpoints", "hash-active"), + ("retire_endpoints", current_hash), + ] + ), + ) + assert response.status_code == 302 + + with client.application.app_context(): + meta = db.session.get(RequestMeta, "req-diff") + assert json.loads(meta.endpoints_to_retire) == ["hash-active"] + assert json.loads(meta.endpoints_to_unretire) == ["hash-retired"] diff --git a/tests/unit/blueprints/datamanager/controllers/test_transform.py b/tests/unit/blueprints/datamanager/controllers/test_transform.py index 8b090ec..8c1fa0b 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_transform.py +++ b/tests/unit/blueprints/datamanager/controllers/test_transform.py @@ -1,10 +1,17 @@ import json +from datetime import date +from unittest.mock import patch +from application.utils import compute_hash from application.blueprints.datamanager.controllers.transform import ( + _days_since, _dedup_candidate_form_value, _prepare_duplicate_candidates, + _resolve_existing_endpoints, ) +TRANSFORM_MODULE = "application.blueprints.datamanager.controllers.transform" + def test_dedup_candidate_form_value_builds_redirect_payload(): form_value = _dedup_candidate_form_value( @@ -152,3 +159,69 @@ def test_prepare_duplicate_candidates_disables_redirects_for_excluded_references assert candidates[0]["redirect_can_select"] is False assert candidates[1]["redirect_can_select"] is True + + +def test_days_since_computes_whole_days(): + fixed = (date.today() - date(2020, 1, 1)).days + assert _days_since("2020-01-01T00:00:00Z") == fixed + assert _days_since("2020-01-01") == fixed + + +def test_days_since_returns_none_for_empty_or_invalid(): + assert _days_since("") is None + assert _days_since(None) is None + assert _days_since("not-a-date") is None + + +def test_resolve_existing_endpoints_enriches_sorts_and_flags(): + current_url = "https://example.com/current.csv" + current_hash = compute_hash(current_url) + source_summary = { + "existing_endpoint_for_organisation_dataset": [ + "hash-old", + current_hash, + "hash-new", + ] + } + endpoint_data = { + "hash-old": { + "endpoint_url": "https://example.com/old.csv", + "entry_date": "2026-01-01", + "end_date": "2026-06-01", + }, + current_hash: { + "endpoint_url": current_url, + "entry_date": "2026-03-01", + "end_date": "", + }, + "hash-new": { + "endpoint_url": "https://example.com/new.csv", + "entry_date": "2026-05-01", + "end_date": "", + }, + } + log_data = { + "hash-new": { + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + "latest_200_date": "2026-07-20", + } + } + + with patch( + f"{TRANSFORM_MODULE}.get_endpoint_urls_for_hashes", return_value=endpoint_data + ), patch( + f"{TRANSFORM_MODULE}.get_endpoint_log_summary_for_hashes", return_value=log_data + ): + result = _resolve_existing_endpoints(source_summary, current_url) + + # Sorted by entry-date desc: hash-new (05-01), current (03-01), hash-old (01-01) + assert [r["endpoint"] for r in result] == ["hash-new", current_hash, "hash-old"] + + by_hash = {r["endpoint"]: r for r in result} + assert by_hash["hash-old"]["is_retired"] is True + assert by_hash["hash-old"]["is_current"] is False + assert by_hash[current_hash]["is_current"] is True + assert by_hash["hash-new"]["latest-status"] == "200" + assert by_hash["hash-new"]["days-since-200"] is not None + assert by_hash["hash-old"]["days-since-200"] is None diff --git a/tests/unit/blueprints/datamanager/services/test_endpoint.py b/tests/unit/blueprints/datamanager/services/test_endpoint.py new file mode 100644 index 0000000..48d0eaf --- /dev/null +++ b/tests/unit/blueprints/datamanager/services/test_endpoint.py @@ -0,0 +1,94 @@ +from unittest.mock import MagicMock, patch + +from application.blueprints.datamanager.services.endpoint import ( + get_endpoint_log_summary_for_hashes, + get_endpoint_info_for_hashes, +) + +ENDPOINT_MODULE = "application.blueprints.datamanager.services.endpoint" + + +def _objects_response(rows): + resp = MagicMock() + resp.raise_for_status = MagicMock() + resp.json.return_value = {"rows": rows} + return resp + + +def _array_response(rows): + resp = MagicMock() + resp.raise_for_status = MagicMock() + resp.json.return_value = rows + return resp + + +class TestGetEndpointUrlsForHashes: + def test_empty_input_returns_empty_dict(self): + assert get_endpoint_info_for_hashes([]) == {} + + def test_maps_url_entry_and_end_date(self, app): + rows = [ + { + "endpoint": "hash-a", + "endpoint_url": "https://example.com/a.csv", + "entry_date": "2026-01-01", + "end_date": "", + } + ] + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get") as mock_get: + mock_get.return_value = _objects_response(rows) + result = get_endpoint_info_for_hashes(["hash-a"]) + + assert result == { + "hash-a": { + "endpoint_url": "https://example.com/a.csv", + "entry_date": "2026-01-01", + "end_date": "", + } + } + called_url = mock_get.call_args[0][0] + assert "endpoint.json" in called_url + assert "endpoint__in=hash-a" in called_url + + def test_exception_returns_empty_dict(self, app): + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("boom")): + assert get_endpoint_info_for_hashes(["hash-a"]) == {} + + +class TestGetEndpointLogSummaryForHashes: + def test_empty_input_returns_empty_dict(self): + assert get_endpoint_log_summary_for_hashes([]) == {} + + def test_single_sql_query_maps_rows(self, app): + rows = [ + { + "endpoint": "hash-a", + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + "latest_200_date": "2026-07-20", + } + ] + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get") as mock_get: + mock_get.return_value = _array_response(rows) + result = get_endpoint_log_summary_for_hashes(["hash-a", "hash-b"]) + + assert result == { + "hash-a": { + "latest_status": "200", + "latest_log_entry_date": "2026-07-20", + "latest_200_date": "2026-07-20", + } + } + # A single grouped SQL query is issued for all hashes. + assert mock_get.call_count == 1 + called_url = mock_get.call_args[0][0] + assert ".json?sql=" in called_url + assert "hash-a" in called_url and "hash-b" in called_url + + def test_exception_returns_empty_dict(self, app): + with app.app_context(): + with patch(f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("slow")): + assert get_endpoint_log_summary_for_hashes(["hash-a"]) == {} From 698dec024d3a33b7d9d3a3b914f556b0ab0ea8fd Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 28 Jul 2026 16:42:05 +0100 Subject: [PATCH 2/7] review updates --- application/blueprints/datamanager/services/github.py | 4 ++-- .../b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py | 4 ++-- .../blueprints/datamanager/test_flagged_resources.py | 2 +- .../blueprints/datamanager/test_datamanager_integration.py | 1 + .../unit/blueprints/datamanager/controllers/test_transform.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/application/blueprints/datamanager/services/github.py b/application/blueprints/datamanager/services/github.py index 237aa3f..751b9fc 100644 --- a/application/blueprints/datamanager/services/github.py +++ b/application/blueprints/datamanager/services/github.py @@ -277,8 +277,8 @@ def trigger_add_data_async_workflow( request_id: str, triggered_by: str = "config-manager", github_branch: str = None, - endpoints_to_retire: list = None, - endpoints_to_unretire: list = None, + endpoints_to_retire: list[str] | None = None, + endpoints_to_unretire: list[str] | None = None, ) -> dict: """ Trigger the 'add-data-async-script' workflow in the digital-land/config repository. diff --git a/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py index 0897e2f..7dc3071 100644 --- a/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py +++ b/migrations/versions/b8c9d0e1f2a3_add_request_meta_endpoints_to_unretire.py @@ -1,7 +1,7 @@ """add request_meta endpoints_to_unretire column Revision ID: b8c9d0e1f2a3 -Revises: a7b8c9d0e1f2 +Revises: b9c0d1e2f3a4 Create Date: 2026-07-27 00:00:00.000000 """ @@ -10,7 +10,7 @@ from alembic import op revision = "b8c9d0e1f2a3" -down_revision = "a7b8c9d0e1f2" +down_revision = "b9c0d1e2f3a4" branch_labels = None depends_on = None diff --git a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py index 2ae7b19..14f5d89 100644 --- a/tests/acceptance/blueprints/datamanager/test_flagged_resources.py +++ b/tests/acceptance/blueprints/datamanager/test_flagged_resources.py @@ -542,7 +542,7 @@ def test_assign_entities_check_results_does_not_show_retire_endpoints(client): transform_controller = "application.blueprints.datamanager.controllers.transform" with patch( - f"{transform_controller}.get_endpoint_urls_for_hashes", + f"{transform_controller}.get_endpoint_info_for_hashes", return_value={ "endpoint-a": { "endpoint_url": "https://example.com/data.csv", diff --git a/tests/integration/blueprints/datamanager/test_datamanager_integration.py b/tests/integration/blueprints/datamanager/test_datamanager_integration.py index d0d7cd3..b745d98 100644 --- a/tests/integration/blueprints/datamanager/test_datamanager_integration.py +++ b/tests/integration/blueprints/datamanager/test_datamanager_integration.py @@ -1131,6 +1131,7 @@ def test_diffs_retire_and_unretire_and_protects_current(self, client): [ ("presented_endpoints", "hash-active"), ("presented_endpoints", "hash-retired"), + ("presented_endpoints", current_hash), ("currently_retired", "hash-retired"), ("retire_endpoints", "hash-active"), ("retire_endpoints", current_hash), diff --git a/tests/unit/blueprints/datamanager/controllers/test_transform.py b/tests/unit/blueprints/datamanager/controllers/test_transform.py index 8c1fa0b..a30a37e 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_transform.py +++ b/tests/unit/blueprints/datamanager/controllers/test_transform.py @@ -209,7 +209,7 @@ def test_resolve_existing_endpoints_enriches_sorts_and_flags(): } with patch( - f"{TRANSFORM_MODULE}.get_endpoint_urls_for_hashes", return_value=endpoint_data + f"{TRANSFORM_MODULE}.get_endpoint_info_for_hashes", return_value=endpoint_data ), patch( f"{TRANSFORM_MODULE}.get_endpoint_log_summary_for_hashes", return_value=log_data ): From cfe49c94c433455641c9da6ee76678831d9aecbb Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 28 Jul 2026 16:47:33 +0100 Subject: [PATCH 3/7] lint --- application/blueprints/datamanager/router.py | 11 +++-------- .../blueprints/datamanager/services/test_endpoint.py | 8 ++++++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/application/blueprints/datamanager/router.py b/application/blueprints/datamanager/router.py index 8bc8511..5a60db1 100644 --- a/application/blueprints/datamanager/router.py +++ b/application/blueprints/datamanager/router.py @@ -275,8 +275,7 @@ def check_transform(request_id): def check_transform_post(request_id): - """Store selected endpoints to retire/unretire from the transform page. - """ + """Store selected endpoints to retire/unretire from the transform page.""" checked = request.form.getlist("retire_endpoints") presented = request.form.getlist("presented_endpoints") currently_retired = request.form.getlist("currently_retired") @@ -287,12 +286,8 @@ def check_transform_post(request_id): # to_retire = presented and checked and not currently retired → newly retiring # to_unretire = currently retired and presented and now unchecked → unretiring - to_retire = [ - h for h in presented if h in checked and h not in currently_retired - ] - to_unretire = [ - h for h in currently_retired if h in presented and h not in checked - ] + to_retire = [h for h in presented if h in checked and h not in currently_retired] + to_unretire = [h for h in currently_retired if h in presented and h not in checked] meta = db.session.get(RequestMeta, request_id) if meta is None: diff --git a/tests/unit/blueprints/datamanager/services/test_endpoint.py b/tests/unit/blueprints/datamanager/services/test_endpoint.py index 48d0eaf..5fc2025 100644 --- a/tests/unit/blueprints/datamanager/services/test_endpoint.py +++ b/tests/unit/blueprints/datamanager/services/test_endpoint.py @@ -53,7 +53,9 @@ def test_maps_url_entry_and_end_date(self, app): def test_exception_returns_empty_dict(self, app): with app.app_context(): - with patch(f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("boom")): + with patch( + f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("boom") + ): assert get_endpoint_info_for_hashes(["hash-a"]) == {} @@ -90,5 +92,7 @@ def test_single_sql_query_maps_rows(self, app): def test_exception_returns_empty_dict(self, app): with app.app_context(): - with patch(f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("slow")): + with patch( + f"{ENDPOINT_MODULE}.requests.get", side_effect=Exception("slow") + ): assert get_endpoint_log_summary_for_hashes(["hash-a"]) == {} From ddad153bb32a6fa53410abc0c1070492a1be6e48 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 29 Jul 2026 10:50:40 +0100 Subject: [PATCH 4/7] remove ? --- application/templates/datamanager/check-transform.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/templates/datamanager/check-transform.html b/application/templates/datamanager/check-transform.html index 1c0d98e..601aa1f 100644 --- a/application/templates/datamanager/check-transform.html +++ b/application/templates/datamanager/check-transform.html @@ -44,7 +44,7 @@ - + From 4b76a60a82290089ded27a3c2cd66ee28b7fdf8b Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 29 Jul 2026 13:18:22 +0100 Subject: [PATCH 5/7] change to use reporting_historic_endpoints for log reporting --- .../datamanager/controllers/transform.py | 13 ------ .../datamanager/services/endpoint.py | 44 ++++++++++--------- .../datamanager/check-transform.html | 2 - .../datamanager/controllers/test_transform.py | 18 +------- .../datamanager/services/test_endpoint.py | 27 +++++------- 5 files changed, 36 insertions(+), 68 deletions(-) diff --git a/application/blueprints/datamanager/controllers/transform.py b/application/blueprints/datamanager/controllers/transform.py index 95960c1..3f4b67d 100644 --- a/application/blueprints/datamanager/controllers/transform.py +++ b/application/blueprints/datamanager/controllers/transform.py @@ -1,7 +1,6 @@ import json import logging import re -from datetime import date, datetime import requests from flask import current_app, render_template, request as flask_request @@ -417,17 +416,6 @@ def _date_only(date_str: str) -> str: return date_str[:10] if date_str else "" -def _days_since(date_str: str): - """Whole days between ``date_str`` (ISO ``YYYY-MM-DD...``) and today, or None.""" - if not date_str: - return None - try: - d = datetime.fromisoformat(date_str[:10]).date() - except ValueError: - return None - return (date.today() - d).days - - def _resolve_existing_endpoints( source_summary: dict, current_endpoint_url: str = "" ) -> list: @@ -459,7 +447,6 @@ def _resolve_existing_endpoints( "latest-log-entry-date": _date_only( log_data.get(h, {}).get("latest_log_entry_date", "") ), - "days-since-200": _days_since(log_data.get(h, {}).get("latest_200_date")), "is_retired": bool(endpoint_data.get(h, {}).get("end_date", "")), "is_current": h == current_hash, } diff --git a/application/blueprints/datamanager/services/endpoint.py b/application/blueprints/datamanager/services/endpoint.py index 7cb837e..46d8d0b 100644 --- a/application/blueprints/datamanager/services/endpoint.py +++ b/application/blueprints/datamanager/services/endpoint.py @@ -1,5 +1,4 @@ import logging -from urllib.parse import quote import requests from flask import current_app @@ -48,41 +47,44 @@ def get_endpoint_info_for_hashes(hashes: list) -> dict: def get_endpoint_log_summary_for_hashes(hashes: list) -> dict: """ Given a list of endpoint hashes, returns a dict mapping - {hash: {"latest_status": ..., "latest_log_entry_date": ..., - "latest_200_date": ...}} by querying the datasette ``log`` table. + {hash: {"latest_status": ..., "latest_log_entry_date": ...}}. - Warning the ``log`` table is large and slow + Reads the ``performance/reporting_historic_endpoints`` table (which includes + retired endpoints) instead of the large ``log`` table, avoiding datasette's + SQL time limit. That table holds multiple rows per endpoint, so we keep the + row with the most recent ``latest_log_entry_date`` for each hash. """ if not hashes: return {} datasette_url = current_app.config.get("DATASETTE_BASE_URL") - hash_list = ",".join(f"'{h}'" for h in hashes) - sql = ( - "SELECT l.endpoint AS endpoint, " - "l.status AS latest_status, " - "l.entry_date AS latest_log_entry_date, " - "(SELECT max(l2.entry_date) FROM log l2 " - "WHERE l2.endpoint = l.endpoint AND l2.status = '200') AS latest_200_date " - "FROM log l " - "WHERE l.rowid IN (" - f"SELECT max(rowid) FROM log WHERE endpoint IN ({hash_list}) GROUP BY endpoint)" + # DATASETTE_BASE_URL points at the ``digital-land`` database; the reporting + # table lives in the sibling ``performance`` database on the same host. + datasette_root = datasette_url.rsplit("/", 1)[0] + url = ( + f"{datasette_root}/performance/reporting_historic_endpoints.json" + f"?endpoint__in={','.join(hashes)}" + f"&_shape=objects" + f"&_size=max" ) - url = f"{datasette_url}.json?sql={quote(sql)}&_shape=array&_size=max" result = {} try: response = requests.get(url, timeout=REQUESTS_TIMEOUT) response.raise_for_status() - rows = response.json() - # _shape=array returns a list of row dicts. - for row in rows or []: + data = response.json() + for row in data.get("rows", []): h = row.get("endpoint") - if h: + if not h: + continue + log_date = row.get("latest_log_entry_date") or "" + existing = result.get(h) + # Keep the most recent record per endpoint (dates are ISO, so string + # comparison orders them correctly). + if existing is None or log_date > existing["latest_log_entry_date"]: result[h] = { "latest_status": row.get("latest_status") or "", - "latest_log_entry_date": row.get("latest_log_entry_date") or "", - "latest_200_date": row.get("latest_200_date") or "", + "latest_log_entry_date": log_date, } except Exception as e: logger.error( diff --git a/application/templates/datamanager/check-transform.html b/application/templates/datamanager/check-transform.html index 601aa1f..0f36c67 100644 --- a/application/templates/datamanager/check-transform.html +++ b/application/templates/datamanager/check-transform.html @@ -49,7 +49,6 @@ - @@ -80,7 +79,6 @@ - {% endfor %} diff --git a/tests/unit/blueprints/datamanager/controllers/test_transform.py b/tests/unit/blueprints/datamanager/controllers/test_transform.py index a30a37e..65a8f05 100644 --- a/tests/unit/blueprints/datamanager/controllers/test_transform.py +++ b/tests/unit/blueprints/datamanager/controllers/test_transform.py @@ -1,10 +1,8 @@ import json -from datetime import date from unittest.mock import patch from application.utils import compute_hash from application.blueprints.datamanager.controllers.transform import ( - _days_since, _dedup_candidate_form_value, _prepare_duplicate_candidates, _resolve_existing_endpoints, @@ -161,18 +159,6 @@ def test_prepare_duplicate_candidates_disables_redirects_for_excluded_references assert candidates[1]["redirect_can_select"] is True -def test_days_since_computes_whole_days(): - fixed = (date.today() - date(2020, 1, 1)).days - assert _days_since("2020-01-01T00:00:00Z") == fixed - assert _days_since("2020-01-01") == fixed - - -def test_days_since_returns_none_for_empty_or_invalid(): - assert _days_since("") is None - assert _days_since(None) is None - assert _days_since("not-a-date") is None - - def test_resolve_existing_endpoints_enriches_sorts_and_flags(): current_url = "https://example.com/current.csv" current_hash = compute_hash(current_url) @@ -204,7 +190,6 @@ def test_resolve_existing_endpoints_enriches_sorts_and_flags(): "hash-new": { "latest_status": "200", "latest_log_entry_date": "2026-07-20", - "latest_200_date": "2026-07-20", } } @@ -223,5 +208,4 @@ def test_resolve_existing_endpoints_enriches_sorts_and_flags(): assert by_hash["hash-old"]["is_current"] is False assert by_hash[current_hash]["is_current"] is True assert by_hash["hash-new"]["latest-status"] == "200" - assert by_hash["hash-new"]["days-since-200"] is not None - assert by_hash["hash-old"]["days-since-200"] is None + assert by_hash["hash-new"]["latest-log-entry-date"] == "2026-07-20" diff --git a/tests/unit/blueprints/datamanager/services/test_endpoint.py b/tests/unit/blueprints/datamanager/services/test_endpoint.py index 5fc2025..973f0a2 100644 --- a/tests/unit/blueprints/datamanager/services/test_endpoint.py +++ b/tests/unit/blueprints/datamanager/services/test_endpoint.py @@ -15,13 +15,6 @@ def _objects_response(rows): return resp -def _array_response(rows): - resp = MagicMock() - resp.raise_for_status = MagicMock() - resp.json.return_value = rows - return resp - - class TestGetEndpointUrlsForHashes: def test_empty_input_returns_empty_dict(self): assert get_endpoint_info_for_hashes([]) == {} @@ -63,32 +56,36 @@ class TestGetEndpointLogSummaryForHashes: def test_empty_input_returns_empty_dict(self): assert get_endpoint_log_summary_for_hashes([]) == {} - def test_single_sql_query_maps_rows(self, app): + def test_maps_latest_row_per_endpoint(self, app): + # Two rows for the same endpoint; the most recent log date wins. rows = [ + { + "endpoint": "hash-a", + "latest_status": "404", + "latest_log_entry_date": "2026-07-10", + }, { "endpoint": "hash-a", "latest_status": "200", "latest_log_entry_date": "2026-07-20", - "latest_200_date": "2026-07-20", - } + }, ] with app.app_context(): with patch(f"{ENDPOINT_MODULE}.requests.get") as mock_get: - mock_get.return_value = _array_response(rows) + mock_get.return_value = _objects_response(rows) result = get_endpoint_log_summary_for_hashes(["hash-a", "hash-b"]) assert result == { "hash-a": { "latest_status": "200", "latest_log_entry_date": "2026-07-20", - "latest_200_date": "2026-07-20", } } - # A single grouped SQL query is issued for all hashes. + # A single precomputed-table lookup is issued for all hashes. assert mock_get.call_count == 1 called_url = mock_get.call_args[0][0] - assert ".json?sql=" in called_url - assert "hash-a" in called_url and "hash-b" in called_url + assert "performance/reporting_historic_endpoints.json" in called_url + assert "endpoint__in=hash-a,hash-b" in called_url def test_exception_returns_empty_dict(self, app): with app.app_context(): From 79ad642d605b4214bd16d8cee739cdeb493247a5 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 29 Jul 2026 14:00:22 +0100 Subject: [PATCH 6/7] doc update --- application/blueprints/datamanager/services/endpoint.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/application/blueprints/datamanager/services/endpoint.py b/application/blueprints/datamanager/services/endpoint.py index 46d8d0b..ee295a3 100644 --- a/application/blueprints/datamanager/services/endpoint.py +++ b/application/blueprints/datamanager/services/endpoint.py @@ -48,18 +48,11 @@ def get_endpoint_log_summary_for_hashes(hashes: list) -> dict: """ Given a list of endpoint hashes, returns a dict mapping {hash: {"latest_status": ..., "latest_log_entry_date": ...}}. - - Reads the ``performance/reporting_historic_endpoints`` table (which includes - retired endpoints) instead of the large ``log`` table, avoiding datasette's - SQL time limit. That table holds multiple rows per endpoint, so we keep the - row with the most recent ``latest_log_entry_date`` for each hash. """ if not hashes: return {} datasette_url = current_app.config.get("DATASETTE_BASE_URL") - # DATASETTE_BASE_URL points at the ``digital-land`` database; the reporting - # table lives in the sibling ``performance`` database on the same host. datasette_root = datasette_url.rsplit("/", 1)[0] url = ( f"{datasette_root}/performance/reporting_historic_endpoints.json" From 832dc34283b65daba30c0861db88612446ad5bb9 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 29 Jul 2026 15:06:47 +0100 Subject: [PATCH 7/7] docs update --- docs/datamanager/add-data.md | 18 +++++++++++------- docs/datamanager/architecture.md | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/datamanager/add-data.md b/docs/datamanager/add-data.md index 2ca2b7f..f36806e 100644 --- a/docs/datamanager/add-data.md +++ b/docs/datamanager/add-data.md @@ -61,10 +61,14 @@ redirects to check-transform. ### 4. Check transform (`datamanager.check_transform`, `controllers/transform.py`) `GET /check-transform/` renders `check-transform-loading.html` then -`check-transform.html`: the transformed facts, issue logs, and an entity-growth view, plus the -option to select **endpoints to retire**. `POST /check-transform/` (`check_transform_post`, -`router.py`) stores the chosen `retire_endpoints` on the `RequestMeta` row and redirects to the -entities preview. +`check-transform.html`: the transformed facts, issue logs, and an entity-growth view, plus a +**retire/unretire** table of the org/dataset's existing endpoints (endpoint URL, entry date, +latest status and latest log entry date — enriched from the `endpoint` table and +`performance/reporting_historic_endpoints`). Each checkbox reflects the desired *retired* state: +ticking an active endpoint retires it, unticking a currently-retired one unretires it; the +endpoint being added is shown disabled and cannot be changed. `POST /check-transform/` +(`check_transform_post`, `router.py`) diffs the submission and stores `retire_endpoints` / +`endpoints_to_unretire` on the `RequestMeta` row, then redirects to the entities preview. > **Scope has grown.** This page started as a home for optional pre-commit *actions* (notably > selecting endpoints to retire), but has since become a fully-fledged **comparison of the entities @@ -119,9 +123,9 @@ landing page). The `datamanager` before-request guard redirects to the landing p left behind. - **`authoritative` must be `yes`/`no`.** It is validated on the form and decides whether `entity-organisation.csv` rows are written by the commit workflow. -- **`retire_endpoints` is stored, not applied.** The check-transform POST saves the selected hashes - on the `RequestMeta` row; the actual end-dating happens later in the commit workflow, not in - config-manager. +- **`retire_endpoints` / `endpoints_to_unretire` are stored, not applied.** The check-transform POST + saves the selected hashes on the `RequestMeta` row; the actual end-dating (retire) and end-date + clearing (unretire) happen later in the commit workflow, not in config-manager. - **The stale guard can silently no-op.** It only runs when submitting onto the shared branch *and* a baseline was captured at submission — see [github-add.md](github-add.md#stale-assessment-guard). diff --git a/docs/datamanager/architecture.md b/docs/datamanager/architecture.md index f2a40de..7a18e35 100644 --- a/docs/datamanager/architecture.md +++ b/docs/datamanager/architecture.md @@ -69,7 +69,7 @@ Controllers receive a request context and orchestrate the workflow: validate inp #### `RequestMeta` and `source_flow` -The entities preview and confirm routes live under `/datamanager` but are reused by the assign-entities flow (which redirects into them rather than having its own copies). Because those two endpoints are shared, the URL prefix alone can't tell which process lock applies. So at submission time each flow records `source_flow` (`"add_data"` / `"assign_entities"`) on `RequestMeta` via `record_source_flow`; the router's lock guard and the preview render then read it back — the single source of truth for which flow a request belongs to. +The entities preview and confirm routes live under `/datamanager` but are reused by the assign-entities flow (which redirects into them rather than having its own copies). Because those two endpoints are shared, the URL prefix alone can't tell which process lock applies. So at submission time each flow records `source_flow` (`"add_data"` / `"assign_entities"`) on `RequestMeta` via `record_source_flow`; the router's lock guard and the preview render then read it back — the single source of truth for which flow a request belongs to. `RequestMeta` also carries the check-transform retire/unretire selections (`endpoints_to_retire` / `endpoints_to_unretire`) that the async request's params can't — stored on POST and forwarded to the commit workflow. #### `ControllerError`
Retire?Retire Endpoint URL Entry date Latest statusEntry date Latest status Latest log entry dateDays since 200
{{ ep['entry-date'] or "—" }} {{ ep['latest-status'] or "—" }} {{ ep['latest-log-entry-date'] or "—" }}{{ ep['days-since-200'] if ep['days-since-200'] is not none else "N/A" }}