better context in retire endpoints and unretire option

application/blueprints/datamanager/controllers/preview.py

@@ -99,6 +99,29 @@ def _load_json_list(value: str | None) -> list:
     return loaded if isinstance(loaded, list) else []
 
 
+def _build_endpoint_summary(
+    selected_hashes, existing_endpoints, dataset_id, organisation_code
+):
+    summary = []
+    if not selected_hashes:
+        return summary
+    dataset_display = get_dataset_name(dataset_id, default=dataset_id)
+    org_display = get_organisation_name(organisation_code)
+    for ep in existing_endpoints:
+        ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep
+        ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep
+        if ep_hash in selected_hashes:
+            summary.append(
+                {
+                    "endpoint": ep_hash,
+                    "endpoint-url": ep_url,
+                    "dataset": dataset_display,
+                    "organisation": org_display,
+                }
+            )
+    return summary
+
+
 def _count_excluded_references(params: dict) -> int:
     references = params.get("excluded_references") or []
     if not isinstance(references, list):
@@ -237,6 +260,9 @@ def handle_entities_preview(request_id, req):
     endpoints_to_retire = (
         _load_json_list(request_meta.endpoints_to_retire) if request_meta else []
     )
+    endpoints_to_unretire = (
+        _load_json_list(request_meta.endpoints_to_unretire) if request_meta else []
+    )
     old_entity_rows = pipeline_summary.get("old-entity") or []
     old_entity_redirect_table_params = build_old_entity_redirect_table(old_entity_rows)
     old_entity_redirect_count = (
@@ -250,22 +276,13 @@ def handle_entities_preview(request_id, req):
     if isinstance(existing_endpoints, str):
         existing_endpoints = [existing_endpoints] if existing_endpoints else []
     organisation_code = params.get("organisationName") or params.get("organisation", "")
-    retire_summary = []
-    if endpoints_to_retire:
-        dataset_display = get_dataset_name(dataset_id, default=dataset_id)
-        org_display = get_organisation_name(organisation_code)
-        for ep in existing_endpoints:
-            ep_hash = ep.get("endpoint") if isinstance(ep, dict) else ep
-            ep_url = ep.get("endpoint-url", ep_hash) if isinstance(ep, dict) else ep
-            if ep_hash in endpoints_to_retire:
-                retire_summary.append(
-                    {
-                        "endpoint": ep_hash,
-                        "endpoint-url": ep_url,
-                        "dataset": dataset_display,
-                        "organisation": org_display,
-                    }
-                )
+
+    retire_summary = _build_endpoint_summary(
+        endpoints_to_retire, existing_endpoints, dataset_id, organisation_code
+    )
+    unretire_summary = _build_endpoint_summary(
+        endpoints_to_unretire, existing_endpoints, dataset_id, organisation_code
+    )
 
     # Build entity-organisation CSV preview
     authoritative = params.get("authoritative", False)
@@ -286,6 +303,7 @@ def handle_entities_preview(request_id, req):
         source_flow=source_flow,
         return_url=return_url,
         retire_summary=retire_summary,
+        unretire_summary=unretire_summary,
         old_entity_redirect_table_params=old_entity_redirect_table_params,
         old_entity_redirect_count=old_entity_redirect_count,
         new_count=len(new_entities),
@@ -318,6 +336,9 @@ def handle_add_data_confirm(
     endpoints_to_retire = (
         _load_json_list(request_meta.endpoints_to_retire) if request_meta else []
     )
+    endpoints_to_unretire = (
+        _load_json_list(request_meta.endpoints_to_unretire) if request_meta else []
+    )
 
     # Stale-assessment guard: if the config branch has advanced for this collection
     # since the assessment was taken, the assigned entity numbers may now collide.
@@ -364,6 +385,7 @@ def handle_add_data_confirm(
             triggered_by=f"{session.get('user', {}).get('login', 'unknown')}",
             github_branch=github_branch,
             endpoints_to_retire=endpoints_to_retire,
+            endpoints_to_unretire=endpoints_to_unretire,
         )
     except GitHubWorkflowError as e:
         logger.exception(f"GitHub async workflow error: {e}")

application/blueprints/datamanager/controllers/transform.py

@@ -8,11 +8,15 @@
 from shapely.geometry import mapping
 
 from . import ControllerError
+from application.utils import compute_hash
 from ..services.async_api import fetch_response_details
 from ..services.dataset import get_dataset_name, get_dataset_typology
 from ..services.organisation import get_org_entity, get_organisation_name
 from ..services.doc_crawler import check_endpoint_in_doc, is_gov_uk_url
-from ..services.endpoint import get_endpoint_urls_for_hashes
+from ..services.endpoint import (
+    get_endpoint_log_summary_for_hashes,
+    get_endpoint_info_for_hashes,
+)
 from ..services.planning_data import (
     get_entities_for_organisation_and_dataset,
     get_entity_count_for_organisation_and_dataset,
@@ -407,22 +411,48 @@ def _count_categories(rows: list) -> dict:
     return counts
 
 
-def _resolve_existing_endpoints(source_summary: dict) -> list:
+def _date_only(date_str: str) -> str:
+    """Truncate an ISO datetime (``YYYY-MM-DDThh:mm:ssZ``) to ``YYYY-MM-DD``."""
+    return date_str[:10] if date_str else ""
+
+
+def _resolve_existing_endpoints(
+    source_summary: dict, current_endpoint_url: str = ""
+) -> list:
     existing_endpoints = (
         source_summary.get("existing_endpoint_for_organisation_dataset") or []
     )
     if isinstance(existing_endpoints, str):
         existing_endpoints = [existing_endpoints] if existing_endpoints else []
-    if existing_endpoints:
-        endpoint_data = get_endpoint_urls_for_hashes(existing_endpoints)
-        existing_endpoints = [
-            {
-                "endpoint": h,
-                "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""),
-                "end-date": endpoint_data.get(h, {}).get("end_date", ""),
-            }
-            for h in existing_endpoints
-        ]
+    # The same endpoint hash can appear on more than one source.csv row for an
+    # org/dataset; retiring matches on the hash so it actions every row at once.
+    # Collapse duplicates so the retire table shows one row per endpoint.
+    existing_endpoints = list(dict.fromkeys(existing_endpoints))
+    if not existing_endpoints:
+        return existing_endpoints
+
+    endpoint_data = get_endpoint_info_for_hashes(existing_endpoints)
+    log_data = get_endpoint_log_summary_for_hashes(existing_endpoints)
+    # The endpoint hash is sha256(endpoint_url), so this matches the endpoint
+    # being added when it is already present in the endpoint CSV.
+    current_hash = compute_hash(current_endpoint_url) if current_endpoint_url else None
+
+    existing_endpoints = [
+        {
+            "endpoint": h,
+            "endpoint-url": endpoint_data.get(h, {}).get("endpoint_url", ""),
+            "entry-date": _date_only(endpoint_data.get(h, {}).get("entry_date", "")),
+            "end-date": _date_only(endpoint_data.get(h, {}).get("end_date", "")),
+            "latest-status": log_data.get(h, {}).get("latest_status", ""),
+            "latest-log-entry-date": _date_only(
+                log_data.get(h, {}).get("latest_log_entry_date", "")
+            ),
+            "is_retired": bool(endpoint_data.get(h, {}).get("end_date", "")),
+            "is_current": h == current_hash,
+        }
+        for h in existing_endpoints
+    ]
+    existing_endpoints.sort(key=lambda e: e["entry-date"] or "", reverse=True)
     return existing_endpoints
 
 
@@ -779,7 +809,7 @@ def handle_check_transform(
     response_payload = req.get("response") or {}
     response_data = response_payload.get("data") or {}
     source_summary = response_data.get("source-summary") or {}
-    existing_endpoints = _resolve_existing_endpoints(source_summary)
+    existing_endpoints = _resolve_existing_endpoints(source_summary, endpoint_url)
     pipelines_append_required = source_summary.get("pipelines_append_required")
     pipeline_summary = response_data.get("pipeline-summary") or {}
     show_dedup_tab = is_assign_entities and dataset_id == "conservation-area"

application/blueprints/datamanager/router.py

@@ -16,6 +16,7 @@
 from application.blueprints.base.views import ADD_DATA_LOCK, ASSIGN_ENTITIES_LOCK
 from application.db.models import RequestMeta, ServiceLock
 from application.extensions import db
+from application.utils import compute_hash
 
 from .controllers.form import (
     handle_dashboard_get,
@@ -274,20 +275,45 @@ def check_transform(request_id):
 
 
 def check_transform_post(request_id):
-    """Store selected endpoints to retire from transform page and continue to preview."""
-    hashes = request.form.getlist("retire_endpoints")
+    """Store selected endpoints to retire/unretire from the transform page."""
+    checked = request.form.getlist("retire_endpoints")
+    presented = request.form.getlist("presented_endpoints")
+    currently_retired = request.form.getlist("currently_retired")
+
+    # Never allow the endpoint being added (already in the CSV) to be changed,
+    current_hash = _current_endpoint_hash(request_id)
+    presented = [h for h in presented if h != current_hash]
+
+    # to_retire = presented and checked and not currently retired → newly retiring
+    # to_unretire = currently retired and presented and now unchecked → unretiring
+    to_retire = [h for h in presented if h in checked and h not in currently_retired]
+    to_unretire = [h for h in currently_retired if h in presented and h not in checked]
+
     meta = db.session.get(RequestMeta, request_id)
     if meta is None:
         meta = RequestMeta(
-            request_id=request_id, endpoints_to_retire=json.dumps(hashes)
+            request_id=request_id,
+            endpoints_to_retire=json.dumps(to_retire),
+            endpoints_to_unretire=json.dumps(to_unretire),
         )
         db.session.add(meta)
     else:
-        meta.endpoints_to_retire = json.dumps(hashes)
+        meta.endpoints_to_retire = json.dumps(to_retire)
+        meta.endpoints_to_unretire = json.dumps(to_unretire)
     db.session.commit()
     return redirect(url_for("datamanager.entities_preview", request_id=request_id))
 
 
+def _current_endpoint_hash(request_id):
+    """sha256 of the endpoint URL being added, or None if it can't be resolved."""
+    try:
+        req = fetch_request(request_id)
+    except AsyncAPIError:
+        return None
+    url = (req.get("params") or {}).get("url", "")
+    return compute_hash(url) if url else None
+
+
 def add_data_confirm_async(request_id):
     logger.info(f"Triggering async GitHub workflow for request_id: {request_id}")
     github_branch = request.form.get("github_branch") or None

application/blueprints/datamanager/services/endpoint.py

@@ -8,9 +8,10 @@
 logger = logging.getLogger(__name__)
 
 
-def get_endpoint_urls_for_hashes(hashes: list) -> dict:
+def get_endpoint_info_for_hashes(hashes: list) -> dict:
     """
-    Given a list of endpoint hashes, returns a dict mapping {hash: endpoint_url}
+    Given a list of endpoint hashes, returns a dict mapping
+    {hash: {"endpoint_url": ..., "entry_date": ..., "end_date": ...}}
     by querying the datasette endpoint table.
     """
     if not hashes:
@@ -34,9 +35,53 @@ def get_endpoint_urls_for_hashes(hashes: list) -> dict:
             if h:
                 result[h] = {
                     "endpoint_url": row.get("endpoint_url", ""),
+                    "entry_date": row.get("entry_date") or "",
                     "end_date": row.get("end_date") or "",
                 }
     except Exception as e:
         logger.error(f"Failed to fetch endpoint URLs for hashes: {e}", exc_info=True)
 
     return result
+
+
+def get_endpoint_log_summary_for_hashes(hashes: list) -> dict:
+    """
+    Given a list of endpoint hashes, returns a dict mapping
+    {hash: {"latest_status": ..., "latest_log_entry_date": ...}}.
+    """
+    if not hashes:
+        return {}
+
+    datasette_url = current_app.config.get("DATASETTE_BASE_URL")
+    datasette_root = datasette_url.rsplit("/", 1)[0]
+    url = (
+        f"{datasette_root}/performance/reporting_historic_endpoints.json"
+        f"?endpoint__in={','.join(hashes)}"
+        f"&_shape=objects"
+        f"&_size=max"
+    )
+
+    result = {}
+    try:
+        response = requests.get(url, timeout=REQUESTS_TIMEOUT)
+        response.raise_for_status()
+        data = response.json()
+        for row in data.get("rows", []):
+            h = row.get("endpoint")
+            if not h:
+                continue
+            log_date = row.get("latest_log_entry_date") or ""
+            existing = result.get(h)
+            # Keep the most recent record per endpoint (dates are ISO, so string
+            # comparison orders them correctly).
+            if existing is None or log_date > existing["latest_log_entry_date"]:
+                result[h] = {
+                    "latest_status": row.get("latest_status") or "",
+                    "latest_log_entry_date": log_date,
+                }
+    except Exception as e:
+        logger.error(
+            f"Failed to fetch endpoint log summary for hashes: {e}", exc_info=True
+        )
+
+    return result

application/blueprints/datamanager/services/github.py

@@ -277,7 +277,8 @@ def trigger_add_data_async_workflow(
     request_id: str,
     triggered_by: str = "config-manager",
     github_branch: str = None,
-    endpoints_to_retire: list = None,
+    endpoints_to_retire: list[str] | None = None,
+    endpoints_to_unretire: list[str] | None = None,
 ) -> dict:
     """
     Trigger the 'add-data-async-script' workflow in the digital-land/config repository.
@@ -297,6 +298,11 @@ def trigger_add_data_async_workflow(
                 "retire_endpoints": (
                     ",".join(endpoints_to_retire or []) if endpoints_to_retire else ""
                 ),
+                "unretire_endpoints": (
+                    ",".join(endpoints_to_unretire or [])
+                    if endpoints_to_unretire
+                    else ""
+                ),
                 "environment": current_app.config.get("ENVIRONMENT"),
             },
         }

application/db/models.py

@@ -18,6 +18,9 @@ class RequestMeta(db.Model):
     endpoints_to_retire = db.Column(
         db.Text, nullable=True
     )  # JSON list of endpoint hashes
+    endpoints_to_unretire = db.Column(
+        db.Text, nullable=True
+    )  # JSON list of endpoint hashes to clear the end-date for
     branch_sha = db.Column(
         db.Text, nullable=True
     )  # config-manager-update HEAD SHA when the assessment was submitted