Refactor Assign Entities workflow and update related documentation

application/blueprints/datamanager/controllers/flagged_resources.py

@@ -348,7 +348,12 @@ def _get_resource_organisation(resource, dataset_id):
 
 
 def _submit_assign_entities_request(
-    dataset_input, resource, organisation=None, return_endpoint=None
+    dataset_input,
+    resource,
+    organisation=None,
+    return_endpoint=None,
+    excluded_references=None,
+    selected_redirects=None,
 ):
     dataset_id, collection_id = _resolve_dataset_and_collection(dataset_input)
     organisation = organisation or _get_resource_organisation(resource, dataset_id)
@@ -365,6 +370,10 @@ def _submit_assign_entities_request(
         params["organisation"] = organisation
     if return_endpoint:
         params["return_endpoint"] = return_endpoint
+    if excluded_references is not None:
+        params["excluded_references"] = excluded_references
+    if selected_redirects is not None:
+        params["selected_redirects"] = selected_redirects
     preview_id = submit_request(params)
     record_branch_baseline(preview_id, params["github_branch"])
     return preview_id

application/blueprints/datamanager/controllers/preview.py

@@ -1,6 +1,5 @@
 import json
 import logging
-from datetime import date
 
 from flask import (
     render_template,
@@ -102,8 +101,17 @@ def _load_json_list(value: str | None) -> list:
     return loaded if isinstance(loaded, list) else []
 
 
-def build_old_entity_redirect_table(entity_redirects: list[dict]) -> dict | None:
-    if not entity_redirects:
+def _count_excluded_references(params: dict) -> int:
+    references = params.get("excluded_references") or []
+    if not isinstance(references, list):
+        return 0
+    return len(
+        {str(reference).strip() for reference in references if str(reference).strip()}
+    )
+
+
+def build_old_entity_redirect_table(old_entity_rows: list[dict]) -> dict | None:
+    if not old_entity_rows:
         return None
 
     columns = [
@@ -115,22 +123,30 @@ def build_old_entity_redirect_table(entity_redirects: list[dict]) -> dict | None
         "entry-date",
         "start-date",
     ]
-    entry_date = date.today().isoformat()
     rows = []
-    for redirect in entity_redirects:
+    for old_entity in old_entity_rows:
+        if not isinstance(old_entity, dict):
+            continue
         row = {
-            "old-entity": str(redirect.get("old_entity", "")),
-            "status": "301",
-            "entity": str(redirect.get("entity", "")),
-            "notes": str(
-                redirect.get("notes")
-                or "Redirect duplicate entity selected in Assign Entities"
+            "old-entity": str(
+                old_entity.get("old-entity", "") or old_entity.get("old_entity", "")
+            ),
+            "status": str(old_entity.get("status", "") or ""),
+            "entity": str(old_entity.get("entity", "") or ""),
+            "notes": str(old_entity.get("notes", "") or ""),
+            "end-date": str(
+                old_entity.get("end-date", "") or old_entity.get("end_date", "")
+            ),
+            "entry-date": str(
+                old_entity.get("entry-date", "") or old_entity.get("entry_date", "")
+            ),
+            "start-date": str(
+                old_entity.get("start-date", "") or old_entity.get("start_date", "")
             ),
-            "end-date": "",
-            "entry-date": entry_date,
-            "start-date": "",
         }
         rows.append({"columns": {c: {"value": row[c]} for c in columns}})
+    if not rows:
+        return None
 
     return {
         "columns": columns,
@@ -260,10 +276,13 @@ def handle_entities_preview(request_id, req):
     endpoints_to_retire = (
         _load_json_list(request_meta.endpoints_to_retire) if request_meta else []
     )
-    entity_redirects = (
-        _load_json_list(request_meta.entity_redirects) if request_meta else []
+    old_entity_rows = pipeline_summary.get("old-entity") or []
+    old_entity_redirect_table_params = build_old_entity_redirect_table(old_entity_rows)
+    old_entity_redirect_count = (
+        len(old_entity_redirect_table_params["rows"])
+        if old_entity_redirect_table_params
+        else 0
     )
-    old_entity_redirect_table_params = build_old_entity_redirect_table(entity_redirects)
     existing_endpoints = (
         source_summary_data.get("existing_endpoint_for_organisation_dataset") or []
     )
@@ -306,9 +325,10 @@ def handle_entities_preview(request_id, req):
         source_flow=source_flow,
         return_url=return_url,
         retire_summary=retire_summary,
-        entity_redirects=entity_redirects,
         old_entity_redirect_table_params=old_entity_redirect_table_params,
-        new_count=int(pipeline_summary.get("new-in-resource") or 0),
+        old_entity_redirect_count=old_entity_redirect_count,
+        new_count=len(new_entities),
+        excluded_count=_count_excluded_references(params),
         existing_count=int(pipeline_summary.get("existing-in-resource") or 0),
         endpoint_already_exists=endpoint_already_exists,
         endpoint_url=endpoint_url,
@@ -337,9 +357,6 @@ def handle_add_data_confirm(
     endpoints_to_retire = (
         _load_json_list(request_meta.endpoints_to_retire) if request_meta else []
     )
-    entity_redirects = (
-        _load_json_list(request_meta.entity_redirects) if request_meta else []
-    )
 
     # Stale-assessment guard: if the config branch has advanced for this collection
     # since the assessment was taken, the assigned entity numbers may now collide.
@@ -386,7 +403,6 @@ def handle_add_data_confirm(
             triggered_by=f"{session.get('user', {}).get('login', 'unknown')}",
             github_branch=github_branch,
             endpoints_to_retire=endpoints_to_retire,
-            entity_redirects=entity_redirects,
         )
     except GitHubWorkflowError as e:
         logger.exception(f"GitHub async workflow error: {e}")

application/blueprints/datamanager/controllers/transform.py

@@ -220,6 +220,54 @@ def _build_entities_data(resp_details: list, platform_entities: list) -> dict:
     return {"columns": columns, "rows": rows}
 
 
+def _normalise_excluded_references(excluded_references) -> set:
+    """Return explicitly excluded references from async request params.
+
+    A missing or empty param means no references were excluded from assignment.
+    """
+    if not excluded_references:
+        return set()
+    if not isinstance(excluded_references, list):
+        return set()
+    return {
+        str(reference).strip()
+        for reference in excluded_references
+        if str(reference).strip()
+    }
+
+
+def _entity_selection_form_value(reference: str) -> str:
+    return reference
+
+
+def _add_assign_entities_selection_metadata(
+    entities_data: dict,
+    excluded_references,
+) -> dict:
+    """Add checkbox metadata to Assign Entities rows.
+
+    Only rows categorised as new can be assigned entity numbers. Async request
+    params contain references excluded from assignment, so every selectable row
+    starts checked unless its reference is explicitly excluded.
+    """
+    excluded_reference_set = _normalise_excluded_references(excluded_references)
+
+    for row in entities_data.get("rows", []):
+        fields = row.get("fields") or {}
+        reference = str(fields.get("reference", "")).strip()
+        can_select = row.get("category") == "new" and bool(reference)
+        selected = can_select and reference not in excluded_reference_set
+        row["entity_selection"] = {
+            "can_select": can_select,
+            "selected": selected,
+            "form_value": (
+                _entity_selection_form_value(reference) if reference else ""
+            ),
+        }
+
+    return entities_data
+
+
 def _entity_row_matches_search(row: dict, search_query: str) -> bool:
     if not search_query:
         return True
@@ -235,26 +283,44 @@ def _entity_row_matches_filter(row: dict, category_filter: str) -> bool:
     return row.get("category") == category_filter
 
 
-def _name_similarity_percent(value) -> float:
-    if value in (None, ""):
-        return 0
-    try:
-        similarity = float(str(value).strip().rstrip("%"))
-    except (TypeError, ValueError):
-        return 0
-    return similarity * 100 if 0 < similarity <= 1 else similarity
+def _dedup_candidate_redirect_key(candidate: dict) -> tuple[str, str]:
+    return (
+        str(candidate.get("old_entity", "") or "").strip(),
+        str(candidate.get("entity", "") or "").strip(),
+    )
 
 
-def _dedup_candidate_auto_select(candidate: dict) -> bool:
-    if candidate.get("old_entity_redirects"):
-        return False
+def _old_entity_redirect_key(row: dict) -> tuple[str, str]:
+    return (
+        str(row.get("old-entity", "") or row.get("old_entity", "") or "").strip(),
+        str(row.get("entity", "") or "").strip(),
+    )
 
-    match_type = str(candidate.get("match_type", "")).replace(" ", "_").lower()
-    if match_type == "complete_match":
-        return True
-    if match_type != "single_match":
-        return False
-    return _name_similarity_percent(candidate.get("name_similarity")) > 85
+
+def _dedup_candidate_selected_entity_reference(candidate: dict) -> str:
+    return str(
+        candidate.get("new_reference", "") or candidate.get("reference", "") or ""
+    ).strip()
+
+
+def _dedup_candidate_selected_redirect_key(candidate: dict) -> tuple[str, str]:
+    """Return the key used to compare a Dedup candidate with selected_redirects."""
+    return (
+        _dedup_candidate_selected_entity_reference(candidate),
+        str(candidate.get("old_entity", "") or "").strip(),
+    )
+
+
+def _selected_redirect_key(redirect: dict) -> tuple[str, str]:
+    """Return the async selected_redirects key for a submitted redirect param."""
+    return (
+        str(redirect.get("reference", "") or "").strip(),
+        str(
+            redirect.get("old_entity_number", "")
+            or redirect.get("old_entity", "")
+            or ""
+        ).strip(),
+    )
 
 
 def _dedup_candidate_form_value(candidate: dict) -> str:
@@ -276,11 +342,46 @@ def _dedup_candidate_form_value(candidate: dict) -> str:
     )
 
 
-def _prepare_duplicate_candidates(candidates: list[dict]) -> list[dict]:
+def _prepare_duplicate_candidates(
+    candidates: list[dict],
+    old_entity_rows: list[dict] | None = None,
+    organisation: str = "",
+    excluded_references=None,
+    selected_redirects=None,
+) -> list[dict]:
+    """Prepare Dedup candidates for rendering and selection.
+
+    Async's ``pipeline-summary.old-entity`` is the source of initial redirect
+    preselection. Rows present in old-entity but absent from request
+    ``selected_redirects`` are inferred to be async auto-selected and are locked
+    in the UI so users cannot untick redirects generated by async policy.
+    """
+    preselected_redirects = {
+        key
+        for key in (
+            _old_entity_redirect_key(row)
+            for row in (old_entity_rows or [])
+            if isinstance(row, dict)
+        )
+        if all(key)
+    }
+    selected_redirect_keys = {
+        _selected_redirect_key(redirect)
+        for redirect in (selected_redirects or [])
+        if isinstance(redirect, dict) and all(_selected_redirect_key(redirect))
+    }
+    excluded_reference_set = _normalise_excluded_references(excluded_references)
     return [
         {
             **candidate,
-            "auto_select": _dedup_candidate_auto_select(candidate),
+            "auto_select": _dedup_candidate_redirect_key(candidate)
+            in preselected_redirects,
+            "redirect_locked": _dedup_candidate_redirect_key(candidate)
+            in preselected_redirects
+            and _dedup_candidate_selected_redirect_key(candidate)
+            not in selected_redirect_keys,
+            "redirect_can_select": _dedup_candidate_selected_entity_reference(candidate)
+            not in excluded_reference_set,
             "form_value": _dedup_candidate_form_value(candidate),
         }
         for candidate in candidates
@@ -337,9 +438,12 @@ def _paginate_entity_data(
     entity_page: int,
     entity_search: str,
     entity_filter: str = "",
+    include_selection: bool = False,
+    excluded_references=None,
 ) -> tuple:
     entity_start_offset = (entity_page - 1) * _ROWS_PER_PAGE
     entities_data_full = _build_entities_data(all_resp_details, platform_entities)
+
     # Counts cover every entity, independent of the current search/filter, so the
     # summary boxes always show the full picture.
     category_counts = _count_categories(entities_data_full["rows"])
@@ -366,6 +470,11 @@ def _paginate_entity_data(
         "columns": entities_data_full["columns"],
         "rows": entity_page_rows,
     }
+    if include_selection:
+        entities_data = _add_assign_entities_selection_metadata(
+            entities_data,
+            excluded_references,
+        )
     return (
         entities_data,
         has_next_entity_page,
@@ -618,6 +727,7 @@ def handle_check_transform(
     """
     params = req.get("params") or {}
     organisation_code = params.get("organisationName") or params.get("organisation", "")
+    excluded_references = params.get("excluded_references")
     dataset_id = params.get("dataset", "")
     is_assign_entities = transform_endpoint == "assign_entities.flagged_resource_detail"
     resource_hash = params.get("resource", "")
@@ -664,7 +774,11 @@ def handle_check_transform(
     pipeline_summary = response_data.get("pipeline-summary") or {}
     show_dedup_tab = is_assign_entities and dataset_id == "conservation-area"
     duplicate_candidates = _prepare_duplicate_candidates(
-        pipeline_summary.get("duplicate-candidates") or [] if show_dedup_tab else []
+        pipeline_summary.get("duplicate-candidates") or [] if show_dedup_tab else [],
+        pipeline_summary.get("old-entity") or [],
+        organisation=organisation_code,
+        excluded_references=excluded_references,
+        selected_redirects=params.get("selected_redirects"),
     )
 
     # Calculate pagination for transformed facts and issue logs, and for entities.
@@ -693,6 +807,8 @@ def handle_check_transform(
         entity_page,
         entity_search,
         entity_filter,
+        include_selection=is_assign_entities,
+        excluded_references=excluded_references,
     )
     transformed_table = _build_transform_table(resp_details)
     issue_log_table = _build_issue_log_table(resp_details)
@@ -746,6 +862,7 @@ def handle_check_transform(
         endpoint_url=endpoint_url,
         documentation_url=documentation_url,
         resource_hash=resource_hash,
+        is_assign_entities=is_assign_entities,
         show_dedup_tab=show_dedup_tab,
         duplicate_candidates=duplicate_candidates,
         planning_entity_base_url=(