Big cleanup of old processes

.github/workflows/docs-check.yml

@@ -0,0 +1,116 @@
+name: Docs update reminder
+run-name: Docs check - ${{ github.head_ref || github.ref_name }} by @${{ github.actor }}
+
+# Warn-only check: when a PR makes a substantial code change (more than
+# DOCS_MIN_CHANGED_LINES lines across application/src/migrations/requirements)
+# but does not touch the docs/ folder, post a non-blocking reminder to update
+# the documentation. This never fails the build - it is a nudge, not a gate.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+env:
+  # Minimum number of changed code lines (additions + deletions) before the
+  # docs reminder fires. Small changes are ignored.
+  DOCS_MIN_CHANGED_LINES: 100
+
+jobs:
+  docs-reminder:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Check whether a substantial code change skipped docs
+        id: changes
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          base="origin/${BASE_REF}"
+          git fetch --no-tags --depth=1 origin "${BASE_REF}"
+          numstat="$(git diff --numstat "$base"...HEAD)"
+          echo "Changed files (added / deleted / path):"
+          echo "$numstat"
+
+          docs_changed=false
+          code_lines=0
+          while IFS=$'\t' read -r added deleted path; do
+            [ -z "$path" ] && continue
+            case "$path" in
+              docs/*)
+                docs_changed=true
+                ;;
+              application/*|src/*|migrations/*|requirements/*)
+                # Binary files report '-' for added/deleted; skip them.
+                if [ "$added" != "-" ]; then
+                  code_lines=$(( code_lines + added + deleted ))
+                fi
+                ;;
+            esac
+          done <<< "$numstat"
+
+          echo "Code lines changed: $code_lines (threshold ${DOCS_MIN_CHANGED_LINES})"
+          echo "code_lines=$code_lines" >> "$GITHUB_OUTPUT"
+
+          if [ "$docs_changed" = false ] && [ "$code_lines" -gt "${DOCS_MIN_CHANGED_LINES}" ]; then
+            echo "needs_docs=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "needs_docs=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Warn in job summary
+        if: steps.changes.outputs.needs_docs == 'true'
+        run: |
+          {
+            echo "## 📝 Documentation reminder"
+            echo ""
+            echo "This PR changes **${{ steps.changes.outputs.code_lines }}** lines of code"
+            echo "(threshold ${DOCS_MIN_CHANGED_LINES}) but does not update anything in \`docs/\`."
+            echo "If this change affects behaviour, please update the relevant docs"
+            echo "(e.g. \`docs/datamanager/\`). This check is a reminder only and does **not** block merging."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment on PR
+        if: steps.changes.outputs.needs_docs == 'true'
+        uses: actions/github-script@v7
+        env:
+          CODE_LINES: ${{ steps.changes.outputs.code_lines }}
+          THRESHOLD: ${{ env.DOCS_MIN_CHANGED_LINES }}
+        with:
+          script: |
+            const marker = '<!-- docs-update-reminder -->';
+            const body = `${marker}\n📝 **Documentation reminder**\n\n` +
+              `This PR changes **${process.env.CODE_LINES}** lines of code ` +
+              `(threshold ${process.env.THRESHOLD}) but does not update anything in \`docs/\`. ` +
+              'If this change affects behaviour, please update the relevant docs ' +
+              '(e.g. `docs/datamanager/`).\n\n' +
+              '_This is a non-blocking reminder — it will not prevent merging._';
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+            }

Makefile

@@ -56,12 +56,6 @@ upgrade-db:
 downgrade-db:
 	flask db downgrade
 
-load-data:
-	flask data load --spec 1 --config 1
-
-drop-data:
-	flask data drop
-
 test-unit:
 	@echo "Running Unit tests...."
 	python -m pytest tests/unit/ -v

README.md

@@ -14,14 +14,6 @@ Create or update db schema
 
     make upgrade-db
 
-Load data
-
-    make load-data
-
-Drop local data
-
-    make drop-data
-
 To run the application run:
 
     make run

application/blueprints/datamanager/controllers/transform.py

@@ -8,7 +8,6 @@
 from shapely.geometry import mapping
 
 from . import ControllerError
-from ..config import get_entity_geojson_url, get_entity_search_url
 from ..services.async_api import fetch_response_details
 from ..services.dataset import get_dataset_name, get_dataset_typology
 from ..services.organisation import get_org_entity, get_organisation_name
@@ -22,6 +21,17 @@
 
 logger = logging.getLogger(__name__)
 
+
+def _entity_search_url(dataset_id, reference):
+    base = current_app.config["PLANNING_BASE_URL"]
+    return f"{base}/entity.json?dataset={dataset_id}&reference={reference}"
+
+
+def _entity_geojson_url(reference):
+    base = current_app.config["PLANNING_BASE_URL"]
+    return f"{base}/entity.geojson?reference={reference}"
+
+
 _TRANSFORM_COLS = [
     "entry_number",
     "entity",
@@ -689,7 +699,7 @@ def fetch_boundary_geojson(organisation_code: str) -> dict:
             return empty
         lpa_prefix, lpa_id = organisation_code.split(":", 1)
         resp = requests.get(
-            get_entity_search_url(lpa_prefix, lpa_id), timeout=REQUESTS_TIMEOUT
+            _entity_search_url(lpa_prefix, lpa_id), timeout=REQUESTS_TIMEOUT
         )
         resp.raise_for_status()
         d = resp.json()
@@ -702,7 +712,7 @@ def fetch_boundary_geojson(organisation_code: str) -> dict:
         if not reference:
             return empty
         return requests.get(
-            get_entity_geojson_url(reference), timeout=REQUESTS_TIMEOUT
+            _entity_geojson_url(reference), timeout=REQUESTS_TIMEOUT
         ).json()
     except Exception as e:
         logger.warning("Failed to fetch boundary data for %s: %s", organisation_code, e)

application/blueprints/datamanager/services/async_api.py

@@ -3,18 +3,27 @@
 
 import requests
 
+from config.config import get_request_api_endpoint
+
 from application.extensions import cache
 
-from ..config import (
-    get_async_request_url,
-    get_async_requests_url,
-    get_async_response_details_url,
-)
 from ..utils import REQUESTS_TIMEOUT
 
 logger = logging.getLogger(__name__)
 
 
+def _requests_url() -> str:
+    return f"{get_request_api_endpoint()}/requests"
+
+
+def _request_url(request_id: str) -> str:
+    return f"{get_request_api_endpoint()}/requests/{request_id}"
+
+
+def _response_details_url(request_id: str) -> str:
+    return f"{get_request_api_endpoint()}/requests/{request_id}/response-details"
+
+
 class AsyncAPIError(Exception):
     """Raised when the async request API returns an error."""
 
@@ -36,9 +45,7 @@ def submit_request(params: dict) -> str:
     logger.info("Submitting request to async API")
     logger.debug(json.dumps(payload, indent=2))
 
-    response = requests.post(
-        get_async_requests_url(), json=payload, timeout=REQUESTS_TIMEOUT
-    )
+    response = requests.post(_requests_url(), json=payload, timeout=REQUESTS_TIMEOUT)
 
     logger.info(f"Async API responded with {response.status_code}")
     try:
@@ -70,7 +77,7 @@ def fetch_request(request_id: str) -> dict:
     Returns the parsed JSON response on 200.
     Raises AsyncAPIError on non-200 status.
     """
-    response = requests.get(get_async_request_url(request_id), timeout=REQUESTS_TIMEOUT)
+    response = requests.get(_request_url(request_id), timeout=REQUESTS_TIMEOUT)
 
     if response.status_code != 200:
         raise AsyncAPIError(
@@ -108,7 +115,7 @@ def fetch_response_details(
             min(limit, max_rows - len(all_details)) if max_rows is not None else limit
         )
         try:
-            url = get_async_response_details_url(request_id)
+            url = _response_details_url(request_id)
             params = {"offset": offset, "limit": fetch_limit}
             logger.debug(f"Fetching batch - URL: {url}, Params: {params}")
 

application/blueprints/datamanager/utils/__init__.py

@@ -1,7 +1,5 @@
 import csv
 import logging
-import os
-import re
 from datetime import datetime
 from io import StringIO
 
@@ -17,22 +15,6 @@
 REQUESTS_TIMEOUT = 20  # seconds
 
 
-def get_allowed_override_users() -> set:
-    """Read config/allowed-users.md and return a set of GitHub usernames."""
-    project_root = os.path.dirname(current_app.root_path)
-    path = os.path.join(project_root, "config", "allowed-users.md")
-    try:
-        with open(path, "r") as f:
-            content = f.read()
-        return {
-            m.group(1).strip().lower()
-            for m in re.finditer(r"^[-*]\s+(\S+)", content, re.MULTILINE)
-        }
-    except Exception as e:
-        logger.warning(f"Could not read allowed-users.md: {e}")
-        return set()
-
-
 def handle_error(e):
     logger.exception(f"Error: {e}")
     return render_template("datamanager/error.html", message=str(e)), 500