diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml new file mode 100644 index 00000000..92da456c --- /dev/null +++ b/.github/workflows/docs-check.yml @@ -0,0 +1,116 @@ +name: Docs update reminder +run-name: Docs check - ${{ github.head_ref || github.ref_name }} by @${{ github.actor }} + +# Warn-only check: when a PR makes a substantial code change (more than +# DOCS_MIN_CHANGED_LINES lines across application/src/migrations/requirements) +# but does not touch the docs/ folder, post a non-blocking reminder to update +# the documentation. This never fails the build - it is a nudge, not a gate. + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +env: + # Minimum number of changed code lines (additions + deletions) before the + # docs reminder fires. Small changes are ignored. + DOCS_MIN_CHANGED_LINES: 100 + +jobs: + docs-reminder: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Check whether a substantial code change skipped docs + id: changes + env: + BASE_REF: ${{ github.base_ref }} + run: | + base="origin/${BASE_REF}" + git fetch --no-tags --depth=1 origin "${BASE_REF}" + numstat="$(git diff --numstat "$base"...HEAD)" + echo "Changed files (added / deleted / path):" + echo "$numstat" + + docs_changed=false + code_lines=0 + while IFS=$'\t' read -r added deleted path; do + [ -z "$path" ] && continue + case "$path" in + docs/*) + docs_changed=true + ;; + application/*|src/*|migrations/*|requirements/*) + # Binary files report '-' for added/deleted; skip them. + if [ "$added" != "-" ]; then + code_lines=$(( code_lines + added + deleted )) + fi + ;; + esac + done <<< "$numstat" + + echo "Code lines changed: $code_lines (threshold ${DOCS_MIN_CHANGED_LINES})" + echo "code_lines=$code_lines" >> "$GITHUB_OUTPUT" + + if [ "$docs_changed" = false ] && [ "$code_lines" -gt "${DOCS_MIN_CHANGED_LINES}" ]; then + echo "needs_docs=true" >> "$GITHUB_OUTPUT" + else + echo "needs_docs=false" >> "$GITHUB_OUTPUT" + fi + + - name: Warn in job summary + if: steps.changes.outputs.needs_docs == 'true' + run: | + { + echo "## πŸ“ Documentation reminder" + echo "" + echo "This PR changes **${{ steps.changes.outputs.code_lines }}** lines of code" + echo "(threshold ${DOCS_MIN_CHANGED_LINES}) but does not update anything in \`docs/\`." + echo "If this change affects behaviour, please update the relevant docs" + echo "(e.g. \`docs/datamanager/\`). This check is a reminder only and does **not** block merging." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Comment on PR + if: steps.changes.outputs.needs_docs == 'true' + uses: actions/github-script@v7 + env: + CODE_LINES: ${{ steps.changes.outputs.code_lines }} + THRESHOLD: ${{ env.DOCS_MIN_CHANGED_LINES }} + with: + script: | + const marker = ''; + const body = `${marker}\nπŸ“ **Documentation reminder**\n\n` + + `This PR changes **${process.env.CODE_LINES}** lines of code ` + + `(threshold ${process.env.THRESHOLD}) but does not update anything in \`docs/\`. ` + + 'If this change affects behaviour, please update the relevant docs ' + + '(e.g. `docs/datamanager/`).\n\n' + + '_This is a non-blocking reminder β€” it will not prevent merging._'; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } diff --git a/Makefile b/Makefile index 2f1a399c..a5a5d1fe 100644 --- a/Makefile +++ b/Makefile @@ -56,12 +56,6 @@ upgrade-db: downgrade-db: flask db downgrade -load-data: - flask data load --spec 1 --config 1 - -drop-data: - flask data drop - test-unit: @echo "Running Unit tests...." python -m pytest tests/unit/ -v diff --git a/README.md b/README.md index d10394b3..3a291fe9 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,6 @@ Create or update db schema make upgrade-db -Load data - - make load-data - -Drop local data - - make drop-data - To run the application run: make run diff --git a/application/blueprints/datamanager/config.py b/application/blueprints/datamanager/config.py deleted file mode 100644 index 43bdb720..00000000 --- a/application/blueprints/datamanager/config.py +++ /dev/null @@ -1,38 +0,0 @@ -from flask import current_app - -from config.config import get_request_api_endpoint - -# --- Planning Data API URLs --- -# TODO: This is temporary!, this is helpful to move all url's to be retrieved -# from the overall config.py for Config-manager - - -def get_planning_base_url(): - return current_app.config["PLANNING_BASE_URL"] - - -def get_datasets_url(): - return f"{get_planning_base_url()}/dataset.json?_labels=on&_size=max" - - -def get_entity_search_url(dataset_id, reference): - return f"{get_planning_base_url()}/entity.json?dataset={dataset_id}&reference={reference}" - - -def get_entity_geojson_url(reference): - return f"{get_planning_base_url()}/entity.geojson?reference={reference}" - - -# --- Async Request API URLs --- - - -def get_async_requests_url(): - return f"{get_request_api_endpoint()}/requests" - - -def get_async_request_url(request_id): - return f"{get_request_api_endpoint()}/requests/{request_id}" - - -def get_async_response_details_url(request_id): - return f"{get_request_api_endpoint()}/requests/{request_id}/response-details" diff --git a/application/blueprints/datamanager/controllers/transform.py b/application/blueprints/datamanager/controllers/transform.py index f1322e4c..f2645d53 100644 --- a/application/blueprints/datamanager/controllers/transform.py +++ b/application/blueprints/datamanager/controllers/transform.py @@ -8,7 +8,6 @@ from shapely.geometry import mapping from . import ControllerError -from ..config import get_entity_geojson_url, get_entity_search_url from ..services.async_api import fetch_response_details from ..services.dataset import get_dataset_name, get_dataset_typology from ..services.organisation import get_org_entity, get_organisation_name @@ -22,6 +21,17 @@ logger = logging.getLogger(__name__) + +def _entity_search_url(dataset_id, reference): + base = current_app.config["PLANNING_BASE_URL"] + return f"{base}/entity.json?dataset={dataset_id}&reference={reference}" + + +def _entity_geojson_url(reference): + base = current_app.config["PLANNING_BASE_URL"] + return f"{base}/entity.geojson?reference={reference}" + + _TRANSFORM_COLS = [ "entry_number", "entity", @@ -689,7 +699,7 @@ def fetch_boundary_geojson(organisation_code: str) -> dict: return empty lpa_prefix, lpa_id = organisation_code.split(":", 1) resp = requests.get( - get_entity_search_url(lpa_prefix, lpa_id), timeout=REQUESTS_TIMEOUT + _entity_search_url(lpa_prefix, lpa_id), timeout=REQUESTS_TIMEOUT ) resp.raise_for_status() d = resp.json() @@ -702,7 +712,7 @@ def fetch_boundary_geojson(organisation_code: str) -> dict: if not reference: return empty return requests.get( - get_entity_geojson_url(reference), timeout=REQUESTS_TIMEOUT + _entity_geojson_url(reference), timeout=REQUESTS_TIMEOUT ).json() except Exception as e: logger.warning("Failed to fetch boundary data for %s: %s", organisation_code, e) diff --git a/application/blueprints/datamanager/services/async_api.py b/application/blueprints/datamanager/services/async_api.py index ce9778ee..afcb1bd7 100644 --- a/application/blueprints/datamanager/services/async_api.py +++ b/application/blueprints/datamanager/services/async_api.py @@ -3,18 +3,27 @@ import requests +from config.config import get_request_api_endpoint + from application.extensions import cache -from ..config import ( - get_async_request_url, - get_async_requests_url, - get_async_response_details_url, -) from ..utils import REQUESTS_TIMEOUT logger = logging.getLogger(__name__) +def _requests_url() -> str: + return f"{get_request_api_endpoint()}/requests" + + +def _request_url(request_id: str) -> str: + return f"{get_request_api_endpoint()}/requests/{request_id}" + + +def _response_details_url(request_id: str) -> str: + return f"{get_request_api_endpoint()}/requests/{request_id}/response-details" + + class AsyncAPIError(Exception): """Raised when the async request API returns an error.""" @@ -36,9 +45,7 @@ def submit_request(params: dict) -> str: logger.info("Submitting request to async API") logger.debug(json.dumps(payload, indent=2)) - response = requests.post( - get_async_requests_url(), json=payload, timeout=REQUESTS_TIMEOUT - ) + response = requests.post(_requests_url(), json=payload, timeout=REQUESTS_TIMEOUT) logger.info(f"Async API responded with {response.status_code}") try: @@ -70,7 +77,7 @@ def fetch_request(request_id: str) -> dict: Returns the parsed JSON response on 200. Raises AsyncAPIError on non-200 status. """ - response = requests.get(get_async_request_url(request_id), timeout=REQUESTS_TIMEOUT) + response = requests.get(_request_url(request_id), timeout=REQUESTS_TIMEOUT) if response.status_code != 200: raise AsyncAPIError( @@ -108,7 +115,7 @@ def fetch_response_details( min(limit, max_rows - len(all_details)) if max_rows is not None else limit ) try: - url = get_async_response_details_url(request_id) + url = _response_details_url(request_id) params = {"offset": offset, "limit": fetch_limit} logger.debug(f"Fetching batch - URL: {url}, Params: {params}") diff --git a/application/blueprints/datamanager/utils/__init__.py b/application/blueprints/datamanager/utils/__init__.py index b9845766..67824bec 100644 --- a/application/blueprints/datamanager/utils/__init__.py +++ b/application/blueprints/datamanager/utils/__init__.py @@ -1,7 +1,5 @@ import csv import logging -import os -import re from datetime import datetime from io import StringIO @@ -17,22 +15,6 @@ REQUESTS_TIMEOUT = 20 # seconds -def get_allowed_override_users() -> set: - """Read config/allowed-users.md and return a set of GitHub usernames.""" - project_root = os.path.dirname(current_app.root_path) - path = os.path.join(project_root, "config", "allowed-users.md") - try: - with open(path, "r") as f: - content = f.read() - return { - m.group(1).strip().lower() - for m in re.finditer(r"^[-*]\s+(\S+)", content, re.MULTILINE) - } - except Exception as e: - logger.warning(f"Could not read allowed-users.md: {e}") - return set() - - def handle_error(e): logger.exception(f"Error: {e}") return render_template("datamanager/error.html", message=str(e)), 500 diff --git a/application/blueprints/dataset/__init__.py b/application/blueprints/dataset/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/blueprints/dataset/forms.py b/application/blueprints/dataset/forms.py deleted file mode 100644 index bd79a340..00000000 --- a/application/blueprints/dataset/forms.py +++ /dev/null @@ -1,36 +0,0 @@ -from flask_wtf import FlaskForm -from wtforms import SelectField, StringField, validators - - -def _none_filter(val): - if not val: - return None - return val - - -class EditRuleForm(FlaskForm): - dataset_id = StringField( - "Dataset", validators=[validators.Optional()], filters=[_none_filter] - ) - column = StringField( - "Column", validators=[validators.Optional()], filters=[_none_filter] - ) - endpoint_id = StringField( - "Endpoint", validators=[validators.Optional()], filters=[_none_filter] - ) - field_id = SelectField("Field", choices=[], validate_choice=False) - resource = StringField( - "Resource", validators=[validators.Optional()], filters=[_none_filter] - ) - separator = StringField( - "Separator", validators=[validators.Optional()], filters=[_none_filter] - ) - entry_date = StringField( - "Entry date", validators=[validators.Optional()], filters=[_none_filter] - ) - start_date = StringField( - "Start date", validators=[validators.Optional()], filters=[_none_filter] - ) - end_date = StringField( - "End date", validators=[validators.Optional()], filters=[_none_filter] - ) diff --git a/application/blueprints/dataset/views.py b/application/blueprints/dataset/views.py deleted file mode 100644 index fc688c29..00000000 --- a/application/blueprints/dataset/views.py +++ /dev/null @@ -1,307 +0,0 @@ -import datetime - -from flask import Blueprint, abort, redirect, render_template, request, url_for - -from application.blueprints.dataset.forms import EditRuleForm -from application.data_access.overview.digital_land_queries import ( - get_active_resources, - get_content_type_counts, - get_datasets, - get_latest_collector_run_date, - get_latest_resource, - get_publisher_coverage, - get_resource_count_per_dataset, - get_source_counts, - get_sources, - get_themes, - get_typologies, -) -from application.data_access.overview.entity_queries import get_entity_count -from application.data_access.overview.source_and_resource_queries import ( - get_datasets_summary, - get_monthly_counts, - publisher_counts, -) -from application.db.models import Dataset, PublicationStatus -from application.extensions import db -from application.spec_helpers import ( - PIPELINE_MODELS, - count_pipeline_rules, - get_expected_pipeline_specs, -) -from application.utils import filter_off_btns, index_by, resources_per_publishers - -dataset_bp = Blueprint("dataset", __name__, url_prefix="/dataset") - - -@dataset_bp.get("/") -def index(): - category_datasets = Dataset.query.filter( - Dataset.typology_id == "category", Dataset.collection_id.isnot(None) - ).all() - - datasets = Dataset.query.filter( - Dataset.typology_id != "category", Dataset.collection_id.isnot(None) - ).all() - - return render_template( - "dataset/index.html", - datasets=datasets, - category_datasets=category_datasets, - ) - - -@dataset_bp.get("/") -def dataset(dataset_id): - dataset = Dataset.query.get(dataset_id) - - if dataset is None or dataset.collection_id is None: - return abort(404) - - specification_pipelines = get_expected_pipeline_specs() - - rule_counts = count_pipeline_rules(dataset.collection.pipeline) - - return render_template( - "dataset/dataset.html", - pipeline=dataset.collection.pipeline, - dataset=dataset, - specification_pipelines=specification_pipelines, - rule_counts=rule_counts, - ) - - -@dataset_bp.get("//rules/") -def rule_type(dataset_id, rule_type_name): - limited = False - dataset = Dataset.query.get(dataset_id) - - if dataset is None or dataset.collection_id is None: - return abort(404) - - # # check if name is one of allowable rule types - specification_pipelines = get_expected_pipeline_specs() - if rule_type_name not in specification_pipelines.keys(): - return abort(404) - - rules = getattr(dataset.collection.pipeline, rule_type_name) - - if len(rules) > 1000: - rules = rules[:1000] - limited = True - - return render_template( - "dataset/rules.html", - dataset=dataset, - rule_type_name=rule_type_name, - rule_type_specification=specification_pipelines[rule_type_name], - rules=rules, - limited=limited, - ) - - -@dataset_bp.get("//sources") -def sources(dataset_id): - dataset = Dataset.query.get(dataset_id) - - if dataset is None or dataset.collection_id is None: - return abort(404) - - return render_template( - "dataset/sources.html", - dataset=dataset, - ) - - -def get_rule(id, rule_type): - return PIPELINE_MODELS[rule_type].query.get(id) - - -@dataset_bp.get("//rules//") -def edit_rule(dataset_id, rule_type_name, rule_id): - dataset = Dataset.query.get(dataset_id) - - if dataset is None or dataset.collection_id is None: - return abort(404) - - specification_pipelines = get_expected_pipeline_specs() - if rule_type_name not in specification_pipelines.keys(): - return abort(404) - - if rule_id == "new": - # create empty rule except for dataset - - form = EditRuleForm(dataset_id=dataset.dataset) - form.field_id.choices = [(field.field, field.field) for field in dataset.fields] - rule = {"dataset": dataset.dataset} - else: - rule = get_rule(rule_id, rule_type_name) - if rule is None: - return abort(404) - - form = EditRuleForm(obj=rule, rule_type=rule_type_name) - if hasattr(form, "field_id"): - form.field_id.choices = [ - (field.field, field.field) for field in dataset.fields - ] - if rule.field: - form.field_id.data = rule.field.field - - rule_type_specification = specification_pipelines[rule_type_name] - form_field_names = _get_form_field_names(rule_type_specification) - - return render_template( - "dataset/editrule.html", - form=form, - dataset=dataset, - rule_type_name=rule_type_name, - rule_type_specification=rule_type_specification, - form_field_names=form_field_names, - rule=rule, - ) - - -@dataset_bp.post("//rules//") -def save_rule(dataset_id, rule_type_name, rule_id): - form = EditRuleForm(request.form) - if form.validate_on_submit(): - dataset = Dataset.query.get(dataset_id) - if dataset is None: - return abort(404) - - if rule_id == "new": - rule_class = PIPELINE_MODELS[rule_type_name] - rule = rule_class() - rule.pipeline = dataset.collection.pipeline - else: - rule = get_rule(rule_id, rule_type_name) - if rule is None: - return abort(404) - - form.populate_obj(rule) - rule.pipeline.publication_status = PublicationStatus.DRAFT.name - db.session.add(rule) - db.session.commit() - return redirect( - url_for( - "dataset.rule_type", - dataset_id=dataset_id, - rule_type_name=rule_type_name, - ) - ) - return render_template("dataset/editrule.html", form=form) - - -def _get_form_field_names(rule_type_specification): - field_names = [] - for f in rule_type_specification.fields: - if f.field == rule_type_specification.dataset: - field_names.append(f.field) - elif f.field in ["dataset", "endpoint", "field"]: - field_names.append(f"{f.field}_id") - else: - field_names.append(f.field.replace("-", "_")) - return field_names - - -@dataset_bp.route("/overview/") -def dataset_overview(dataset): - datasets = get_datasets_summary() - dataset_name = dataset - dataset = [v for k, v in datasets.items() if v.get("pipeline") == dataset] - - resources_by_publisher = resources_per_publishers( - get_active_resources(dataset_name) - ) - - # publishers = fetch_publisher_stats(dataset_name) - publishers = publisher_counts(dataset_name) - publisher_splits = {"active": [], "noactive": []} - for k, publisher in publishers.items(): - if publisher["active_resources"] == 0: - publisher_splits["noactive"].append(publisher) - else: - publisher_splits["active"].append(publisher) - - # for the active resource charts - resource_stats = { - "over_one": len( - [p for p in resources_by_publisher if len(resources_by_publisher[p]) > 1] - ), - "one": len( - [p for p in resources_by_publisher if len(resources_by_publisher[p]) == 1] - ), - "zero": len(publisher_splits["noactive"]), - } - resource_counts = index_by("pipeline", get_resource_count_per_dataset()) - - resource_count = ( - resource_counts[dataset_name]["resources"] - if resource_counts.get(dataset_name) - else 0 - ) - - sources_no_doc_url, query_url = get_sources( - limit=500, filter={"pipeline": dataset_name} - ) - - try: - # wrapping in try/except because datasette occasionally timesout - content_type_counts = sorted( - get_content_type_counts(dataset=dataset_name), - key=lambda x: x["resource_count"], - reverse=True, - ) - except Exception as e: - print(e) - content_type_counts = [] - - blank_sources, bls_query = get_sources( - limit=500, - filter={"pipeline": dataset_name}, - only_blanks=True, - ) - - return render_template( - "dataset/performance.html", - name=dataset_name, - dataset=dataset[0] if len(dataset) else "", - latest_resource=get_latest_resource(dataset_name), - monthly_counts=get_monthly_counts(pipeline=dataset_name), - publishers=publisher_splits, - today=datetime.datetime.utcnow().isoformat()[:10], - entity_count=get_entity_count(pipeline=dataset_name), - resource_count=resource_count, - coverage=get_publisher_coverage(dataset_name), - resource_stats=resource_stats, - sources_no_doc_url=sources_no_doc_url, - content_type_counts=content_type_counts, - latest_logs=get_latest_collector_run_date(dataset=dataset_name), - blank_sources=blank_sources, - source_count=get_source_counts(pipeline=dataset_name), - ) - - -@dataset_bp.route("/overview") -def datasets(): - filters = {} - # if request.args.get("active"): - # filters["active"] = request.args.get("active") - if request.args.get("theme"): - filters["theme"] = request.args.get("theme") - if request.args.get("typology"): - filters["typology"] = request.args.get("typology") - - if len(filters.keys()): - dataset_records = get_datasets(filter=filters) - else: - dataset_records = get_datasets() - - return render_template( - "dataset/index.html", - datasets=dataset_records, - filters=filters, - filter_btns=filter_off_btns(filters), - themes=get_themes(), - typologies=get_typologies(), - ) diff --git a/application/blueprints/endpoint/__init__.py b/application/blueprints/endpoint/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/blueprints/endpoint/views.py b/application/blueprints/endpoint/views.py deleted file mode 100644 index 5fac7da6..00000000 --- a/application/blueprints/endpoint/views.py +++ /dev/null @@ -1,25 +0,0 @@ -from flask import Blueprint, jsonify, request - -from application.db.models import Endpoint - -endpoint_bp = Blueprint("endpoint", __name__, url_prefix="/endpoint") - - -@endpoint_bp.route("/") -def endpoint_json(endpoint): - endpoint = Endpoint.query.get(endpoint) - if endpoint: - return jsonify(endpoint), 200 - return {}, 404 - - -@endpoint_bp.route("/search", methods=["POST"]) -def search_json(): - data = request.json - if "endpoint_url" in data.keys(): - endpoint = Endpoint.query.filter( - Endpoint.endpoint_url == data["endpoint_url"] - ).first() - if endpoint: - return jsonify(endpoint), 200 - return {}, 404 diff --git a/application/blueprints/publisher/views.py b/application/blueprints/publisher/views.py deleted file mode 100644 index e5450ac3..00000000 --- a/application/blueprints/publisher/views.py +++ /dev/null @@ -1,137 +0,0 @@ -import datetime - -from flask import Blueprint, render_template - -from application.data_access.overview.api_queries import get_organisation_entity -from application.data_access.overview.digital_land_queries import ( - get_grouped_source_counts, - get_organisation_sources, - get_organisation_stats, - get_publishers, - get_resource_count_per_dataset, -) -from application.data_access.overview.entity_queries import ( - get_datasets_organisation_has_used_enddates, - get_organisation_entity_count, -) -from application.utils import index_by, index_with_list - -publisher_pages = Blueprint("publisher", __name__, url_prefix="/organisation") - - -def publisher_info(): - # returns all publishers, even empty - publisher_source_counts = get_publishers() - # return just the publishers we have data for - publisher_stats = get_organisation_stats() - empty_stats = {"resources": 0, "active": 0, "endpoints": 0, "pipelines": 0} - publishers = {} - for pub, stats in publisher_source_counts.items(): - if pub in publisher_stats.keys(): - publishers[pub] = {**stats, **publisher_stats[pub]} - else: - publishers[pub] = {**stats, **empty_stats} - return publishers - - -def split_publishers(organisations): - lpas = { - publisher: organisations[publisher] - for publisher in organisations.keys() - if "local-authority" in publisher - } - dev_corps = { - publisher: organisations[publisher] - for publisher in organisations.keys() - if "development-corporation" in publisher - } - national_parks = { - publisher: organisations[publisher] - for publisher in organisations.keys() - if "national-park" in publisher - } - other = { - publisher: organisations[publisher] - for publisher in organisations.keys() - if not any( - s in publisher - for s in ["local-authority", "development-corporation", "national-park"] - ) - } - return { - "Development corporation": dev_corps, - "National parks": national_parks, - "Other publishers": other, - "Local authorities": lpas, - } - - -@publisher_pages.route("/") -def organisation(): - publishers = publisher_info() - active_publishers = { - k: publisher - for k, publisher in publishers.items() - if publisher["sources_with_endpoint"] > 0 - } - publishers_with_no_data = { - k: publisher - for k, publisher in publishers.items() - if publisher["sources_with_endpoint"] == 0 - } - - return render_template( - "organisation/index.html", - publishers=split_publishers(active_publishers), - today=datetime.datetime.utcnow().isoformat()[:10], - none_publishers=split_publishers(publishers_with_no_data), - ) - - -@publisher_pages.route("//") -def organisation_performance(prefix, org_id): - id = prefix + ":" + org_id - organisation = get_organisation_entity(prefix, org_id) - resource_counts = get_resource_count_per_dataset(id) - source_counts = get_grouped_source_counts(id) - sources = index_with_list("pipeline", get_organisation_sources(id)) - - missing_datasets = [ - dataset for dataset in source_counts if dataset["sources_with_endpoint"] == 0 - ] - - data = {"datasets": index_by("pipeline", resource_counts)} - data["total_resources"] = sum( - [data["datasets"][d]["resources"] for d in data["datasets"].keys()] - ) - - # TO FIX: I'm not sure this is working - erroneous_sources = [] - for dataset in data["datasets"].keys(): - for source in sources[dataset]: - if source["endpoint"] == "": - erroneous_sources.append(source) - - # setup dict to capture datasets with data from secondary sources - data["data_from_secondary"] = {} - - # add entity counts to dataset data - entity_counts = get_organisation_entity_count(organisation=id) - for dn, count in entity_counts.items(): - if dn in data["datasets"].keys(): - data["datasets"][dn]["entity_count"] = count - else: - # add dataset to list from secondary sources - data["data_from_secondary"].setdefault(dn, {"pipeline": dn}) - data["data_from_secondary"][dn]["entity_count"] = count - - return render_template( - "organisation/performance.html", - organisation=organisation[0], - data=data, - sources_per_dataset=source_counts, - missing_datasets=missing_datasets, - enddates=get_datasets_organisation_has_used_enddates(id), - erroneous_sources=erroneous_sources, - entity_counts=entity_counts, - ) diff --git a/application/blueprints/report/views.py b/application/blueprints/report/views.py deleted file mode 100644 index 6cbdb5d1..00000000 --- a/application/blueprints/report/views.py +++ /dev/null @@ -1,476 +0,0 @@ -from urllib.parse import unquote - -from flask import Blueprint, abort, redirect, render_template, request, send_file -from flask.helpers import url_for - -from application.data_access.endpoint.endpoint_queries import ( - get_endpoint_details, - get_resources, -) -from application.data_access.odp_summaries.conformance import ( - get_odp_conformance_summary, -) -from application.data_access.odp_summaries.issue import ( - get_odp_issue_summary, - get_odp_issues_by_issue_type, -) -from application.data_access.odp_summaries.status import get_odp_status_summary -from application.data_access.odp_summaries.utils import generate_odp_summary_csv -from application.data_access.overview.datasette_queries import ( - fetch_resource_from_dataset, -) -from application.data_access.overview.digital_land_queries import ( - fetch_total_resource_count, - get_content_type_counts, - get_grouped_source_counts, - get_log_summary, - get_organisation_stats, - get_publisher_coverage, - get_resource, - get_resource_count_per_dataset, - get_source_counts, - get_sources, -) -from application.data_access.overview.entity_queries import ( - get_entity_count, - get_grouped_entity_count, -) -from application.data_access.overview.issue_summary import ( - get_full_issue_summary, - get_full_issue_summary_for_csv, -) -from application.data_access.overview.source_and_resource_queries import ( - get_datasets_summary, - get_monthly_counts, - get_new_resources, -) -from application.data_access.overview.utils import generate_overview_issue_summary_csv -from application.data_access.summary_queries import ( - get_contributions_and_erroring_endpoints, - get_contributions_and_errors_by_day, - get_endpoint_errors_and_successes_by_week, - get_endpoints_added_by_week, - get_internal_issues_by_week, - get_issue_counts, -) -from application.utils import ( - create_dict, - filter_off_btns, - index_by, - recent_dates, - yesterday, -) - -report_bp = Blueprint("reporting", __name__, url_prefix="/reporting") - - -@report_bp.get("/") -@report_bp.get("/overview") -def overview(): - contributions_and_errors_by_day_df = get_contributions_and_errors_by_day() - ( - summary_contributions, - summary_endpoint_errors, - ) = get_contributions_and_erroring_endpoints(contributions_and_errors_by_day_df) - errors, warnings = get_issue_counts() - endpoints_added_timeseries = get_endpoints_added_by_week() - ( - endpoint_successes_timeseries, - endpoint_successes_percentages_timeseries, - endpoint_errors_percentages_timeseries, - ) = get_endpoint_errors_and_successes_by_week(contributions_and_errors_by_day_df) - - internal_errors_timeseries = get_internal_issues_by_week() - - summary_metrics = { - "contributions": summary_contributions, - "endpoint_errors": summary_endpoint_errors, - "errors": errors, - "warnings": warnings, - } - graphs = { - "endpoints_added_timeseries": endpoints_added_timeseries, - "endpoint_successes_timeseries": endpoint_successes_timeseries, - "endpoint_successes_percentages_timeseries": endpoint_successes_percentages_timeseries, - "endpoint_errors_percentages_timeseries": endpoint_errors_percentages_timeseries, - "internal_errors_timeseries": internal_errors_timeseries, - } - - issue_summary = get_full_issue_summary() - - return render_template( - "reporting/overview.html", - summary_metrics=summary_metrics, - graphs=graphs, - issue_summary=issue_summary, - ) - - -@report_bp.get("/odp-summary/status") -def odp_status_summary(): - dataset_types = request.args.getlist("dataset_type") - cohorts = request.args.getlist("cohort") - odp_statuses_summary = get_odp_status_summary(dataset_types, cohorts) - - return render_template( - "reporting/odp_status_summary.html", odp_statuses_summary=odp_statuses_summary - ) - - -@report_bp.get("/odp-summary/issue") -def odp_issue_summary(): - dataset_types = request.args.getlist("dataset_type") - cohorts = request.args.getlist("cohort") - odp_issues_summary = get_odp_issue_summary(dataset_types, cohorts) - - return render_template( - "reporting/odp_issue_summary.html", odp_issues_summary=odp_issues_summary - ) - - -@report_bp.get("/odp-summary/conformance") -def odp_conformance_summary(): - dataset_types = request.args.getlist("dataset_type") - cohorts = request.args.getlist("cohort") - odp_conformance_summary, conformance_df = get_odp_conformance_summary( - dataset_types, cohorts - ) - return render_template( - "reporting/odp_conformance_summary.html", - odp_conformance_summary=odp_conformance_summary, - ) - - -@report_bp.get("/download") -def download_csv(): - type = request.args.get("type") - dataset_types = request.args.getlist("dataset_type") - cohorts = request.args.getlist("cohort") - if type == "odp-status": - odp_statuses_summary = get_odp_status_summary(dataset_types, cohorts) - file_path = generate_odp_summary_csv(odp_statuses_summary) - return send_file(file_path, download_name="odp-status.csv") - if type == "odp-issue": - odp_issues_by_type_summary = get_odp_issues_by_issue_type( - dataset_types, cohorts - ) - file_path = generate_odp_summary_csv(odp_issues_by_type_summary) - return send_file(file_path, download_name="odp-issue.csv") - if type == "odp-conformance": - odp_conformance_summary, conformance_df = get_odp_conformance_summary( - dataset_types, cohorts - ) - file_path = generate_odp_summary_csv(conformance_df) - return send_file(file_path, download_name="odp-conformance.csv") - if type == "endpoint_dataset_issue_type_summary": - overview_issue_summary = get_full_issue_summary_for_csv() - file_path = generate_overview_issue_summary_csv(overview_issue_summary) - return send_file(file_path, download_name="overview_issue_summary.csv") - - -@report_bp.get("endpoint//") -def endpoint_details(endpoint_hash, pipeline): - endpoint_details = get_endpoint_details(endpoint_hash, pipeline) - return render_template( - "reporting/endpoint_details.html", endpoint_details=endpoint_details - ) - - -@report_bp.get("/overview-of-datasets") -def overview_of_datasets(): - gs_datasets = get_datasets_summary() - entity_counts = get_grouped_entity_count() - content_type_counts = sorted( - get_content_type_counts(), - key=lambda x: x["resource_count"], - reverse=True, - ) - - return render_template( - "overview/performance.html", - datasets=gs_datasets, - stats=get_monthly_counts(), - publisher_count=get_publisher_coverage(), - source_counts=get_grouped_source_counts(groupby="dataset"), - entity_count=get_entity_count(), - datasets_with_data_count=len(entity_counts.keys()), - resource_count=fetch_total_resource_count(), - content_type_counts=content_type_counts, - new_resources=get_new_resources(dates=recent_dates(7)), - ) - - -@report_bp.route("/resource") -def resources(): - filters = {} - if request.args.get("pipeline"): - filters["pipeline"] = request.args.get("pipeline") - if request.args.get("content_type"): - filters["content_type"] = unquote(request.args.get("content_type")) - if request.args.get("organisation"): - filters["organisation"] = request.args.get("organisation") - if request.args.get("resource"): - filters["resource"] = request.args.get("resource") - - resources_per_dataset = index_by("pipeline", get_resource_count_per_dataset()) - - if len(filters.keys()): - resource_records_results = get_resources(filters=filters) - else: - resource_records_results = get_resources() - - content_type_counts = sorted( - get_content_type_counts(), - key=lambda x: x["resource_count"], - reverse=True, - ) - - columns = resource_records_results[0].keys() if resource_records_results else [] - resource_results = [create_dict(columns, row) for row in resource_records_results] - - return render_template( - "resource/index.html", - by_dataset=resources_per_dataset, - resource_count=fetch_total_resource_count(), - content_type_counts=content_type_counts, - datasets=get_grouped_entity_count(), - resources=resource_results, - filters=filters, - filter_btns=filter_off_btns(filters), - organisations=get_organisation_stats(), - ) - - -@report_bp.route("/logs") -def logs(): - if ( - request.args.get("log-date-day") - and request.args.get("log-date-month") - and request.args.get("log-date-year") - ): - log_year = request.args.get("log-date-year") - log_month = request.args.get("log-date-month") - log_day = request.args.get("log-date-day") - d = f"{log_year}-{log_month}-{log_day}" - return redirect(url_for("base.log", date=d)) - - summary = get_log_summary() - - return render_template( - "logs/logs.html", - summary=summary, - resources=get_new_resources(), - yesterday=yesterday(string=True), - endpoint_count=sum([status["count"] for status in summary]), - ) - - -@report_bp.route("/content-type") -def content_types(): - pipeline = request.args.get("pipeline") - - content_type_counts = sorted( - ( - get_content_type_counts(dataset=pipeline) - if pipeline - else get_content_type_counts() - ), - key=lambda x: x["resource_count"], - reverse=True, - ) - - if pipeline: - return render_template( - "content_type/index.html", - content_type_counts=content_type_counts, - pipeline=pipeline, - ) - - return render_template( - "content_type/index.html", content_type_counts=content_type_counts - ) - - -def paramify(url): - # there was a problem if the url to search on included url params - # this can be avoid if all & are replaced with %26 - url = url.replace("&", "%26") - # replace spaces (' ' or '%20' ) with %2520 - datasette automatically decoded %20 - url = url.replace(" ", "%2520") - return url.replace("%20", "%2520") - - -@report_bp.route("/source") -def sources(): - filters = {} - if request.args.get("pipeline"): - filters["pipeline"] = request.args.get("pipeline") - if request.args.get("organisation") is not None: - filters["organisation"] = request.args.get("organisation") - if request.args.get("endpoint_url"): - filters["endpoint_url"] = paramify(request.args.get("endpoint_url")) - if request.args.get("endpoint_"): - filters["endpoint_"] = request.args.get("endpoint_") - if request.args.get("source"): - filters["source"] = request.args.get("source") - if request.args.get("documentation_url") is not None: - filters["documentation_url"] = request.args.get("documentation_url") - include_blanks = False - if request.args.get("include_blanks") is not None: - include_blanks = request.args.get("include_blanks") - - if len(filters.keys()): - source_records, query_url = get_sources( - filter=filters, include_blanks=include_blanks - ) - else: - source_records, query_url = get_sources(include_blanks=include_blanks) - - return render_template( - "source/index.html", - datasets=get_grouped_source_counts(groupby="dataset"), - counts=get_source_counts()[0], - sources=source_records, - filters=filters, - filter_btns=filter_off_btns(filters), - organisations=get_grouped_source_counts(groupby="organisation"), - query_url=query_url, - include_blanks=include_blanks, - ) - - -@report_bp.route("/resource/") -def resource(resource): - resource_data = get_resource(resource) - if not resource_data: - return abort(404) - dataset = resource_data[0]["pipeline"].split(";")[0] - return render_template( - "resource/resource.html", - resource=resource_data, - info_page=url_for("base.resource_info", resource=resource), - resource_counts=fetch_resource_from_dataset(dataset, resource), - ) - - -@report_bp.route("/source/") -def source(source): - source_data, q = get_sources(filter={"source": source}) - if len(source_data) == 0: - # if no source record return check if blank one exists - source_data, q = get_sources(filter={"source": source}, include_blanks=True) - resource_result = get_resources(filters={"source": source}) - - columns = resource_result[0].keys() if resource_result else [] - - return render_template( - "source/source.html", - source=source_data[0], - resources=[create_dict(columns, row) for row in resource_result], - ) - - -# @report_bp.get("/dataset") -# def datasets(dataset_id): -# dataset = Dataset.query.get(dataset_id) - -# if dataset is None or dataset.collection_id is None: -# return abort(404) - -# specification_pipelines = get_expected_pipeline_specs() - -# rule_counts = count_pipeline_rules(dataset.collection.pipeline) - -# return render_template( -# "dataset/dataset.html", -# pipeline=dataset.collection.pipeline, -# dataset=dataset, -# specification_pipelines=specification_pipelines, -# rule_counts=rule_counts, -# ) - - -# @report_bp.get("dataset/") -# def dataset(dataset_id, rule_type_name): -# limited = False -# dataset = Dataset.query.get(dataset_id) - -# if dataset is None or dataset.collection_id is None: -# return abort(404) - -# # # check if name is one of allowable rule types -# specification_pipelines = get_expected_pipeline_specs() -# if rule_type_name not in specification_pipelines.keys(): -# return abort(404) - -# rules = getattr(dataset.collection.pipeline, rule_type_name) - -# if len(rules) > 1000: -# rules = rules[:1000] -# limited = True - -# return render_template( -# "dataset/rules.html", -# dataset=dataset, -# rule_type_name=rule_type_name, -# rule_type_specification=specification_pipelines[rule_type_name], -# rules=rules, -# limited=limited, -# ) - - -# @report_bp.get("/organisation") -# def organisations(dataset_id): -# dataset = Dataset.query.get(dataset_id) - -# if dataset is None or dataset.collection_id is None: -# return abort(404) - -# return render_template( -# "dataset/sources.html", -# dataset=dataset, -# ) - - -# @report_bp.get("/organisation/") -# def organisation(dataset_id, rule_type_name, rule_id): -# dataset = Dataset.query.get(dataset_id) - -# if dataset is None or dataset.collection_id is None: -# return abort(404) - -# specification_pipelines = get_expected_pipeline_specs() -# if rule_type_name not in specification_pipelines.keys(): -# return abort(404) - -# if rule_id == "new": -# # create empty rule except for dataset - -# form = EditRuleForm(dataset_id=dataset.dataset) -# form.field_id.choices = [(field.field, field.field) for field in dataset.fields] -# rule = {"dataset": dataset.dataset} -# else: -# rule = get_rule(rule_id, rule_type_name) -# if rule is None: -# return abort(404) - -# form = EditRuleForm(obj=rule, rule_type=rule_type_name) -# if hasattr(form, "field_id"): -# form.field_id.choices = [ -# (field.field, field.field) for field in dataset.fields -# ] -# if rule.field: -# form.field_id.data = rule.field.field - -# rule_type_specification = specification_pipelines[rule_type_name] -# form_field_names = _get_form_field_names(rule_type_specification) - -# return render_template( -# "dataset/editrule.html", -# form=form, -# dataset=dataset, -# rule_type_name=rule_type_name, -# rule_type_specification=rule_type_specification, -# form_field_names=form_field_names, -# rule=rule, -# ) diff --git a/application/blueprints/schema/__init__.py b/application/blueprints/schema/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/blueprints/schema/views.py b/application/blueprints/schema/views.py deleted file mode 100644 index 63faeb48..00000000 --- a/application/blueprints/schema/views.py +++ /dev/null @@ -1,35 +0,0 @@ -import string -from collections import OrderedDict - -from flask import Blueprint, render_template - -from application.db.models import Dataset - -schema_bp = Blueprint("schema", __name__, url_prefix="/schema") - - -@schema_bp.get("/") -def index(): - datasets = ( - Dataset.query.filter( - Dataset.typology_id.not_in(["specification", "value", "entity", "category"]) - ) - .order_by(Dataset.name) - .all() - ) - grouped_datasets = OrderedDict() - - for letter in string.ascii_uppercase: - group = [d for d in datasets if d.dataset[0].upper() == letter] - if len(group) > 0: - grouped_datasets[letter] = group - - return render_template( - "schema/index.html", schemas=datasets, grouped_datasets=grouped_datasets - ) - - -@schema_bp.get("/") -def schema(dataset_id): - schema = Dataset.query.get(dataset_id) - return render_template("schema/schema.html", schema=schema) diff --git a/application/blueprints/source/__init__.py b/application/blueprints/source/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/blueprints/source/forms.py b/application/blueprints/source/forms.py deleted file mode 100644 index b87dc060..00000000 --- a/application/blueprints/source/forms.py +++ /dev/null @@ -1,63 +0,0 @@ -from flask_wtf import FlaskForm -from wtforms import RadioField, SelectField, StringField, TextAreaField, ValidationError -from wtforms.validators import URL, DataRequired - -from application.db.models import Dataset - - -def same_collection(form, field): - datasets_ids = field.data.split(";") - if len(datasets_ids) > 1: - datasets = Dataset.query.filter(Dataset.dataset.in_(datasets_ids)).all() - collections = set([d.collection for d in datasets]) - if len(collections) > 1: - raise ValidationError( - "All the datasets you select must belong to the same collection" - ) - - -class SearchForm(FlaskForm): - source = StringField( - "Source", validators=[DataRequired(message="Enter a source hash")] - ) - - -class EditSourceForm(FlaskForm): - documentation_url = StringField("Documentation url") - start_date = StringField("Start date") - attribution = SelectField("Attribution", choices=[]) - licence = SelectField("Licence", choices=[]) - - -class NewSourceForm(EditSourceForm): - endpoint_url = StringField( - "Url", - validators=[ - DataRequired(message="Please provide a url"), - URL(message="Please provide a valid URL"), - ], - ) - dataset = StringField( - "Dataset", - validators=[ - DataRequired(message="Please provide a dataset"), - same_collection, - ], - ) - organisation = SelectField( - "Organisation", - validators=[DataRequired(message="Please provide an organisation ID")], - choices=[], - ) - - -class ArchiveForm(FlaskForm): - confirm = RadioField( - "Are you sure you want to archive the source", - validators=[DataRequired("You must select one")], - choices=[ - ("Yes", "I want to archive this source"), - ("No", "I don't want to archive this source"), - ], - ) - notes = TextAreaField("Why are you archiving this source?") diff --git a/application/blueprints/source/source.html b/application/blueprints/source/source.html deleted file mode 100644 index b5f92aff..00000000 --- a/application/blueprints/source/source.html +++ /dev/null @@ -1,118 +0,0 @@ -{% extends "source/base.html" %} - -{% block dl_breadcrumbs %} -{{ govukBreadcrumbs({ - "items": [ - { - "text": "Sources", - "href": url_for('base.sources') - }, - { - "text": source['source'] - } - ] -}) }} -{% endblock %} - -{% block content %} - -
-

Source

- -
-
-
-

{{ source['source'] }}

-
-
-
Organisation
-
{{ source['organisation']}}
-
-
-
Endpoint
-
- {%- if source['endpoint_url'] %} - {{ source['endpoint_url'] }} - (Endpoint identifier is {{ source['endpoint']|truncate(15) }})
- {% endif -%} -
-
-
Documentation url
-
{% if source['documentation_url'] %} - {{ source['documentation_url'] }} - {% else %} - Missing - {% endif %}
-
-
-
Attribution
-
{% if source['attribution'] %} - {{ source['attribution'] }} - {% else %} - Missing - {% endif %}
-
-
-
Licence
-
{% if source['licence'] %} - {{ source['licence'] }} - {% else %} - Missing - {% endif %}
-
-
-
Entry date
-
{{ source['entry_date'] }}
-
-
-
Start date
-
{{ source['start_date'] }}
-
-
-
End date
-
{{ source['end_date'] }}
-
-
-
-
-
- -
-
-

Resources

-

{{ resources|length }} resource{{ " has" if resources|length == 1 else "s have" }} been collected from this source.

- {% if resources|length > 0 %} -
    - {% for resource in resources %} -
  • -

    Resource {{ resource['resource']|truncate(15) }}

    -
    -
    -
    Dataset
    -
    {{ resource['pipeline'] }}
    -
    -
    -
    Content type
    -
    {{ resource['content_type'] }}
    -
    -
    -
    Collected on
    -
    {{ resource['start_date'] }}
    -
    -
    -
    • - {% endfor %} -
- {% endif %} - -
-
-
-{% endblock %} - -{% block pageScripts %} -{{ super() }} - -{% endblock pageScripts %} diff --git a/application/blueprints/source/views.py b/application/blueprints/source/views.py deleted file mode 100644 index 00a013b1..00000000 --- a/application/blueprints/source/views.py +++ /dev/null @@ -1,270 +0,0 @@ -from datetime import datetime - -# from digital_land.api import DigitalLandApi -from flask import Blueprint, redirect, render_template, request, url_for - -from application.blueprints.source.forms import ( - ArchiveForm, - EditSourceForm, - NewSourceForm, - SearchForm, -) -from application.db.models import ( - Attribution, - Dataset, - Endpoint, - Licence, - Organisation, - Source, -) -from application.extensions import db -from application.utils import ( - check_url_reachable, - compute_hash, - compute_md5_hash, - login_required, -) - -source_bp = Blueprint("source", __name__, url_prefix="/source") - - -def organisation_choices(): - organisations = Organisation.query.order_by(Organisation.name).all() - return [("", "")] + [(o.organisation, o.name) for o in organisations] - - -def dataset_choices(): - datasets = ( - Dataset.query.filter(Dataset.typology != "specification") - .order_by(Dataset.name) - .all() - ) - return [("", "")] + [(d.dataset, d.name) for d in datasets] - - -def get_datasets(s, sep=","): - ids = s.split(sep) - return Dataset.query.filter(Dataset.dataset.in_(ids)).all() - - -def create_source_data(form, _type="new"): - if _type == "new": - return { - "endpoint_url": form.endpoint_url.data, - "organisation": form.organisation.data, - "dataset": form.dataset.data, - "documentation_url": form.documentation_url.data, - "licence": form.licence.data, - "attribution": form.attribution.data, - "start_date": form.start_date.data, - } - else: - return { - "documentation_url": form.documentation_url.data, - "licence": form.licence.data, - "attribution": form.attribution.data, - "start_date": form.start_date.data, - } - - -def dataset_str_to_objs(s): - source_datasets = s.split(";") - return Dataset.query.filter(Dataset.dataset.in_(source_datasets)).all() - - -def create_or_update_endpoint(data): - endpoint_url = data.get("endpoint_url").strip() - hashed_url = compute_hash(endpoint_url) - endpoint = Endpoint.query.get(hashed_url) - if endpoint is None: - endpoint = Endpoint( - endpoint=hashed_url, - endpoint_url=endpoint_url, - entry_date=datetime.now().isoformat(), - ) - datasets = Dataset.query.filter( - Dataset.dataset.in_(data.get("dataset").split(";")) - ).all() - organisation = Organisation.query.get(data.get("organisation")) - collection = datasets[0].collection - source_key = ( - f"{collection.collection}|{organisation.organisation}|{endpoint.endpoint}" - ) - source_key = compute_md5_hash(source_key) - # check if source exists - can happen if user refreshes finish page - source = Source.query.get(source_key) - if source is None: - source = Source( - source=source_key, - organisation=organisation, - entry_date=datetime.now().isoformat(), - collection=collection, - datasets=datasets, - ) - source.update(data) - db.session.add(source) - endpoint.sources.append(source) - return endpoint - - -@source_bp.route("/", methods=["GET", "POST"]) -def search(): - form = SearchForm() - if form.validate_on_submit(): - source_hash = form.source.data.strip() - source = Source.query.get(source_hash) - if source: - return redirect(url_for("source.source", source_hash=source.source)) - form.source.errors.append("We don't recognise that hash, try another") - sources = ( - Source.query.filter(Source.entry_date != None) # noqa: E711 - .order_by(Source.entry_date.desc()) - .limit(10) - .all() - ) - return render_template("source/search.html", form=form, sources=sources) - - -def clean_dataset_string(s): - ds = s.replace(",", ";").split(";") - return ";".join([d.strip() for d in ds]) - - -@source_bp.get("/add") -@login_required -def add(): - if request.args: - form = NewSourceForm(request.args) - else: - form = NewSourceForm() - - organisations = Organisation.query.order_by(Organisation.name).all() - form.organisation.choices = [("", "")] + [ - (o.organisation, o.name) for o in organisations - ] - datasets = ( - Dataset.query.filter(Dataset.typology_id != "specification") - .order_by(Dataset.name) - .all() - ) - form.attribution.choices = [("", "")] + [ - (attribution.attribution, attribution.attribution) - for attribution in Attribution.query.all() - ] - form.licence.choices = [("", "")] + [ - (licence.licence, licence.text) for licence in Licence.query.all() - ] - - if request.args and not request.args.get("_change") and form.validate(): - endpoint_hash = compute_hash(form.endpoint_url.data.strip()) - endpoint = Endpoint.query.get(endpoint_hash) - url_reachable = check_url_reachable(form.endpoint_url.data.strip()) - source_params = { - **request.args, - **{"dataset": clean_dataset_string(request.args.get("dataset"))}, - } - query_params = {"url_reachable": url_reachable, **source_params} - if endpoint is not None: - # will need to update when/if user can put multiple datasets - existing_source = endpoint.get_matching_source( - form.organisation.data, form.dataset.data - ) - if existing_source is not None: - query_params["existing_source"] = existing_source - - return redirect(url_for("source.summary", **query_params)) - - return render_template("source/create.html", form=form, datasets=datasets) - - -@source_bp.get("/add/summary") -def summary(): - form = NewSourceForm(request.args) - url_reachable = request.args.get("url_reachable", None) - existing_source_id = request.args.get("existing_source", None) - existing_source = ( - Source.query.get(existing_source_id) if existing_source_id is not None else None - ) - if existing_source: - datasets = existing_source.datasets - organisation = existing_source.organisation - else: - datasets = dataset_str_to_objs(form.dataset.data) - organisation = Organisation.query.get(form.organisation.data) - return render_template( - "source/summary.html", - sources=[form.data], - existing_source=existing_source, - url_reachable=url_reachable, - organisation=organisation, - datasets=datasets, - form=form, - ) - - -@source_bp.get("/add/finish") -@login_required -def finish(): - # user is not at the end of the add a source journey - if len(request.args) == 0: - return redirect(url_for("source.add")) - - form = NewSourceForm(request.args) - existing_source = request.args.get("existing_source", None) - if existing_source is None: - endpoint = create_or_update_endpoint(form.data) - db.session.add(endpoint) - source = endpoint.sources[-1] - else: - source = Source.query.get(existing_source) - source.update(form.data) - db.session.add(source) - db.session.commit() - return render_template("source/finish.html", source=source) - - -@source_bp.route("") -def source(source_hash): - source = Source.query.get(source_hash) - return render_template("source/source.html", source=source) - - -@source_bp.route("/edit", methods=["GET", "POST"]) -@login_required -def edit(source_hash): - source = Source.query.get(source_hash) - form = EditSourceForm(obj=source) - form.licence.choices = [ - (licence.licence, licence.text) for licence in Licence.query.all() - ] - form.attribution.choices = [ - (attribution.attribution, attribution.text) - for attribution in Attribution.query.all() - ] - - if form.validate_on_submit(): - # if endpoint, org or dataset have changed then has the source changed or is it a new one - # so ignore those for now - params = create_source_data(form, _type="edit") - params["existing_source"] = source.source - params["url_reachable"] = True - return redirect(url_for("source.summary", **params)) - - cancel_href = url_for("source.source", source_hash=source.source) - if request.referrer and url_for("source.summary") in request.referrer: - cancel_href = request.referrer - return render_template( - "source/edit.html", source=source, form=form, cancel_href=cancel_href - ) - - -@source_bp.route("/archive") -def archive(source_hash): - source = Source.query.get(source_hash) - form = ArchiveForm() - if form.validate_on_submit(): - # do something with the answer - pass - return render_template( - "source/archive.html", source=source, form=form, referrer=request.referrer - ) diff --git a/application/commands.py b/application/commands.py deleted file mode 100644 index 4ae28376..00000000 --- a/application/commands.py +++ /dev/null @@ -1,160 +0,0 @@ -import base64 -import logging -import os -import sys - -import github -from flask.cli import AppGroup - -from application.db.models import Collection, PublicationStatus -from application.extensions import db -from application.publish.models import ( - ColumnModel, - CombineModel, - ConcatModel, - ConvertModel, - DefaultModel, - DefaultValueModel, - EndpointModel, - FilterModel, - PatchModel, - SkipModel, - SourceModel, - TransformModel, -) -from application.utils import csv_dict_to_string - -logging.basicConfig(stream=sys.stdout) -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -publish_cli = AppGroup("publish") - - -PUBLISH_MODEL_CLASSES = { - "column": ColumnModel, - "combine": CombineModel, - "concat": ConcatModel, - "conver": ConvertModel, - "default": DefaultModel, - "default_model": DefaultValueModel, - "patch": PatchModel, - "skip": SkipModel, - "transform": TransformModel, - "filter": FilterModel, -} - - -@publish_cli.command("changes") -def publish_config(): - app_id = os.getenv("GITHUB_APP_ID") - private_key = base64.b64decode(os.getenv("GITHUB_APP_PRIVATE_KEY")).decode("utf-8") - branch = os.getenv("CONFIG_REPO_BRANCH") - - g = github.GithubIntegration(app_id, private_key) - token = g.get_access_token(g.get_installation("digital-land", "config").id).token - gh = github.Github(login_or_token=token) - repo = gh.get_repo("digital-land/config") - - for collection in Collection.query.order_by(Collection.name).all(): - if collection.publication_status == PublicationStatus.DRAFT: - logger.info(f"Publish sources and endpoints for {collection.collection}") - _publish_collection_config(collection, repo, branch) - collection.publication_status = PublicationStatus.PUBLISHED - db.session.add(collection) - db.session.commit() - else: - logger.info(f"Collection {collection.collection} has no updates to publish") - - if collection.pipeline.publication_status == PublicationStatus.DRAFT: - logger.info("Publish rules for pipeline", collection.pipeline.pipeline) - _publish_pipeline_config(collection.pipeline, repo, branch) - collection.pipeline.publication_status = PublicationStatus.PUBLISHED - db.session.add(collection) - db.session.commit() - else: - logger.info( - f"Pipeline {collection.pipeline.pipeline} has no updates to publish" - ) - - -def _publish_collection_config(collection, repo, branch_name): - branch = repo.get_branch(branch_name) - branch_sha = branch.commit.sha - base_tree = repo.get_git_tree(sha=branch_sha) - - sources = [] - for source in collection.sources: - sources.append(SourceModel.from_orm(source).dict(by_alias=True)) - - data = csv_dict_to_string(sources) - blob = repo.create_git_blob(data, "utf-8") - - path = f"collection/{collection.collection}/source.csv" - source_element = github.InputGitTreeElement( - path=path, mode="100644", type="blob", sha=blob.sha - ) - - endpoints = [] - for endpoint in collection.endpoints: - endpoints.append(EndpointModel.from_orm(endpoint).dict(by_alias=True)) - - data = csv_dict_to_string(endpoints) - blob = repo.create_git_blob(data, "utf-8") - - path = f"collection/{collection.collection}/endpoint.csv" - endpoint_element = github.InputGitTreeElement( - path=path, mode="100644", type="blob", sha=blob.sha - ) - - tree = repo.create_git_tree([source_element, endpoint_element], base_tree) - parent = repo.get_git_commit(sha=branch_sha) - message = f"Commit update of sources for {collection.collection}" - commit = repo.create_git_commit(message, tree, [parent]) - branch_refs = repo.get_git_ref(f"heads/{branch_name}") - branch_refs.edit(sha=commit.sha) - logger.info(f"Commited collection config - commit sha: {commit.sha}") - - -def _publish_pipeline_config(pipeline, repo, branch_name): - branch = repo.get_branch(branch_name) - branch_sha = branch.commit.sha - base_tree = repo.get_git_tree(sha=branch_sha) - elements = [] - - for rule_type, rules in pipeline.get_pipeline_rules().items(): - if rules: - to_publish = [] - publish_model = PUBLISH_MODEL_CLASSES.get(rule_type, None) - if publish_model is None: - logger.info( - "Can't publish rule type {{rule_type}}. No model defined yet" - ) - continue - - logger.info(f"Publish {len(rules)} {rule_type} rules") - for rule in rules: - to_publish.append(publish_model.from_orm(rule).dict(by_alias=True)) - - data = csv_dict_to_string(to_publish) - blob = repo.create_git_blob(data, "utf-8") - - path = f"pipeline/{pipeline.pipeline}/{rule_type}.csv" - element = github.InputGitTreeElement( - path=path, mode="100644", type="blob", sha=blob.sha - ) - elements.append(element) - else: - logger.info(f"No {rule_type} to publish for {pipeline.pipeline}") - - if elements: - tree = repo.create_git_tree(elements, base_tree) - parent = repo.get_git_commit(sha=branch_sha) - message = f"Commit update of pipeline config for {pipeline.pipeline}" - commit = repo.create_git_commit(message, tree, [parent]) - branch_refs = repo.get_git_ref(f"heads/{branch}") - branch_refs.edit(sha=commit.sha) - logger.info( - f"Commited pipeline {pipeline.pipeline} config - commit sha: {commit.sha}" - ) diff --git a/application/data_access/endpoint/endpoint_queries.py b/application/data_access/endpoint/endpoint_queries.py deleted file mode 100644 index 3723dbc5..00000000 --- a/application/data_access/endpoint/endpoint_queries.py +++ /dev/null @@ -1,98 +0,0 @@ -from application.data_access.datasette_utils import get_datasette_query - - -def get_endpoint_details(endpoint_hash, pipeline): - logs_df = get_logs(endpoint_hash) - logs_headers = list( - map( - lambda x: {"text": x, "classes": "reporting-table-header"}, - logs_df.columns.values.tolist(), - ) - ) - logs_rows = [] - for row in logs_df.values.tolist(): - logs_rows.append( - list(map(lambda x: {"text": x, "classes": "reporting-table-cell"}, row)) - ) - - resources_df = get_resources(endpoint_hash) - resources_headers = list( - map( - lambda x: {"text": x, "classes": "reporting-table-header"}, - resources_df.columns.values.tolist(), - ) - ) - resources_rows = [] - for row in resources_df.values.tolist(): - resources_rows.append( - list(map(lambda x: {"text": x, "classes": "reporting-table-cell"}, row)) - ) - - endpoint_info_df = get_endpoint_info(endpoint_hash, pipeline) - endpoint_info = endpoint_info_df.to_dict(orient="records") - return { - "logs_headers": logs_headers, - "logs_rows": logs_rows, - "resources_headers": resources_headers, - "resources_rows": resources_rows, - "endpoint_info": endpoint_info[0], - } - - -def get_logs(endpoint_hash): - sql = f""" - select - substring(entry_date,1,10) as entry_date, - case - when (status = '') then exception - else status - end as status, - resource - from - log - where - endpoint = '{endpoint_hash}' - order by - entry_date desc - """ - return get_datasette_query("digital-land", sql) - - -def get_resources(endpoint_hash): - sql = f""" - select - r.resource, - r.start_date, - r.end_date - from - resource r - inner join resource_endpoint re on re.resource = r.resource - where - re.endpoint = '{endpoint_hash}' - order by - r.start_date desc - """ - return get_datasette_query("digital-land", sql) - - -def get_endpoint_info(endpoint_hash, dataset): - sql = f""" - select - sp.pipeline, - o.name as organisation_name, - s.organisation, - s.documentation_url, - e.endpoint, - e.endpoint_url, - e.start_date, - substring(e.entry_date,1,10) as entry_date - from - endpoint e - inner join source s on s.endpoint = e.endpoint - inner join source_pipeline sp on sp.source = s.source - inner join organisation o on o.organisation = replace(s.organisation, '-eng', '') - where - s.endpoint = '{endpoint_hash}' - AND sp.pipeline = '{dataset}' - """ - return get_datasette_query("digital-land", sql) diff --git a/application/data_access/odp_summaries/conformance.py b/application/data_access/odp_summaries/conformance.py deleted file mode 100644 index 1dc45ff2..00000000 --- a/application/data_access/odp_summaries/conformance.py +++ /dev/null @@ -1,455 +0,0 @@ -import json - -import numpy as np -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query -from application.data_access.odp_summaries.utils import get_provisions - -SPATIAL_DATASETS = [ - "article-4-direction-area", - "conservation-area", - "listed-building-outline", - "tree-preservation-zone", - "tree", -] -DOCUMENT_DATASETS = [ - "article-4-direction", - "conservation-area-document", - "tree-preservation-order", -] - -# Separate variable for all datasets as arbitrary ordering required -ALL_DATASETS = [ - "article-4-direction", - "article-4-direction-area", - "conservation-area", - "conservation-area-document", - "listed-building-outline", - "tree-preservation-order", - "tree-preservation-zone", - "tree", -] - -# Configs that are passed to the front end for the filters -DATASET_TYPES = [ - {"name": "Spatial", "id": "spatial"}, - {"name": "Document", "id": "document"}, -] - -COHORTS = [ - {"name": "RIPA Beta", "id": "RIPA-Beta"}, - {"name": "RIPA BOPS", "id": "RIPA-BOPS"}, - {"name": "ODP Track 1", "id": "ODP-Track1"}, - {"name": "ODP Track 2", "id": "ODP-Track2"}, - {"name": "ODP Track 3", "id": "ODP-Track3"}, - {"name": "ODP Track 4", "id": "ODP-Track4"}, -] - - -def get_column_field_summary(dataset_clause, offset): - sql = f""" - SELECT edrs.*, rle.licence - FROM endpoint_dataset_resource_summary AS edrs - LEFT JOIN ( - SELECT endpoint, licence, dataset - FROM reporting_latest_endpoints - ) AS rle ON edrs.endpoint = rle.endpoint and edrs.dataset = rle.dataset - LEFT JOIN ( - SELECT endpoint, end_date as endpoint_end_date, dataset - FROM endpoint_dataset_summary - ) as eds on edrs.endpoint = eds.endpoint and edrs.dataset = eds.dataset - WHERE edrs.resource != '' - and eds.endpoint_end_date='' - and ({dataset_clause}) - limit 1000 offset {offset} - """ - column_field_df = get_datasette_query("performance", sql) - - return column_field_df - - -def get_issue_summary(dataset_clause, offset): - sql = f""" - select * from endpoint_dataset_issue_type_summary edrs - where ({dataset_clause}) - limit 1000 offset {offset} - """ - issue_summary_df = get_datasette_query("performance", sql) - return issue_summary_df - - -def get_odp_conformance_summary(dataset_types, cohorts): - params = { - "cohorts": COHORTS, - "dataset_types": DATASET_TYPES, - "selected_dataset_types": dataset_types, - "selected_cohorts": cohorts, - } - if dataset_types == ["spatial"]: - datasets = SPATIAL_DATASETS - elif dataset_types == ["document"]: - datasets = DOCUMENT_DATASETS - else: - datasets = ALL_DATASETS - dataset_clause = " or ".join( - ("edrs.pipeline = '" + str(dataset) + "'" for dataset in datasets) - ) - - provision_df = get_provisions(cohorts, COHORTS) - - # Download column field summary table - # Use pagination in case rows returned > 1000 - pagination_incomplete = True - offset = 0 - column_field_df_list = [] - while pagination_incomplete: - column_field_df = get_column_field_summary(dataset_clause, offset) - column_field_df_list.append(column_field_df) - pagination_incomplete = len(column_field_df) == 1000 - offset += 1000 - if len(column_field_df_list) == 0: - return {"params": params, "rows": [], "headers": []} - column_field_df = pd.concat(column_field_df_list) - - column_field_df = pd.merge( - column_field_df, provision_df, on=["organisation", "cohort"], how="inner" - ) - - # Download issue summary table - pagination_incomplete = True - offset = 0 - issue_df_list = [] - while pagination_incomplete: - issue_df = get_issue_summary(dataset_clause, offset) - issue_df_list.append(issue_df) - pagination_incomplete = len(issue_df) == 1000 - offset += 1000 - issue_df = pd.concat(issue_df_list) - - dataset_field_df = get_dataset_field() - - # remove fields that are auto-created in the pipeline from the dataset_field file to avoid mis-counting - # ("entity", "organisation", "prefix", "point" for all but tree, and "entity", "organisation", "prefix" for tree) - dataset_field_df = dataset_field_df[ - (dataset_field_df["dataset"] != "tree") - & ( - ~dataset_field_df["field"].isin( - ["entity", "organisation", "prefix", "point"] - ) - ) - | (dataset_field_df["dataset"] == "tree") - & (~dataset_field_df["field"].isin(["entity", "organisation", "prefix"])) - ] - - # Filter out fields not in spec - column_field_df["mapping_field"] = column_field_df.replace({'"', ""}).apply( - lambda row: [ - field - for field in ( - row["mapping_field"].split(";") if row["mapping_field"] else "" - ) - if field - in dataset_field_df[dataset_field_df["dataset"] == row["dataset"]][ - "field" - ].values - ], - axis=1, - ) - column_field_df["non_mapping_field"] = column_field_df.replace({'"', ""}).apply( - lambda row: [ - field - for field in ( - row["non_mapping_field"].split(";") if row["non_mapping_field"] else "" - ) - if field - in dataset_field_df[dataset_field_df["dataset"] == row["dataset"]][ - "field" - ].values - ], - axis=1, - ) - - # Map entity errors to reference field - issue_df["field"] = issue_df["field"].replace("entity", "reference") - # Filter out issues for fields not in dataset field (specification) - issue_df["field"] = issue_df.apply( - lambda row: ( - row["field"] - if row["field"] - in dataset_field_df[dataset_field_df["dataset"] == row["dataset"]][ - "field" - ].values - else None - ), - axis=1, - ) - - # Create field matched and field supplied scores - column_field_df["field_matched"] = column_field_df.apply( - lambda row: len(row["mapping_field"]) if row["mapping_field"] else 0, axis=1 - ) - column_field_df["field_supplied"] = column_field_df.apply( - lambda row: row["field_matched"] - + (len(row["non_mapping_field"]) if row["non_mapping_field"] else 0), - axis=1, - ) - column_field_df["field"] = column_field_df.apply( - lambda row: len( - dataset_field_df[dataset_field_df["dataset"] == row["dataset"]] - ), - axis=1, - ) - - # Check for fields which have error issues - results_issues = [ - issue_df[ - (issue_df["resource"] == r["resource"]) & (issue_df["severity"] == "error") - ] - for index, r in column_field_df.iterrows() - ] - results_issues_df = pd.concat(results_issues) - - # Create fields with errors column - column_field_df["field_errors"] = column_field_df.apply( - lambda row: len( - results_issues_df[row["resource"] == results_issues_df["resource"]] - ), - axis=1, - ) - - # Create endpoint ID column to track multiple endpoints per organisation-dataset - column_field_df["endpoint_no."] = ( - column_field_df.groupby(["organisation", "dataset"]).cumcount() + 1 - ) - column_field_df["endpoint_no."] = column_field_df["endpoint_no."].astype(str) - - # group by and aggregate for final summaries - final_count = ( - column_field_df.groupby( - [ - "organisation", - "organisation_name", - "cohort", - "dataset", - "licence", - "endpoint", - "endpoint_no.", - "resource", - "latest_log_entry_date", - "cohort_start_date", - ] - ) - .agg( - { - "field": "sum", - "field_supplied": "sum", - "field_matched": "sum", - "field_errors": "sum", - } - ) - .reset_index() - ) - - final_count["field_error_free"] = ( - final_count["field_supplied"] - final_count["field_errors"] - ) - final_count["field_error_free"] = final_count["field_error_free"].replace(-1, 0) - - # add string fields for [n fields]/[total fields] style counts - final_count["field_supplied_count"] = ( - final_count["field_supplied"].astype(int).map(str) - + "/" - + final_count["field"].map(str) - ) - final_count["field_error_free_count"] = ( - final_count["field_error_free"].astype(int).map(str) - + "/" - + final_count["field"].map(str) - ) - final_count["field_matched_count"] = ( - final_count["field_matched"].astype(int).map(str) - + "/" - + final_count["field"].map(str) - ) - - # create % columns - final_count["field_supplied_pct"] = ( - final_count["field_supplied"] / final_count["field"] - ) - final_count["field_error_free_pct"] = ( - final_count["field_error_free"] / final_count["field"] - ) - final_count["field_matched_pct"] = ( - final_count["field_matched"] / final_count["field"] - ) - - final_count.reset_index(drop=True, inplace=True) - final_count.sort_values( - ["cohort_start_date", "cohort", "organisation_name", "dataset"], inplace=True - ) - - provisions_with_100_pct_match = final_count[final_count["field_matched_pct"] == 1.0] - percent_100_field_match = ( - round(len(provisions_with_100_pct_match) / len(final_count) * 100, 1) - if len(final_count) - else 0 - ) - - out_cols = [ - "cohort", - "organisation_name", - "organisation", - "dataset", - "licence", - "endpoint_no.", - "field_supplied_count", - "field_supplied_pct", - "field_matched_count", - "field_matched_pct", - ] - - csv_out_cols = [ - "organisation", - "organisation_name", - "cohort", - "dataset", - "licence", - "endpoint", - "endpoint_no.", - "resource", - "latest_log_entry_date", - "field", - "field_supplied", - "field_matched", - "field_errors", - "field_error_free", - "field_supplied_pct", - "field_error_free_pct", - "field_matched_pct", - ] - - headers = [ - *map( - lambda column: { - "text": make_pretty(column).title(), - "classes": "reporting-table-header", - }, - out_cols, - ) - ] - - rows = [ - [ - { - "text": make_pretty(cell), - "classes": "reporting-table-cell " + get_background_class(cell), - } - for cell in r - ] - for index, r in final_count[out_cols].iterrows() - ] - - # Calculate overview stats - overview_datasets = [ - "article-4-direction-area", - "conservation-area", - "listed-building-outline", - "tree", - "tree-preservation-zone", - ] - overview_stats_df = pd.DataFrame() - overview_stats_df["dataset"] = overview_datasets - overview_stats_df = overview_stats_df.merge( - final_count[["dataset", "field_supplied_pct"]][ - final_count["field_supplied_pct"] < 0.5 - ] - .groupby("dataset") - .count(), - on="dataset", - how="left", - ).rename(columns={"field_supplied_pct": "< 50%"}) - overview_stats_df = overview_stats_df.merge( - final_count[["dataset", "field_supplied_pct"]][ - (final_count["field_supplied_pct"] >= 0.5) - & (final_count["field_supplied_pct"] < 0.8) - ] - .groupby("dataset") - .count(), - on="dataset", - how="left", - ).rename(columns={"field_supplied_pct": "50% - 80%"}) - overview_stats_df = overview_stats_df.merge( - final_count[["dataset", "field_supplied_pct"]][ - final_count["field_supplied_pct"] >= 0.8 - ] - .groupby("dataset") - .count(), - on="dataset", - how="left", - ).rename(columns={"field_supplied_pct": "> 80%"}) - overview_stats_df.replace(np.nan, 0, inplace=True) - overview_stats_df = overview_stats_df.astype( - { - "< 50%": int, - "50% - 80%": int, - "> 80%": int, - } - ) - - stats_headers = [ - *map( - lambda column: { - "text": column.title(), - "classes": "reporting-table-header", - }, - overview_stats_df.columns.values, - ) - ] - stats_rows = [ - [{"text": cell, "classes": "reporting-table-cell"} for cell in r] - for index, r in overview_stats_df.iterrows() - ] - return { - "headers": headers, - "rows": rows, - "stats_headers": stats_headers, - "stats_rows": stats_rows, - "params": params, - "percent_100_field_match": percent_100_field_match, - }, final_count[csv_out_cols] - - -def make_pretty(text): - if type(text) is float: - # text is a float, make a percentage - return str((round(100 * text))) + "%" - elif "_" in text: - # text is a column name - return text.replace("_", " ").replace("pct", "%").replace("count", "") - return text - - -def get_background_class(text): - if type(text) is float: - group = int((text * 100) / 10) - if group == 10: - return "reporting-100-background" - else: - return "reporting-" + str(group) + "0-" + str(group + 1) + "0-background" - return "" - - -def get_dataset_field(): - specification_df = pd.read_csv( - "https://raw.githubusercontent.com/digital-land/specification/main/specification/specification.csv" - ) - rows = [] - for index, row in specification_df.iterrows(): - specification_dicts = json.loads(row["json"]) - for dict in specification_dicts: - dataset = dict["dataset"] - fields = [field["field"] for field in dict["fields"]] - for field in fields: - rows.append({"dataset": dataset, "field": field}) - return pd.DataFrame(rows) diff --git a/application/data_access/odp_summaries/issue.py b/application/data_access/odp_summaries/issue.py deleted file mode 100644 index 838a2b65..00000000 --- a/application/data_access/odp_summaries/issue.py +++ /dev/null @@ -1,346 +0,0 @@ -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query -from application.data_access.odp_summaries.utils import get_provisions - -SPATIAL_DATASETS = [ - "article-4-direction-area", - "conservation-area", - "listed-building-outline", - "tree-preservation-zone", - "tree", -] -DOCUMENT_DATASETS = [ - "article-4-direction", - "conservation-area-document", - "tree-preservation-order", -] - -# Separate variable for all datasets as arbitrary ordering required -ALL_DATASETS = [ - "article-4-direction", - "article-4-direction-area", - "conservation-area", - "conservation-area-document", - "listed-building-outline", - "tree-preservation-order", - "tree-preservation-zone", - "tree", -] - -# Configs that are passed to the front end for the filters -DATASET_TYPES = [ - {"name": "Spatial", "id": "spatial"}, - {"name": "Document", "id": "document"}, -] - -COHORTS = [ - {"name": "RIPA Beta", "id": "RIPA-Beta"}, - {"name": "RIPA BOPS", "id": "RIPA-BOPS"}, - {"name": "ODP Track 1", "id": "ODP-Track1"}, - {"name": "ODP Track 2", "id": "ODP-Track2"}, - {"name": "ODP Track 3", "id": "ODP-Track3"}, - {"name": "ODP Track 4", "id": "ODP-Track4"}, -] - - -def get_provision_summary(dataset_clause, offset): - sql = f""" - SELECT - organisation, - organisation_name, - dataset, - active_endpoint_count, - error_endpoint_count, - count_issue_error_internal, - count_issue_error_external, - count_issue_warning_internal, - count_issue_warning_external, - count_issue_notice_internal, - count_issue_notice_external - FROM - provision_summary - {dataset_clause} - limit 1000 offset {offset} - """ - return get_datasette_query("performance", sql) - - -def get_full_provision_summary(datasets): - dataset_clause = "WHERE " + " or ".join( - ("dataset = '" + str(dataset) + "'" for dataset in datasets) - ) - - pagination_incomplete = True - offset = 0 - provision_summary_df_list = [] - while pagination_incomplete: - provision_summary_df = get_provision_summary(dataset_clause, offset) - provision_summary_df_list.append(provision_summary_df) - pagination_incomplete = len(provision_summary_df) == 1000 - offset += 1000 - return pd.concat(provision_summary_df_list) - - -def get_odp_issue_summary(dataset_types, cohorts): - if dataset_types == ["spatial"]: - datasets = SPATIAL_DATASETS - elif dataset_types == ["document"]: - datasets = DOCUMENT_DATASETS - else: - datasets = ALL_DATASETS - provisions_df = get_provisions(cohorts, COHORTS) - provision_summary_df = get_full_provision_summary(datasets) - provision_issue_df = provisions_df.merge( - provision_summary_df, how="left", on="organisation" - ) - provision_issue_df = provision_issue_df.sort_values( - ["cohort_start_date", "cohort", "organisation_name"] - ) - # Get list of organisations to iterate over - organisation_cohorts_df = provision_issue_df.drop_duplicates( - subset=["cohort", "organisation"] - ) - - rows = [] - for idx, df_row in organisation_cohorts_df.iterrows(): - row = [] - row.append({"text": df_row["cohort"], "classes": "reporting-table-cell"}) - row.append( - {"text": df_row["organisation_name"], "classes": "reporting-table-cell"} - ) - issues_df = provision_issue_df[ - provision_issue_df["organisation"] == df_row["organisation"] - ] - for dataset in datasets: - issues_df_row = issues_df[issues_df["dataset"] == dataset] - # Check if any endpoints exist - if ( - issues_df_row["active_endpoint_count"].values[0] > 0 - or issues_df_row["error_endpoint_count"].values[0] > 0 - ): - issues = [] - # Now check for individual issue severities - if issues_df_row["error_endpoint_count"].values[0] > 0: - issues.insert(0, "Endpoint broken") - classes = "reporting-table-cell reporting-null-background" - if ( - issues_df_row["count_issue_warning_internal"].values[0] > 0 - or issues_df_row["count_issue_warning_external"].values[0] > 0 - ): - issues.insert(0, "Warning") - classes = "reporting-table-cell reporting-medium-background" - if ( - issues_df_row["count_issue_error_internal"].values[0] > 0 - or issues_df_row["count_issue_error_external"].values[0] > 0 - ): - issues.insert(0, "Error") - classes = "reporting-table-cell reporting-bad-background" - if ( - issues_df_row["count_issue_notice_internal"].values[0] > 0 - or issues_df_row["count_issue_notice_external"].values[0] > 0 - ): - issues.insert(0, "Notice") - classes = "reporting-table-cell reporting-bad-background" - - if issues == []: - text = "No issues" - classes = "reporting-table-cell reporting-good-background" - else: - text = ", ".join(issues) - else: - text = "No endpoint" - classes = "reporting-table-cell reporting-null-background" - - row.append({"text": text, "classes": classes}) - rows.append(row) - - # Calculate overview stats - - # Total issues for percentage calculation - total_issues = ( - provision_issue_df[ - [ - "count_issue_error_internal", - "count_issue_error_external", - "count_issue_warning_internal", - "count_issue_warning_external", - "count_issue_notice_internal", - "count_issue_notice_external", - ] - ] - .sum() - .sum() - ) - total_internal = 0 - total_external = 0 - - stats_rows = [] - # Loop over severities counting internal, external, percentages - for severity in ["warning", "error", "notice"]: - internal = provision_issue_df[f"count_issue_{severity}_internal"].sum() - total_internal += internal - external = provision_issue_df[f"count_issue_{severity}_external"].sum() - total_external += external - total = internal + external - total_percentage = str(int((total / total_issues) * 100)) + "%" - classes = ( - "reporting-medium-background" - if severity == "warning" - else "reporting-bad-background" - ) - stats_rows.append( - [ - { - "text": severity.title(), - "classes": classes + " reporting-table-cell", - }, - { - "text": total, - "classes": "reporting-table-cell", - }, - { - "text": total_percentage, - "classes": "reporting-table-cell", - }, - { - "text": internal, - "classes": "reporting-table-cell", - }, - { - "text": external, - "classes": "reporting-table-cell", - }, - ] - ) - - # Add totals row - stats_rows.append( - [ - {"text": "Total", "classes": "reporting-table-cell"}, - {"text": total_issues, "classes": "reporting-table-cell"}, - {"text": "", "classes": "reporting-table-cell"}, - {"text": total_internal, "classes": "reporting-table-cell"}, - {"text": total_external, "classes": "reporting-table-cell"}, - ] - ) - - endpoints_with_no_issues_count = len( - provision_issue_df[ - (provision_issue_df["count_issue_error_internal"] == 0) - & (provision_issue_df["count_issue_error_external"] == 0) - & (provision_issue_df["count_issue_warning_internal"] == 0) - & (provision_issue_df["count_issue_warning_external"] == 0) - & (provision_issue_df["count_issue_notice_internal"] == 0) - & (provision_issue_df["count_issue_notice_external"] == 0) - & (provision_issue_df["active_endpoint_count"] > 0) - & (provision_issue_df["error_endpoint_count"] == 0) - ] - ) - total_endpoints = len( - provision_issue_df[provision_issue_df["active_endpoint_count"] > 0] - ) - - stats_headers = [ - {"text": "Issue Severity"}, - {"text": "Count"}, - {"text": "% Count"}, - {"text": "Internal"}, - {"text": "External"}, - ] - headers = [ - {"text": "Cohort", "classes": "reporting-table-header"}, - {"text": "Organisation", "classes": "reporting-table-header"}, - *map( - lambda dataset: {"text": dataset, "classes": "reporting-table-header"}, - datasets, - ), - ] - params = { - "cohorts": COHORTS, - "dataset_types": DATASET_TYPES, - "selected_dataset_types": dataset_types, - "selected_cohorts": cohorts, - } - return { - "rows": rows, - "headers": headers, - "stats_headers": stats_headers, - "stats_rows": stats_rows, - "endpoints_no_issues": { - "count": endpoints_with_no_issues_count, - "total_endpoints": total_endpoints, - }, - "params": params, - } - - -def get_issue_summary_by_issue_type(dataset_clause, offset): - sql = f""" - SELECT - * - FROM - endpoint_dataset_issue_type_summary edits - LEFT JOIN ( - SELECT endpoint, end_date as endpoint_end_date, latest_status, - latest_exception, entry_date as endpoint_entry_date - FROM endpoint_dataset_summary - ) as eds on edits.endpoint = eds.endpoint - {dataset_clause} - and eds.endpoint_end_date = '' - limit 1000 offset {offset} - """ - return get_datasette_query("performance", sql) - - -def get_odp_issues_by_issue_type(dataset_types, cohorts): - # Separate method for issue download as granularity required is different from the issue summary - provisions_df = get_provisions(cohorts, COHORTS) - if dataset_types == ["spatial"]: - datasets = SPATIAL_DATASETS - elif dataset_types == ["document"]: - datasets = DOCUMENT_DATASETS - else: - datasets = ALL_DATASETS - - dataset_clause = "WHERE " + " or ".join( - ("edits.dataset = '" + str(dataset) + "'" for dataset in datasets) - ) - - pagination_incomplete = True - offset = 0 - provision_summary_df_list = [] - while pagination_incomplete: - issue_summary_df = get_issue_summary_by_issue_type(dataset_clause, offset) - provision_summary_df_list.append(issue_summary_df) - pagination_incomplete = len(issue_summary_df) == 1000 - offset += 1000 - issue_summary_df = pd.concat(provision_summary_df_list) - - provision_issue_df = provisions_df.merge( - issue_summary_df, how="inner", on=["organisation", "cohort"] - ) - - return provision_issue_df[ - [ - "organisation", - "cohort", - "organisation_name", - "pipeline", - "issue_type", - "severity", - "responsibility", - "count_issues", - "collection", - "endpoint", - "endpoint_url", - "latest_status", - "latest_exception", - "resource", - "latest_log_entry_date", - "endpoint_entry_date", - "endpoint_end_date", - "resource_start_date", - "resource_end_date", - ] - ] diff --git a/application/data_access/odp_summaries/status.py b/application/data_access/odp_summaries/status.py deleted file mode 100644 index 435a4b32..00000000 --- a/application/data_access/odp_summaries/status.py +++ /dev/null @@ -1,203 +0,0 @@ -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query -from application.data_access.odp_summaries.utils import get_provisions - -SPATIAL_DATASETS = [ - "article-4-direction-area", - "conservation-area", - "listed-building-outline", - "tree-preservation-zone", - "tree", -] -DOCUMENT_DATASETS = [ - "article-4-direction", - "conservation-area-document", - "tree-preservation-order", -] - -# Separate variable for all datasets as arbitrary ordering required -ALL_DATASETS = [ - "article-4-direction", - "article-4-direction-area", - "conservation-area", - "conservation-area-document", - "listed-building-outline", - "tree-preservation-order", - "tree-preservation-zone", - "tree", -] - -# Configs that are passed to the front end for the filters -DATASET_TYPES = [ - {"name": "Spatial", "id": "spatial"}, - {"name": "Document", "id": "document"}, -] - -COHORTS = [ - {"name": "RIPA Beta", "id": "RIPA-Beta"}, - {"name": "RIPA BOPS", "id": "RIPA-BOPS"}, - {"name": "ODP Track 1", "id": "ODP-Track1"}, - {"name": "ODP Track 2", "id": "ODP-Track2"}, - {"name": "ODP Track 3", "id": "ODP-Track3"}, - {"name": "ODP Track 4", "id": "ODP-Track4"}, -] - - -def get_odp_status_summary(dataset_types, cohorts): - filtered_cohorts = [ - x for x in cohorts if cohorts[0] in [cohort["id"] for cohort in COHORTS] - ] - provision_cohort_data = get_provisions(cohorts, COHORTS) - sql = """ - select - rle.organisation, - rle.collection, - rle.pipeline, - rle.endpoint, - rle.endpoint_url, - rle.licence, - rle.latest_status as status, - rle.days_since_200, - rle.latest_exception as exception, - rle.resource, - rle.latest_log_entry_date, - rle.endpoint_entry_date, - rle.endpoint_end_date, - rle.resource_start_date, - rle.resource_end_date - from reporting_latest_endpoints rle - """ - status_df = get_datasette_query("performance", sql) - status_df["organisation"] = status_df["organisation"].str.replace("-eng", "") - result = pd.merge( - status_df, - provision_cohort_data, - left_on="organisation", - right_on="organisation", - how="right", - ) - result = result.sort_values(["cohort_start_date", "cohort", "name"]) - if filtered_cohorts: - result = result[result["cohort"].isin(filtered_cohorts)] - rows = [] - if result is not None: - organisation_cohort_dict_list = ( - result[["organisation", "cohort", "name"]] - .drop_duplicates() - .to_dict(orient="records") - ) - if dataset_types == ["spatial"]: - datasets = SPATIAL_DATASETS - elif dataset_types == ["document"]: - datasets = DOCUMENT_DATASETS - else: - datasets = ALL_DATASETS - for organisation_cohort_dict in organisation_cohort_dict_list: - rows.append( - create_status_row( - organisation_cohort_dict["organisation"], - organisation_cohort_dict["cohort"], - organisation_cohort_dict["name"], - result, - datasets, - ) - ) - # Calculate overview stats - percentages = 0.0 - datasets_added = 0 - lpa_all_data_provided = 0 - lpa_some_data_provided = 0 - for row in rows: - percentages += float(row[-1]["text"].strip("%")) / 100 - if row[-1]["text"] != "0%": - lpa_some_data_provided += 1 - if row[-1]["text"] == "100%": - lpa_all_data_provided += 1 - for cell in row: - if cell.get("data", None) and cell["text"] != "No endpoint": - datasets_added += 1 - datasets_added = str(datasets_added) - max_datasets = len(rows) * len(datasets) - number_of_lpas = len(rows) - average_percentage_added = str(int(100 * (percentages / len(rows))))[:2] + "%" - - headers = [ - {"text": "Cohort", "classes": "reporting-table-header"}, - {"text": "Organisation", "classes": "reporting-table-header"}, - *map( - lambda dataset: {"text": dataset, "classes": "reporting-table-header"}, - datasets, - ), - {"text": "% provided", "classes": "reporting-table-header"}, - ] - params = { - "cohorts": COHORTS, - "dataset_types": DATASET_TYPES, - "selected_dataset_types": dataset_types, - "selected_cohorts": cohorts, - } - return { - "rows": rows, - "headers": headers, - "percentage_datasets_added": average_percentage_added, - "number_of_lpas": number_of_lpas, - "datasets_added": datasets_added, - "max_datasets": max_datasets, - "params": params, - "lpa_some_data_provided": lpa_some_data_provided, - "lpa_all_data_provided": lpa_all_data_provided, - } - else: - return None - - -def create_status_row(organisation, cohort, name, status_df, datasets): - row = [] - row.append({"text": cohort, "classes": "reporting-table-cell"}) - row.append({"text": name, "classes": "reporting-table-cell"}) - provided_score = 0 - for dataset in datasets: - df_row = status_df[ - (status_df["organisation"] == organisation) - & (status_df["pipeline"] == dataset) - ] - if len(df_row) != 0: - provided_score += 1 - days_since_200 = df_row["days_since_200"].values[0] - if days_since_200 < 5: - text = "Endpoint added" - classes = "reporting-good-background reporting-table-cell" - else: - text = "Endpoint broken" - classes = "reporting-bad-background reporting-table-cell" - else: - text = "No endpoint" - classes = "reporting-null-background reporting-table-cell" - - endpoint_hash = df_row["endpoint"] - if len(endpoint_hash) > 0: - html = ( - f'' - + text - + "" - ) - else: - html = "" - - row.append( - { - "text": text, - "html": html, - "classes": classes, - "data": ( - df_row.fillna("").to_dict(orient="records") - if (len(df_row) != 0) - else {} - ), - } - ) - # Calculate % of endpoints provided - provided_percentage = str(int(provided_score / len(datasets) * 100)) + "%" - row.append({"text": provided_percentage, "classes": "reporting-table-cell"}) - return row diff --git a/application/data_access/odp_summaries/utils.py b/application/data_access/odp_summaries/utils.py deleted file mode 100644 index de73685d..00000000 --- a/application/data_access/odp_summaries/utils.py +++ /dev/null @@ -1,137 +0,0 @@ -import tempfile - -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query - - -def generate_odp_summary_csv(odp_summary): - dataset_pipelines = { - "article-4-direction": ["article-4-direction", "article-4-direction-area"], - "conservation-area": ["conservation-area", "conservation-area-document"], - "listed-building": ["listed-building-outline"], - "tree-preservation-order": [ - "tree-preservation-order", - "tree-preservation-zone", - "tree", - ], - } - organisation_data = get_organisation_code() - org_lookup = dict(zip(organisation_data["name"], organisation_data["organisation"])) - - # Can convert any Dataframe to csv - if type(odp_summary) is pd.DataFrame: - with tempfile.NamedTemporaryFile(mode="w", delete=False) as csvfile: - odp_summary.to_csv(csvfile, index=False) - return csvfile.name - # Else assume odp summary structure - else: - dfs = [] - for row in odp_summary["rows"]: - cohort = row[0]["text"] - name = row[1]["text"] - data_found = False - - for cell in row: - try: - data = cell.get("data", []) - if data: - df = pd.DataFrame.from_records(data) - dfs.append(df) - data_found = True - except Exception: - pass - - # If no data found, placeholder rows - if not data_found: - placeholder_rows = [] - for collection, pipelines in dataset_pipelines.items(): - for pipeline in pipelines: - placeholder_row = { - "organisation": org_lookup.get(name, "LPA code not found"), - "cohort": cohort, - "name": name, - "collection": collection, - "pipeline": pipeline, - "endpoint": "No endpoint added", - "endpoint_url": "", - "licence": "", - "status": "", - "days_since_200": "", - "exception": "", - "resource": "", - "latest_log_entry_date": "", - "endpoint_entry_date": "", - "endpoint_end_date": "", - "resource_start_date": "", - "resource_end_date": "", - "cohort_start_date": "", - } - placeholder_rows.append(placeholder_row) - - dfs.append(pd.DataFrame(placeholder_rows)) - - if dfs: - output_df = pd.concat(dfs, ignore_index=True) - - columns_order = ["organisation", "cohort", "name"] + [ - col - for col in output_df.columns - if col not in ["organisation", "cohort", "name"] - ] - output_df = output_df[columns_order] - - with tempfile.NamedTemporaryFile(mode="w", delete=False) as csvfile: - output_df.to_csv(csvfile, index=False) - return csvfile.name - - -def get_provisions(selected_cohorts, all_cohorts): - filtered_cohorts = [ - x - for x in selected_cohorts - if selected_cohorts[0] in [cohort["id"] for cohort in all_cohorts] - ] - cohort_clause = ( - "AND (" - + " or ".join("c.cohort = '" + str(n) + "'" for n in filtered_cohorts) - + ")" - if filtered_cohorts - else "" - ) - sql = f""" - SELECT - p.cohort, - p.organisation, - c.start_date as cohort_start_date, - org.name as name - FROM - provision p - INNER JOIN - cohort c on c.cohort = p.cohort - JOIN organisation org - WHERE - p.provision_reason = "expected" - AND p.project == "open-digital-planning" - {cohort_clause} - AND org.organisation == p.organisation - GROUP BY - p.organisation - ORDER BY - cohort_start_date, - p.cohort - """ - provision_df = get_datasette_query("digital-land", sql) - return provision_df - - -def get_organisation_code(): - sql = """ -select - organisation, - name -from - organisation - """ - code_data = get_datasette_query("digital-land", sql) - return code_data diff --git a/application/data_access/overview/api_queries.py b/application/data_access/overview/api_queries.py deleted file mode 100644 index 6278bc67..00000000 --- a/application/data_access/overview/api_queries.py +++ /dev/null @@ -1,28 +0,0 @@ -# from application.caching import get - -from application.data_access.datasette_utils import get_datasette_query - -API_URL = "https://www.digital-land.info/entity.json" - - -# TODO move to local db -def get_entities(parameters): - params = "" - for p, v in parameters.items(): - prefix = "?" if params == "" else "&" - params = params + prefix + f"{p}={v}" - url = f"{API_URL}{params}" - rows = get_datasette_query("digital-land", url) - # result = get(url, format="json") - return rows["entities"] - - -def get_organisation_entity(prefix, reference): - return get_entities({"dataset": prefix, "reference": reference}) - - -def get_organisation_entity_number(prefix, reference): - results = get_organisation_entity(prefix, reference) - if len(results): - return results[0]["entity"] - return None diff --git a/application/data_access/overview/datasette_queries.py b/application/data_access/overview/datasette_queries.py deleted file mode 100644 index 7178ecf9..00000000 --- a/application/data_access/overview/datasette_queries.py +++ /dev/null @@ -1,29 +0,0 @@ -# from application.caching import get -from application.data_access.datasette_utils import get_datasette_query -from application.utils import create_dict - -DATASETTE_URL = "https://datasette.planning.data.gov.uk" - - -# TODO - this data is not in digital land db but each dataset has own -# database so unless we download every sqlite db for each dataset -# this will have to carry on using datasette for the moment -def fetch_resource_from_dataset(database_name, resource): - query_lines = [ - "SELECT", - "*", - "FROM", - "dataset_resource", - "WHERE", - f"resource = '{resource}'", - ] - query_str = " ".join(query_lines) - rows = get_datasette_query("digital-land", query_str) - - # return [dict(zip(columns, row)) for row in rows.to_numpy()] - return create_dict(rows["columns"], rows["rows"][0]) - - -def fetch_entry_count(database_name, resource): - r = fetch_resource_from_dataset(database_name, resource) - return r["entry_count"] diff --git a/application/data_access/overview/entity_queries.py b/application/data_access/overview/entity_queries.py deleted file mode 100644 index 3253b60c..00000000 --- a/application/data_access/overview/entity_queries.py +++ /dev/null @@ -1,81 +0,0 @@ -import json -import urllib.request - -from application.data_access.datasette_utils import get_datasette_query -from application.data_access.overview.api_queries import get_organisation_entity_number -from application.utils import split_organisation_id - - -def get_grouped_entity_count(dataset=None, organisation_entity=None): - query_lines = [ - "select count(prefix)as count,", - "prefix", - "FROM", - "lookup", - ] - if organisation_entity: - query_lines.append("WHERE") - query_lines.append(f"organisation_entity = '{organisation_entity}'") - if dataset: - if "WHERE" not in query_lines: - query_lines.append("WHERE") - else: - query_lines.append("AND") - query_lines.append(f"dataset = '{dataset}'") - else: - query_lines.append("GROUP BY") - query_lines.append("prefix") - - query_str = " ".join(query_lines) - - rows = get_datasette_query("digital-land", f"""{query_str}""") - if rows is not None and not rows.empty: - return {row["prefix"]: row["count"] for index, row in rows.iterrows()} - - return {} - - -def get_entity_count(pipeline=None): - url = f"https://www.planning.data.gov.uk/entity.json{f'?prefix={pipeline}' if pipeline else ''}" - with urllib.request.urlopen(url) as response: - data = json.load(response) - return data["count"] - - -def get_organisation_entity_count(organisation, dataset=None): - prefix, ref = split_organisation_id(organisation) - return get_grouped_entity_count( - dataset=dataset, - organisation_entity=get_organisation_entity_number(prefix, ref), - ) - - -def get_datasets_organisation_has_used_enddates(organisation): - prefix, ref = split_organisation_id(organisation) - organisation_entity_num = get_organisation_entity_number(prefix, ref) - if not organisation_entity_num: - return None - query_lines = [ - "SELECT", - "dataset", - "FROM", - "entity_end_date_counts", - "WHERE", - '("end_date" is not null and "end_date" != "")', - "AND", - f'("organisation_entity" = {organisation_entity_num})', - "GROUP BY", - "dataset", - ] - query_str = " ".join(query_lines) - rows = get_datasette_query("entity", query_str) - if len(rows) > 0: - return [dataset[0] for dataset in rows] - return [] - # columns = rows.columns.tolist() - # with SqliteDatabase(entity_stats_db_path) as db: - # rows = db.execute(query_str).fetchall() - # if rows: - # return [dataset[0] for dataset in rows] - - # return [] diff --git a/application/data_access/overview/issue_summary.py b/application/data_access/overview/issue_summary.py deleted file mode 100644 index 360d26f5..00000000 --- a/application/data_access/overview/issue_summary.py +++ /dev/null @@ -1,229 +0,0 @@ -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query - - -def get_full_issue_summary(): - pagination_incomplete = True - offset = 0 - issue_summary_df_list = [] - while pagination_incomplete: - issue_summary_df = get_issue_summary(offset) - issue_summary_df_list.append(issue_summary_df) - pagination_incomplete = len(issue_summary_df) == 1000 - offset += 1000 - issues_df = pd.concat(issue_summary_df_list) - - # Convert DataFrame to a list of dictionaries (rows) - rows = issues_df.to_dict(orient="records") - - # Define severity counts structure - issue_severity_counts = [ - { - "display_severity": "No issues", - "severity": "", - "total_count": 0, - "total_count_percentage": 0.00, - "internal_count": 0, - "internal_count_percentage": 0.00, - "external_count": 0, - "external_count_percentage": 0.00, - "classes": "reporting-good-background", - }, - { - "display_severity": "Info", - "severity": "info", - "total_count": 0, - "total_count_percentage": 0.00, - "internal_count": 0, - "internal_count_percentage": 0.00, - "external_count": 0, - "external_count_percentage": 0.00, - "classes": "reporting-good-background", - }, - { - "display_severity": "Warning", - "severity": "warning", - "total_count": 0, - "total_count_percentage": 0.00, - "internal_count": 0, - "internal_count_percentage": 0.00, - "external_count": 0, - "external_count_percentage": 0.00, - "classes": "reporting-medium-background", - }, - { - "display_severity": "Error", - "severity": "error", - "total_count": 0, - "total_count_percentage": 0.00, - "internal_count": 0, - "internal_count_percentage": 0.00, - "external_count": 0, - "external_count_percentage": 0.00, - "classes": "reporting-bad-background", - }, - { - "display_severity": "Notice", - "severity": "notice", - "total_count": 0, - "total_count_percentage": 0.00, - "internal_count": 0, - "internal_count_percentage": 0.00, - "external_count": 0, - "external_count_percentage": 0.00, - "classes": "reporting-bad-background", - }, - ] - - # Calculate metrics from rows - total_issues = 0 - endpoints_with_no_issues_count = 0 - total_endpoints = len(rows) - - for row in rows: - severity = row.get("severity", "") - issues_count = row.get("count_issues", 0) - responsibility = row.get("responsibility", "") - internal_count = 0 - external_count = 0 - - # Accumulate severity count - if responsibility == "internal": - internal_count += int(issues_count) - elif responsibility == "external": - external_count += int(issues_count) - total_count = internal_count + external_count - if severity: - for issue_severity in issue_severity_counts: - if issue_severity["severity"] == severity: - issue_severity["internal_count"] += int(internal_count) - issue_severity["external_count"] += int(external_count) - issue_severity["total_count"] += total_count - total_issues += total_count - else: - endpoints_with_no_issues_count += 1 - - # Compute totals/percentages - total_internal = sum(issue["internal_count"] for issue in issue_severity_counts) - total_external = sum(issue["external_count"] for issue in issue_severity_counts) - - # Add issue_severity row - stats_rows = [] - for issue_severity in issue_severity_counts: - if issue_severity["internal_count"] > 0: - issue_severity["internal_count_percentage"] = round( - (issue_severity["internal_count"] / total_issues) * 100, 2 - ) - - if issue_severity["external_count"] > 0: - issue_severity["external_count_percentage"] = round( - (issue_severity["external_count"] / total_issues) * 100, 2 - ) - - if issue_severity["total_count"] > 0: - issue_severity["total_count_percentage"] = round( - (issue_severity["total_count"] / total_issues) * 100, 2 - ) - - stats_rows.append( - [ - { - "text": issue_severity["display_severity"], - "classes": issue_severity["classes"] + " reporting-table-cell", - }, - { - "text": f"{issue_severity['internal_count']} ({issue_severity['internal_count_percentage']}%)", - "classes": "reporting-table-cell", - }, - { - "text": f"{issue_severity['external_count']} ({issue_severity['external_count_percentage']}%)", - "classes": "reporting-table-cell", - }, - { - "text": f"{issue_severity['total_count']} ({issue_severity['total_count_percentage']}%)", - "classes": "reporting-table-cell", - }, - ] - ) - - # Add totals row, the bottom - stats_rows.append( - [ - {"text": "Total", "classes": "reporting-table-cell"}, - { - "text": f"{total_internal} ({round((total_internal/total_issues)*100, 2)}%)", - "classes": "reporting-table-cell", - }, - { - "text": f"{total_external} ({round((total_external/total_issues)*100, 2)}%)", - "classes": "reporting-table-cell", - }, - {"text": total_issues, "classes": "reporting-table-cell"}, - ] - ) - - # Define headers - stats_headers = [ - {"text": "Severity"}, - {"text": "Internal (%)"}, - {"text": "External (%)"}, - {"text": "Total (%)"}, - ] - - return { - "issue_severity_counts": issue_severity_counts, - "stats_headers": stats_headers, - "stats_rows": stats_rows, - "endpoints_no_issues": { - "count": endpoints_with_no_issues_count, - "total_endpoints": total_endpoints, - }, - } - - -def get_full_issue_summary_for_csv(): - pagination_incomplete = True - offset = 0 - issue_summary_df_list = [] - while pagination_incomplete: - issue_summary_df = get_issue_summary_for_csv(offset) - issue_summary_df_list.append(issue_summary_df) - pagination_incomplete = len(issue_summary_df) == 1000 - offset += 1000 - issue_summary_df = pd.concat(issue_summary_df_list) - - issue_summary_df = issue_summary_df[issue_summary_df["count_issues"].notna()] - - return issue_summary_df[ - [ - "organisation", - "organisation_name", - "pipeline", - "issue_type", - "severity", - "responsibility", - "count_issues", - "collection", - "endpoint", - "endpoint_url", - "resource", - "latest_log_entry_date", - "resource_start_date", - "resource_end_date", - ] - ] - - -def get_issue_summary(offset): - sql = f""" - select count_issues, severity, responsibility from endpoint_dataset_issue_type_summary limit 1000 offset {offset} - """ - return get_datasette_query("performance", sql) - - -def get_issue_summary_for_csv(offset): - sql = f""" - select * from endpoint_dataset_issue_type_summary limit 1000 offset {offset} - """ - return get_datasette_query("performance", sql) diff --git a/application/data_access/overview/source_and_resource_queries.py b/application/data_access/overview/source_and_resource_queries.py deleted file mode 100644 index f1e9db75..00000000 --- a/application/data_access/overview/source_and_resource_queries.py +++ /dev/null @@ -1,401 +0,0 @@ -import datetime -import re - -import pandas as pd - -from application.data_access.datasette_utils import get_datasette_query -from application.data_access.overview.digital_land_queries import ( - get_datasets, - get_monthly_source_counts, -) -from application.utils import index_by, month_dict, months_since, this_month, yesterday - - -# returns organisation counts per dataset -def publisher_coverage(): - # TODO handle when pipeline is not None - sql = """ - SELECT - source_pipeline.pipeline, - count(DISTINCT source.organisation), - count(DISTINCT provision.organisation), - CASE - WHEN COUNT(DISTINCT provision.organisation) = 0 THEN COUNT(DISTINCT source.organisation) - ELSE COUNT(DISTINCT provision.organisation) - END AS expected_publishers, - COUNT( - DISTINCT CASE - WHEN source.endpoint != '' - and source.organisation!='government-organisation:D1342' - THEN source.organisation - END - ) AS publishers - FROM - source - INNER JOIN source_pipeline ON source.source = source_pipeline.source - LEFT JOIN provision on source_pipeline.pipeline=provision.dataset - and - provision.end_date=="" - GROUP BY - source_pipeline.pipeline - """ - - rows = get_datasette_query("digital-land", f"{sql}") - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def resources_by_dataset(): - # TODO handle when pipeline is not None - sql = """ - SELECT - count(DISTINCT resource.resource) AS total_resources, - count( - DISTINCT CASE - WHEN resource.end_date == '' THEN resource.resource - WHEN strftime('%Y%m%d', resource.end_date) >= strftime('%Y%m%d', 'now') THEN resource.resource - END - ) AS active_resources, - count( - DISTINCT CASE - WHEN resource.end_date != '' - AND strftime('%Y%m%d', resource.end_date) <= strftime('%Y%m%d', 'now') THEN resource.resource - END - ) AS ended_resources, - source_pipeline.pipeline - FROM - resource - INNER JOIN resource_endpoint ON resource.resource = resource_endpoint.resource - INNER JOIN source ON source.endpoint = resource_endpoint.endpoint - INNER JOIN source_pipeline ON source.source = source_pipeline.source - GROUP BY - source_pipeline.pipeline - """ - rows = get_datasette_query("digital-land", f"{sql}") - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def first_and_last_resource(): - # used by get_datasets_summary - sql = """ - SELECT - resource.resource, - MAX(resource.start_date) AS latest, - MIN(resource.start_date) AS first, - source_pipeline.pipeline - FROM - resource - INNER JOIN resource_endpoint ON resource.resource = resource_endpoint.resource - INNER JOIN source ON resource_endpoint.endpoint = source.endpoint - INNER JOIN source_pipeline ON source.source = source_pipeline.source - GROUP BY - source_pipeline.pipeline - ORDER BY - resource.start_date DESC""" - - rows = get_datasette_query("digital-land", sql) - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def latest_endpoint_entry_date(): - # used by get_datasets_summary - sql = """ - select pipeline,substr(MAX(endpoint_entry_date), 1, - instr(MAX(endpoint_entry_date), 'T') - 1) as latest_endpoint - from reporting_latest_endpoints group by pipeline""" - - rows = get_datasette_query("digital-land", sql) - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def active_and_total_endpoints(): - # used by get_datasets_summary - sql = """ - SELECT - pipeline, - COUNT(DISTINCT endpoint) AS total, - COUNT(CASE WHEN status == '200' THEN status END) as active - FROM ( - SELECT - pipeline, - endpoint, - status - FROM - reporting_historic_endpoints - WHERE - endpoint_end_date = "" - GROUP BY - pipeline, endpoint - ) AS subquery - GROUP BY - pipeline;""" - - rows = get_datasette_query("digital-land", sql) - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def get_typology(): - # used by get_datasets_summary - sql = """ - select dataset as pipeline, typology from dataset group by dataset""" - - rows = get_datasette_query("digital-land", sql) - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def get_spec_filename(dataset_urls): - if pd.isna(dataset_urls) or not isinstance(dataset_urls, str): - return [] - - urls = dataset_urls.split(";") - - # extract dataset name from specification file URL - filenames = [ - re.search(r"/([^/]+)\.md", url).group(1) - for url in urls - if re.search(r"/([^/]+)\.md", url) - ] - - return filenames - - -def get_frequency(): - # used by get_datasets_summary - df = pd.read_csv( - "https://design.planning.data.gov.uk/planning-consideration/planning-considerations.csv" - ) - - if "datasets" not in df.columns or "frequency-of-updates" not in df.columns: - raise ValueError( - "CSV must contain 'datasets' and 'frequency-of-updates' columns" - ) - - df["frequency-of-updates"].fillna("", inplace=True) - df["pipeline"] = df["datasets"].apply(get_spec_filename) - df = df.explode("pipeline") - columns = ["pipeline", "frequency-of-updates"] - data = df[columns].to_dict(orient="records") - - return data - - -def get_datasets_summary(): - # get all the datasets listed with their active status - all_datasets = index_by("dataset", get_datasets()) - - missing = [] - - # add the publisher coverage numbers - dataset_coverage = publisher_coverage() - for d in dataset_coverage: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add the total resource count - dataset_resource_counts = resources_by_dataset() - for d in dataset_resource_counts: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add the first and last resource dates - dataset_resource_dates = first_and_last_resource() - for d in dataset_resource_dates: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add the most recent endpoint entry date - dataset_endpoint_entry_date = latest_endpoint_entry_date() - for d in dataset_endpoint_entry_date: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add endpoints coverage - dataset_endpoint_entry_date = active_and_total_endpoints() - for d in dataset_endpoint_entry_date: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add typology - dataset_endpoint_entry_date = get_typology() - for d in dataset_endpoint_entry_date: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - # add frequency of updates - dataset_frequency = get_frequency() - for d in dataset_frequency: - if all_datasets.get(d["pipeline"]): - all_datasets[d["pipeline"]] = {**all_datasets[d["pipeline"]], **d} - else: - missing.append(d["pipeline"]) - - return all_datasets - - -def get_monthly_resource_counts(pipeline=None): - if not pipeline: - sql = """SELECT - strftime('%Y-%m', resource.start_date) AS yyyy_mm, - count(distinct resource.resource) AS count - FROM - resource - WHERE - resource.start_date != "" - GROUP BY - yyyy_mm - ORDER BY - yyyy_mm""" - - else: - sql = """ - SELECT - strftime('%Y-%m', resource.start_date) AS yyyy_mm, - count(distinct resource.resource) AS count - FROM - resource - INNER JOIN resource_endpoint ON resource.resource = resource_endpoint.resource - INNER JOIN endpoint ON resource_endpoint.endpoint = endpoint.endpoint - INNER JOIN source ON resource_endpoint.endpoint = source.endpoint - INNER JOIN source_pipeline ON source.source = source_pipeline.source - WHERE - resource.start_date != "" - AND source_pipeline.pipeline = :pipeline - GROUP BY - yyyy_mm - ORDER BY - yyyy_mm""" - - if pipeline: - rows = get_datasette_query("digital-land", sql, {"pipeline": pipeline}) - else: - rows = get_datasette_query("digital-land", sql) - columns = rows.columns.tolist() - - return [dict(zip(columns, row)) for row in rows.to_numpy()] - - -def get_monthly_counts(pipeline=None): - source_counts = get_monthly_source_counts(pipeline) - resource_counts = get_monthly_resource_counts(pipeline) - - # handle if either are empty - if not bool(source_counts): - return None - - first_source_month_str = source_counts[0]["yyyy_mm"] - first_resource_month_str = ( - resource_counts[0]["yyyy_mm"] if bool(resource_counts) else this_month() - ) - - earliest = ( - first_source_month_str - if first_source_month_str < first_resource_month_str - else first_resource_month_str - ) - start_date = datetime.datetime.strptime(earliest, "%Y-%m") - months_since_start = months_since(start_date) - all_months = month_dict(months_since_start) - - counts = {} - for k, v in {"resources": resource_counts, "sources": source_counts}.items(): - d = all_months.copy() - for row in v: - if row["yyyy_mm"] in d.keys(): - d[row["yyyy_mm"]] = d[row["yyyy_mm"]] + row["count"] - # needs to be in tuple form - counts[k] = [(k, v) for k, v in d.items()] - counts["months"] = list(all_months.keys()) - - return counts - - -def get_new_resources(dates=[yesterday(string=True)]): - if len(dates) == 1: - sql = f"""SELECT - DISTINCT resource, start_date - FROM resource - WHERE start_date = '{dates[0]}' - ORDER BY start_date""" - else: - sql = """SELECT - DISTINCT resource, start_date - FROM resource - WHERE start_date IN %(dates)s - ORDER BY start_date - """ % {"dates": tuple(dates)} - - rows = get_datasette_query("digital-land", sql) - - return rows - - -def publisher_counts(pipeline): - # returns resource, active resource, endpoints, sources, active source, latest resource date and days since update - - sql = """ - SELECT - organisation.name, - source.organisation, - organisation.end_date AS organisation_end_date, - COUNT(DISTINCT resource.resource) AS resources, - COUNT( - DISTINCT CASE - WHEN resource.end_date == '' THEN resource.resource - WHEN strftime('%Y%m%d', resource.end_date) >= strftime('%Y%m%d', 'now') THEN resource.resource - END - ) AS active_resources, - COUNT(DISTINCT resource_endpoint.endpoint) AS endpoints, - COUNT(DISTINCT source.source) AS sources, - COUNT( - DISTINCT CASE - WHEN source.end_date == '' THEN source.source - WHEN strftime('%Y%m%d', source.end_date) >= strftime('%Y%m%d', 'now') THEN source.source - END - ) AS active_sources, - MAX(resource.start_date), - Cast ( - ( - julianday('now') - julianday(MAX(resource.start_date)) - ) AS INTEGER - ) AS days_since_update - FROM - resource - INNER JOIN resource_endpoint ON resource.resource = resource_endpoint.resource - INNER JOIN endpoint ON resource_endpoint.endpoint = endpoint.endpoint - INNER JOIN source ON resource_endpoint.endpoint = source.endpoint - INNER JOIN source_pipeline ON source.source = source_pipeline.source - INNER JOIN organisation ON replace(source.organisation, '-eng', '') = organisation.organisation - WHERE - source_pipeline.pipeline = :pipeline - GROUP BY - source.organisation""" - - rows = get_datasette_query("digital-land", sql, {"pipeline": pipeline}) - columns = rows.columns.tolist() - - organisations = [dict(zip(columns, row)) for row in rows.to_numpy()] - - return index_by("organisation", organisations) diff --git a/application/data_access/overview/utils.py b/application/data_access/overview/utils.py deleted file mode 100644 index 3d731254..00000000 --- a/application/data_access/overview/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -import tempfile - -import pandas as pd - - -def generate_overview_issue_summary_csv(overview_issue_summary): - # Can convert any Dataframe to csv - if type(overview_issue_summary) is pd.DataFrame: - with tempfile.NamedTemporaryFile(mode="w", delete=False) as csvfile: - overview_issue_summary.to_csv(csvfile, index=False) - return csvfile.name - # Else assume overview issue summary structure - else: - dfs = [] - for row in overview_issue_summary["rows"]: - for cell in row: - try: - data = cell["data"] - dfs.append(pd.DataFrame.from_records(data)) - except Exception: - pass - output_df = pd.concat(dfs) - columns_order = ["organisation", "name"] + [ - col for col in output_df.columns if col not in ["organisation", "name"] - ] - output_df = output_df[columns_order] - with tempfile.NamedTemporaryFile(mode="w", delete=False) as csvfile: - output_df.to_csv(csvfile, index=False) - return csvfile.name diff --git a/application/data_access/summary_queries.py b/application/data_access/summary_queries.py deleted file mode 100644 index f466db9c..00000000 --- a/application/data_access/summary_queries.py +++ /dev/null @@ -1,222 +0,0 @@ -from datetime import datetime, timezone - -import pandas as pd - -from application.data_access.datasette_utils import generate_weeks, get_datasette_query - - -def get_contributions_and_errors(offset): - sql = f""" - select - count(*) as count, - case when (status = '200') then '200' else 'Not 200' end as status, - strftime('%Y',entry_date) as year, - strftime('%m',entry_date) as month, - strftime('%W',entry_date) as week, - strftime('%d',entry_date) as day, - substr(entry_date,1,10) as entry_date - from log l - where year > '2018' - group by substr(entry_date,1,10), case when status = '200' then status else 'not_200' end - limit 1000 offset {offset} - """ - return get_datasette_query("digital-land", sql) - - -def get_contributions_and_errors_by_day(): - # Use pagination in case rows returned > 1000 - pagination_incomplete = True - offset = 0 - contributions_and_errors_df_list = [] - while pagination_incomplete: - contributions_and_errors_df = get_contributions_and_errors(offset) - contributions_and_errors_df_list.append(contributions_and_errors_df) - pagination_incomplete = len(contributions_and_errors_df) == 1000 - offset += 1000 - contributions_and_errors_df = pd.concat(contributions_and_errors_df_list) - return contributions_and_errors_df - - -def get_issue_counts(): - sql = """ - select - it.severity, count(*) as count - from issue i - inner join issue_type it - where i.issue_type = it.issue_type - group by it.severity - """ - issues_df = get_datasette_query("digital-land", sql) - if issues_df is not None: - errors = issues_df[issues_df["severity"] == "error"].iloc[0]["count"] - warning = issues_df[issues_df["severity"] == "warning"].iloc[0]["count"] - return errors, warning - else: - return None - - -def get_internal_issues_by_week(): - """ - returns a list of weeks and unknown entity counts, - to include "invalid organisation" when integrated into table - """ - sql = """SELECT COUNT(*) as count, [entry-date] as date, "issue-type" - FROM operational_issue - WHERE "issue-type" = "unknown entity" - GROUP BY [entry-date] - ORDER BY date ASC""" - - internal_issues_df = get_datasette_query("digital-land", sql) - - if internal_issues_df.empty: - print("No data returned.") - return None - - start_date = datetime(2024, 9, 27).date() - end_date = datetime.now(timezone.utc).date() - - internal_issues_df["date"] = pd.to_datetime(internal_issues_df["date"]) - # set start of the week to Monday - internal_issues_df["week_start"] = internal_issues_df["date"] - pd.to_timedelta( - internal_issues_df["date"].dt.dayofweek, unit="D" - ) - - weekly_issues = ( - internal_issues_df.groupby("week_start")["count"].sum().reset_index() - ) - - all_weeks = pd.DataFrame( - { - "week_start": pd.date_range( - start=start_date - pd.Timedelta(days=start_date.weekday()), - end=end_date, - freq="W-MON", - ) - } - ) - - all_weeks_issues = pd.merge(all_weeks, weekly_issues, on="week_start", how="left") - - all_weeks_issues["count"].fillna(0, inplace=True) - - all_weeks_issues.rename(columns={"week_start": "date"}, inplace=True) - - all_weeks_issues["date"] = all_weeks_issues["date"].dt.strftime("%Y-%m-%d") - - weekly_issues_list = all_weeks_issues.to_dict(orient="records") - return weekly_issues_list - - -def get_contributions_and_erroring_endpoints(contributions_and_errors_df): - if contributions_and_errors_df is not None: - contributions_df = contributions_and_errors_df[ - contributions_and_errors_df["status"] == "200" - ].reindex() - errors_df = contributions_and_errors_df[ - contributions_and_errors_df["status"] != "200" - ].reindex() - contributions = contributions_df["count"].tolist() - contributions_dates = contributions_df["entry_date"].tolist() - errors = errors_df["count"].tolist() - errors_dates = errors_df["entry_date"].tolist() - return {"dates": contributions_dates, "contributions": contributions}, { - "dates": errors_dates, - "errors": errors, - } - else: - return None - - -def get_endpoints_added_by_week(): - sql = """ - select - strftime('%Y',entry_date) as year, - strftime('%m',entry_date) as month, - strftime('%W',entry_date) as week, - strftime('%d',entry_date) as day, - count(endpoint), - substr(entry_date,1,10) as entry_date - from endpoint - where year > '2018' - group by year, week - """ - endpoints_added_df = get_datasette_query("digital-land", sql) - if endpoints_added_df is not None: - endpoints_added_df["week"] = pd.to_numeric(endpoints_added_df["week"]) - endpoints_added_df["year"] = pd.to_numeric(endpoints_added_df["year"]) - # min_entry_date = endpoints_added_df["entry_date"].min() - # Use hardcoded start date as data does not begin at the same date across all three graphs - min_entry_date = "2019-08-21" - dates = generate_weeks(date_from=min_entry_date) - endpoints_added = [] - for date in dates: - current_date_data_df = endpoints_added_df[ - (endpoints_added_df["week"] == date["week_number"]) - & (endpoints_added_df["year"] == date["year_number"]) - ] - if len(current_date_data_df) != 0: - count = current_date_data_df["count(endpoint)"].values[0] - else: - count = 0 - endpoints_added.append( - {"date": date["date"].strftime("%d/%m/%Y"), "count": count} - ) - return endpoints_added - else: - return None - - -def get_endpoint_errors_and_successes_by_week(contributions_and_errors_by_day_df): - contributions_and_errors_by_day_df["year"] = pd.to_numeric( - contributions_and_errors_by_day_df["year"] - ) - contributions_and_errors_by_day_df["week"] = pd.to_numeric( - contributions_and_errors_by_day_df["week"] - ) - contributions_and_errors_by_week_df = ( - contributions_and_errors_by_day_df.groupby(["year", "week", "status"])["count"] - .sum() - .reset_index() - ) - min_entry_date = contributions_and_errors_by_day_df["entry_date"].min() - dates = generate_weeks(date_from=min_entry_date) - successes_by_week = [] - successes_percentages_by_week = [] - errors_percentages_by_week = [] - for date in dates: - current_date_data_df = contributions_and_errors_by_week_df[ - (contributions_and_errors_by_week_df["week"] == date["week_number"]) - & (contributions_and_errors_by_week_df["year"] == date["year_number"]) - ] - if len(current_date_data_df) > 0: - successes_df = current_date_data_df[current_date_data_df["status"] == "200"] - if len(successes_df) > 0: - successes = successes_df["count"].values[0] - else: - successes = 0 - errors_df = current_date_data_df[ - current_date_data_df["status"] == "Not 200" - ] - if len(errors_df) > 0: - errors = errors_df["count"].values[0] - else: - errors = 0 - if errors == 0 and successes == 0: - success_percentage = 0 - error_percentage = 0 - else: - success_percentage = successes / (successes + errors) * 100 - error_percentage = errors / (successes + errors) * 100 - else: - successes = 0 - errors = 0 - successes_by_week.append( - {"date": date["date"].strftime("%d/%m/%Y"), "count": successes} - ) - successes_percentages_by_week.append( - {"date": date["date"].strftime("%d/%m/%Y"), "count": success_percentage} - ) - errors_percentages_by_week.append( - {"date": date["date"].strftime("%d/%m/%Y"), "count": error_percentage} - ) - return successes_by_week, successes_percentages_by_week, errors_percentages_by_week diff --git a/application/data_commands.py b/application/data_commands.py deleted file mode 100644 index 77deea5f..00000000 --- a/application/data_commands.py +++ /dev/null @@ -1,389 +0,0 @@ -import csv -import glob -import logging -import os -import pathlib -import tempfile -from datetime import datetime -from zipfile import ZipFile - -import click -import psycopg2 -import requests -from flask.cli import AppGroup - -from application.db.models import ( - Collection, - Dataset, - Endpoint, - Pipeline, - PublicationStatus, - Source, -) - -logger = logging.getLogger(__name__) - -# Create handlers -info_handler = logging.StreamHandler() -error_handler = logging.FileHandler("error.log") -info_handler.setLevel(logging.INFO) -error_handler.setLevel(logging.ERROR) - -logger.addHandler(info_handler) -logger.addHandler(error_handler) - -data_cli = AppGroup("data") - -DIGITAL_LAND_DATASETTE = "https://datasette.planning.data.gov.uk/digital-land" -DIGITAL_LAND_RAW_GITHUB_URL = "https://raw.githubusercontent.com/digital-land" - -specification_tables = [ - "attribution", - "licence", - "typology", - "datatype", - "field", - "collection", - "organisation", - "dataset", - "dataset_field", -] - - -foreign_key_columns = set( - [ - "dataset", - "endpoint", - "datatype", - "typology", - "dataset", - "field", - "licence", - "attribution", - "organisation", - "pipeline", - "collection", - ] -) - - -""" -These commands were created during development as the means to seed the application with -data from the central config repo. As long as users edit the config repo directly then -these commands can be used to drop and re seed the data from the config repo. - -If/when this application becomes the sole source of configuration edits then this -file should be removed. -""" - - -@data_cli.command("load") -@click.option("--spec", default=False, help="Load just specification tables") -@click.option("--config", default=False, help="Load config tables") -def load_data(spec, config): - from application.extensions import db - - if spec: - for table in specification_tables: - _load_specification_data(db, table) - - if config: - _load_config(db) - - -@data_cli.command("drop") -def drop_data(): - from application.extensions import db - - for table in reversed(db.metadata.sorted_tables): - delete = table.delete() - try: - db.session.execute(delete) - db.session.commit() - except Exception as e: - logger.error(e) - - -def _load_specification_data(db, table): - url = f"{DIGITAL_LAND_DATASETTE}/{table}.json?_shape=array" - logger.info(f"loading from {url}") - - records = [] - while url: - resp = requests.get(url) - try: - url = resp.links.get("next").get("url") - except AttributeError: - url = None - records.extend(resp.json()) - - for record in records: - if table in ["attribution", "licence"]: - if "entity" in record.keys(): - del record["entity"] - - for key, val in record.items(): - if not val: - record[key] = None - continue - if "date" in key: - if not val: - record[key] = None - else: - try: - record[key] = datetime.strptime(val, "%Y-%m-%d").date() - except Exception as e: - logger.exception(e) - record[key] = None - - insert_record = {} - for key, val in record.items(): - if key == "rowid": - continue - if key != table and key in foreign_key_columns: - f_key = f"{key}_id" - insert_record[f_key] = val - else: - insert_record[key] = val - - if table == "collection": - insert_record["publication_status"] = PublicationStatus.PUBLISHED.name - try: - t = db.metadata.tables.get(table) - insert = t.insert().values(**insert_record) - db.session.execute(insert) - db.session.commit() - except Exception as e: - logger.exception(e) - logger.exception(f"error loading {table} with data {record}") - - logger.info(f"inserted {len(records)} records into {table}") - - -def _load_config(db): - config_zip_url = ( - "https://github.com/digital-land/config/archive/refs/heads/main.zip" - ) - - with tempfile.TemporaryDirectory() as tmp_dir: - logger.info(f"Created temp directory {tmp_dir}") - - resp = requests.get(config_zip_url) - zip_file_path = os.path.join(tmp_dir, "main.zip") - - with open(zip_file_path, "wb") as f: - f.write(resp.content) - - with ZipFile(zip_file_path) as config_zip: - config_zip.extractall(path=tmp_dir) - - pipeline_dir = os.path.join(tmp_dir, "config-main", "pipeline") - collection_dir = os.path.join(tmp_dir, "config-main", "collection") - - pipelines = os.listdir(pipeline_dir) - - for p in pipelines: - pipeline_path = os.path.join(pipeline_dir, p) - collection_path = os.path.join(collection_dir, p) - - logger.info(f"pipleline path {pipeline_path}") - - pipeline = Pipeline.query.get(p) - if pipeline is None: - name = p.replace("-", " ").capitalize() - collection = Collection.query.get(p) - pipeline = Pipeline( - pipeline=p, - name=name, - collection=collection, - publication_status=PublicationStatus.PUBLISHED.name, - ) - db.session.add(pipeline) - db.session.commit() - - endpoint_file = f"{collection_path}/endpoint.csv" - - if os.path.exists(endpoint_file): - with open(endpoint_file, "r") as f: - reader = csv.DictReader(f) - for row in reader: - endpoint_id = row["endpoint"] - if Endpoint.query.get(endpoint_id) is None: - insert_copy = _get_insert_copy(row, "endpoint") - try: - endpoint = Endpoint(**insert_copy) - db.session.add(endpoint) - db.session.commit() - except Exception as e: - logger.error(f"could not insert endpoint {row}") - logger.error(e) - - source_file = f"{collection_path}/source.csv" - if os.path.exists(source_file): - with open(source_file, "r") as f: - reader = csv.DictReader(f) - for row in reader: - source_id = row["source"] - endpoint_id = row["endpoint"].strip() - if endpoint_id == "": - message = f"Can't add source without endpoint {row}" - logger.error(message) - continue - source = Source.query.get(source_id) - dataset_pipelines = row.get("pipelines").split(";") - datasets = Dataset.query.filter( - Dataset.dataset.in_(dataset_pipelines) - ).all() - if source is None: - # TODO - fix licence and attribution missing from reference data - insert_copy = _get_insert_copy( - row, "source", skip_fields=["pipelines"] - ) - if ( - not insert_copy.get("source") - or insert_copy.get("attribution_id") == "wikidata" - ): - message = f"source {insert_copy} for pipeline {p} has no source id" - logger.error(message) - continue - try: - source = Source(**insert_copy) - for d in datasets: - if d: - source.datasets.append(d) - db.session.add(source) - db.session.commit() - except Exception as e: - message = f"could not save source {insert_copy} for pipeline {p}" - logger.error(e) - logger.error(message) - db.session.rollback() - continue - - # # load the rest of the config files - files = [ - file - for file in glob.glob(f"{pipeline_path}/*.csv") - if pathlib.Path(file).name - not in ["source.csv", "endpoint.csv", "lookup.csv"] - ] - - for file in files: - path = pathlib.Path(file) - logger.info(f" processing {path.name} for {p}") - table_name = path.stem.replace("-", "_") - table = db.metadata.tables.get(table_name) - if table is not None: - with open(file) as f: - reader = csv.DictReader(f) - for row in reader: - try: - insert_record = _get_insert_copy(row, path.stem) - insert_record["pipeline_id"] = p - insert = table.insert().values(**insert_record) - db.session.execute(insert) - db.session.commit() - logger.info(f"inserted: {insert}") - except Exception as e: - logger.error(e) - logger.error( - f"could not insert into table {table_name}: {insert_record}" - ) - db.session.rollback() - - lookup_files = glob.glob(f"{pipeline_path}/lookup.csv") - - for file in lookup_files: - path = pathlib.Path(file) - logger.info(f" processing {path.name} for {p}") - with open(file) as f: - reader = csv.reader(f) - fields = next(reader) - fields = [field.replace("-", "_") for field in fields] - fieldnames = [] - for field in fields: - if field in ["organisation", "endpoint"]: - field = f"{field}_id" - fieldnames.append(field) - if path.stem == "lookup": - fieldnames.extend(["pipeline_id", "version_id"]) - else: - fieldnames.extend( - [ - "dataset_id", - "pipeline_id", - "version_id", - ] - ) - rows = list(reader) - update_header_path = f"{file}.tmp" - with open(update_header_path, "w") as out: - writer = csv.writer(out) - writer.writerow(fieldnames) - for row in rows: - if path.stem == "lookup": - row.extend([p, 0]) - else: - row.extend([p, p, 0]) - writer.writerow(row) - - # run pg copy for lookup files due to number of records - - connection = psycopg2.connect(os.getenv("DATABASE_URL")) - cursor = connection.cursor() - with open(update_header_path) as f: - # cursor.copy_from(f, "lookup", columns=fieldnames, sep=",") - copy_command = f"COPY lookup({','.join(fieldnames)}) FROM STDIN WITH HEADER CSV" - cursor.copy_expert(copy_command, f) - connection.commit() - connection.close() - - logger.info("done") - - -def _get_insert_copy(row, current_file_key, skip_fields=[]): - insert_copy = {} - for key, val in row.items(): - if key in skip_fields: - continue - if key != current_file_key and key in foreign_key_columns: - k = f"{key}_id" - else: - k = key - k = k.replace("-", "_") - if val is None or val.strip() == "": - insert_copy[k] = None - else: - insert_copy[k] = val - if "date" in k: - insert_copy[k] = _parse_date(val, row) - return insert_copy - - -def _parse_date(value, row): - if value.strip() == "": - return None - try: - date = datetime.strptime(value, "%Y-%m-%d").date() - return date - except Exception as e: - logger.exception(e) - logger.exception(row) - - try: - date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").date() - return date - except Exception as e: - logger.exception(e) - logger.exception(row) - - try: - date = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S:%fZ").date() - return date - except Exception as e: - logger.exception(e) - logger.exception(row) - - logger.error(f"no date parsed for {row} using value {value}") - - return None diff --git a/application/db/models.py b/application/db/models.py index c4430580..ea7edb98 100644 --- a/application/db/models.py +++ b/application/db/models.py @@ -1,438 +1,8 @@ -import enum from datetime import datetime -from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy.orm import declarative_mixin, declared_attr - from application.extensions import db -class PublicationStatus(enum.Enum): - DRAFT = "DRAFT" - PUBLISHED = "PUBLISHED" - - -@declarative_mixin -class VersionedMixin: - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() - - version_id = db.Column(db.Integer, nullable=False, default=0) - - __mapper_args__ = {"version_id_col": version_id} - - -class DateModel(db.Model): - __abstract__ = True - - entry_date = db.Column(db.Date) - start_date = db.Column(db.Date) - end_date = db.Column(db.Date) - - -source_dataset = db.Table( - "source_dataset", - db.Column("dataset_id", db.Text, db.ForeignKey("dataset.dataset")), - db.Column("source_id", db.Text, db.ForeignKey("source.source")), -) - -# read only models - i.e. read only copy of some specification tables - -dataset_field = db.Table( - "dataset_field", - db.Column( - "dataset_id", db.Text, db.ForeignKey("dataset.dataset"), primary_key=True - ), - db.Column("field_id", db.Text, db.ForeignKey("field.field"), primary_key=True), - db.Column("hint", db.Text), - db.Column("guidance", db.Text), - db.Column("entry_date", db.TIMESTAMP), - db.Column("start_date", db.Date), - db.Column("end_date", db.Date), -) - - -class Collection(DateModel): - collection = db.Column(db.Text, primary_key=True) - name = db.Column(db.Text) - pipeline = db.relationship("Pipeline", uselist=False, back_populates="collection") - datasets = db.relationship("Dataset", back_populates="collection") - sources = db.relationship("Source", back_populates="collection") - - publication_status = db.Column( - db.Enum(PublicationStatus, name="publication_status"), - default=PublicationStatus.DRAFT, - ) - - @property - def endpoints(self): - endpoints = set([]) - for s in self.sources: - endpoints.add(s.endpoint) - return endpoints - - -class Organisation(DateModel): - organisation = db.Column(db.Text, primary_key=True) - addressbase_custodian = db.Column(db.Text) - billing_authority = db.Column(db.Text) - census_area = db.Column(db.Text) - combined_authority = db.Column(db.Text) - company = db.Column(db.Text) - entity = db.Column(db.BigInteger) - esd_inventory = db.Column(db.Text) - local_authority_type = db.Column(db.Text) - local_resilience_forum = db.Column(db.Text) - name = db.Column(db.Text) - official_name = db.Column(db.Text) - opendatacommunities_uri = db.Column(db.Text) - parliament_thesaurus = db.Column(db.Text) - prefix = db.Column(db.Text) - reference = db.Column(db.Text) - region = db.Column(db.Text) - shielding_hub = db.Column(db.Text) - statistical_geography = db.Column(db.Text) - twitter = db.Column(db.Text) - website = db.Column(db.Text) - wikidata = db.Column(db.Text) - wikipedia = db.Column(db.Text) - sources = db.relationship( - "Source", backref="organisation", lazy=True, order_by="Source.entry_date" - ) - - -class Dataset(DateModel): - dataset = db.Column(db.Text, primary_key=True) - attribution_id = db.Column(db.Text, db.ForeignKey("attribution.attribution")) - collection_id = db.Column(db.Text, db.ForeignKey("collection.collection")) - collection = db.relationship("Collection", back_populates="datasets") - description = db.Column(db.Text) - key_field = db.Column(db.Text) - entity_minimum = db.Column(db.BigInteger) - entity_maximum = db.Column(db.BigInteger) - licence_id = db.Column(db.Text, db.ForeignKey("licence.licence")) - name = db.Column(db.Text) - paint_options = db.Column(JSON) - plural = db.Column(db.Text) - prefix = db.Column(db.Text) - text = db.Column(db.Text) - typology_id = db.Column(db.Text, db.ForeignKey("typology.typology")) - typology = db.relationship("Typology") - wikidata = db.Column(db.Text) - wikipedia = db.Column(db.Text) - fields = db.relationship("Field", secondary=dataset_field, lazy="subquery") - - sources = db.relationship( - "Source", secondary=source_dataset, lazy="subquery", back_populates="datasets" - ) - - -class Typology(DateModel): - typology = db.Column(db.Text, primary_key=True, nullable=False) - name = db.Column(db.Text) - description = db.Column(db.Text) - text = db.Column(db.Text) - plural = db.Column(db.Text) - wikidata = db.Column(db.Text) - wikipedia = db.Column(db.Text) - fields = db.relationship("Field", backref="typology", lazy=True) - - -class Attribution(DateModel): - attribution = db.Column(db.Text, primary_key=True) - text = db.Column(db.Text) - - -class Licence(DateModel): - licence = db.Column(db.Text, primary_key=True) - text = db.Column(db.Text) - - -class Field(DateModel): - field = db.Column(db.Text, primary_key=True, nullable=False) - cardinality = db.Column(db.Text) - description = db.Column(db.Text) - guidance = db.Column(db.Text) - hint = db.Column(db.Text) - name = db.Column(db.Text) - parent_field = db.Column(db.Text) - replacement_field = db.Column(db.Text) - text = db.Column(db.Text) - uri_template = db.Column(db.Text) - wikidata_property = db.Column(db.Text) - datatype_id = db.Column(db.Text, db.ForeignKey("datatype.datatype"), nullable=True) - typology_id = db.Column(db.Text, db.ForeignKey("typology.typology"), nullable=True) - - -class Datatype(DateModel): - datatype = db.Column(db.Text, primary_key=True, nullable=False) - name = db.Column(db.Text) - text = db.Column(db.Text) - fields = db.relationship("Field", backref="datatype", lazy=True) - - -# end read only models - - -class Pipeline(db.Model): - pipeline = db.Column(db.Text, primary_key=True) - name = db.Column(db.Text) - - collection_id = db.Column(db.Text, db.ForeignKey("collection.collection")) - collection = db.relationship("Collection", back_populates="pipeline") - - column = db.relationship("Column", backref="pipeline") - combine = db.relationship("Combine", backref="pipeline") - concat = db.relationship("Concat", backref="pipeline") - convert = db.relationship("Convert", backref="pipeline") - default = db.relationship("Default", backref="pipeline") - default_value = db.relationship("DefaultValue", backref="pipeline") - filter = db.relationship("Filter", backref="pipeline") - lookup = db.relationship("Lookup", backref="pipeline") - patch = db.relationship("Patch", backref="pipeline") - skip = db.relationship("Skip", backref="pipeline") - transform = db.relationship("Transform", backref="pipeline") - - publication_status = db.Column( - db.Enum(PublicationStatus, name="publication_status"), - default=PublicationStatus.DRAFT, - ) - - # No lookups handled here yet - need a better way to process - # them as there are so many - def get_pipeline_rules(self): - return { - "column": self.column, - "combine": self.combine, - "concat": self.concat, - "convert": self.convert, - "default": self.default, - "default_value": self.default_value, - "filter": self.filter, - "patch": self.patch, - "skip": self.skip, - "transform": self.transform, - } - - -class Source(DateModel, VersionedMixin): - source = db.Column(db.Text, primary_key=True) - attribution_id = db.Column(db.Text, db.ForeignKey("attribution.attribution")) - documentation_url = db.Column(db.Text) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=False) - licence_id = db.Column(db.Text, db.ForeignKey("licence.licence")) - organisation_id = db.Column(db.Text, db.ForeignKey("organisation.organisation")) - collection_id = db.Column(db.Text, db.ForeignKey("collection.collection")) - collection = db.relationship("Collection", back_populates="sources") - - attribution = db.relationship("Attribution") - licence = db.relationship("Licence") - - datasets = db.relationship( - "Dataset", secondary=source_dataset, lazy="subquery", back_populates="sources" - ) - - @property - def pipelines(self): - datasets = [d.dataset for d in self.datasets] - if not datasets: - return None - if len(datasets) == 1: - return datasets[0] - else: - return ";".join(datasets) - - def update(self, data): - for key, val in data.items(): - if hasattr(self, key) and key not in [ - "datasets", - "organisation", - "collection", - ]: - if val == "": - val = None - if key in ["attribution", "licence"]: - setattr(self, f"{key}_id", val) - else: - setattr(self, key, val) - self.collection.publication_status = PublicationStatus.DRAFT.name - - -class Endpoint(DateModel, VersionedMixin): - endpoint = db.Column(db.Text, primary_key=True) - endpoint_url = db.Column(db.Text) - parameters = db.Column(db.Text) - plugin = db.Column(db.Text) - - sources = db.relationship( - "Source", backref="endpoint", lazy=True, order_by="Source.entry_date" - ) - - def to_csv_dict(self): - return { - "endpoint": self.endpoint, - "endpoint-url": self.endpoint_url, - "parameters": self.parameters, - "plugin": self.plugin, - "entry-date": self.entry_date, - "start-date": self.start_date, - "end-date": self.end_date, - } - - -class Column(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - column = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class Combine(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - separator = db.Column(db.Text) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class Concat(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - fields = db.Column(db.Text) - separator = db.Column(db.Text) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class Convert(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - plugin = db.Column(db.Text) - parameters = db.Column(db.Text) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - - -class Default(DateModel, VersionedMixin): - __tablename__ = "_default" - - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - default_field = db.Column(db.Text) - entry_number = db.Column(db.BigInteger) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class DefaultValue(DateModel, VersionedMixin): - __tablename__ = "default_value" - - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field")) - entry_number = db.Column(db.BigInteger) - value = db.Column(db.Text) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class Lookup(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - organisation_id = db.Column( - db.Text, db.ForeignKey("organisation.organisation"), nullable=True - ) - resource = db.Column(db.Text) - entity = db.Column(db.BigInteger) - entry_number = db.Column(db.BigInteger) - prefix = db.Column(db.Text) - reference = db.Column(db.Text) - value = db.Column(db.Text) - - endpoint = db.relationship("Endpoint") - organisation = db.relationship("Organisation") - - -class Patch(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - entry_number = db.Column(db.BigInteger) - pattern = db.Column(db.Text) - value = db.Column(db.Text) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - -class Skip(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - pattern = db.Column(db.Text) - entry_number = db.Column(db.BigInteger) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - - -class Transform(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - replacement_field = db.Column(db.Text) - entry_number = db.Column(db.BigInteger) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") - - class ServiceLock(db.Model): __tablename__ = "service_lock" @@ -454,18 +24,3 @@ class RequestMeta(db.Model): check_request_id = db.Column( db.Text, nullable=True ) # check-results request this assessment came from, for re-run routing - - -class Filter(DateModel, VersionedMixin): - id = db.Column(db.Integer, primary_key=True) - pipeline_id = db.Column(db.Text, db.ForeignKey("pipeline.pipeline"), nullable=False) - dataset_id = db.Column(db.Text, db.ForeignKey("dataset.dataset"), nullable=True) - endpoint_id = db.Column(db.Text, db.ForeignKey("endpoint.endpoint"), nullable=True) - resource = db.Column(db.Text) - field_id = db.Column(db.Text, db.ForeignKey("field.field"), nullable=True) - pattern = db.Column(db.Text) - entry_number = db.Column(db.BigInteger) - - dataset = db.relationship("Dataset") - endpoint = db.relationship("Endpoint") - field = db.relationship("Field") diff --git a/application/factory.py b/application/factory.py index 82609664..e6c46045 100644 --- a/application/factory.py +++ b/application/factory.py @@ -65,7 +65,6 @@ def create_app(config_filename): register_templates(app) register_filters(app) register_extensions(app) - register_commands(app) # get_specification(app) @@ -81,22 +80,10 @@ def register_blueprints(app): app.register_blueprint(base) - from application.blueprints.source.views import source_bp - - app.register_blueprint(source_bp) - - from application.blueprints.endpoint.views import endpoint_bp - - app.register_blueprint(endpoint_bp) - from application.blueprints.auth.views import auth_bp app.register_blueprint(auth_bp) - from application.blueprints.dataset.views import dataset_bp - - app.register_blueprint(dataset_bp) - from application.blueprints.datamanager.router import ( assign_entities_bp, datamanager_bp, @@ -105,18 +92,6 @@ def register_blueprints(app): app.register_blueprint(datamanager_bp) app.register_blueprint(assign_entities_bp) - from application.blueprints.schema.views import schema_bp - - app.register_blueprint(schema_bp) - - from application.blueprints.report.views import report_bp - - app.register_blueprint(report_bp) - - from application.blueprints.publisher.views import publisher_pages - - app.register_blueprint(publisher_pages) - def register_context_processors(app): """ @@ -230,11 +205,3 @@ def register_templates(app): ] ) app.jinja_loader = multi_loader - - -def register_commands(app): - from application.commands import publish_cli - from application.data_commands import data_cli - - app.cli.add_command(data_cli) - app.cli.add_command(publish_cli) diff --git a/application/publish/__init__.py b/application/publish/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/publish/models.py b/application/publish/models.py deleted file mode 100644 index 5d28a03e..00000000 --- a/application/publish/models.py +++ /dev/null @@ -1,164 +0,0 @@ -from datetime import date -from typing import Any, Optional - -from pydantic import BaseModel -from pydantic.utils import GetterDict - - -def _to_kebab_case(field_name: str) -> str: - return field_name.replace("_", "-") - - -# this all feels a bit hacky - is there a better way to customise from_orm? -class RelatedObjectKeyGetter(GetterDict): - def get(self, key: str, default: Any) -> Any: - if key in { - "dataset", - "attribution", - "organisation", - "licence", - "endpoint", - "field", - "collection", - }: - if self._obj.__tablename__ != key: - obj = getattr(self._obj, key, None) - if obj: - return getattr(obj, key, None) - return super().get(key, default) - - -class ConfigBaseModel(BaseModel): - class Config: - alias_generator = _to_kebab_case - allow_population_by_field_name = True - orm_mode = True - arbitrary_types_allowed = True - getter_dict = RelatedObjectKeyGetter - - -class SourceModel(ConfigBaseModel): - source: str - attribution: Optional[str] - collection: Optional[str] - documentation_url: Optional[str] - endpoint: Optional[str] - licence: Optional[str] - organisation: Optional[str] - pipelines: str - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class EndpointModel(ConfigBaseModel): - endpoint: str - endpoint_url: str - parameters: Optional[str] - plugin: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class ConfigPipelineModel(ConfigBaseModel): - dataset: Optional[str] - endpoint: Optional[str] - resource: Optional[str] - - -class ColumnModel(ConfigPipelineModel): - column: Optional[str] - field: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class CombineModel(ConfigPipelineModel): - field: Optional[str] - separator: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class ConcatModel(ConfigPipelineModel): - field: Optional[str] - fields: Optional[str] - separator: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class ConvertModel(ConfigPipelineModel): - pluguin: Optional[str] - parameters: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class DefaultModel(ConfigPipelineModel): - field: Optional[str] - default_field: Optional[str] - entry_number: Optional[int] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class DefaultValueModel(ConfigPipelineModel): - field: Optional[str] - entry_number: Optional[int] - value: Optional[str] - - -class LookupModel(ConfigPipelineModel): - organisation: Optional[str] - entity: Optional[int] - entry_number: Optional[int] - prefix: Optional[str] - reference: Optional[str] - value: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class PatchModel(ConfigPipelineModel): - field: Optional[str] - entry_number: Optional[int] - prefix: Optional[str] - pattern: Optional[str] - value: Optional[str] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class SkipModel(ConfigPipelineModel): - pattern: Optional[str] - entry_number: Optional[int] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class TransformModel(ConfigPipelineModel): - field: Optional[str] - replacement_field: Optional[str] - entry_number: Optional[int] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] - - -class FilterModel(ConfigPipelineModel): - field: Optional[str] - pattern: Optional[str] - entry_number: Optional[int] - entry_date: Optional[date] - start_date: Optional[date] - end_date: Optional[date] diff --git a/application/spec_helpers.py b/application/spec_helpers.py deleted file mode 100644 index 794ccd16..00000000 --- a/application/spec_helpers.py +++ /dev/null @@ -1,54 +0,0 @@ -from application.db.models import ( - Column, - Combine, - Concat, - Convert, - Dataset, - Default, - DefaultValue, - Filter, - Lookup, - Patch, - Skip, - Transform, -) - -PIPELINE_MODELS = { - "column": Column, - "combine": Combine, - "concat": Concat, - "convert": Convert, - "default": Default, - "default-value": DefaultValue, - "filter": Filter, - "lookup": Lookup, - "patch": Patch, - "skip": Skip, - "transform": Transform, -} - - -# hard code names of pipeline specifications until there -# is a way to extract the list from specification -PIPELINE_SPECIFICATIONS = list(PIPELINE_MODELS.keys()) - - -def get_expected_pipeline_specs(): - datasets = Dataset.query.filter(Dataset.dataset.in_(PIPELINE_SPECIFICATIONS)).all() - specs = {spec: dataset for spec, dataset in zip(PIPELINE_SPECIFICATIONS, datasets)} - return specs - - -def count_pipeline_rules(pipeline): - exclude_list = ["lookup"] - rule_types = [ - rule_type_name.replace("-", "_") for rule_type_name in PIPELINE_SPECIFICATIONS - ] - s = sum( - [ - len(getattr(pipeline, n)) - for n in rule_types - if n not in exclude_list and hasattr(pipeline, n) - ] - ) - return {"pipeline": s, "lookup": len(pipeline.lookup)} diff --git a/application/templates/components/filter-group/macro.jinja b/application/templates/components/filter-group/macro.jinja deleted file mode 100644 index 87d7955a..00000000 --- a/application/templates/components/filter-group/macro.jinja +++ /dev/null @@ -1,17 +0,0 @@ -{% macro dlFilterGroup(params) %} -
-

{{ params.title }}

- {%- if params.selected %}{{ params.selected }} selected{% endif -%} - - - -
- -

- {{ params.title }} -

- - {{ caller() }} -
-
-{% endmacro %} diff --git a/application/templates/components/helpers.jinja b/application/templates/components/helpers.jinja deleted file mode 100644 index 097a7e8d..00000000 --- a/application/templates/components/helpers.jinja +++ /dev/null @@ -1 +0,0 @@ -{% macro random_int(len) -%}{% for n in range(len) %}{{- [0,1,2,3,4,5,6,7,8,9]|random -}}{% endfor %}{%- endmacro %} diff --git a/application/templates/components/resource-card.html b/application/templates/components/resource-card.html deleted file mode 100644 index d398bda1..00000000 --- a/application/templates/components/resource-card.html +++ /dev/null @@ -1,14 +0,0 @@ -{% macro summaryResourceCard(params) %} - - -{% endmacro %} diff --git a/application/templates/components/rule-table-header.html b/application/templates/components/rule-table-header.html deleted file mode 100644 index 6743d7fd..00000000 --- a/application/templates/components/rule-table-header.html +++ /dev/null @@ -1,18 +0,0 @@ -{% macro ruleTableHead(params) %} - - - Dataset - {% for field in params.fields %} - {% if field.field not in ['dataset', 'start-date', 'end-date', 'entry-date'] %} - {{ field.name }} - {% endif %} - {% endfor %} - Entry date - Start date - End date - {% if params.editable == 'true' %} - - {% endif %} - - -{% endmacro %} diff --git a/application/templates/components/source-card.html b/application/templates/components/source-card.html deleted file mode 100644 index 6b6c3516..00000000 --- a/application/templates/components/source-card.html +++ /dev/null @@ -1,227 +0,0 @@ -{% macro summaryField(params) %} -
- {% if not params.value or params.value == "" %} - You should add a {{ params.prop }} - {% else %} - {{ params.value }} - {% endif %} -
-{% endmacro %} - -{% macro summaryEditedField(params) %} -{% set original = params.value.original if params.value.original is not none else '' %} -
- {% if original != params.value.edited %} -
- Current: {{ params.value.original if params.value.original }} -
-
- Your edit: {{ params.value.edited if params.value.edited }} -
- {% else %} - {{ original }} - {% endif %} - {% if not params.value.edited or params.value.edited == "" %} - You should add a {{ params.prop }} - {% endif %} -
-{% endmacro %} - -{% macro fullSourceCard(params) %} -{% set existingSourceChangeHTML %} - - {% endset %} -
-
-
-

Source

-
- {% if params.existing_source %} - Edit - {%- else %} - Change - {% endif -%} -
-
-
- -
- -
-
-
-
-
- Url -
-
- {{ params.existing_source.endpoint.endpoint_url if params.existing_source else params.source.endpoint_url }} - {% if not params.existing_source %} - {{ 'Url can be reached' if params.reachable_url else "Can't reach this url" }} - {% endif %} -
-
-
-
- Dataset -
-
- {% if params.existing_source %} - {% for dataset in params.existing_source.datasets %} - {{ dataset.name }}{{ ', ' if not loop.last else '' }} - {% endfor %} - {% else %} - {{ params.source.dataset }} - {% endif %} -
-
-
-
- Organisation -
-
- {{ params.existing_source.organisation.name if params.existing_source else params.source.organisation }} -
-
-
-
- Documentation url -
-
- {% if params.existing_source %} - {% set original = params.existing_source.documentation_url if params.existing_source.documentation_url is not none else '' %} - {% if params.source.documentation_url != original %} -
- Current: {{ original }} -
-
- Your edit: {{ params.source.documentation_url if params.source.documentation_url }} -
- {% else %} - {{ params.source.documentation_url if params.source.documentation_url }} - {% endif %} - {% else %} - {{ params.source.documentation_url if params.source.documentation_url }} - {% endif %} -
-
-
-
- Licence -
- {% if params.existing_source %} - {{- summaryEditedField({ - "value": { - "original": params.existing_source['licence'], - "edited": params.source['licence'] - }, - "prop": "licence" - }) -}} - {% else %} - {{- summaryField({ - "value": params.source['licence'], - "prop": "licence" - }) -}} - {% endif %} -
-
-
- Attribution -
- {% if params.existing_source %} - {{- summaryEditedField({ - "value": { - "original": params.existing_source['attribution'], - "edited": params.source['attribution'] - }, - "prop": "attribution" - }) -}} - {% else %} - {{- summaryField({ - "value": params.source['attribution'], - "prop": "attribution" - }) -}} - {% endif %} -
-
-
- Start date -
-
- {% if params.existing_source %} - {% set original = params.existing_source.start_date.strftime("%Y-%m-%d") if params.existing_source.start_date is not none else '' %} - {% if params.source.start_date == original %} - {{ params.source.start_date if params.source.start_date }} - {% else %} -
- Current: {{ original }} -
-
- Your edit: {{ params.source.start_date if params.source.start_date }} -
- {% endif %} - {% else %} - {{ params.source.start_date if params.source.start_date }} - {% endif %} -
-
-
-
-
-
-
- -{% endmacro %} - - -{% macro summarySourceCard(params) %} -
-
-
-

Source {{ params.source.source }}

-
-
- -
- -
-
-
-
-
- Organisation -
-
- {{ params.source.organisation.name }} -
-
-
-
- Datasets -
-
- {% for dataset in params.source.datasets %} - {{ dataset.name }}{{ ', ' if not loop.last else '' }} - {% endfor %} -
-
-
-
- Url -
-
- {{ params.source.endpoint.endpoint_url }} -
-
- -
-
-
-
-
- Source added on {{ params.source.entry_date }} -
-
- -{% endmacro %} diff --git a/application/templates/content_type/index.html b/application/templates/content_type/index.html deleted file mode 100644 index 3e10d82a..00000000 --- a/application/templates/content_type/index.html +++ /dev/null @@ -1,67 +0,0 @@ -{% extends "layouts/layout.html" %} -{% block page_title %}Flask prototyping index{% endblock %} - - -{% block dl_breadcrumbs %} -{{ govukBreadcrumbs({ - "items": [ - { - "text": "Data operations", - "href": "/" - }, - { - "text": "Sources" - } - ] -}) }} -{% endblock %} - -{% block content %} - -
-

Content types

- -

Showing {{ content_type_counts|length }} content-type{{ "" if content_type_counts|length == 1 else "s" }}

- {% if pipeline -%} -
-
- Filter: - - xremove filtering by true - {{ pipeline }} - -
-
- {% endif %} - -
-{% endblock %} - -{% block footer %} -
-
-
-
- {# need a macro for this bit, param would set the href #} - - - - Back to top - -
-
-
-
-{{ super() }} -{% endblock %} diff --git a/application/templates/datamanager/assign-entities-check-results.html b/application/templates/datamanager/assign-entities-check-results.html index dcb29867..4a08ac0b 100644 --- a/application/templates/datamanager/assign-entities-check-results.html +++ b/application/templates/datamanager/assign-entities-check-results.html @@ -1,4 +1,4 @@ -{% extends 'components/check-transform-base.html' %} +{% extends 'datamanager/components/check-transform-base.html' %} {% block pageTitle %}{{ dataset_display }} – {{ organisation_display }} – Assign Entities Resource Details{% endblock %} diff --git a/application/templates/datamanager/check-transform.html b/application/templates/datamanager/check-transform.html index a4b98c0d..e9bf0440 100644 --- a/application/templates/datamanager/check-transform.html +++ b/application/templates/datamanager/check-transform.html @@ -1,4 +1,4 @@ -{% extends 'components/check-transform-base.html' %} +{% extends 'datamanager/components/check-transform-base.html' %} {% block pageTitle %}{{ dataset_display }} – {{ organisation_display }} – Provision Comparison{% endblock %} diff --git a/application/templates/components/check-transform-base.html b/application/templates/datamanager/components/check-transform-base.html similarity index 100% rename from application/templates/components/check-transform-base.html rename to application/templates/datamanager/components/check-transform-base.html diff --git a/application/templates/dataset/dataset.html b/application/templates/dataset/dataset.html deleted file mode 100644 index 983b1319..00000000 --- a/application/templates/dataset/dataset.html +++ /dev/null @@ -1,130 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% from 'components/rule-table-header.html' import ruleTableHead %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Datasets -
    3. -
  3. - {{dataset.name}} -
    4. -
-
-{% endblock beforeContent %} - -{% block content %} - -Dataset -

{{dataset.name}}

- -

This dataset is the yield of the {{ dataset.collection.name }} pipeline. The pipeline has been configured to collect, process and yield data that matches the {{ dataset.name }} schema.

- -
-
-
-
-

- Configuration summary -

-
-
-
-
-
Typology
-
- {{ dataset.typology.name }} -
-
-
-
Sources
-
{{ dataset.sources|length }}
-
-
-
Pipeline rules
-
{{ rule_counts.pipeline }}
-
-
-
Lookup rules
-
{{ rule_counts.lookup }}
-
-
-
Collection
-
{{ dataset.collection.name }}
-
-
-
-
-
-
- -
-

Pipeline rules

- -
- - {% for rule_type_name in specification_pipelines -%} -
-
-

- - {{ pipeline[rule_type_name]|length }} {{ rule_type_name }} rules - -

-
-
-

{{ specification_pipelines[rule_type_name].description }}

- {% if pipeline[rule_type_name]|length > 0 %} - {% set rows = pipeline[rule_type_name]|length if pipeline[rule_type_name]|length < 6 else 5 %} -

Showing {{ rows }} {{ rule_type_name }} rule{{ 's' if rows > 1 }}

- - {{ ruleTableHead({ "fields": specification_pipelines[rule_type_name].fields }) }} - - {% for n in range(rows) %} - {% set rule = pipeline[rule_type_name][n] %} - - - {%- for field in specification_pipelines[rule_type_name].fields %} - {%- if field.field not in ['dataset', 'start-date', 'end-date', 'entry-date'] -%} - - {%- endif -%} - {% endfor -%} - - - - - {% endfor %} - -
{{ rule['dataset'].dataset if rule['dataset'] }}{% if rule|attr(field.field) %}{{ rule|render_field_value(field.field) }}{% endif %}{{ rule['entry-date'] }}{{ rule['start-date'] }}{{ rule['end-date'] }}
-

See all {{ rule_type_name }} rules

- {% endif %} -
-
- {%- endfor %} - -
-
- -
-

Sources

- - - + add new source - - - {% set limit = 20 if dataset.sources|length > 10 else dataset.sources|length %} -

Showing {{ limit }} source{{'s' if limit != 1 }}.

-

See table of all sources for {{ dataset.name }} dataset.

- -
- -{% endblock content %} diff --git a/application/templates/dataset/editrule.html b/application/templates/dataset/editrule.html deleted file mode 100644 index ba95425e..00000000 --- a/application/templates/dataset/editrule.html +++ /dev/null @@ -1,93 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% from 'components/rule-table-header.html' import ruleTableHead %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Datasets -
    3. -
  3. - {{dataset.name}} -
    4. -
  4. - {{ rule_type_name|capitalize }} rules -
    5. -
  5. {{ rule.id|default('new') }}
    6. -
-
-{% endblock beforeContent %} - -{% block content %} - -{{ dataset.name }} - {{ rule_type_name }} rules -

{% if rule.id %}Edit rule #{{ rule.id }}{% else %}Add new rule{% endif %}

- -
-
- -
- {{ form.hidden_tag() }} -
- - {{ form.dataset_id(class="govuk-input govuk-input--width-10 app-text-input__readonly", readonly=True) }} -
- {% for field_name in form_field_names %} - {% if field_name not in ["dataset_id", "start_date", "end_date", "entry_date"] %} - {% set input = form[field_name] %} -
- {{ input.label(class="govuk-label") }} - {% if input.type == "SelectField" %} - {{ input(class="govuk-select govuk-input--width-10") }} - {% else %} - {{ input(class="govuk-input govuk-input--width-10") }} - {% endif %} -
- {% endif %} - {% endfor %} - -

The dates determine when this rule will be used by the pipeline and when it is no longer needed.

-
- {{ form.entry_date.label(class="govuk-label") }} - {% if rule.entry_date %} - {{ form.entry_date(class="govuk-input govuk-input--width-10", readonly=True)}} - {% else %} - {{ form.entry_date(class="govuk-input govuk-input--width-10")}} - {% endif %} -
-
- {{ form.start_date.label(class="govuk-label") }} - {% if rule.start_date %} - {{ form.start_date(class="govuk-input govuk-input--width-10", readonly=True)}} - {% else %} - {{ form.start_date(class="govuk-input govuk-input--width-10")}} - {% endif %} -
-
- {{ form.end_date.label(class="govuk-label") }} - {% if rule.end_date %} - {{ form.end_date(class="govuk-input govuk-input--width-10", readonly=True)}} - {% else %} - {{ form.end_date(class="govuk-input govuk-input--width-10")}} - {% endif %} -
-
- - - Cancel -
-
- -
-
-{% endblock content %} diff --git a/application/templates/dataset/index.html b/application/templates/dataset/index.html deleted file mode 100644 index ff454894..00000000 --- a/application/templates/dataset/index.html +++ /dev/null @@ -1,54 +0,0 @@ -{% extends 'layouts/base.html' %} -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Datasets -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - -

Datasets - pipeline configuration

- -

Datasets

- -
-
-

These datasets are the product of pipelines that collect and process source data.

-
-
- -

Configuration for {{ datasets|length }} datasets

- - - -

Category type datasets

- -
-
-

These datasets are category data managed by DLUHC. Category data is maintained in registers - and should not need to be transformed by a pipeline.

-
-
- -

Configuration for {{ category_datasets|length }} datasets.

- - - -{% endblock content %} diff --git a/application/templates/dataset/performance.html b/application/templates/dataset/performance.html deleted file mode 100644 index 358a18ac..00000000 --- a/application/templates/dataset/performance.html +++ /dev/null @@ -1,428 +0,0 @@ -{% extends "layouts/base.html" %} - -{% from 'macro/resources-sources-by-month.html' import sourcesAndResourcesByMonthChart, sourcesAndResourcesByMonthJS %} -{% from 'tables/org-table-on-dataset.html' import orgTableOnDataset %} -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - {{ name|replace("-", " ")|capitalize }} -
    3. -
-
-{% endblock beforeContent %} -{% block dl_breadcrumbs %} -{{ govukBreadcrumbs({ - "items": [ - { - "text": "Datasets", - "href": url_for('dataset.datasets') - }, - { - "text": name|replace("-", " ")|capitalize - } - ] -}) }} -{% endblock %} - -{% block content %} -
-
-

{{ name|replace("-", " ")|capitalize }} summary

-
-
-
-

- {% if entity_count is not none -%} - {{ appDataItem({ - "label": "Entity" if entity_count == 1 else "Entities", - "value": entity_count|commanum - }) }} - {%- endif %} -

-
-
-
-
-

- {% if coverage -%} -
-

Organisations

- {{ coverage['active'] }}/{{ coverage['total'] }} -
- {%- endif %} -

-
-
-

- {% if resource_count is not none %} - {{ appDataItem({ - "label": "Total Resources", - "value": resource_count|default('0') - }) }} - {%- endif %} -

-
-
-

- {% if source_count is not none %} - {% set sourceLinkHTML %}{{ source_count[0]['active']|default('0') }}{% endset %} - {{ appDataItem({ - "label": "Active sources", - "value": { - "html": sourceLinkHTML - }, - "explainer": { - "summary": "What is an active source?", - "text": "It is a source with an endpoint (a url) that the digital land collector visits looking for resources. A source can become inactive if an end-date is added." - } - }) }} - {%- endif %} -

-
-
-
-
-
-
- Dataset performance -

{{ name|replace("-", " ")|capitalize }}

- -

Dataset info

-
- - {% if dataset['name'] %} -
-
- Name -
-
- {{ dataset['name'] }} -
-
- {% endif %} - - {% if dataset['project'] %} -
-
- Project -
-
- {{ dataset['project'] }} -
-
- {% endif %} - - {% if dataset['pipeline'] %} -
-
- Pipeline -
-
- {{ dataset['pipeline'] }} -
-
- {% endif %} - - {% if dataset['data-provider'] %} -
-
- Data provider -
-
- {{ dataset['data-provider'] }} -
-
- {% endif %} - - {% if dataset['category'] %} -
-
- Policy area -
-
- {{ dataset['category'] }} -
-
- {% endif %} - - {% if latest_resource %} -
-
- Date collected newest resource -
-
- {% if latest_resource | count == 0 %} -
Unable to get latest resource data
- {% else %} - {{ latest_resource[0]['start_date'] }} -
({{ latest_resource['resource']|truncate(15) }} was the resource collected)
- {% endif %} -
-
- {% endif %} - - {% if latest_logs.get(name) is not none %} -
-
- Collector last ran on -
-
- {{ latest_logs.get(name)['latest_attempt'] }} -
-
- {% endif %} - -
- -
-

- Contents -

- - -
- {{ sourcesAndResourcesByMonthChart({}) }} - -

Resource content-types

-
-
-
- {{ content_type_counts|length }} -

Content types

-
-
- - - What does this tell us? - - -
- The higher the number of different content-types, the more variety there is in the types of resource we have to process. It is easier to process, extract and combine data from the same type of resource. -
-
-
-
- {% if content_type_counts|length > 0 %} -
    -

    Most common types

    - {# only want to show most common types #} - {% set items = content_type_counts|length if content_type_counts|length < 4 else 4 %} - {% for n in range(items) %} -
  • - {% if content_type_counts[n]['content_type'] %} - {{ content_type_counts[n]['content_type'] }}: - {% else %} - No content-type - {% endif %} - {{ content_type_counts[n]['resource_count'] }} resource{{ "" if content_type_counts[n]['resource_count'] == 1 else "s" }} -
    • - {% endfor %} -
- {% else %} -

Error fetching content types from datasette.

- {% endif %} -
-
-
- -
- {% if publishers %} -

Organisations

- -
-
-

- Chart showing the number of active resources per publishers. Ideally each publisher would have only 1 active resource, then we can be confident this is the resource containing the latest data. -

-
-
-
-
-
- -

Who has an active resource?

-
-
-
-

- - {{ publishers['active']|length }} publishers with an active resource - -

-
-
- {{ orgTableOnDataset({ - "caption": "Organisations with an active resource", - "organisations": publishers['active'], - "today": today - }) }} -
-
-
-
-

- - {{ publishers['noactive']|length }} publishers with NO active resource - -

-
-
- {{ orgTableOnDataset({ - "caption": "Organisations without an active resource", - "organisations": publishers['noactive'], - "today": today - }) }} -
-
- {% if blank_sources|length %} -
-
-

- - No data for {{ blank_sources|length }} expected publishers - Missing - -

-
-
-
    - {% for source in blank_sources %} - {%- set prefix = source['organisation'].split(":")[0] %} - {%- set org_id = source['organisation'].split(":")[1] %} -
  • - {{ source['name'] }} -
    • - {% endfor %} -
-
-
- {% endif %} -
- {% endif %} -
-
- -{% endblock content %} - - -{% block pageScripts %} -{{ super() }} -{% include 'overview/high-charts-scripts.html' %} - - - - - -{% if monthly_counts %} -{{ sourcesAndResourcesByMonthJS({ - "months": monthly_counts['months']|tojson, - "resources": monthly_counts['resources']|tojson, - "sources": monthly_counts['sources']|tojson -}) }} -{% else %} -

No sources and resources for {{name|replace("-", " ")|capitalize}}.

-{% endif %} - - -{% if resource_stats %} - -{% endif %} -{% endblock pageScripts %} diff --git a/application/templates/dataset/rules.html b/application/templates/dataset/rules.html deleted file mode 100644 index ecff92fb..00000000 --- a/application/templates/dataset/rules.html +++ /dev/null @@ -1,67 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% from 'components/rule-table-header.html' import ruleTableHead %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Datasets -
    3. -
  3. - {{dataset.name}} -
    4. -
  4. {{ rule_type_name|capitalize }} rules
    5. -
-
-{% endblock beforeContent %} - -{% block content %} - -{{ dataset.name }} -

{{ rule_type_name|capitalize }} rules

- - - Add new rule - - - -

Showing {{ rules|length }} {{ rule_type_name }} rule{{ '' if rules|length == 1 else 's' }}

-{% if limited %} -
- - - Warning - More than 1000 {{rule_type_name}} rules have been configured. We can't display them at this time. - -
-{% endif %} - - {{ ruleTableHead({ "fields": rule_type_specification.fields, "editable": "true" }) }} - - {% if rules|length > 0 %} - {% for rule in rules %} - - - {%- for field in rule_type_specification.fields %} - {%- if field.field not in ['dataset', 'start-date', 'end-date', 'entry-date'] -%} - - {%- endif -%} - {% endfor -%} - - - - - - {% endfor %} - {% endif %} - -
{{ rule['dataset'].dataset if rule['dataset'] }}{% if rule|attr(field.field) %}{{ rule|render_field_value(field.field) }}{% endif %}{% if rule.entry_date %}{{ rule.entry_date }}{% endif %}{% if rule.start_date %}{{ rule.start_date }}{% endif %}{% if rule.end_date %}{{ rule.end_date }}{% endif %} - Edit pipeline rule - Delete pipeline rule -
- -{% endblock content %} diff --git a/application/templates/dataset/sources.html b/application/templates/dataset/sources.html deleted file mode 100644 index 1f6c11c2..00000000 --- a/application/templates/dataset/sources.html +++ /dev/null @@ -1,55 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% from 'components/rule-table-header.html' import ruleTableHead %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Datasets -
    3. -
  3. - {{dataset.name}} -
    4. -
  4. Sources
    5. -
-
-{% endblock beforeContent %} - -{% block content %} - -{{ dataset.name }} -

Sources

- - - Add new source - - - -

Showing {{ dataset.sources|length }} source{{'s' if limit != 1 }}

- - - - - - - - - - - - {% for source in dataset.sources %} - - - - - - - {% endfor %} - -
HashEndpoint urlOrganisationEnd date
{{ source.source|truncate(13) }}{{ source.endpoint.endpoint_url|truncate(100) }}{{ source.organisation.organisation }}{{ source.end_date if source.end_date }}
- -{% endblock content %} diff --git a/application/templates/layouts/layout.html b/application/templates/layouts/layout.html deleted file mode 100644 index 66be18a4..00000000 --- a/application/templates/layouts/layout.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "digital-land-frontend/layouts/base.jinja" %} -{% from 'macro/primary-nav.html' import primaryNavigation %} -{% from 'macro/data-item.html' import appDataItem %} - -{% block stylesheets %} - -{% endblock %} - -{% block mastHead %} - -{% block primaryNav %} -{% include 'partials/primary-navigation.html' %} -{% endblock primaryNav %} -{% endblock %} - - -{% block pageScripts %} - -{% if info_page %} - -{% endif %} -{% endblock %} diff --git a/application/templates/macro/primary-nav.html b/application/templates/macro/primary-nav.html deleted file mode 100644 index 6c6ad141..00000000 --- a/application/templates/macro/primary-nav.html +++ /dev/null @@ -1,17 +0,0 @@ -{% macro primaryNavigation(params) %} -
-
-
- -
-
-
-{% endmacro %} diff --git a/application/templates/macro/resources-sources-by-month.html b/application/templates/macro/resources-sources-by-month.html deleted file mode 100644 index 19a45923..00000000 --- a/application/templates/macro/resources-sources-by-month.html +++ /dev/null @@ -1,84 +0,0 @@ -{% macro sourcesAndResourcesByMonthChart(params) %} -

{{ params.heading|default("Sources and resources by month") }}

-
-
-

- {% if params.caption %} - {{ params.caption }} - {% else %} - A chart showing the number of sources Digital land have added each month and the number of new resources collected each month. - {% endif %} -

-
-{% endmacro %} - -{% macro sourcesAndResourcesByMonthJS(params) %} - -{% endmacro %} diff --git a/application/templates/overview/high-charts-scripts.html b/application/templates/overview/high-charts-scripts.html deleted file mode 100644 index 1a082ca9..00000000 --- a/application/templates/overview/high-charts-scripts.html +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/application/templates/overview/main-dataset-table.html b/application/templates/overview/main-dataset-table.html deleted file mode 100644 index 82f717fc..00000000 --- a/application/templates/overview/main-dataset-table.html +++ /dev/null @@ -1,183 +0,0 @@ -
- - - - - - - - - - - - - - - - - - {% for k, dataset in datasets.items() %} - - - - - - - - - - - - - - {% endfor %} - -
- Table of datasets -
- Name - - Typology - - Organisations (current/expected) - - Started - - Endpoints (active/total) - - Most recent endpoint - - Active Resources - - Days since update - - Frequency of updates -
- {% set dataset_url_str = dataset.name|lower|replace(" ", "-") %} - {{ dataset.name }} - - {{ dataset['typology'] }} - - {{ dataset['publishers'] }} - / {{ dataset['expected_publishers'] }} - - {% if dataset['first'] %} {{ dataset['first'] }} {% elif - dataset.pipeline %} - - No data - {% else %} - Backlog - {% endif %} - - {{ dataset['active'] }} - / {{ dataset['total'] }} - - {{ dataset['latest_endpoint'] }} - - {{ dataset['active_resources'] }} - - {{ dataset['latest']|days_since if dataset['latest'] }} - - {{dataset['frequency-of-updates']}} -
-
-
- Backlog - Datasets we know about but have not started collecting data. -
-
- No data - Datasets with a pipeline but no resources collected. -
-
-
diff --git a/application/templates/overview/performance.html b/application/templates/overview/performance.html deleted file mode 100644 index ee099f2f..00000000 --- a/application/templates/overview/performance.html +++ /dev/null @@ -1,331 +0,0 @@ -{% extends "layouts/base.html" %} - -{% set containerClasses = 'reporting-page' %} -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Overview of Datasets -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} -
-
-
-

Summary

-
-
-
-
-

Entities

-

{{ entity_count|commanum }}

-
- - - What is an entity? - - -
- An entity is a unit of data with a distinct and independent purpose, such as an organisation, document, location, policy, etc -
-
-
-
-
-
-
-
-

Datasets

-

{{ datasets|length }}

-
-
-
-
-

Sources

-

{{ source_counts | sum(attribute='sources_with_endpoint') }}

-
-
-
-
-

Resources

-

{{ resource_count }}

-
-
-
-
-

Organisations

-

{{ publisher_count['active'] }}/{{ publisher_count['total'] }}

-
- - - What does this tell us? - - -
- It measures the number of organisations we have collected data from against the number we expect to collect data from. Ideally these numbers would be the same. -
-
-
-
-
-
-
-
-
-
-
-
-

- Contents -

- - -
-

Overview of datasets

-

The datasets used to make planning related decisions. Digital land are either collecting some data for each dataset listed or plan to do so in the future.

- {% include 'overview/main-dataset-table.html' %} -
- -
-

Sources and resources

- -

Overview of sources

- -
-
-
- {{ source_counts | sum(attribute='sources_with_endpoint') }} -

Total

-
-
- - - What does this tell us? - - -
- This is the number of places Digital land attempts to collect data from each night. The number shows the challenge anyone looking for planning data faces. -
-
-
-
-
- {{ source_counts | sum(attribute='sources_missing_document_url') }} -

Without a documentation URL

-
-
- - - What does this tell us? - - -
- When we add a source we usually record a documentation-url. This is the place where we found the source. It helps us answer questions about where we found the source and why we are collecting it. Without it we are less confident about the origin of the source and therefore the authority of the data. -
-
-
-
-
    - {% for pipeline in source_counts %} - {%- if pipeline['sources_missing_document_url'] > 0 %} -
  • - {{ pipeline['pipeline'] }}: {{ pipeline['sources_missing_document_url'] }} -
    • - {% endif -%} - {% endfor %} -
-
-
-
- ⊻see breakdown by dataset - ⊼close breakdown -
-
-
-
- -

Sources and resources by month

-
-
-

- A chart showing the number of sources Digital land have added each month and the number of new resources collected each month. - enhanced readability. -

-
- -

Overview of resources

-
-
-
- {{ resource_count }} -

Resources

-
-
-
-
- {{ content_type_counts|length }} -

Content types

-
- -
-
-
    - {# only want to show most common types #} - {% for n in range(10) %} -
  • - {% if content_type_counts[n]['content_type'] %} - {{ content_type_counts[n]['content_type'] }}: - {% else %} - No content-type - {% endif %} - {{ content_type_counts[n]['resource_count'] }} resource{{ "" if content_type_counts[n]['resource_count'] == 1 else "s" }} -
    • - {% endfor %} -
-
-
-
- ⊻see top 10 content types - ⊼close breakdown -
-
-
-
-
- -
-

Overview of publishers

-
-
-
- {{ publisher_count['active'] }}/{{ publisher_count['total'] }} -

Publisher coverage

-
-
-
-
- -
-
-
-{% endblock content %} - -{% block pageScripts %} -{% include 'overview/high-charts-scripts.html' %} - - - - - - -{% endblock pageScripts %} diff --git a/application/templates/partials/primary-navigation.html b/application/templates/partials/primary-navigation.html deleted file mode 100644 index 3492ea35..00000000 --- a/application/templates/partials/primary-navigation.html +++ /dev/null @@ -1,31 +0,0 @@ -{{ primaryNavigation({ - "items": [ - { - "text": "Overview", - "href": url_for('reporting.overview_of_datasets'), - "active": True - }, - { - "text": "Datasets", - "href": url_for('dataset.datasets') - }, - { - "text": "Organisations", - "href": url_for('publisher.organisation') - }, - { - "text": "Sources", - "href": url_for('reporting.sources') - }, - { - "text": "Resources", - "href": url_for('reporting.resources') - }, - { - "text": "Logs", - "href": url_for('reporting.logs') - }, - - ] -}) -}} diff --git a/application/templates/reporting/endpoint_details.html b/application/templates/reporting/endpoint_details.html deleted file mode 100644 index 91ea7f64..00000000 --- a/application/templates/reporting/endpoint_details.html +++ /dev/null @@ -1,85 +0,0 @@ -{% extends 'layouts/base.html' %} {% set containerClasses = 'reporting-page' %} -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - ODP Summary -
    3. -
  3. Endpoint Details
    4. -
-
-{% endblock beforeContent %} {% block content %} -
-
-
-
Endpoint Details
-
-

- Documentation URL: - {% if endpoint_details.endpoint_info.documentation_url %} - {{ endpoint_details.endpoint_info.documentation_url }} - {% else %} No documentation URL available {% endif %} -

-

- Endpoint: {{endpoint_details.endpoint_info.endpoint}} -

-

- Endpoint URL: {{endpoint_details.endpoint_info.endpoint_url}} -

-

- Organisation: {{endpoint_details.endpoint_info.organisation_name}} -

-

- Organisation Code: {{endpoint_details.endpoint_info.organisation}} -

-

- Dataset: {{endpoint_details.endpoint_info.pipeline}} -

-

- Start Date: {{endpoint_details.endpoint_info.start_date}} -

-

- Entry Date: {{endpoint_details.endpoint_info.entry_date}} -

-
-
-
-
Resource History
-
- {{ govukTable({ "head": endpoint_details.resources_headers, "rows": - endpoint_details.resources_rows, "classes": - "govuk-table--small-text-until-tablet reporting-table" }) }} -
-
-
- -
-

Log History

-
- {{ govukTable({ "head": endpoint_details.logs_headers, "rows": - endpoint_details.logs_rows, "classes": "reporting-table" }) }} -
-
-
-{% endblock content %} diff --git a/application/templates/reporting/odp_conformance_summary.html b/application/templates/reporting/odp_conformance_summary.html deleted file mode 100644 index 1f5c8ba4..00000000 --- a/application/templates/reporting/odp_conformance_summary.html +++ /dev/null @@ -1,162 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from "govuk_frontend_jinja/components/table/macro.html" import govukTable %} -{%- from "components/filter-group/macro.jinja" import dlFilterGroup %} - -{% set containerClasses = 'reporting-page' %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - ODP Summary -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - -
-
-
- Status -
-
- Issue -
-
- Conformance -
-
- -
- -
-
-
- -

- Filter dataset type and cohort -

- - -
-
- {% call dlFilterGroup({ - "title": "Dataset type:", - "is_open": True, - "selected": odp_conformance_summary.params.selected_dataset_types|length - }) %} -
- {% for dataset_type in odp_conformance_summary.params.dataset_types %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- {% call dlFilterGroup({ - "title": "Cohort:", - "is_open": True, - "selected": odp_conformance_summary.params.selected_cohorts|length - }) %} -
- {% for cohort in odp_conformance_summary.params.cohorts %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- -
- - - Download Current Table - -
-
-
-
-

No. of LPAs supplying fields as per specification

- {{ govukTable({ - "head": odp_conformance_summary.stats_headers, - "rows": odp_conformance_summary.stats_rows, - "classes": "reporting-table" - }) }} - -

- Conformance to the technical specifications, where all expected fields have been provided, is {{ odp_conformance_summary.percent_100_field_match }}%. -

-
-
-
- -
- {% if odp_conformance_summary.rows == [] %} -

No data for current filtering

- {% else %} - {{ govukTable({ - "head": odp_conformance_summary.headers, - "rows": odp_conformance_summary.rows, - "classes": "reporting-table" - }) }} - {% endif %} -
- - - -{% endblock content %} diff --git a/application/templates/reporting/odp_issue_summary.html b/application/templates/reporting/odp_issue_summary.html deleted file mode 100644 index 5528b488..00000000 --- a/application/templates/reporting/odp_issue_summary.html +++ /dev/null @@ -1,155 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from "govuk_frontend_jinja/components/table/macro.html" import govukTable %} -{%- from "components/filter-group/macro.jinja" import dlFilterGroup %} - -{% set containerClasses = 'reporting-page' %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - ODP Summary -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - -
-
-
- Status -
-
- Issue -
-
- Conformance -
-
- -
- -
-
-
- -

- Filter dataset type and cohort -

- - -
-
- {% call dlFilterGroup({ - "title": "Dataset type:", - "is_open": True, - "selected": odp_issues_summary.params.selected_dataset_types|length - }) %} -
- {% for dataset_type in odp_issues_summary.params.dataset_types %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- {% call dlFilterGroup({ - "title": "Cohort:", - "is_open": True, - "selected": odp_issues_summary.params.selected_cohorts|length - }) %} -
- {% for cohort in odp_issues_summary.params.cohorts %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- -
- - - Download Current Table - -
-
-
-
-

Overview stats:

- {{ govukTable({ - "head": odp_issues_summary.stats_headers, - "rows": odp_issues_summary.stats_rows, - "classes": "reporting-table" - }) }} -

{{odp_issues_summary.endpoints_no_issues.count}} / {{odp_issues_summary.endpoints_no_issues.total_endpoints}} datasets with no issues

-
-
-
- -
- {{ govukTable({ - "head": odp_issues_summary.headers, - "rows": odp_issues_summary.rows, - "classes": "reporting-table" - }) }} -
- - - -{% endblock content %} diff --git a/application/templates/reporting/odp_status_summary.html b/application/templates/reporting/odp_status_summary.html deleted file mode 100644 index 2048b43d..00000000 --- a/application/templates/reporting/odp_status_summary.html +++ /dev/null @@ -1,178 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from "govuk_frontend_jinja/components/table/macro.html" import govukTable %} -{%- from "components/filter-group/macro.jinja" import dlFilterGroup %} - -{% set containerClasses = 'reporting-page' %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - ODP Summary -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - -
- -
-
- Status -
-
- Issue -
-
- Conformance -
-
- -
- -
-
-
- -

- Filter dataset type and cohort -

- - -
-
- {% call dlFilterGroup({ - "title": "Dataset type:", - "is_open": True, - "selected": odp_statuses_summary.params.selected_dataset_types|length - }) %} -
- {% for dataset_type in odp_statuses_summary.params.dataset_types %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- {% call dlFilterGroup({ - "title": "Cohort:", - "is_open": True, - "selected": odp_statuses_summary.params.selected_cohorts|length - }) %} -
- {% for cohort in odp_statuses_summary.params.cohorts %} -
- - -
- {% endfor %} -
- {% endcall %} -
-
- -
- - - Download Current Table - -
-
-
-
-

Overview stats:

-
-

No. of LPAs: -

{{odp_statuses_summary.number_of_lpas}} -

-
-
-

No. of LPAs with some : -

- {{odp_statuses_summary.lpa_some_data_provided}}

-
-
-

No. of LPAs with 100% data: -

- {{odp_statuses_summary.lpa_all_data_provided}}

-
-
-

No. of datasets added: -

- {{odp_statuses_summary.datasets_added}}/{{odp_statuses_summary.max_datasets}}

-
-
-

% of datasets added:

-

{{ odp_statuses_summary.percentage_datasets_added }}

-
-
-
-
- -
- {{ govukTable({ - "head": odp_statuses_summary.headers, - "rows": odp_statuses_summary.rows, - "classes": "reporting-table" - }) }} -
- - - -{% endblock content %} diff --git a/application/templates/reporting/overview.html b/application/templates/reporting/overview.html deleted file mode 100644 index f2e2277b..00000000 --- a/application/templates/reporting/overview.html +++ /dev/null @@ -1,162 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from "govuk_frontend_jinja/components/table/macro.html" import govukTable %} -{% block beforeContent %} - -
-
    -
  1. - Home -
    2. -
  2. - Reporting -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - - -
-
-

Overview

-
-
-

Endpoint Collection Stats

-
- - -
-
- - -
-
-

Resources downloaded:

-

Endpoint errors:

-
-
- - Download Issues Summary Table - -
- -
-
-

Issues:

- {{ govukTable({ - "head": issue_summary.stats_headers, - "rows": issue_summary.stats_rows, - "classes": "reporting-table app-table-issue-summary" - }) }} -

{{issue_summary.endpoints_no_issues.count}} / {{issue_summary.endpoints_no_issues.total_endpoints}} endpoints with no issues

-
-
- -

Endpoints added per week:

-
- -
- - -

Resources downloaded per week:

-
- -
- -

Endpoint errors and successes per week:

-
- -
- -

Interal errors per week:

-
- -
- - - - - - - -{% endblock content %} diff --git a/application/templates/schema/index.html b/application/templates/schema/index.html deleted file mode 100644 index 613f3825..00000000 --- a/application/templates/schema/index.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends 'layouts/base.html' %} -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Schemas -
    3. -
-
-{% endblock beforeContent %} - -{% block content %} - -
-
-

Schemas

-
-
- {{ - govukDetails({ - 'summaryText': 'What is a Schema?', - 'text': 'Schemas define the shape of the data. We configure one pipeline for each schema. Schemas are defined in the specification.' - }) - }} -
-
-

Total schemas: {{ schemas|length }}

-
-
-
-
-
- - {% for letter, datasets in grouped_datasets.items() %} -
-
-

{{ letter }}

-
-
-
    - {% for d in datasets %} -
  • - {{ d.name }} -
    • - {% endfor %} -
-
-
-
-
-
- {% endfor %} - -{% endblock content %} diff --git a/application/templates/schema/schema.html b/application/templates/schema/schema.html deleted file mode 100644 index af2fd876..00000000 --- a/application/templates/schema/schema.html +++ /dev/null @@ -1,208 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block beforeContent %} -
-
    -
  1. - Home -
    2. -
  2. - Schemas -
    3. -
  3. - {{schema.name}} -
    4. -
-
-{% endblock beforeContent %} - -{% block content %} - -Dataset schema -

{{schema.name}}

- -

This schema defines the shape of data the platform will collect for the {{ schema.name }} dataset.

- -{% if schema.pipeline %} -

- View the {{ schema.pipeline.name }} pipeline configuration for this schema. -

-{% endif %} - -
-
- -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- attribution - - {{ schema.attribution }} -
- collection - - {{ schema.collection.collection }} -
- dataset - - {{ schema.dataset }} -
- description - - {{ schema.description }} -
- end-date - - {{ schema.end_date }} -
- entity-maximum - - {{ schema.entity_maximum }} -
- entity-minimum - - {{ schema.entity_minimum }} -
- entry-date - - {{ schema.entry_date }} -
- fields - -
    - {% for field in schema.fields %}
  1. {{ field.field }}
    2. {% endfor %} -
-
- key-field - - {{ schema.key_field }} -
- licence - - {{ schema.licence }} -
- name - - {{ schema.name }} -
- paint-options - - {{ schema.paint_options }} -
- plural - - {{ schema.plural }} -
- prefix - - {{ schema.prefix }} -
- start-date - - {{ schema.start_date }} -
- themes - - {{ schema.themes }} -
- typology - - {{ schema.typology.typology }} -
- wikidata - - {{ schema.wikidata }} -
- wikipedia - - {{ schema.wikipedia }} -
-
-
-
-
- -{% endblock content %} diff --git a/application/templates/source/archive.html b/application/templates/source/archive.html deleted file mode 100644 index d4271f84..00000000 --- a/application/templates/source/archive.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block content %} -

Are you sure you want to archive this source?

- -
-
-
-
- {% for radio_option in form.confirm %} -
- {% if radio_option.data == "Yes" %} - {{ radio_option(required=False, class="govuk-radios__input", data_aria_controls="conditional-notes") }} - {% else %} - {{ radio_option(required=False, class="govuk-radios__input") }} - {% endif %} - -
- {% if radio_option.data == "Yes" %} -
-
- - {{ form.notes(required=False, class="govuk-textarea") }} -
-
- {% endif %} - {% endfor %} -
-
-
- - - Cancel -
-
-
-{% endblock content %} diff --git a/application/templates/source/create.html b/application/templates/source/create.html deleted file mode 100644 index 1dcbec5d..00000000 --- a/application/templates/source/create.html +++ /dev/null @@ -1,138 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block pageStylesheets %} - -{% endblock pageStylesheets %} - -{% block content %} -

Create a source entry

- -
- {{ form.hidden_tag() }} -
-
-
- - {% if form.endpoint_url.errors %} -

- {% for error in form.endpoint_url.errors %}Error: {{ error }}{% endfor %} -

- {% endif %} - {{ form.endpoint_url(required=False, class="govuk-input")}} -
- We have this endpoint url already. -
-
-
-
-
-
-
- -
- The datasets the source provides data for -
- {% if form.dataset.errors %} -

- {% for error in form.dataset.errors %}Error: {{ error }}{% endfor %} -

- {% endif %} -
- {{ form.dataset(required=False, class="govuk-input app-select-datasets__input")}} -
- -
- Selected datasets -
    -
    -
    -
    -
    -
    - -
    - The name of the organisation the source belongs to -
    - {% if form.organisation.errors %} -

    - {% for error in form.organisation.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.organisation(required=False, class="govuk-select govuk-!-width-two-thirds")}} -
    -
    - -
    - A link to a page explaining the data source -
    - {% if form.documentation_url.errors %} -

    - {% for error in form.documentation_url.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.documentation_url(required=False, class="govuk-input")}} -
    -
    - -
    - If available, add the licence the data was published under -
    - {% if form.licence.errors %} -

    - {% for error in form.licence.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.licence(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    - -
    - If available, add the attribution for the data -
    - {% if form.attribution.errors %} -

    - {% for error in form.attribution.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.attribution(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    - -
    - If known, add the date of publication. Dates should be in the YYYY-MM-DD format. -
    - {% if form.start_date.errors %} -

    - {% for error in form.start_date.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.start_date(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    -
    - - -
    - - - Cancel -
    -
    - -{% endblock content %} - -{% block pageScripts %} - - -{% endblock pageScripts %} diff --git a/application/templates/source/edit.html b/application/templates/source/edit.html deleted file mode 100644 index 12512f92..00000000 --- a/application/templates/source/edit.html +++ /dev/null @@ -1,123 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block content %} -

    Edit source

    -{% if source %} - -
    -
    - {% if source.entry_date %} -

    This source was created on {{ source.entry_date }}.

    - {% endif %} - - {% if source.end_date %} -

    This source has been archived. It was archived on {{ source.end_date }}

    - {% endif %} -
    -
    -
    - - -
    -
    -
    - - -
    - {{ form.hidden_tag() }} -
    -
    -
    - Source -
    -
    - {{ source.source }} -
    -
    - -
    -
    - Endpoint -
    -
    - {{ source.endpoint.endpoint_url }} -
    -
    - -
    -
    - Organisation -
    -
    - {% if source.organisation %} - {{ source.organisation.name }} - ({{ source.organisation.organisation }}) - {% else %} - No organisation - {% endif %} -
    -
    - -
    -
    - Datasets -
    -
    - {% for dataset in source.datasets %} - {{ dataset.dataset }}{{ ', ' if not loop.last else '' }} - {% endfor %} -
    -
    - -
    -
    - Documentation url -
    -
    - {{ form.documentation_url(required=False, class="govuk-input")}} -
    -
    - -
    -
    - Licence -
    -
    - {{ form.licence(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    - -
    -
    - Attribution -
    -
    - {{ form.attribution(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    - -
    -
    - Start date -
    -
    - {{ form.start_date(required=False, class="govuk-input govuk-!-width-one-half")}} -
    -
    - -
    - -
    - - - Cancel -
    -
    -{% endif %} -{% endblock content %} diff --git a/application/templates/source/finish.html b/application/templates/source/finish.html deleted file mode 100644 index 4a50b32d..00000000 --- a/application/templates/source/finish.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block content %} -
    -
    -
    -

    - Source saved -

    -
    - -

    We have saved the source.

    -

    - See source record -

    - -

    The collector will attempt to collect from the {{ source.collection.name }} collection repo name next time it runs.

    - -

    What do you want to do next?

    - -

    I have more sources to add

    -

    - Add another source. -

    - -

    I have finished adding sources

    - -
    -
    -{% endblock content %} diff --git a/application/templates/source/index.html b/application/templates/source/index.html deleted file mode 100644 index 7bf4f5f7..00000000 --- a/application/templates/source/index.html +++ /dev/null @@ -1,289 +0,0 @@ -{% extends "source/base.html" %} - -{% from 'macro/data-item.html' import appDataItem %} -{% from "digital-land-frontend/components/filter-group/macro.jinja" import dlFilterGroup %} -{% from "components/helpers.jinja" import random_int %} - -{% from 'macro/remove-filter-button.html' import removeFilterButton %} - -{% block dl_breadcrumbs %}{% endblock %} - -{% block main %} -
    -
    -

    Sources summary

    -
    -
    -
    -
    -
    - {{ appDataItem({ - "label": "Total", - "value": counts['sources'] - }) }} -
    -
    - {{ appDataItem({ - "label": "Old", - "value": counts['inactive'] - }) }} -
    -
    - {{ appDataItem({ - "label": "Datasets", - "value": counts['pipelines'] - }) }} -
    -
    -
    -
    -
    -
    -{% block content %} -

    Sources

    - -
    -
    -
    -

    Filters

    - - {%- if datasets %} -
    - {% call dlFilterGroup({ - "title": "Dataset", - "is_open": True if request.args['pipeline'] else False, - "selected": request.args.getlist('pipeline')|length if request.args['pipeline'] else 0 - }) %} -
    - {%- set randomID_input = random_int(5) %} -
    - - -
    - -
    - How many showing - -
    - {% for dataset in datasets %} -
    - - -
    - {% endfor %} -
    -
    -
    - {% endcall %} -
    - {% endif -%} - - - {%- if organisations %} -
    - {% call dlFilterGroup({ - "title": "Publisher", - "is_open": True if request.args['organisation'] else False, - "selected": request.args.getlist('organisation')|length if request.args['organisation'] else 0 - }) %} -
    - {%- set randomID_input = random_int(5) %} -
    - - -
    - -
    - How many showing - -
    - {% for organisation in organisations %} -
    - - -
    - {% endfor %} -
    -
    -
    - {% endcall %} -
    - {% endif -%} - - -
    -

    Documentation URL

    -
    -
    -
    -
    - - -
    -
    -
    -
    -
    - - -
    -

    Include blanks

    -
    -
    -
    -
    - - -
    -
    -
    -
    -
    - - -

    Search

    - - {% from "govuk_frontend_jinja/components/input/macro.html" import govukInput %} -
    - {{ govukInput({ - "label": { - "text": "By source" - }, - "id": "source", - "name": "source", - "value": filters['source'] if filters['source'] else "" - }) }} -
    - -
    - {{ govukInput({ - "label": { - "text": "By endpoint" - }, - "id": "endpoint_", - "name": "endpoint_", - "value": filters['endpoint_'] if filters['endpoint_'] else "" - }) }} -
    -
    - {{ govukInput({ - "label": { - "text": "By URL" - }, - "id": "endpoint_url", - "name": "endpoint_url", - "value": filters['endpoint_url'] if filters['endpoint_url'] else "" - }) }} -
    -
    - - Clear -
    -
    -
    -
    -
    -
    - Showing {{ sources|length }} source{{ "" if total_results == 1 else "s" }} - {% if sources|length == 100 %}Results are limited to 100{% endif %} -
    - {% if query_url %} - See on datasette - {% endif %} -
    -
    - {% if filters -%} -
    - Filter: - {% for filter in filter_btns %} - - xremove filtering by {{ filter['value'] }} - {{ filter['value'] }} - - {% endfor %} -
    - {% endif %} -
    - Include blanks: - {{ include_blanks }} -
    -
    -
      - {% for source in sources %} -
    • -
      -

      - Source {{ source['source'] }}

      - {%- if source['end_date'] or source['documentation_url'] == "" -%} -
      - {% if source['end_date'] %}Historical{% endif %} - {% if source['documentation_url'] == "" %}No documentation url{% endif %} - {% if source['endpoint'] == "" %}No endpoint{% endif %} -
      - {% endif -%} -
      -
      -
      -
      Organisation
      -
      {{ source['name'] }}
      -
      -
      -
      Dataset
      -
      {{ source['pipeline'] }}
      -
      -
      -
      • - {% endfor %} -
    -
    -
    - -{% endblock content %} -
    -{% endblock main %} - -{% block footer %} -
    -
    -
    -
    - {# need a macro for this bit, param would set the href #} - - - - Back to top - -
    -
    -
    -
    -{{ super() }} -{% endblock %} - -{% block pageScripts %} -{{ super() }} - - - -{% endblock pageScripts %} diff --git a/application/templates/source/search.html b/application/templates/source/search.html deleted file mode 100644 index 8eff84f1..00000000 --- a/application/templates/source/search.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from 'components/source-card.html' import summarySourceCard %} - -{% block content %} -

    Find source

    - -
    -
    - -
    - {{ form.hidden_tag() }} -
    - - {% if form.source.errors %} -

    - {% for error in form.source.errors %}Error: {{ error }}{% endfor %} -

    - {% endif %} - {{ form.source(required=False, class="govuk-input")}} -
    -
    - -
    -
    - -
    -
    -
    - -
    -
    -

    Recently added sources

    -
      - {% for source in sources %} -
    1. - {{ summarySourceCard({ - "source": source - }) }} -
      2. - {% endfor %} -
    -
    -
    - -{% endblock content %} diff --git a/application/templates/source/source.html b/application/templates/source/source.html deleted file mode 100644 index c4dcc5d6..00000000 --- a/application/templates/source/source.html +++ /dev/null @@ -1,124 +0,0 @@ -{% extends 'layouts/base.html' %} - -{% block beforeContent %} -{{ super() }} -Source search -{% endblock beforeContent %} - -{% block content %} -

    Source record

    -{% if source %} -
    - -
    - -
    -
    -
    - Source -
    -
    - {{ source.source }} -
    -
    -
    -
    - Endpoint -
    -
    - {{ source.endpoint.endpoint_url|makelink }} - ({{ source.endpoint.endpoint|truncate(13) }}) -
    -
    -
    -
    - Organisation -
    -
    - {% if source.organisation %} - {{ source.organisation.name }} - ({{ source.organisation.organisation }}) - {% else %} - No organisation recorded - {% endif %} -
    -
    -
    -
    - Documentation url -
    -
    - {% if source.documentation_url %} - {{ source.documentation_url|makelink }} - {% endif %} -
    -
    -
    -
    - Datasets -
    -
    - {% for dataset in source.datasets %} - {{ dataset.name }}{{ ', ' if not loop.last else '' }} - {% endfor %} -
    -
    -
    -
    - Attribution -
    -
    - {{ source.attribution.text if source.attribution }} -
    -
    -
    -
    - Licence -
    -
    - {{ source.licence.text if source.licence }} -
    -
    -
    -
    - Entry date -
    -
    - {{ source.entry_date if source.entry_date }} -
    -
    -
    -
    - Start date -
    -
    - {{ source.start_date if source.start_date }} -
    -
    -
    -
    - End date -
    -
    - {{ source.end_date if source.end_date }} -
    -
    -
    -
    - Status -
    -
    - {{ source.publication_status.value if source.publication_status }} -
    -
    -
    - -{% endif %} -{% endblock content %} diff --git a/application/templates/source/summary.html b/application/templates/source/summary.html deleted file mode 100644 index 0a9ef7c2..00000000 --- a/application/templates/source/summary.html +++ /dev/null @@ -1,38 +0,0 @@ -{% extends 'layouts/base.html' %} -{% from 'components/source-card.html' import fullSourceCard %} - -{% block content %} - -

    Check source details before submitting

    - -{% if existing_source %} -

    You are editing a source that already exists.

    -{% endif %} - -{{ fullSourceCard({ - 'source': sources[0], - "reachable_url": url_reachable, - "existing_source": existing_source, - "form": form -}) }} - -
    - {{ form.hidden_tag() }} - {% for name, value in form.data.items() %} - - {% endfor %} - {% if existing_source %} - - {% endif %} -
    - {% if existing_source %} - - {% else %} - 1 }}" /> - {% endif %} - Cancel -
    -
    - - -{% endblock content %} diff --git a/application/templates/tables/org-table-on-dataset.html b/application/templates/tables/org-table-on-dataset.html deleted file mode 100644 index b579435c..00000000 --- a/application/templates/tables/org-table-on-dataset.html +++ /dev/null @@ -1,38 +0,0 @@ -{% macro orgTableOnDataset(params) %} -
    - - {% if params.caption %}{% endif %} - - - - - - - - - - - - {% for org in params.organisations %} - {% set inactive_sources = org['sources']|to_int - org['active_sources']|to_int %} - {% set inactive_resources = org['resources']|to_int - org['active_resources']|to_int %} - {% set orgIdPieces = org['organisation'].split(':') %} - - - - - - - - - {% endfor %} - -
    {{ params.caption }}
    NameActive sourcesInactive sourcesTotal resourcesActive resourcesDays since update
    {{ org['name'] }} - {% if org['organisation_end_date'] and org['organisation_end_date'] < params.today %} - - Dissolved - - {% endif %} - {{ org['active_sources'] }}{{ inactive_sources }}{{ org['resources']|to_int }}{{ org['active_resources'] }}{{ org['days_since_update'] }}
    -
    -{% endmacro %} diff --git a/config/allowed-users.md b/config/allowed-users.md deleted file mode 100644 index 30776f52..00000000 --- a/config/allowed-users.md +++ /dev/null @@ -1,12 +0,0 @@ -# Allowed users - -Users listed here may override blocking check failures and proceed to add data. -Add one GitHub username per line under the list below. - -## List - -- pooleycodes -- Swati-Dash -- Ben-Hodgkiss -- gibahjoe - diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..c00d5c0a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,60 @@ +# Manage service β€” documentation + +The Manage service (config-manager) is a Flask app for the "add data" and +"assign entities" operator workflows. After the [dead-code cleanup](#removed-blueprints), +the service consists of three active Flask blueprints: + +| Blueprint | URL prefix | Purpose | +|---|---|---| +| `base` | `/` | Landing page (`index`), service-lock toggles | +| `auth` | `/auth` | GitHub OAuth login and admin-team checks | +| `datamanager` (+ `assign_entities`) | `/datamanager`, `/assign-entities` | The add-data and assign-entities operator workflows | + +All operator functionality is reached from the landing page (`base.index`) and +lives in the `datamanager` blueprint. It is the only blueprint with substantial +logic, and the only one with documentation and tests. + +## Contents + +- [`datamanager/architecture.md`](datamanager/architecture.md) β€” structure of the + `datamanager` blueprint (router, controllers, services, utils) and how to work within it +- [`datamanager/add-data.md`](datamanager/add-data.md) β€” the **Add data** user flow, step by step +- [`datamanager/assign-entities.md`](datamanager/assign-entities.md) β€” the **Assign entities** + (batch) flow end to end, and how it shares steps with Add data +- [`datamanager/github-add.md`](datamanager/github-add.md) β€” the GitHub workflow-dispatch that adds + data to the `config` repo, and the stale-assessment guard that runs at confirmation + +## Removed blueprints + +The following blueprints were **registered but unreachable from the live UI** and were +removed during the cleanup (no page or nav linked to them; they were only reachable by +typing a URL directly): + +- `source`, `dataset`, `report` (`/reporting`), `schema`, `endpoint`, `publisher` + +Their views, templates, `data_access` query modules, static JS bundles, and the old +cross-blueprint navigation (`layouts/layout.html`, `macro/primary-nav.html`) were removed +along with them. If any of this functionality is needed again, recover it from git history. + +## Removed config database + CLI + +Those blueprints were also the only readers/writers of the **pipeline/config schema** +in Postgres (collections, datasets, sources, endpoints, pipeline rules, etc.). With the +editing UI gone, that schema and its tooling were retired: + +- The config ORM models were removed from [`application/db/models.py`](../application/db/models.py), + which now defines only `ServiceLock` and `RequestMeta` β€” the two tables the running + service actually uses. +- The `flask data load` / `flask data drop` (dev seeding) and `flask publish changes` + (Postgres β†’ `digital-land/config` repo) CLI commands were removed, along with + `application/publish/` (the pydantic export models) and the `PyGithub` dependency. +- Migration [`a7b8c9d0e1f2_drop_config_tables`](../migrations/versions/a7b8c9d0e1f2_drop_config_tables.py) + drops the 24 config tables and the `publication_status` enum. It runs automatically via + `flask db upgrade` on deploy. **This is destructive and has no downgrade** β€” recover the + schema from migration `f8c9f47c8797` in git history if it is ever needed again. + +## Keeping docs up to date + +A non-blocking GitHub Actions check ([`.github/workflows/docs-check.yml`](../.github/workflows/docs-check.yml)) +posts a reminder on pull requests that change application code without updating anything in +`docs/`. It is a nudge, not a merge gate β€” update the relevant docs when behaviour changes. diff --git a/docs/datamanager/add-data.md b/docs/datamanager/add-data.md new file mode 100644 index 00000000..2ca2b7f3 --- /dev/null +++ b/docs/datamanager/add-data.md @@ -0,0 +1,132 @@ +# Add data β€” user flow + +The **Add data** flow (the "Add data" card on the landing page) lets an operator add a new or +updated data endpoint for a dataset + organisation (a provision). The tool runs the endpoint through the async +check/transform pipeline, previews the entities that would be created (the rows that are appended into the lookup.csv), and β€” on confirmation β€” +commits the resulting config to the `digital-land/config` repo as a PR request. + +All routes are on the `datamanager` blueprint (`/datamanager`, login-required, blocked while the +Add Data [service lock](#service-lock) is held). View functions live in `router.py` and delegate to +the controllers named below. + +## Code map + +| Area | File | Role | +| --- | --- | --- | +| Routes | `application/blueprints/datamanager/router.py` | Add-data routes, thin view functions, retire-endpoints POST | +| Dashboard / form | `application/blueprints/datamanager/controllers/form.py` | Dashboard GET/POST, CSV import, add-data form, preview submission | +| Check results | `application/blueprints/datamanager/controllers/check.py` | Check-results display and column-mapping resubmit | +| Transform | `application/blueprints/datamanager/controllers/transform.py` | Transformed facts, issue logs, entity-growth view (shared with Assign Entities) | +| Preview / confirm | `application/blueprints/datamanager/controllers/preview.py` | Entities preview, stale-assessment guard, GitHub dispatch | +| Async client | `application/blueprints/datamanager/services/async_api.py` | Submit/fetch async requests and response details | +| GitHub dispatch | `application/blueprints/datamanager/services/github.py` | Triggers the config repo commit workflow | +| Dataset / org lookups | `application/blueprints/datamanager/services/{dataset,organisation}.py` | Dataset/collection ids, organisation names and entity numbers | + +## The steps + +``` +dashboard -> initial form β†’ check-results β†’ add-data form (only if needed) β†’ check-transform β†’ entities-preview β†’ confirm β†’ success +``` + +### 1. Dashboard (`datamanager.dashboard_get` / `dashboard_add`, `controllers/form.py`) + +`GET /` renders `dashboard_add.html`: the operator picks a **dataset** and **organisation** and +enters the **endpoint URL** (or uses `GET/POST /import`, `handle_dashboard_add_import`, to paste a +CSV). `POST /` validates the form and calls `submit_request` (`services/async_api.py`) to create a +**check** request on the async API, then redirects to the check-results page for that `request_id`. + +A dashboard can also be pre-filled from an existing request (`fetch_request`), which supports magic +links that jump straight into the flow. + +### 2. Check results (`datamanager.check_results`, `controllers/check.py`) + +`GET /check-results/` renders `check-results-loading.html` while the async check is +pending (the page polls), then `check-results.html` once complete: converted rows and any issues, +with an inline **column-mapping** UI. From here the operator can: + +- **Re-run the check** with corrected column mappings β€” `POST /check-results/` + (`handle_check_resubmit`) submits a new check request and redirects back to check-results. +- **Proceed to Add data** β€” links to the add-data form. + +### 3. Add-data form (`datamanager.add_data`, `controllers/form.py`) + +`GET/POST /add-data/` renders `add-data.html` to collect the remaining fields β€” +`documentation_url`, `licence`, `start_date`, `authoritative`, and whether the endpoint is new. +These are held in `session["add_data_fields"]`; if they are already all present the form is skipped +and submission happens directly. On submit, `_submit_add_data_preview` calls `submit_request` to +create the **preview** request, records the config-branch baseline for the +[stale-assessment guard](github-add.md#stale-assessment-guard) (`record_branch_baseline`), and +redirects to check-transform. + +### 4. Check transform (`datamanager.check_transform`, `controllers/transform.py`) + +`GET /check-transform/` renders `check-transform-loading.html` then +`check-transform.html`: the transformed facts, issue logs, and an entity-growth view, plus the +option to select **endpoints to retire**. `POST /check-transform/` (`check_transform_post`, +`router.py`) stores the chosen `retire_endpoints` on the `RequestMeta` row and redirects to the +entities preview. + +> **Scope has grown.** This page started as a home for optional pre-commit *actions* (notably +> selecting endpoints to retire), but has since become a fully-fledged **comparison of the entities +> in the resource against the entities already on the platform**. Those two concerns now share one +> page; it will likely need **tabs** to separate them β€” one for **actions** (e.g. retire endpoints) +> and one for the **platform provision comparison**. + +> `transform.py` is shared with the [Assign Entities](assign-entities.md) flow and +> branches on the calling endpoint; see that doc for the differences. + +### 5. Entities preview (`datamanager.entities_preview`, `controllers/preview.py`) + +`GET /add-data//entities` renders `entities_preview.html`: a final review of the new +entities, any oldβ†’new entity redirects, and the organisation summary. This is the confirmation +point. + +> **Platform vs lookup β€” a common confusion.** Check transform (step 4) decides what is "new" by +> comparing the resource against the **platform API** (what is actually live). This preview decides +> what is "new" by comparing only against the **`lookup.csv`** file (what config already knows). So +> an entity can flag as new on check transform (not yet on the platform) but *not* appear as new +> here because it already exists in the lookup β€” and vice versa. Only a genuinely new lookup entry +> shows here, because this page reflects the rows that would be appended to `lookup.csv`. + +### 6. Confirm β†’ commit (`datamanager.add_data_confirm_async`, `controllers/preview.py`) + +`POST /add-data//confirm-async` runs `handle_add_data_confirm`, which applies the +[stale-assessment guard](github-add.md#stale-assessment-guard) (waits for any in-flight commit +workflow, then fails closed if the config branch moved for this collection) and, if clear, calls +`trigger_add_data_async_workflow` to dispatch the GitHub Action that commits the config. Success +renders `add-data-success.html`; a stale result renders `add-data-stale.html` with a "Re-run +transform" action. + +The commit workflow itself β€” payload, branch behaviour, and which CSVs it writes β€” is documented in +[github-add.md](github-add.md). + +## Service lock + +The Add Data flow is disabled while a `ServiceLock` named `add_data` is held (toggled from the +landing page). The `datamanager` before-request guard redirects to the landing page with a +`add_data_blocked_by` note while the lock is set. + +## Gotchas + +- **Two async requests per journey.** The dashboard submits a **check** request; the add-data form + submits a separate **preview** request. They have different `request_id`s, and the URL switches + from `/check-results/` to `/check-transform/` and `/add-data//…` accordingly. +- **The add-data form can be skipped.** If `session["add_data_fields"]` already has every field + (`_has_all_add_data_fields`), the form is bypassed and the preview is submitted directly β€” so a + magic-link/prefilled journey can jump straight past step 3. +- **Re-running the check makes a new id.** Resubmitting with corrected column mappings + (`handle_check_resubmit`) creates a *new* check request and redirects to its id; the old id is + left behind. +- **`authoritative` must be `yes`/`no`.** It is validated on the form and decides whether + `entity-organisation.csv` rows are written by the commit workflow. +- **`retire_endpoints` is stored, not applied.** The check-transform POST saves the selected hashes + on the `RequestMeta` row; the actual end-dating happens later in the commit workflow, not in + config-manager. +- **The stale guard can silently no-op.** It only runs when submitting onto the shared branch *and* + a baseline was captured at submission β€” see [github-add.md](github-add.md#stale-assessment-guard). + +## Related + +- [Assign Entities architecture](assign-entities.md) β€” the sibling flow that shares the transform/preview/commit steps. +- [github-add.md](github-add.md) β€” the GitHub commit workflow and the stale-assessment guard. +- [architecture.md](architecture.md) β€” the datamanager blueprint structure. diff --git a/docs/datamanager/architecture.md b/docs/datamanager/architecture.md index f581210f..370ca2a8 100644 --- a/docs/datamanager/architecture.md +++ b/docs/datamanager/architecture.md @@ -6,14 +6,12 @@ This document describes the structure of the `datamanager` blueprint and how to ## Directory layout -> Docs for this blueprint live in `docs/datamanager/` (this file, plus `github-add.md` and -> `stale-check.md`). Assign Entities has its own workflow docs in -> `docs/assign-entities/architecture.md`. +> Docs for this blueprint live in `docs/datamanager/` (this file, plus `add-data.md`, +> `assign-entities.md`, and `github-add.md`). ``` application/blueprints/datamanager/ β”œβ”€β”€ router.py # Blueprint definition, URL rules, auth guard -β”œβ”€β”€ config.py # External API URL builders β”œβ”€β”€ controllers/ β”‚ β”œβ”€β”€ __init__.py # ControllerError exception β”‚ β”œβ”€β”€ form.py # Dashboard GET/POST, import, add-data form diff --git a/docs/assign-entities/architecture.md b/docs/datamanager/assign-entities.md similarity index 97% rename from docs/assign-entities/architecture.md rename to docs/datamanager/assign-entities.md index 0034bf65..b5a10e6b 100644 --- a/docs/assign-entities/architecture.md +++ b/docs/datamanager/assign-entities.md @@ -30,7 +30,7 @@ itself. It passes selected references to async, then displays and commits whatev | Routes | `application/blueprints/datamanager/router.py` | Assign Entities routes, selection POST handling, replacement async request | | Start/import | `application/blueprints/datamanager/controllers/flagged_resources.py` | Direct resource/dataset entry, flagged-resources CSV import, async request submission | | Results page | `application/blueprints/datamanager/controllers/transform.py` | Builds Entities and Dedup tab data from async response | -| Shared results template | `application/templates/components/check-transform-base.html` | Entities tab table, entity checkboxes, selection count, shared button state | +| Shared results template | `application/templates/datamanager/components/check-transform-base.html` | Entities tab table, entity checkboxes, selection count, shared button state | | Dedup template | `application/templates/datamanager/assign-entities-check-results.html` | Dedup tab, duplicate redirect checkboxes, hidden changed flag | | Preview | `application/blueprints/datamanager/controllers/preview.py` | Displays async-generated lookup, entity-organisation, and old-entity rows | | GitHub dispatch | `application/blueprints/datamanager/services/github.py` | Triggers the config repo workflow with the completed async request id | @@ -355,6 +355,6 @@ suite before merging changes that touch shared templates or GitHub dispatch beha ## Related -- [Datamanager GitHub workflow](../datamanager/github-add.md) -- [Datamanager stale-assessment check](../datamanager/stale-check.md) -- [Datamanager blueprint architecture](../datamanager/architecture.md) +- [Datamanager GitHub workflow](github-add.md) +- [Datamanager stale-assessment guard](github-add.md#stale-assessment-guard) +- [Datamanager blueprint architecture](architecture.md) diff --git a/docs/datamanager/github-add.md b/docs/datamanager/github-add.md index 6ddc7c33..c91c8743 100644 --- a/docs/datamanager/github-add.md +++ b/docs/datamanager/github-add.md @@ -2,28 +2,19 @@ When a user confirms an add-data (or assign-entities) submission, config-manager triggers a GitHub Action in the `digital-land/config` repo that commits the assessed data onto a config branch and -opens/updates a PR. This document describes that trigger and the workflow it runs. +opens/updates a PR. The trigger lives in `services/github.py` (`trigger_add_data_async_workflow`, called from -`controllers/preview.py`). The workflow itself is `.github/workflows/add-data-async-script.yml` -("Add Data From Async API") in `digital-land/config`, running `bin/add_data.py`. - -## How it works - -1. Fetches the full request from the async API using `request_id`. -2. Validates the request `status` is `COMPLETE` with no error. -3. Resolves the target branch (see [Branch behaviour](#branch-behaviour)). -4. Appends rows to the relevant collection/pipeline CSVs (see [CSV files updated](#csv-files-updated)). -5. Commits and creates or updates a PR against `main`. +`controllers/preview.py`). The workflow is `.github/workflows/add-data-async-script.yml` in +`digital-land/config`, running `bin/add_data.py`. It fetches the request from the async API, +validates `status` is `COMPLETE` with no error, appends rows to the relevant collection/pipeline +CSVs, then commits and creates/updates a PR against `main`. ## Triggering the workflow -**Endpoint:** `POST {GITHUB_API_BASE_URL}/repos/digital-land/config/dispatches` - -**Headers:** -- `Accept: application/vnd.github+json` -- `Authorization: Bearer ` -- `X-GitHub-Api-Version: 2022-11-28` +**Endpoint:** `POST {GITHUB_API_BASE_URL}/repos/digital-land/config/dispatches` β€” sent as a GitHub +App (`GITHUB_APP_ID`, `GITHUB_APP_INSTALLATION_ID`, `GITHUB_APP_PRIVATE_KEY`), with +`X-GitHub-Api-Version: 2022-11-28`. **Payload:** ```json @@ -39,82 +30,26 @@ The trigger lives in `services/github.py` (`trigger_add_data_async_workflow`, ca } ``` -| Field | Required | Purpose | -| --- | --- | --- | -| `event_type` | yes | Must be `add-data-async-script` | -| `client_payload.request_id` | yes | ID of a `COMPLETE` async request | -| `client_payload.triggered_by` | no | Who/what triggered it (used in commit/PR content) | -| `client_payload.branch` | no | Target branch β€” see [Branch behaviour](#branch-behaviour) | -| `client_payload.retire_endpoints` | no | Comma-separated endpoint hashes to end-date | -| `client_payload.environment` | no | Async API environment: `development` \| `staging` \| `production` (default `staging`) | +Only `event_type` (must be `add-data-async-script`) and `client_payload.request_id` (a `COMPLETE` +async request) are required. `branch` defaults per [Branch behaviour](#branch-behaviour); +`retire_endpoints` end-dates the matching endpoint/source rows; `environment` (`development` | +`staging` | `production`, default `staging`) selects the async API base URL the workflow reads. The branch config-manager sends is its `CONFIG_REPO_BRANCH` setting β€” `config-manager-update` in production, `test-config-manager-update` in development. ## Branch behaviour -`bin/add_data.py` (`resolve_branch`) is branch-agnostic; the `branch` parameter controls how it -creates or updates branches and PRs. - -- **No `branch`** β€” a new branch `add-data-async/{collection}-{timestamp}` is created and a PR is - opened against `main`. -- **`branch` given, open PR exists** β€” checks out the branch, appends on top, and updates the - existing PR body with the new submission label. This batches multiple submissions into one PR. -- **`branch` given, branch exists but no open PR** β€” checks out the branch, appends, opens a new - PR against `main`. -- **`branch` given, branch does not exist** β€” creates a fresh branch with that name and opens a - PR against `main`. -- **Test mode** (`--test`) β€” commits to a `test/{branch}` branch as a **draft** PR that must not - be merged. config-manager does not send this today; it uses a dedicated test branch instead. - -Only a PR from `config-manager-update β†’ main` is auto-merged -(`auto-merge-config-manager.yml`); other branch names (e.g. `test-config-manager-update`) open a -normal PR that is never auto-merged. - -## Commit messages and PR content - -Each submission produces a commit and PR label: - -``` -add-{dataset}-{organisation}-{triggered_by} -``` - -e.g. `add-article-4-direction-area-local-authority:SKP-matt`. When batched onto one branch, the PR -body accumulates all labels. - -## Async API service - -The workflow fetches request data from `{ASYNC_API_BASE_URL}/requests/{request_id}`, where the base -URL is resolved from the `environment` in the payload: +`bin/add_data.py` (`resolve_branch`) is branch-agnostic; the `branch` parameter controls branch/PR +creation: -| environment | base URL | -| --- | --- | -| `development` | `http://development-pub-async-api-lb-…elb.amazonaws.com` | -| `staging` (default) | `http://staging-pub-async-api-lb-…elb.amazonaws.com` | -| `production` | `http://production-pub-async-api-lb-…elb.amazonaws.com` | +- **No `branch`** β€” creates `add-data-async/{collection}-{timestamp}` and opens a PR against `main`. +- **`branch` given** β€” checks out (or creates) it, appends on top, and updates the open PR if there + is one or opens a new one. This batches multiple submissions into a single PR, whose body + accumulates one label per submission: `add-{dataset}-{organisation}-{triggered_by}`. -### Expected response shape - -```json -{ - "params": { - "collection": "article-4-direction", - "dataset": "article-4-direction-area", - "organisation": "local-authority:SKP", - "column_mapping": { "geom": "geometry" }, - "authoritative": true - }, - "status": "COMPLETE", - "response": { - "data": { - "endpoint-summary": { "new_endpoint_entry": {}, "endpoint_url_in_endpoint_csv": false }, - "source-summary": { "new_source_entry": {}, "documentation_url_in_source_csv": false }, - "pipeline-summary": { "new-entities": [], "entity-organisation": [], "old-entity": [] } - }, - "error": null - } -} -``` +Only a `config-manager-update β†’ main` PR is auto-merged (`auto-merge-config-manager.yml`); other +branch names (e.g. `test-config-manager-update`) open a normal PR that is never auto-merged. ## CSV files updated @@ -127,25 +62,68 @@ URL is resolved from the `environment` in the payload: | `pipeline/{collection}/entity-organisation.csv` | `pipeline-summary.entity-organisation` | `params.authoritative` true, and not an overlap/error | | `pipeline/{collection}/old-entity.csv` | `pipeline-summary.old-entity` | array non-empty | -Retiring endpoints (`retire_endpoints`) does not append rows β€” it sets `end-date` on the matching -rows in `collection/{collection}/endpoint.csv` and `source.csv`. - -## Authentication - -config-manager triggers the dispatch as a GitHub App (`services/github.py`), using -`GITHUB_APP_ID`, `GITHUB_APP_INSTALLATION_ID`, and `GITHUB_APP_PRIVATE_KEY`. The workflow in the -config repo authenticates separately to fetch the request and push commits. - -## Error handling - The workflow fails if `request_id` is empty, the request cannot be fetched, its status is not `COMPLETE`, the response contains an error, the `collection` has no matching `collection/` and `pipeline/` directories, or there are no changes to commit. +## Stale-assessment guard + +Adding data is two steps: **assess** (the async worker reads the shared branch, assigns free entity +numbers, and freezes them into the result) and **confirm** (the user reviews, then the workflow +above commits the frozen numbers). The review page can sit open a long time; if another submission +advances the shared branch in between, the frozen numbers can collide β€” the same entity number used +twice, surfacing later as a failed PR a developer has to unpick by hand. This guard blocks the +confirm when that has happened. + +**Baseline at submission.** `record_branch_baseline` (`controllers/preview.py`, called from both +`_submit_add_data_preview` in `controllers/form.py` and `_submit_assign_entities_request` in +`controllers/flagged_resources.py`) records the branch HEAD SHA the assessment is based on onto +`RequestMeta.branch_sha`. It: + +- skips new-branch submissions (no shared state to race); +- baselines against `main` when `config-manager-update` does not exist yet (it is created lazily by + the first commit), via `get_config_baseline_sha` (`services/github.py`) β€” matching what the async + worker reads when the branch is absent; +- reads only the branch HEAD (one API call, no waiting) so submit stays fast; +- **fails open** β€” any error is logged and skipped rather than blocking the submission. + +**Check at confirmation.** Before triggering the commit workflow, `handle_add_data_confirm` +(`controllers/preview.py`) first **waits for any in-flight `add-data-async-script` workflow to +finish** (`wait_for_add_data_workflow_idle`, `services/github.py`) so it compares a settled branch, +bounded by `ADD_DATA_WORKFLOW_WAIT_TIMEOUT`; the confirm page shows a "Submitting…" panel during the +wait. It then compares the baseline via `config_branch_changed_for_collection` +(`GET /repos/digital-land/config/compare/{baseline_sha}...{head}`, where `head` is the shared branch +or `main`): + +- returns "changed" only if a changed file lives under `pipeline/{collection}/`, so other + collections do not block each other; +- **fails closed** β€” treats the branch as changed on a diverged/force-pushed history, a truncated + (>=300-file) diff, or any API error, so a possibly-stale result is never let through. + +If the branch changed, the confirm is blocked and the user sees +`templates/datamanager/add-data-stale.html` with a "Re-run transform" action, routing back to +`datamanager.check_results` for the recorded `check_request_id` (or the flow's start page). The +guard only runs when submitting onto the shared branch **and** a baseline was captured, so requests +predating this feature pass through unchanged. + +### Configuration (`config/config.py`) + +| Setting | Default | Purpose | +| --- | --- | --- | +| `CONFIG_REPO_BRANCH` | `config-manager-update` (prod), `test-config-manager-update` (dev) | Shared branch to commit to / check against | +| `ADD_DATA_WORKFLOW_WAIT_TIMEOUT` | `60` (seconds) | Max wait at confirm for an in-flight workflow before comparing | +| `ADD_DATA_WORKFLOW_POLL_INTERVAL` | `5` (seconds) | Poll interval while waiting | + +The gunicorn `--timeout` in the `Procfile` (120s) is kept comfortably above the wait timeout so a +web worker is not killed while the confirm request waits. + +> **Note:** there is no CI check for duplicate entity numbers before the auto-merged PR lands, and +> the commit workflow has no concurrency group β€” both are worth adding as defence-in-depth but are +> out of scope for this guard. + ## Related -- [stale-check.md](stale-check.md) β€” how config-manager avoids committing stale entity numbers when - the branch advances between assessment and confirmation. -- [Assign Entities architecture](../assign-entities/architecture.md) β€” how Assign Entities +- [add-data.md](add-data.md) β€” the Add data user flow that leads to this commit workflow. +- [Assign Entities architecture](assign-entities.md) β€” how Assign Entities selections are sent to async. - [architecture.md](architecture.md) β€” the datamanager blueprint structure. diff --git a/docs/datamanager/stale-check.md b/docs/datamanager/stale-check.md deleted file mode 100644 index 938b9897..00000000 --- a/docs/datamanager/stale-check.md +++ /dev/null @@ -1,103 +0,0 @@ -# Add-data stale-assessment check - -## What problem this solves - -Adding data is a two-step, human-in-the-loop flow: - -1. **Assess** β€” the user submits a transform. The async worker reads the current state of the - shared config branch (`config-manager-update`), works out which entity numbers are free, and - assigns new ones. These numbers are frozen into the stored result. -2. **Confirm** β€” the user reviews the result and confirms. A GitHub Action then commits the frozen - numbers onto the shared branch. - -The review page can sit open for a long time. If another submission advances the shared branch in -the meantime, the frozen entity numbers can collide with numbers that were assigned in between β€” -the same entity number ends up used twice, which only surfaces later as a merge conflict / failed -PR that a developer has to unpick by hand. - -The stale-check detects that the branch moved for this collection between assessment and -confirmation, and blocks the confirm with a prompt to re-run. - -## How it works - -### 1. Capture a baseline at submission - -When a transform is submitted, `record_branch_baseline` (`controllers/preview.py`) records the -config branch HEAD SHA the assessment is based on, on the `RequestMeta` row (`branch_sha` column). -It is called from both submission sites: - -- `_submit_add_data_preview` (`controllers/form.py`) β€” add-data flow -- `_submit_assign_entities_request` (`controllers/flagged_resources.py`) β€” assign-entities flow - -Behaviour: - -- **New-branch submissions** (no shared branch) are skipped β€” there is no shared state to race. -- **Lazy branch / main fallback.** `config-manager-update` is created lazily by the first - add-data commit, so early in a cycle it does not exist and reading its HEAD 404s. In that case - `get_config_baseline_sha` (`services/github.py`) baselines against `main` instead β€” which is - what the async worker reads when the branch is absent. Without this the baseline would be empty - and the check would silently do nothing. -- **Fast.** Submission only reads the branch HEAD (one API call) β€” it does not wait on any - in-flight workflow, so the submit request never hangs. The workflow-idle wait happens at confirm - (the decision point) instead. -- **Fails open.** Any error reading the SHA is logged and skipped rather than blocking the - submission. - -### 2. Check at confirmation - -Before triggering the commit workflow, `handle_add_data_confirm` (`controllers/preview.py`) first -**waits for any in-flight `add-data-async-script` workflow to finish** (`wait_for_add_data_workflow_idle` -/ `add_data_workflow_running`, `services/github.py`) so the comparison reads a settled branch and -not a mid-push state. This is a no-op in the common case and bounded by -`ADD_DATA_WORKFLOW_WAIT_TIMEOUT`; on timeout it proceeds and the fail-closed compare below is the -backstop. During this wait the confirm page shows a "Submitting…" loading panel -(`templates/datamanager/entities_preview.html`) so it never looks stuck. - -It then compares the baseline against the current branch state via -`config_branch_changed_for_collection` (`services/github.py`): - -- Uses the GitHub compare API, `GET /repos/digital-land/config/compare/{baseline_sha}...{head}`. -- `head` is the shared branch if it exists, otherwise `main` (mirroring the baseline fallback). -- Returns "changed" only if a changed file lives under `pipeline/{collection}/`, so submissions - to other collections do not block each other. -- **Fails closed** β€” treats the branch as changed on a diverged/force-pushed history, a truncated - (>=300-file) diff, or any API error, so a possibly-stale result is never let through. - -If the branch changed for the collection, the confirm is blocked and the user is shown -`templates/datamanager/add-data-stale.html` with a "Re-run transform" action. When the assessment -came from a check-results page (the add-data flow records its `check_request_id` on `RequestMeta`), -that action routes back to `datamanager.check_results` for that id so the user can re-transform; -otherwise it falls back to the flow's start page or home. The commit workflow is only triggered -when the branch has not changed. - -The check only runs when submitting onto the shared branch **and** a baseline was captured β€” so -requests created before this feature (no `branch_sha`) pass through unchanged. - -## Configuration - -In `config/config.py`: - -| Setting | Default | Purpose | -| --- | --- | --- | -| `CONFIG_REPO_BRANCH` | `config-manager-update` (prod), `test-config-manager-update` (dev) | Shared branch to commit to / check against | -| `ADD_DATA_WORKFLOW_WAIT_TIMEOUT` | `60` (seconds) | Max wait at confirm for an in-flight workflow before comparing | -| `ADD_DATA_WORKFLOW_POLL_INTERVAL` | `5` (seconds) | Poll interval while waiting | - -The gunicorn `--timeout` in the `Procfile` (120s) is kept comfortably above the wait timeout so a -web worker is not killed while the confirm request waits. - -## Design intent - -- Correctness lives in the confirm-time check, which **waits for the workflow to settle** and then - **fails closed**. -- The submission-side pieces (baseline capture with the main fallback) only exist to keep the - baseline accurate; they **fail open** so they can never block a submission over an infrastructure - hiccup, and they do no blocking work so the submit stays fast. - -## Related - -The stored numbers are committed by the `add-data-async-script` workflow in the `digital-land/config` -repo (`bin/add_data.py`), which is branch-agnostic and operates on whatever branch is passed to it. -There is no CI check for duplicate entity numbers before that PR auto-merges, and the commit -workflow has no concurrency group β€” both are worth adding as defence-in-depth but are out of scope -for this check. diff --git a/migrations/versions/a7b8c9d0e1f2_drop_config_tables.py b/migrations/versions/a7b8c9d0e1f2_drop_config_tables.py new file mode 100644 index 00000000..ccb8dedb --- /dev/null +++ b/migrations/versions/a7b8c9d0e1f2_drop_config_tables.py @@ -0,0 +1,71 @@ +"""drop unused config tables + +Removes the pipeline/config schema that was only read and written by the +removed source/dataset/endpoint/report blueprints and the retired +``flask data load`` / ``flask publish changes`` CLI commands. The running +service only uses ``service_lock`` and ``request_meta``. + +The tables are dropped with ``IF EXISTS ... CASCADE`` so the migration is +robust to environments where a subset was already absent and does not depend +on foreign-key drop ordering. + +Revision ID: a7b8c9d0e1f2 +Revises: 7b8c9d0e1f2a +Create Date: 2026-07-24 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7b8c9d0e1f2" +down_revision = "7b8c9d0e1f2a" +branch_labels = None +depends_on = None + + +# Config tables in reverse creation order (children before parents). CASCADE +# makes the ordering non-critical, but we keep it dependency-safe regardless. +CONFIG_TABLES = [ + "transform", + "source_dataset", + "skip", + "patch", + "lookup", + "filter", + "default_value", + "dataset_field", + "convert", + "concat", + "combine", + "column", + "_default", + "source", + "pipeline", + "field", + "dataset", + "typology", + "organisation", + "licence", + "endpoint", + "datatype", + "collection", + "attribution", +] + + +def upgrade(): + for table in CONFIG_TABLES: + op.execute(f'DROP TABLE IF EXISTS "{table}" CASCADE') + op.execute("DROP TYPE IF EXISTS publication_status") + + +def downgrade(): + # These tables and the associated ORM models were permanently removed. + # Recreating them faithfully is out of scope; recover the schema from the + # initial migration (f8c9f47c8797) and subsequent revisions in git history + # if it is ever needed again. + raise NotImplementedError( + "Downgrade is not supported: the config tables were permanently removed. " + "Recover from migration f8c9f47c8797 in git history if required." + ) diff --git a/requirements/requirements.in b/requirements/requirements.in index d4f6703f..fa9d688d 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -13,7 +13,6 @@ is-safe-url flask-talisman digital_land @ git+https://github.com/digital-land/pipeline.git#egg=digital-land sentry-sdk[flask] -PyGithub -pydantic +PyJWT[crypto] flask-sslify pandas diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f534d6ef..0cb48ed0 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -46,7 +46,6 @@ cffi==2.0.0 # via # cryptography # pygit2 - # pynacl charset-normalizer==3.4.4 # via requests click==8.3.1 @@ -217,19 +216,13 @@ pyarrow==23.0.1 pycparser==3.0 # via cffi pydantic==2.12.5 - # via - # -r requirements/requirements.in - # digital-land + # via digital-land pydantic-core==2.41.5 # via pydantic pygit2==1.18.2 # via digital-land -pygithub==2.8.1 - # via -r requirements/requirements.in pyjwt[crypto]==2.11.0 - # via pygithub -pynacl==1.6.2 - # via pygithub + # via -r requirements/requirements.in pyparsing==3.3.2 # via rdflib pyproj==3.7.1 @@ -259,7 +252,6 @@ requests==2.32.5 # digital-land # esridump # moto - # pygithub # responses responses==0.26.0 # via moto @@ -299,7 +291,6 @@ typing-extensions==4.15.0 # flexparser # pydantic # pydantic-core - # pygithub # sqlalchemy # typing-inspect # typing-inspection @@ -313,7 +304,6 @@ tzdata==2025.3 urllib3==2.6.3 # via # botocore - # pygithub # requests # responses # sentry-sdk diff --git a/rollup.config.js b/rollup.config.js index 17df2ee6..4e434143 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,39 +1,4 @@ module.exports = [ - { - input: "src/javascripts/application-add-source.js", - output: { - file: "application/static/javascripts/application-add-source.js", - format: "iife", - }, - }, - { - input: "src/javascripts/app-background-check.js", - output: { - file: "application/static/javascripts/app-background-check.js", - format: "iife", - }, - }, - { - input: "src/javascripts/app-resource-mapping.js", - output: { - file: "application/static/javascripts/app-resource-mapping.js", - format: "iife", - }, - }, - { - input: "src/javascripts/reporting-summary.js", - output: { - file: "application/static/javascripts/reporting-summary.js", - // format: 'iife' - }, - }, - { - input: "src/javascripts/utilities/timeseries-chart.js", - output: { - file: "application/static/javascripts/utilities/timeseries-chart.js", - // format: 'iife' - }, - }, { input: "src/javascripts/map.js", output: { diff --git a/src/javascripts/app-background-check.js b/src/javascripts/app-background-check.js deleted file mode 100644 index 84cd80b4..00000000 --- a/src/javascripts/app-background-check.js +++ /dev/null @@ -1,4 +0,0 @@ -import ResourceCheck from './components/resource-check' - -const $mappingLink = document.querySelector('[data-module="resource-check"]') -const mappingModule = new ResourceCheck($mappingLink).init({}) diff --git a/src/javascripts/app-resource-mapping.js b/src/javascripts/app-resource-mapping.js deleted file mode 100644 index 36035786..00000000 --- a/src/javascripts/app-resource-mapping.js +++ /dev/null @@ -1,9 +0,0 @@ -import CollapsibleSection from './components/collapsible-section' - -const $collapsibles = document.querySelectorAll("[data-module='app-collapsible']") -$collapsibles.forEach(function ($el) { - const $section = $el.querySelector('.dl-collapsible') - console.log($section) - const $triggers = $el.querySelector('.expanding-line-break') - const collapsibleComponent = new CollapsibleSection($section, $triggers).init() -}) diff --git a/src/javascripts/application-add-source.js b/src/javascripts/application-add-source.js deleted file mode 100644 index 790c7410..00000000 --- a/src/javascripts/application-add-source.js +++ /dev/null @@ -1,64 +0,0 @@ -/* global accessibleAutocomplete */ - -import { fetchWithArgs } from './utilities/fetchWithArgs' -import SelectDatasets from './components/select-datasets' - -// set up the auto complete inputs -accessibleAutocomplete.enhanceSelectElement({ - selectElement: document.querySelector('#organisation') -}) - -// accessibleAutocomplete.enhanceSelectElement({ -// selectElement: document.querySelector('#dataset') -// }) - -const $selectDatasetComponent = document.querySelector("[data-module='select-datasets']") -const $selectDatasetModule = new SelectDatasets($selectDatasetComponent).init({}) - -window.$selectDatasetModule = $selectDatasetModule - -function appFetch (url, fetchParams, _callback) { - fetchWithArgs(url, fetchParams) - .then(data => { - if (_callback) { - _callback(data) - } else { - console.log(data) // JSON data parsed by `data.json()` call - } - }) - .catch(function (err) { - console.log('Error', err.response.status) - }) -} - -const endpointSearchUrl = '/endpoint/search' - -const $endpointInputContainer = document.querySelector('[data-module="endpoint-input"]') -const $endpointInput = $endpointInputContainer.querySelector('input[name="endpoint_url"]') -const $endpointWarning = $endpointInputContainer.querySelector('.app-input__warning') -const $endpointWarningCount = $endpointWarning.querySelector('.app-input__warning__endpoint-count') -$endpointWarning.classList.add('js-hidden') -$endpointInput.addEventListener('input', function (e) { - $endpointWarning.classList.add('js-hidden') - console.log('on input should hide any info/warnings') -}) - -function handleReturnedEndpoint (data) { - if (data) { - $endpointWarning.classList.remove('js-hidden') - console.log('handling response') - console.log(data) - const gram = (data.sources.length === 1) ? $endpointWarningCount.dataset.singular : $endpointWarningCount.dataset.plural; - const message = `${data.sources.length} ${gram} this endpoint` - $endpointWarningCount.textContent = message - } -} - -$endpointInput.addEventListener('change', function (e) { - console.log('on change check if seen endpoint url before') - const url = e.target.value - appFetch(endpointSearchUrl, { endpoint_url: url.trim() }, handleReturnedEndpoint) -}) -console.log($endpointInput) - -window.appFetch = appFetch diff --git a/src/javascripts/components/collapsible-section.js b/src/javascripts/components/collapsible-section.js deleted file mode 100644 index 1b319396..00000000 --- a/src/javascripts/components/collapsible-section.js +++ /dev/null @@ -1,89 +0,0 @@ -function CollapsibleSection ($section, $trigger) { - this.$section = $section - this.$trigger = $trigger -} - -CollapsibleSection.prototype.init = function () { - if (!this.$section || !this.$trigger) { - return undefined - } - - const btns = this.getBtns() - - // show triggers - this.$trigger.classList.remove('js-hidden') - const boundClickHandler = this.clickHandler.bind(this) - - // make sure it is closed by default - this.close() - - btns.forEach(function ($btn) { - $btn.addEventListener('click', boundClickHandler) - }) - - const that = this - this.$section.addEventListener('animationend', function (e) { - if (e.animationName === 'opening-panel') { - console.log('open animation') - that.$section.classList.remove('opening') - that.$section.dataset.collapsible = 'open' - } else if (e.animationName === 'closing-panel') { - console.log('close animation') - that.$section.classList.remove('closing') - that.$section.dataset.collapsible = 'closed' - } - }) - - return this -} - -CollapsibleSection.prototype.getBtns = function () { - this.$openBtn = this.$trigger.querySelector('.open-section') - this.$closeBtn = this.$trigger.querySelector('.close-section') - - return [this.$openBtn, this.$closeBtn] -} - -CollapsibleSection.prototype.clickHandler = function (e) { - const $clickedBtn = e.target - if ($clickedBtn.dataset.action === 'open-section') { - this.open() - } else { - this.close() - } -} - -CollapsibleSection.prototype.open = function (animate = true) { - this.$openBtn.classList.add('js-hidden') - this.$closeBtn.classList.remove('js-hidden') - if (animate) { - this.$section.classList.add('opening') - } else { - this.$section.dataset.collapsible = 'open' - } -} - -CollapsibleSection.prototype.close = function (animate = true) { - this.$closeBtn.classList.add('js-hidden') - this.$openBtn.classList.remove('js-hidden') - if (animate) { - this.$section.classList.add('closing') - } else { - this.$section.dataset.collapsible = 'closed' - } -} - -// CollapsibleSection.prototype.animationHandler = function(e) { -// categoriesContainer.addEventListener('animationend', function(e) { -// console.log(e); -// if(e.animationName === "opening-panel") { -// categoriesContainer.classList.remove('opening'); -// categoriesContainer.classList.remove('collapsed'); -// } else if (e.animationName === "closing-panel") { -// categoriesContainer.classList.remove('closing'); -// categoriesContainer.classList.add('collapsed'); -// } -// }); -// } - -export default CollapsibleSection diff --git a/src/javascripts/components/resource-check.js b/src/javascripts/components/resource-check.js deleted file mode 100644 index 7f7592e1..00000000 --- a/src/javascripts/components/resource-check.js +++ /dev/null @@ -1,43 +0,0 @@ -function ResourceCheck ($module) { - this.$module = $module -} - -ResourceCheck.prototype.init = function (params) { - if (this.hasBeenChecked()) { - this.display() - } else { - this.hide() - this.performCheck() - } - - return this -} - -ResourceCheck.prototype.hasBeenChecked = function () { - if (this.$module.dataset.checkPerformed === 'true') { - return true - } - return false -} - -ResourceCheck.prototype.performCheck = function () { - const hash = this.$module.dataset.resourceHash - const boundCheckCompleteHandler = this.checkCompleteHandler.bind(this) - window.fetch(`/resource/${hash}/check`) - .then(response => response.json()) - .then(boundCheckCompleteHandler) -} - -ResourceCheck.prototype.checkCompleteHandler = function (data) { - this.display() -} - -ResourceCheck.prototype.display = function () { - this.$module.classList.remove('js-hidden') -} - -ResourceCheck.prototype.hide = function () { - this.$module.classList.add('js-hidden') -} - -export default ResourceCheck diff --git a/src/javascripts/components/select-datasets.js b/src/javascripts/components/select-datasets.js deleted file mode 100644 index 6bb9cc3d..00000000 --- a/src/javascripts/components/select-datasets.js +++ /dev/null @@ -1,152 +0,0 @@ -/* global accessibleAutocomplete */ - -import { empty } from '../utilities/DOM-helpers' - -function SelectDatasets ($module) { - this.$module = $module -} - -SelectDatasets.prototype.init = function (params) { - this.setupOptions(params) - this.selectedNames = [] - this.selectedValues = [] - this.selected = {} - - // register elements - this.$input = this.$module.querySelector(this.options.inputSelector) - this.$summaryContainer = this.$module.querySelector(this.options.containerSelector) - this.$select = this.$module.querySelector(this.options.selectSelector) - this.$summary = this.$module.querySelector(this.options.listSelector) - - this.setInitalSelection() - - this.displayControls() - this.setUpAutoComplete() - - const boundOnDeselectClickHandler = this.onDeselectClickHandler.bind(this) - this.$summary.addEventListener('click', boundOnDeselectClickHandler) - - return this -} - -SelectDatasets.prototype.addSelectedOption = function (name, dataset) { - if (!Object.prototype.hasOwnProperty.call(this.selected, dataset)) { - this.selected[dataset] = name - } - this.refreshUI() -} - -SelectDatasets.prototype.createDeselectButton = function (name) { - const $btn = document.createElement('button') - $btn.classList.add(this.options.btnClass) - $btn.textContent = 'x' - const $hiddenSpan = document.createElement('span') - $hiddenSpan.classList.add('govuk-visually-hidden') - $hiddenSpan.textContent = `Deselect ${name}` - $btn.appendChild($hiddenSpan) - return $btn -} - -SelectDatasets.prototype.createSelectedItem = function (dataset, name) { - const $item = document.createElement('li') - $item.classList.add(this.options.itemClass) - $item.textContent = name - $item.dataset.dataset = dataset - $item.appendChild(this.createDeselectButton(name)) - return $item -} - -SelectDatasets.prototype.deselect = function (dataset) { - // remove from list of selected - if (Object.prototype.hasOwnProperty.call(this.selected, dataset)) { - delete this.selected[dataset] - } - this.refreshUI() -} - -SelectDatasets.prototype.displayControls = function () { - this.$input.classList.add('js-hidden') - this.$summaryContainer.classList.remove('js-hidden') -} - -SelectDatasets.prototype.displaySelected = function () { - // remove all items - empty(this.$summary) - // repopulate with selected items - const that = this - for (const selectedDataset in this.selected) { - that.$summary.appendChild(that.createSelectedItem(selectedDataset, that.selected[selectedDataset])) - } -} - -SelectDatasets.prototype.onDeselectClickHandler = function (e) { - e.preventDefault() - if (e.target.tagName === 'BUTTON') { - const $item = e.target.closest('.' + this.options.itemClass) - this.deselect($item.dataset.dataset) - } -} - -SelectDatasets.prototype.setUpAutoComplete = function () { - const boundOptionSelectedHandler = this.optionSelectedHandler.bind(this) - accessibleAutocomplete.enhanceSelectElement({ - selectElement: this.$select, - onConfirm: boundOptionSelectedHandler, - confirmOnBlur: false - }) -} - -SelectDatasets.prototype.optionSelectedHandler = function (o) { - const val = this.$select.querySelector(`[data-option-selector="${o}"]`).value - this.addSelectedOption(o, val) -} - -SelectDatasets.prototype.refreshUI = function () { - this.updateInput() - this.displaySelected() -} - -SelectDatasets.prototype.setInitalSelection = function () { - const datasetsString = this.$input.value - const that = this - if (datasetsString) { - const datasets = datasetsString.split(';') - datasets.forEach(function (dataset) { - that.selectedValues.push(dataset) - const $opt = that.$select.querySelector(`[value="${dataset}"]`) - that.selectedNames.push($opt.textContent) - that.selected[dataset] = $opt.textContent - }) - } - // display the initially selected datasets - this.displaySelected() -} - -SelectDatasets.prototype.setupOptions = function (params) { - this.options = { - btnClass: params.btnClass || 'app-select-datasets__btn', - containerSelector: params.containerSelector || '.app-select-datasets__container', - inputSelector: params.inputSelector || '.app-select-datasets__input', - itemClass: params.itemClass || 'app-select-datasets__item', - selectSelector: params.selectSelector || '.app-select-datasets__select', - listSelector: params.listSelector || '.app-select-datasets__list' - } -} - -SelectDatasets.prototype.updateInput = function () { - const selectedDatasets = [] - for (const dataset in this.selected) { - selectedDatasets.push(dataset) - } - this.$input.value = selectedDatasets.join(';') -} - -// SelectDatasets.prototype.onlyUnique = function (value, index, self) { -// return self.indexOf(value) === index -// } - -SelectDatasets.prototype.getSelect = function () { - return this.$select -} - -export default SelectDatasets diff --git a/src/javascripts/reporting-summary.js b/src/javascripts/reporting-summary.js deleted file mode 100644 index cd5665b9..00000000 --- a/src/javascripts/reporting-summary.js +++ /dev/null @@ -1,71 +0,0 @@ -class SummaryMetrics { - constructor(summary_metrics) { - this.summary_metrics = summary_metrics; - const dateFromSelector = document.querySelector("#date-from-selector") - const dateToSelector = document.querySelector("#date-to-selector") - dateFromSelector.addEventListener("change", this.updateMetrics.bind(this)) - dateToSelector.addEventListener("change", this.updateMetrics.bind(this)) - } - - init() { - const today = new Date(); - const date = today.toISOString().split('T')[0]; - const contributions_index = this.summary_metrics.contributions.dates.indexOf(date); - const errors_index = this.summary_metrics.endpoint_errors.dates.indexOf(date); - document.getElementById('contributions').innerHTML = "Resources downloaded: ".concat(this.summary_metrics.contributions.contributions[contributions_index]); - document.getElementById('errors').innerHTML = "Endpoint errors: ".concat(this.summary_metrics.endpoint_errors.errors[errors_index]); - document.getElementById('date-from-selector').value = date - document.getElementById('date-to-selector').value = date - return this - } - - updateMetrics (e) { - if (e != undefined) { - const today = new Date(); - const date = today.toISOString().split('T')[0]; - - const dateFromSelector = document.querySelector("#date-from-selector") - let date_from = String(dateFromSelector.value) - const dateToSelector = document.querySelector("#date-to-selector") - let date_to = String(dateToSelector.value) - - if (!date_from) { - date_from = date - } - if (!date_to) { - date_to = date - } - - // Find dates in lists so we know indexes of data - let contributions_date_from_index = this.summary_metrics.contributions.dates.indexOf(date_from) - let contributions_date_to_index = this.summary_metrics.contributions.dates.indexOf(date_to) - // If date outside data range use max/min - if (contributions_date_from_index == -1) {contributions_date_from_index = 0} - if (contributions_date_to_index == -1) {contributions_date_to_index = Object.keys(this.summary_metrics.contributions.dates).length - 1} - - let errors_date_from_index = this.summary_metrics.endpoint_errors.dates.indexOf(date_from) - let errors_date_to_index = this.summary_metrics.endpoint_errors.dates.indexOf(date_to) - if (errors_date_from_index == -1) {errors_date_from_index = 0} - if (errors_date_to_index == -1) {errors_date_to_index = Object.keys(this.summary_metrics.endpoint_errors.dates).length - 1} - - - let total_contributions; - let total_errors; - let summary_text; - if (date_from == date_to) { - total_contributions = this.summary_metrics.contributions.contributions[contributions_date_from_index] - total_errors = this.summary_metrics.endpoint_errors.errors[errors_date_from_index] - summary_text = "Today's Summary" - } else { - total_contributions = this.summary_metrics.contributions.contributions.slice(contributions_date_from_index, contributions_date_to_index+1).reduce((a,b)=>a+b) - total_errors = this.summary_metrics.endpoint_errors.errors.slice(errors_date_from_index, errors_date_to_index+1).reduce((a,b)=>a+b) - summary_text = date_from.concat(" to ").concat(date_to).concat(" Summary") - } - - document.getElementById('contributions').innerHTML = "Resources downloaded: ".concat(total_contributions); - document.getElementById('errors').innerHTML = "Endpoint errors: ".concat(total_errors); - } - } -} - -export { SummaryMetrics as default }; diff --git a/src/javascripts/utilities/DOM-helpers.js b/src/javascripts/utilities/DOM-helpers.js deleted file mode 100644 index 1e368d3a..00000000 --- a/src/javascripts/utilities/DOM-helpers.js +++ /dev/null @@ -1,3 +0,0 @@ -export function empty (element) { - element.textContent = '' -} diff --git a/src/javascripts/utilities/fetchWithArgs.js b/src/javascripts/utilities/fetchWithArgs.js deleted file mode 100644 index dc8c53a6..00000000 --- a/src/javascripts/utilities/fetchWithArgs.js +++ /dev/null @@ -1,24 +0,0 @@ -// based on https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch -// Example POST method implementation: -export async function fetchWithArgs (url = '', data = {}) { - // Default options are marked with * - const response = await fetch (url, { - method: 'POST', // *GET, POST, PUT, DELETE, etc. - mode: 'cors', // no-cors, *cors, same-origin - cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached - credentials: 'same-origin', // include, *same-origin, omit - headers: { - 'Content-Type': 'application/json' - // 'Content-Type': 'application/x-www-form-urlencoded', - }, - redirect: 'follow', // manual, *follow, error - referrerPolicy: 'no-referrer', // no-referrer, *no-referrer-when-downgrade, origin, origin-when-cross-origin, same-origin, strict-origin, strict-origin-when-cross-origin, unsafe-url - body: JSON.stringify(data) // body data type must match "Content-Type" header - }); - if (!response.ok) { - const err = new Error("Not 2xx response"); - err.response = response; - throw err; - } - return response.json(); // parses JSON response into native JavaScript objects -} diff --git a/src/javascripts/utilities/timeseries-chart.js b/src/javascripts/utilities/timeseries-chart.js deleted file mode 100644 index 31a0189b..00000000 --- a/src/javascripts/utilities/timeseries-chart.js +++ /dev/null @@ -1,53 +0,0 @@ -class TimeseriesChart { - constructor (options) { - const {xAxisTitle, xAxisKey, xMax, yAxisTitle, yAxisKey, yMax, datasets, htmlId, type, stacked} = options - this.xAxisTitle = xAxisTitle - this.xAxisKey = xAxisKey - this.xMax = xMax - this.yAxisTitle = yAxisTitle - this.yAxisKey = yAxisKey - this.yMax = yMax - this.datasets = datasets - this.htmlId = htmlId - this.type = type - this.stacked = stacked - } - - init() { - const graph_ctx = document.getElementById(this.htmlId); - - return new Chart(graph_ctx, { - type: this.type, - data: { - datasets: this.datasets - }, - options: { - parsing: { - xAxisKey: this.xAxisKey, - yAxisKey: this.yAxisKey - }, - scales: { - y: { - title: { - display: true, - text: this.yAxisTitle, - }, - max: this.yMax, - beginAtZero: true, - stacked: this.stacked - }, - x: { - title: { - display: true, - text: this.xAxisTitle - }, - max: this.xMax, - stacked: this.stacked - } - } - } - }); - } -} - -export {TimeseriesChart as default}