diff --git a/config/default.yaml b/config/default.yaml index 22aee2c08..7ddd0f94c 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -140,7 +140,7 @@ features: enabled: true provisionBasedDatasets: # This feature will load datasets based on provisionReasons - enabled: false + enabled: true nonAuthPages: # This feature enables pages to show non-authoritative datasets # If off, it will still check for local-plan datasets. diff --git a/src/filters/makeDatasetSlugToReadableNameFilter.js b/src/filters/makeDatasetSlugToReadableNameFilter.js index 241f3798e..d99b3dcde 100644 --- a/src/filters/makeDatasetSlugToReadableNameFilter.js +++ b/src/filters/makeDatasetSlugToReadableNameFilter.js @@ -12,16 +12,20 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => { * A filter function that takes a dataset slug as input and returns its corresponding readable name. * * @param {string} slug - The dataset slug to look up. + * @param {boolean} [capitalize=false] - Whether to capitalize the first letter. * @returns {string} The readable name corresponding to the provided slug. * @throws {Error} - If the provided slug is not found in the dataset name mapping. */ - return (slug) => { + const capitalizeFirst = (str) => str.charAt(0).toUpperCase() + str.slice(1) + const lowercaseFirst = (str) => str.charAt(0).toLowerCase() + str.slice(1) + + return (slug, capitalize = false) => { const name = datasetNameMapping.get(slug) if (!name) { // ToDo: work out what to do here? potentially update it with data from datasette logger.debug(`can't find a name for ${slug}`) - return slug + return capitalize ? capitalizeFirst(slug) : lowercaseFirst(slug) } - return name + return capitalize ? capitalizeFirst(name) : lowercaseFirst(name) } } diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index ec5f91205..88ee2fab4 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -166,6 +166,8 @@ export const fetchEntitiesPlatformDb = fetchMany({ * @param {Object} req.params - Route parameters * @param {string} req.params.dataset - Dataset name * @param {string} [req.authority] OUT parameter - Set to 'authoritative', 'some', or '' (empty string) + * @param {{entity_count: number}} [req.entityCount] OUT parameter - Set to count from API data + * @param {Array} [req.alternateEntityList] OUT parameter - List of alternate entity IDs when quality is 'some' * @param {Object} res - Response object * @param {Function} next - Next middleware function */ @@ -201,6 +203,12 @@ export const prepareAuthority = async (req, res, next) => { if (authoritativeResult.formattedData && authoritativeResult.formattedData.length > 0) { req.authority = 'authoritative' + // Set record count to only show authoritative count if authoritative data exists + const count = authoritativeResult?.data?.count + if (count !== undefined) { + req.entityCount = { entity_count: count } + } + logger.info(`Authoritative data found with count ${count}, skipping non-authoritative check`) return next() } @@ -216,6 +224,11 @@ export const prepareAuthority = async (req, res, next) => { // Set list of alternate entities provided in req for later use const rows = someResult.formattedData || [] req.alternateEntityList = rows.map(({ entity }) => entity) + // Also set record count here if available + const someCount = someResult?.data?.count + if (someCount !== undefined) { + req.entityCount = { entity_count: someCount } + } } else { req.authority = '' } @@ -384,8 +397,9 @@ export const pullOutDatasetSpecification = (req, res, next) => { } const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset) if (!datasetSpecification) { - logger.error('Dataset specification not found', { dataset: req.dataset.dataset }) - return next(new MiddlewareError('Dataset specification not found', 404)) + logger.info('Dataset specification not found, clearing specification and falling back to dataset fields', { dataset: req.dataset.dataset }) + req.specification = null + return next() } req.specification = datasetSpecification next() @@ -786,28 +800,18 @@ export const fetchEntityIssueCounts = fetchMany({ query: ({ req }) => { const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : '' return ` - WITH unique_issues AS ( - SELECT DISTINCT - i.dataset, - i.field, - i.issue_type, - i.entity - FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type - WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') - AND COALESCE(entity, '') <> '' - AND (i.end_date = '' OR i.end_date IS NULL) - AND it.responsibility = 'external' - AND it.severity = 'error' - ${datasetClause} - ) SELECT - dataset, - field, - issue_type, - COUNT(*) AS count - FROM unique_issues - GROUP BY field, issue_type, dataset + i.dataset, + i.field, + i.issue_type, + COUNT(DISTINCT i.entity) AS count + FROM issue i + WHERE i.resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') + AND i.entity IS NOT NULL AND i.entity <> '' + AND (i.end_date = '' OR i.end_date IS NULL) + AND i.issue_type IN (SELECT it.issue_type FROM issue_type it WHERE it.responsibility = 'external' AND it.severity = 'error') + ${datasetClause} + GROUP BY i.field, i.issue_type, i.dataset ` }, result: 'entityIssueCounts' @@ -1161,3 +1165,16 @@ export const setAvailableDatasets = async (req, res, next) => { req.availableDatasets = await CONSTANTS.availableDatasets() next() } + +/** + * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! + * + * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb + */ +export const fetchEntityIssueCountsPerformanceDb = fetchMany({ + query: ({ params }) => { + return performanceDbApi.fetchEntityIssueCounts(params.lpa, params.dataset) + }, + result: 'entityIssueCounts', + dataset: FetchOptions.performanceDb +}) diff --git a/src/middleware/dataset-failed-expectation-details.middleware.js b/src/middleware/dataset-failed-expectation-details.middleware.js index b7d58281d..297e0ee51 100644 --- a/src/middleware/dataset-failed-expectation-details.middleware.js +++ b/src/middleware/dataset-failed-expectation-details.middleware.js @@ -75,7 +75,7 @@ const preparePaginationInfo = (req, res, next) => { * @returns {undefined} */ const prepareTemplateParams = (req, res, next) => { - const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination } = req + const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination, parsedParams } = req const entityAugmented = prepareEntityForTable(entity) @@ -100,6 +100,7 @@ const prepareTemplateParams = (req, res, next) => { } }) }, + pageNumber: parsedParams.pageNumber, dataRange, pagination } diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index 0ad4f13a6..c6c6de415 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,8 +4,8 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' -import { fetchOne, fetchMany, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' +import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' +import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' import { types } from '../utils/logging.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entryIssueCounts, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req + const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req let endpointErrorIssues = 0 const endpoints = sources @@ -219,8 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { if (authority === 'some') { taskCount = 1 } else { - taskCount = (entryIssueCounts ? entryIssueCounts.length : 0) + - (entityIssueCounts ? entityIssueCounts.length : 0) + + taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) + endpointErrorIssues + (expectationOutOfBounds.length > 0 ? 1 : 0) } @@ -264,14 +263,14 @@ export default [ fetchColumnSummary, fetchResources, fetchSources, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, fetchSpecification, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, ...processSpecificationMiddlewares, // setNoticesFromSourceKey('resources'), // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) - fetchEntityCount, + // Currently fallback entity count all records if authority entity count fails + onlyIf(req => req.entityCount === undefined, fetchEntityCount), prepareDatasetOverviewTemplateParams, getDatasetOverview, logPageError diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index 5ecdacf03..cf63c434c 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -15,11 +15,10 @@ import { logPageError, fetchResources, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, fetchEntitiesPlatformDb } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate } from './middleware.builders.js' +import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js' import * as v from 'valibot' import { splitByLeading } from '../utils/table.js' @@ -41,7 +40,7 @@ export const fetchEntities = fetchMany({ }) export const setRecordCount = (req, res, next) => { - req.recordCount = req?.entityCount?.count || 0 + req.recordCount = req?.entityCount?.entity_count ?? req?.entityCount?.count ?? 0 next() } @@ -82,10 +81,10 @@ export const constructTableParams = (req, res, next) => { } export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, entryIssueCounts, authority, alternateSources, uniqueDatasetFields } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req // Hard code task count for 'some' authority - const taskCount = authority !== 'some' ? entityIssueCounts.length + entryIssueCounts.length : 1 + const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 // Build the fields query parameter and download url const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0 ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&') @@ -121,17 +120,17 @@ export default [ fetchDatasetInfo, fetchResources, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, - fetchEntityCount, + ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API + + onlyIf(req => req.entityCount === undefined, fetchEntityCount), // fallback setRecordCount, getSetDataRange(config.tablePageLength), show404IfPageNumberNotInRange, - ...processAuthoritativeMiddlewares, // Used to see if alternative or authoritative, and update entites fetch accordingly - fetchEntitiesPlatformDb, // This technically fetches twice from entities table, could be refactored later + fetchEntitiesPlatformDb, // Fetches entities filtered by authority quality extractJsonFieldFromEntities, replaceUnderscoreInEntities, diff --git a/src/middleware/entityIssueDetails.middleware.js b/src/middleware/entityIssueDetails.middleware.js index 5a53ac44b..866431cd4 100644 --- a/src/middleware/entityIssueDetails.middleware.js +++ b/src/middleware/entityIssueDetails.middleware.js @@ -60,7 +60,7 @@ export const getIssueField = (text, html, classes) => { }, value: { html: html ? html.toString() : '', - originalValue: html // we don't want any markup here + originalValue: html != null ? html.toString() : undefined }, classes } diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 05bb38cca..922e13e91 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,9 +4,8 @@ * @description Middleware for oragnisation (LPA) overview page */ -import performanceDbApi from '../services/performanceDbApi.js' -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate, parallel } from './middleware.builders.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js' +import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' import logger from '../utils/logger.js' @@ -15,19 +14,6 @@ import platformApi from '../services/platformApi.js' import { types } from '../utils/logging.js' import config from '../../config/index.js' -/** - * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! - * - * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb - */ -const fetchEntityIssueCountsPerformanceDb = fetchMany({ - query: ({ params }) => { - return performanceDbApi.fetchEntityIssueCounts(params.lpa) - }, - result: 'entityIssueCounts', - dataset: FetchOptions.performanceDb -}) - const fetchProvisions = fetchMany({ query: ({ params }) => { return /* sql */ `select dataset, project, provision_reason diff --git a/src/routes/schemas.js b/src/routes/schemas.js index b717c27d8..262bd9e4b 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -211,7 +211,7 @@ export const OrgDatasetTaskList = v.strictObject({ organisation: OrgField, authority: v.string(), dataset: v.strictObject({ - dataset: v.optional(NonEmptyString), + dataset: NonEmptyString, name: NonEmptyString, collection: NonEmptyString }) diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index f21a2bb83..cf903bec7 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -251,18 +251,20 @@ export default { }, // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb - fetchEntityIssueCounts (datasetId) { + fetchEntityIssueCounts (lpa, dataset) { + const datasetClause = dataset ? `AND dataset = '${dataset}'` : '' return /* sql */ ` - SELECT + SELECT dataset, issue_type, field, - COUNT(*) AS count - FROM endpoint_dataset_issue_type_summary - WHERE organisation = '${datasetId}' - AND severity = 'error' - AND responsibility = 'external' + COUNT(*) AS count + FROM endpoint_dataset_issue_type_summary + WHERE organisation = '${lpa}' + AND severity = 'error' + AND responsibility = 'external' AND (resource_end_date = '' OR resource_end_date IS NULL) + ${datasetClause} GROUP BY dataset, issue_type, field` }, diff --git a/src/utils/datasetSubjectLoader.js b/src/utils/datasetSubjectLoader.js index 8f4baba36..492508b02 100644 --- a/src/utils/datasetSubjectLoader.js +++ b/src/utils/datasetSubjectLoader.js @@ -147,7 +147,7 @@ export async function buildDataSubjects () { // Use existing datasetSlugToReadableName to create lookup of dataset keys to readable names const nameMap = {} for (const key of datasetKeys) { - nameMap[key] = datasetSlugToReadableName(key) + nameMap[key] = datasetSlugToReadableName(key, true) } return makeDatasetSubjectMap(nameMap) diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index b7b251d22..2742f7b0f 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -55,13 +55,21 @@ {% elif errorMessage == "The URL must be accessible" %}

You must host the URL on a server which does not block access due to set permissions.

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

+ {% elif errorMessage == "URL must be the data layer" %} +

The URL you have provided is an ArcGIS link, which is not the data layer.

+

The link to the data layer ends with a forward slash, followed by a number. For example, /8.

+

If you believe you have provided the information correctly and the problem persists, contact us at digitalland@communities.gov.uk

{% else %}

Error: {{ errorMessage }}

{% endif %} -

Please try again or contact support if the problem persists.

+ {# Specific case where we don't want duplication of contact reasons #} + {% if errorMessage != "URL must be the data layer" %} +

Please try again or contact support if the problem persists.

+ {% endif %} diff --git a/src/views/components/dataset-banner.html b/src/views/components/dataset-banner.html index e3bbd694b..7bc8ed5a6 100644 --- a/src/views/components/dataset-banner.html +++ b/src/views/components/dataset-banner.html @@ -1,11 +1,11 @@ {% macro datasetBanner(orgName, datasetName) %} {% if orgName and datasetName %} - {{ orgName }} — {{ datasetName | datasetSlugToReadableName }} + {{ orgName }} — {{ datasetName | datasetSlugToReadableName(true) }} {% elif orgName %} {{ orgName }} {% elif datasetName %} - {{ datasetName | datasetSlugToReadableName }} + {{ datasetName | datasetSlugToReadableName(true) }} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/src/views/includes/_dataset-page-header.html b/src/views/includes/_dataset-page-header.html index f06685203..5f929868b 100644 --- a/src/views/includes/_dataset-page-header.html +++ b/src/views/includes/_dataset-page-header.html @@ -1,10 +1,10 @@
{% if authority and authority === "some" %} {{ organisation.name }} -

Review alternative source of {{ dataset.name }} data

+

Review alternative source of {{ dataset.dataset | datasetSlugToReadableName }} data

{% else %} {{ organisation.name }} -

{{ dataset.name }}

+

{{ dataset.dataset | datasetSlugToReadableName(true) }}

{% if notice %} {{ deadlineNotice(dataset.dataset, notice, organisation) }} diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index aadb6663c..f4af56cf4 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -29,7 +29,7 @@ {% set serviceType = 'Manage' %} {% set pageName %} - {{organisation.name}} - {{dataset.name}} - Dataset overview + {{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Dataset overview {% endset %} {% set urlStyle = 'text-overflow: ellipsis; overflow: hidden; white-space: nowrap; width: 400px; display: block;' %} @@ -56,7 +56,7 @@ href: '/organisations/' + organisation.organisation }, { - text: dataset.name + text: dataset.dataset | datasetSlugToReadableName(true) } ] }) }} @@ -225,10 +225,10 @@

Dataset details

Dataset actions