Arcgis Detail, Auth Data Only, Capitalization and No Dataset Spec Fallback

config/default.yaml

@@ -140,7 +140,7 @@ features:
     enabled: true
   provisionBasedDatasets:
     # This feature will load datasets based on provisionReasons
-    enabled: false
+    enabled: true
   nonAuthPages:
     # This feature enables pages to show non-authoritative datasets
     # If off, it will still check for local-plan datasets.

src/filters/makeDatasetSlugToReadableNameFilter.js

@@ -12,16 +12,20 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => {
    * A filter function that takes a dataset slug as input and returns its corresponding readable name.
    *
    * @param {string} slug - The dataset slug to look up.
+   * @param {boolean} [capitalize=false] - Whether to capitalize the first letter.
    * @returns {string} The readable name corresponding to the provided slug.
    * @throws {Error} - If the provided slug is not found in the dataset name mapping.
    */
-  return (slug) => {
+  const capitalizeFirst = (str) => str.charAt(0).toUpperCase() + str.slice(1)
+  const lowercaseFirst = (str) => str.charAt(0).toLowerCase() + str.slice(1)
+
+  return (slug, capitalize = false) => {
     const name = datasetNameMapping.get(slug)
     if (!name) {
       // ToDo: work out what to do here? potentially update it with data from datasette
       logger.debug(`can't find a name for ${slug}`)
-      return slug
+      return capitalize ? capitalizeFirst(slug) : lowercaseFirst(slug)
     }
-    return name
+    return capitalize ? capitalizeFirst(name) : lowercaseFirst(name)
   }
 }

src/middleware/common.middleware.js

@@ -166,6 +166,8 @@ export const fetchEntitiesPlatformDb = fetchMany({
  * @param {Object} req.params - Route parameters
  * @param {string} req.params.dataset - Dataset name
  * @param {string} [req.authority] OUT parameter - Set to 'authoritative', 'some', or '' (empty string)
+ * @param {{entity_count: number}} [req.entityCount] OUT parameter - Set to count from API data
+ * @param {Array<number>} [req.alternateEntityList] OUT parameter - List of alternate entity IDs when quality is 'some'
  * @param {Object} res - Response object
  * @param {Function} next - Next middleware function
  */
@@ -201,6 +203,12 @@ export const prepareAuthority = async (req, res, next) => {
 
     if (authoritativeResult.formattedData && authoritativeResult.formattedData.length > 0) {
       req.authority = 'authoritative'
+      // Set record count to only show authoritative count if authoritative data exists
+      const count = authoritativeResult?.data?.count
+      if (count !== undefined) {
+        req.entityCount = { entity_count: count }
+      }
+      logger.info(`Authoritative data found with count ${count}, skipping non-authoritative check`)
       return next()
     }
 
@@ -216,6 +224,11 @@ export const prepareAuthority = async (req, res, next) => {
       // Set list of alternate entities provided in req for later use
       const rows = someResult.formattedData || []
       req.alternateEntityList = rows.map(({ entity }) => entity)
+      // Also set record count here if available
+      const someCount = someResult?.data?.count
+      if (someCount !== undefined) {
+        req.entityCount = { entity_count: someCount }
+      }
     } else {
       req.authority = ''
     }
@@ -384,8 +397,9 @@ export const pullOutDatasetSpecification = (req, res, next) => {
   }
   const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset)
   if (!datasetSpecification) {
-    logger.error('Dataset specification not found', { dataset: req.dataset.dataset })
-    return next(new MiddlewareError('Dataset specification not found', 404))
+    logger.info('Dataset specification not found, clearing specification and falling back to dataset fields', { dataset: req.dataset.dataset })
+    req.specification = null
+    return next()
   }
   req.specification = datasetSpecification
   next()
@@ -786,28 +800,18 @@ export const fetchEntityIssueCounts = fetchMany({
   query: ({ req }) => {
     const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : ''
     return `
-      WITH unique_issues AS (
-        SELECT DISTINCT
-          i.dataset,
-          i.field,
-          i.issue_type,
-          i.entity
-        FROM issue i
-        LEFT JOIN issue_type it ON i.issue_type = it.issue_type
-        WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}')
-          AND COALESCE(entity, '') <> ''
-          AND (i.end_date = '' OR i.end_date IS NULL)
-          AND it.responsibility = 'external'
-          AND it.severity = 'error'
-          ${datasetClause}
-      )
       SELECT
-        dataset,
-        field,
-        issue_type,
-        COUNT(*) AS count
-      FROM unique_issues
-      GROUP BY field, issue_type, dataset
+        i.dataset,
+        i.field,
+        i.issue_type,
+        COUNT(DISTINCT i.entity) AS count
+      FROM issue i
+      WHERE i.resource IN ('${req.resources.map(resource => resource.resource).join("', '")}')
+        AND i.entity IS NOT NULL AND i.entity <> ''
+        AND (i.end_date = '' OR i.end_date IS NULL)
+        AND i.issue_type IN (SELECT it.issue_type FROM issue_type it WHERE it.responsibility = 'external' AND it.severity = 'error')
+        ${datasetClause}
+      GROUP BY i.field, i.issue_type, i.dataset
     `
   },
   result: 'entityIssueCounts'
@@ -1161,3 +1165,16 @@ export const setAvailableDatasets = async (req, res, next) => {
   req.availableDatasets = await CONSTANTS.availableDatasets()
   next()
 }
+
+/**
+ * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together!
+ *
+ * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb
+ */
+export const fetchEntityIssueCountsPerformanceDb = fetchMany({
+  query: ({ params }) => {
+    return performanceDbApi.fetchEntityIssueCounts(params.lpa, params.dataset)
+  },
+  result: 'entityIssueCounts',
+  dataset: FetchOptions.performanceDb
+})

src/middleware/dataset-failed-expectation-details.middleware.js

@@ -75,7 +75,7 @@ const preparePaginationInfo = (req, res, next) => {
  * @returns {undefined}
  */
 const prepareTemplateParams = (req, res, next) => {
-  const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination } = req
+  const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination, parsedParams } = req
 
   const entityAugmented = prepareEntityForTable(entity)
 
@@ -100,6 +100,7 @@ const prepareTemplateParams = (req, res, next) => {
         }
       })
     },
+    pageNumber: parsedParams.pageNumber,
     dataRange,
     pagination
   }

src/middleware/datasetOverview.middleware.js

@@ -4,8 +4,8 @@
  * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview)
  */
 
-import { fetchDatasetPlatformInfo, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js'
-import { fetchOne, fetchMany, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
+import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js'
+import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import logger from '../utils/logger.js'
 import { types } from '../utils/logging.js'
@@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({
  * @param {Function} next - Express next middleware function
  */
 export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
-  const { orgInfo, entityCount, sources, dataset, entryIssueCounts, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req
+  const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req
 
   let endpointErrorIssues = 0
   const endpoints = sources
@@ -219,8 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
   if (authority === 'some') {
     taskCount = 1
   } else {
-    taskCount = (entryIssueCounts ? entryIssueCounts.length : 0) +
-    (entityIssueCounts ? entityIssueCounts.length : 0) +
+    taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) +
     endpointErrorIssues +
     (expectationOutOfBounds.length > 0 ? 1 : 0)
   }
@@ -264,14 +263,14 @@ export default [
   fetchColumnSummary,
   fetchResources,
   fetchSources,
-  fetchEntityIssueCounts,
-  fetchEntryIssueCounts,
+  fetchEntityIssueCountsPerformanceDb,
   fetchSpecification,
   isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop,
   ...processAuthoritativeMiddlewares,
   ...processSpecificationMiddlewares,
   // setNoticesFromSourceKey('resources'), // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824)
-  fetchEntityCount,
+  // Currently fallback entity count all records if authority entity count fails
+  onlyIf(req => req.entityCount === undefined, fetchEntityCount),
   prepareDatasetOverviewTemplateParams,
   getDatasetOverview,
   logPageError

src/middleware/dataview.middleware.js

@@ -15,11 +15,10 @@ import {
   logPageError,
   fetchResources,
   fetchEntityCount,
-  fetchEntityIssueCounts,
-  fetchEntryIssueCounts,
+  fetchEntityIssueCountsPerformanceDb,
   fetchEntitiesPlatformDb
 } from './common.middleware.js'
-import { fetchMany, FetchOptions, renderTemplate } from './middleware.builders.js'
+import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js'
 import * as v from 'valibot'
 import { splitByLeading } from '../utils/table.js'
 
@@ -41,7 +40,7 @@ export const fetchEntities = fetchMany({
 })
 
 export const setRecordCount = (req, res, next) => {
-  req.recordCount = req?.entityCount?.count || 0
+  req.recordCount = req?.entityCount?.entity_count ?? req?.entityCount?.count ?? 0
   next()
 }
 
@@ -82,10 +81,10 @@ export const constructTableParams = (req, res, next) => {
 }
 
 export const prepareTemplateParams = (req, res, next) => {
-  const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, entryIssueCounts, authority, alternateSources, uniqueDatasetFields } = req
+  const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req
 
   // Hard code task count for 'some' authority
-  const taskCount = authority !== 'some' ? entityIssueCounts.length + entryIssueCounts.length : 1
+  const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1
   // Build the fields query parameter and download url
   const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0
     ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&')
@@ -121,17 +120,17 @@ export default [
   fetchDatasetInfo,
 
   fetchResources,
-  fetchEntityIssueCounts,
-  fetchEntryIssueCounts,
+  fetchEntityIssueCountsPerformanceDb,
 
-  fetchEntityCount,
+  ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API
+
+  onlyIf(req => req.entityCount === undefined, fetchEntityCount), // fallback
   setRecordCount,
 
   getSetDataRange(config.tablePageLength),
   show404IfPageNumberNotInRange,
 
-  ...processAuthoritativeMiddlewares, // Used to see if alternative or authoritative, and update entites fetch accordingly
-  fetchEntitiesPlatformDb, // This technically fetches twice from entities table, could be refactored later
+  fetchEntitiesPlatformDb, // Fetches entities filtered by authority quality
   extractJsonFieldFromEntities,
   replaceUnderscoreInEntities,
 

src/middleware/entityIssueDetails.middleware.js

@@ -60,7 +60,7 @@ export const getIssueField = (text, html, classes) => {
     },
     value: {
       html: html ? html.toString() : '',
-      originalValue: html // we don't want any markup here
+      originalValue: html != null ? html.toString() : undefined
     },
     classes
   }

src/middleware/lpa-overview.middleware.js

@@ -4,9 +4,8 @@
  * @description Middleware for oragnisation (LPA) overview page
  */
 
-import performanceDbApi from '../services/performanceDbApi.js'
-import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets } from './common.middleware.js'
-import { fetchMany, FetchOptions, renderTemplate, parallel } from './middleware.builders.js'
+import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js'
+import { fetchMany, renderTemplate, parallel } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import _ from 'lodash'
 import logger from '../utils/logger.js'
@@ -15,19 +14,6 @@ import platformApi from '../services/platformApi.js'
 import { types } from '../utils/logging.js'
 import config from '../../config/index.js'
 
-/**
- * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together!
- *
- * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb
- */
-const fetchEntityIssueCountsPerformanceDb = fetchMany({
-  query: ({ params }) => {
-    return performanceDbApi.fetchEntityIssueCounts(params.lpa)
-  },
-  result: 'entityIssueCounts',
-  dataset: FetchOptions.performanceDb
-})
-
 const fetchProvisions = fetchMany({
   query: ({ params }) => {
     return /* sql */ `select dataset, project, provision_reason

src/routes/schemas.js

@@ -211,7 +211,7 @@ export const OrgDatasetTaskList = v.strictObject({
   organisation: OrgField,
   authority: v.string(),
   dataset: v.strictObject({
-    dataset: v.optional(NonEmptyString),
+    dataset: NonEmptyString,
     name: NonEmptyString,
     collection: NonEmptyString
   })

src/services/performanceDbApi.js

@@ -251,18 +251,20 @@ export default {
   },
 
   // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb
-  fetchEntityIssueCounts (datasetId) {
+  fetchEntityIssueCounts (lpa, dataset) {
+    const datasetClause = dataset ? `AND dataset = '${dataset}'` : ''
     return /* sql */ `
-    SELECT 
+    SELECT
     dataset,
     issue_type,
     field,
-    COUNT(*) AS count 
-    FROM endpoint_dataset_issue_type_summary 
-    WHERE organisation = '${datasetId}'
-        AND severity = 'error' 
-        AND responsibility = 'external' 
+    COUNT(*) AS count
+    FROM endpoint_dataset_issue_type_summary
+    WHERE organisation = '${lpa}'
+        AND severity = 'error'
+        AND responsibility = 'external'
         AND (resource_end_date = '' OR resource_end_date IS NULL)
+        ${datasetClause}
     GROUP BY dataset, issue_type, field`
   },
 

src/utils/datasetSubjectLoader.js

@@ -147,7 +147,7 @@ export async function buildDataSubjects () {
   // Use existing datasetSlugToReadableName to create lookup of dataset keys to readable names
   const nameMap = {}
   for (const key of datasetKeys) {
-    nameMap[key] = datasetSlugToReadableName(key)
+    nameMap[key] = datasetSlugToReadableName(key, true)
   }
 
   return makeDatasetSubjectMap(nameMap)

src/views/check/error-redirect.html

@@ -55,13 +55,21 @@
     {% elif errorMessage == "The URL must be accessible" %}
       <p class="govuk-body" id="bad-upload" tabindex="-1">You must host the URL on a server which does not block access due to set permissions.</p>
       <p class="govuk-body" id="bad-upload" tabindex="-1">Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.</p>
+    {% elif errorMessage == "URL must be the data layer" %}
+      <p class="govuk-body" id="bad-upload" tabindex="-1">The URL you have provided is an ArcGIS link, which is not the data layer.</p>
+      <p class="govuk-body" id="bad-upload" tabindex="-1">The link to the data layer ends with a forward slash, followed by a number. For example, /8.</p>
+      <p class="govuk-body" id="bad-upload" tabindex="-1">If you believe you have provided the information correctly and the problem persists, contact us at <a
+        href="mailto:digitalland@communities.gov.uk">digitalland@communities.gov.uk</a></p>
     {% else %}
       <p class="govuk-body" id="bad-upload" tabindex="-1">
         Error: {{ errorMessage }}
       </p>
     {% endif %}
-    <p class="govuk-body">Please <a href="/check/upload-method">try again</a> or <a
-        href="mailto:digitalland@communities.gov.uk">contact support</a> if the problem persists.</p>
+    {# Specific case where we don't want duplication of contact reasons #}
+    {% if errorMessage != "URL must be the data layer" %}
+      <p class="govuk-body">Please <a href="/check/upload-method">try again</a> or <a
+          href="mailto:digitalland@communities.gov.uk">contact support</a> if the problem persists.</p>
+    {% endif %}
   </div>
 </div>
 

src/views/components/dataset-banner.html

@@ -1,11 +1,11 @@
 {% macro datasetBanner(orgName, datasetName) %}
 <span class="govuk-caption-xl" data-testid="dataset-banner">
   {% if orgName and datasetName %}
-  {{ orgName }} — {{ datasetName | datasetSlugToReadableName }}
+  {{ orgName }} — {{ datasetName | datasetSlugToReadableName(true) }}
   {% elif orgName %}
   {{ orgName }}
   {% elif datasetName %}
-  {{ datasetName | datasetSlugToReadableName }}
+  {{ datasetName | datasetSlugToReadableName(true) }}
   {% endif %}
 </span>
 {% endmacro %}

src/views/includes/_dataset-page-header.html

@@ -1,10 +1,10 @@
 <div class="govuk-grid-column-two-thirds">
 {% if authority and authority === "some" %}
   <span class="govuk-caption-xl">{{ organisation.name }}</span>
-  <h1 class="govuk-heading-xl">Review alternative source of {{ dataset.name }} data</h1>
+  <h1 class="govuk-heading-xl">Review alternative source of {{ dataset.dataset | datasetSlugToReadableName }} data</h1>
 {% else %}
   <span class="govuk-caption-xl">{{ organisation.name }}</span>
-  <h1 class="govuk-heading-xl">{{ dataset.name }}</h1>
+  <h1 class="govuk-heading-xl">{{ dataset.dataset | datasetSlugToReadableName(true) }}</h1>
 
   {% if notice %}
     {{ deadlineNotice(dataset.dataset, notice, organisation) }}