From fb90128bf1f423de57ac9650752641e3e11f96c7 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Fri, 6 Feb 2026 12:48:06 +0000 Subject: [PATCH 1/9] detail on why arcgis may have failed --- config/default.yaml | 2 +- src/views/check/error-redirect.html | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 22aee2c08..7ddd0f94c 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -140,7 +140,7 @@ features: enabled: true provisionBasedDatasets: # This feature will load datasets based on provisionReasons - enabled: false + enabled: true nonAuthPages: # This feature enables pages to show non-authoritative datasets # If off, it will still check for local-plan datasets. diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index b7b251d22..cd655f6f5 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -55,13 +55,21 @@ {% elif errorMessage == "The URL must be accessible" %}

You must host the URL on a server which does not block access due to set permissions.

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

+ {% elif errorMessage == "URL must be the data layer" %} +

The URL you have provided is an ArcGIS link, which is not the data layer.

+

The link to the data layer ends with a forward slash, followed by a number. For example, /8.

+

If you believe you have provided the information correctly and the the problem persists, contact us at digitalland@communities.gov.uk

{% else %}

Error: {{ errorMessage }}

{% endif %} -

Please try again or contact support if the problem persists.

+ {# Specific case where we don't want duplication of contact reasons #} + {% if errorMessage != "URL must be the data layer" %} +

Please try again or contact support if the problem persists.

+ {% endif %} From f8946bef5d0fcd190c66cf57be37a7358eef129d Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 10 Feb 2026 10:48:56 +0000 Subject: [PATCH 2/9] capilization of provisions, record count for auth only, table to pull through only auth or some, work with no dataset but collection --- .../makeDatasetSlugToReadableNameFilter.js | 10 +++++++--- src/middleware/common.middleware.js | 18 ++++++++++++++++-- src/middleware/datasetOverview.middleware.js | 5 +++-- src/middleware/dataview.middleware.js | 11 ++++++----- src/utils/datasetSubjectLoader.js | 2 +- src/views/components/dataset-banner.html | 4 ++-- src/views/includes/_dataset-page-header.html | 4 ++-- src/views/organisations/dataset-overview.html | 8 ++++---- src/views/organisations/datasetTaskList.html | 4 ++-- src/views/organisations/dataview.html | 2 +- src/views/organisations/get-started.html | 2 +- src/views/organisations/overview.html | 4 ++-- src/views/submit/check-answers.html | 2 +- src/views/submit/confirmation.html | 2 +- .../authoritative_data.playwright.test.js | 4 ++-- test/unit/datasetSubjectLoader.test.js | 10 +++++----- test/unit/datasetTaskListPage.test.js | 8 +++++--- ...makeDatasetSlugToReadableNameFilter.test.js | 16 +++++++++++----- test/unit/middleware/common.middleware.test.js | 10 ++++------ .../organisations/dataset-overview.test.js | 16 ++++++++++------ .../organisations/get-startedPage.test.js | 8 +++++--- .../organisations/issueDetailsPage.test.js | 6 +++--- 22 files changed, 94 insertions(+), 62 deletions(-) diff --git a/src/filters/makeDatasetSlugToReadableNameFilter.js b/src/filters/makeDatasetSlugToReadableNameFilter.js index 241f3798e..d99b3dcde 100644 --- a/src/filters/makeDatasetSlugToReadableNameFilter.js +++ b/src/filters/makeDatasetSlugToReadableNameFilter.js @@ -12,16 +12,20 @@ export const makeDatasetSlugToReadableNameFilter = (datasetNameMapping) => { * A filter function that takes a dataset slug as input and returns its corresponding readable name. * * @param {string} slug - The dataset slug to look up. + * @param {boolean} [capitalize=false] - Whether to capitalize the first letter. * @returns {string} The readable name corresponding to the provided slug. * @throws {Error} - If the provided slug is not found in the dataset name mapping. */ - return (slug) => { + const capitalizeFirst = (str) => str.charAt(0).toUpperCase() + str.slice(1) + const lowercaseFirst = (str) => str.charAt(0).toLowerCase() + str.slice(1) + + return (slug, capitalize = false) => { const name = datasetNameMapping.get(slug) if (!name) { // ToDo: work out what to do here? potentially update it with data from datasette logger.debug(`can't find a name for ${slug}`) - return slug + return capitalize ? capitalizeFirst(slug) : lowercaseFirst(slug) } - return name + return capitalize ? capitalizeFirst(name) : lowercaseFirst(name) } } diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index ec5f91205..265189810 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -166,6 +166,8 @@ export const fetchEntitiesPlatformDb = fetchMany({ * @param {Object} req.params - Route parameters * @param {string} req.params.dataset - Dataset name * @param {string} [req.authority] OUT parameter - Set to 'authoritative', 'some', or '' (empty string) + * @param {{entity_count: number}} [req.entityCount] OUT parameter - Set to count from API data + * @param {Array} [req.alternateEntityList] OUT parameter - List of alternate entity IDs when quality is 'some' * @param {Object} res - Response object * @param {Function} next - Next middleware function */ @@ -201,6 +203,12 @@ export const prepareAuthority = async (req, res, next) => { if (authoritativeResult.formattedData && authoritativeResult.formattedData.length > 0) { req.authority = 'authoritative' + // Set record count to only show authoritative count if authoritative data exists + const count = authoritativeResult?.data?.count + if (count !== undefined) { + req.entityCount = { entity_count: count } + } + console.log(`Authoritative data found with count ${count}, skipping non-authoritative check`) return next() } @@ -216,6 +224,11 @@ export const prepareAuthority = async (req, res, next) => { // Set list of alternate entities provided in req for later use const rows = someResult.formattedData || [] req.alternateEntityList = rows.map(({ entity }) => entity) + // Also set record count here if available + const someCount = someResult?.data?.count + if (someCount !== undefined) { + req.entityCount = { entity_count: someCount } + } } else { req.authority = '' } @@ -384,8 +397,9 @@ export const pullOutDatasetSpecification = (req, res, next) => { } const datasetSpecification = collectionSpecifications.find((spec) => spec.dataset === req.dataset.dataset) if (!datasetSpecification) { - logger.error('Dataset specification not found', { dataset: req.dataset.dataset }) - return next(new MiddlewareError('Dataset specification not found', 404)) + logger.info('Dataset specification not found, clearing specification and falling back to dataset fields', { dataset: req.dataset.dataset }) + req.specification = null + return next() } req.specification = datasetSpecification next() diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index 0ad4f13a6..080a7427b 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -5,7 +5,7 @@ */ import { fetchDatasetPlatformInfo, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' -import { fetchOne, fetchMany, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' +import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' import { types } from '../utils/logging.js' @@ -271,7 +271,8 @@ export default [ ...processAuthoritativeMiddlewares, ...processSpecificationMiddlewares, // setNoticesFromSourceKey('resources'), // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) - fetchEntityCount, + // Currently fallback entity count all records if authority entity count fails + onlyIf(req => req.entityCount === undefined, fetchEntityCount), prepareDatasetOverviewTemplateParams, getDatasetOverview, logPageError diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index 5ecdacf03..49b9b7651 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -19,7 +19,7 @@ import { fetchEntryIssueCounts, fetchEntitiesPlatformDb } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate } from './middleware.builders.js' +import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js' import * as v from 'valibot' import { splitByLeading } from '../utils/table.js' @@ -41,7 +41,7 @@ export const fetchEntities = fetchMany({ }) export const setRecordCount = (req, res, next) => { - req.recordCount = req?.entityCount?.count || 0 + req.recordCount = req?.entityCount?.entity_count ?? req?.entityCount?.count ?? 0 next() } @@ -124,14 +124,15 @@ export default [ fetchEntityIssueCounts, fetchEntryIssueCounts, - fetchEntityCount, + ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API + + onlyIf(req => req.entityCount === undefined, fetchEntityCount), // fallback setRecordCount, getSetDataRange(config.tablePageLength), show404IfPageNumberNotInRange, - ...processAuthoritativeMiddlewares, // Used to see if alternative or authoritative, and update entites fetch accordingly - fetchEntitiesPlatformDb, // This technically fetches twice from entities table, could be refactored later + fetchEntitiesPlatformDb, // Fetches entities filtered by authority quality extractJsonFieldFromEntities, replaceUnderscoreInEntities, diff --git a/src/utils/datasetSubjectLoader.js b/src/utils/datasetSubjectLoader.js index 8f4baba36..492508b02 100644 --- a/src/utils/datasetSubjectLoader.js +++ b/src/utils/datasetSubjectLoader.js @@ -147,7 +147,7 @@ export async function buildDataSubjects () { // Use existing datasetSlugToReadableName to create lookup of dataset keys to readable names const nameMap = {} for (const key of datasetKeys) { - nameMap[key] = datasetSlugToReadableName(key) + nameMap[key] = datasetSlugToReadableName(key, true) } return makeDatasetSubjectMap(nameMap) diff --git a/src/views/components/dataset-banner.html b/src/views/components/dataset-banner.html index e3bbd694b..7bc8ed5a6 100644 --- a/src/views/components/dataset-banner.html +++ b/src/views/components/dataset-banner.html @@ -1,11 +1,11 @@ {% macro datasetBanner(orgName, datasetName) %} {% if orgName and datasetName %} - {{ orgName }} — {{ datasetName | datasetSlugToReadableName }} + {{ orgName }} — {{ datasetName | datasetSlugToReadableName(true) }} {% elif orgName %} {{ orgName }} {% elif datasetName %} - {{ datasetName | datasetSlugToReadableName }} + {{ datasetName | datasetSlugToReadableName(true) }} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/src/views/includes/_dataset-page-header.html b/src/views/includes/_dataset-page-header.html index f06685203..5f929868b 100644 --- a/src/views/includes/_dataset-page-header.html +++ b/src/views/includes/_dataset-page-header.html @@ -1,10 +1,10 @@
{% if authority and authority === "some" %} {{ organisation.name }} -

Review alternative source of {{ dataset.name }} data

+

Review alternative source of {{ dataset.dataset | datasetSlugToReadableName }} data

{% else %} {{ organisation.name }} -

{{ dataset.name }}

+

{{ dataset.dataset | datasetSlugToReadableName(true) }}

{% if notice %} {{ deadlineNotice(dataset.dataset, notice, organisation) }} diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index aadb6663c..f4af56cf4 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -29,7 +29,7 @@ {% set serviceType = 'Manage' %} {% set pageName %} - {{organisation.name}} - {{dataset.name}} - Dataset overview + {{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Dataset overview {% endset %} {% set urlStyle = 'text-overflow: ellipsis; overflow: hidden; white-space: nowrap; width: 400px; display: block;' %} @@ -56,7 +56,7 @@ href: '/organisations/' + organisation.organisation }, { - text: dataset.name + text: dataset.dataset | datasetSlugToReadableName(true) } ] }) }} @@ -225,10 +225,10 @@

Dataset details

Dataset actions

  • - How to prepare and provide your {{ dataset.name }} data + How to prepare and provide your {{ dataset.dataset | datasetSlugToReadableName }} data
  • - Check {{ dataset.name }} dataset + Check {{ dataset.dataset | datasetSlugToReadableName }} dataset
    • {% if dataset.dataset | getDatasetGuidanceUrl %}
  • diff --git a/src/views/organisations/datasetTaskList.html b/src/views/organisations/datasetTaskList.html index 4f809d6b8..87fcd6fda 100644 --- a/src/views/organisations/datasetTaskList.html +++ b/src/views/organisations/datasetTaskList.html @@ -28,7 +28,7 @@ href: '/organisations/' + organisation.organisation }, { - text: dataset.name + text: dataset.dataset | datasetSlugToReadableName(true) } ] }) }} @@ -59,7 +59,7 @@

    {% if taskList.length == 0 %} -

    There are no issues with {{ organisation.name }}'s {{ dataset.name }} dataset.

    +

    There are no issues with {{ organisation.name }}'s {{ dataset.dataset | datasetSlugToReadableName }} dataset.

    {% else %} diff --git a/src/views/organisations/dataview.html b/src/views/organisations/dataview.html index 9d1719503..b89989b4a 100644 --- a/src/views/organisations/dataview.html +++ b/src/views/organisations/dataview.html @@ -30,7 +30,7 @@ href: '/organisations/' + organisation.organisation }, { - text: dataset.name | capitalize + text: dataset.dataset | datasetSlugToReadableName(true) } ] }) }} diff --git a/src/views/organisations/get-started.html b/src/views/organisations/get-started.html index 7aa1e093b..f75d1fc8b 100644 --- a/src/views/organisations/get-started.html +++ b/src/views/organisations/get-started.html @@ -45,7 +45,7 @@ {% if authority and authority === "some" %}

    How to improve {{ organisation.name }}'s data

    {% else %} -

    How to prepare and provide your {{ dataset.name }} data

    +

    How to prepare and provide your {{ dataset.dataset | datasetSlugToReadableName }} data

    {% endif %}
    {% if dataset.status == 'Not submitted' %} - {{dataset.dataset | datasetSlugToReadableName}} + {{dataset.dataset | datasetSlugToReadableName(true)}} {% else %} - {{dataset.dataset | datasetSlugToReadableName}} + {{dataset.dataset | datasetSlugToReadableName(true)}} {% endif %} diff --git a/src/views/submit/check-answers.html b/src/views/submit/check-answers.html index 11c693e24..ada2e50ba 100644 --- a/src/views/submit/check-answers.html +++ b/src/views/submit/check-answers.html @@ -43,7 +43,7 @@

    text: "Dataset" }, value: { - text: values['dataset'] | datasetSlugToReadableName + text: values['dataset'] | datasetSlugToReadableName(true) } }, { diff --git a/src/views/submit/confirmation.html b/src/views/submit/confirmation.html index 3a5eac937..93d208b3f 100644 --- a/src/views/submit/confirmation.html +++ b/src/views/submit/confirmation.html @@ -4,7 +4,7 @@ {% from "govuk/components/panel/macro.njk" import govukPanel %} {% set serviceType = 'Submit' %} -{% set pageName = (values['dataset'] | datasetSlugToReadableName) + " provided" %} +{% set pageName = (values['dataset'] | datasetSlugToReadableName(true)) + " provided" %} {% block content %} diff --git a/test/integration/authoritative_data.playwright.test.js b/test/integration/authoritative_data.playwright.test.js index fe0d90e19..273d2810a 100644 --- a/test/integration/authoritative_data.playwright.test.js +++ b/test/integration/authoritative_data.playwright.test.js @@ -14,13 +14,13 @@ test('The LPA Dashboard reflects both auth and non-auth data, and the non-auth d // Navigate to LPA Dashboard await page.goto('/organisations/local-authority:SLF') - await expect(page.getByText('Data is not from an authoritative source')).toBeVisible() + await expect(page.getByText('Data is not from an authoritative source').first()).toBeVisible() // Go to that dataset page await page.getByRole('link', { name: 'Conservation area document' }).click() // Check that this is shown in the title with review alternate data - await expect(page.getByRole('heading', { level: 1 })).toContainText('Review alternative source of Conservation area document data') + await expect(page.getByRole('heading', { level: 1 })).toContainText('Review alternative source of conservation area document data') // Go to task list await page.locator('a.govuk-service-navigation__link', { hasText: 'Task list' }).click() await expect(page.getByText('Provide authoritative data')).toBeVisible() diff --git a/test/unit/datasetSubjectLoader.test.js b/test/unit/datasetSubjectLoader.test.js index 325123b6f..3e89cf693 100644 --- a/test/unit/datasetSubjectLoader.test.js +++ b/test/unit/datasetSubjectLoader.test.js @@ -189,8 +189,8 @@ describe('datasetSubjectLoader', () => { const result = await buildDataSubjects() expect(fetchDatasetsFromProvisions).toHaveBeenCalled() - expect(datasetSlugToReadableName).toHaveBeenCalledWith('article-4-direction') - expect(datasetSlugToReadableName).toHaveBeenCalledWith('tree') + expect(datasetSlugToReadableName).toHaveBeenCalledWith('article-4-direction', true) + expect(datasetSlugToReadableName).toHaveBeenCalledWith('tree', true) expect(result).toHaveProperty('article-4-direction') expect(result).toHaveProperty('tree-preservation-order') @@ -211,9 +211,9 @@ describe('datasetSubjectLoader', () => { await buildDataSubjects() expect(logger.warn).toHaveBeenCalledWith('buildDataSubjects: Error fetching dataset keys roll back to defaults') - expect(datasetSlugToReadableName).toHaveBeenCalledWith('article-4-direction') - expect(datasetSlugToReadableName).toHaveBeenCalledWith('conservation-area') - expect(datasetSlugToReadableName).toHaveBeenCalledWith('tree') + expect(datasetSlugToReadableName).toHaveBeenCalledWith('article-4-direction', true) + expect(datasetSlugToReadableName).toHaveBeenCalledWith('conservation-area', true) + expect(datasetSlugToReadableName).toHaveBeenCalledWith('tree', true) }) }) diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js index 7c1a130e4..2c9d80eee 100644 --- a/test/unit/datasetTaskListPage.test.js +++ b/test/unit/datasetTaskListPage.test.js @@ -11,19 +11,20 @@ const seed = new Date().getTime() describe(`Dataset Task List Page (seed: ${seed})`, () => { const params = mocker(OrgDatasetTaskList, seed) + const html = nunjucks.render('organisations/datasetTaskList.html', params) const dom = new jsdom.JSDOM(html) const document = dom.window.document runGenericPageTests(html, { - pageTitle: `${params.organisation.name} - ${params.dataset.name} - Task list - Check and provide planning data`, breadcrumbs: [{ text: 'Home', href: '/' }, { text: 'Organisations', href: '/organisations' }, { text: 'mock org', href: `/organisations/${params.organisation.organisation}` }, { text: 'Article 4 direction area' }] }) it('Renders the correct headings', () => { expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name) - expect(document.querySelector('h1').textContent).toContain(params.dataset.name) + // Filter returns slug in tests (no mapping loaded), check slug passes through + expect(document.querySelector('h1').textContent).toContain(params.dataset.dataset) }) it('Renders the dataset navigation links correctly', () => { @@ -72,7 +73,8 @@ describe(`Dataset Task List Page (seed: ${seed})`, () => { authority }) - const paragraphText = `There are no issues with ${organisation.name}'s ${dataset.name} dataset.` + // Filter returns slug in tests (no mapping loaded) + const paragraphText = `There are no issues with ${organisation.name}'s ${dataset.dataset} dataset.` expect(html).toContain(paragraphText) }) diff --git a/test/unit/makeDatasetSlugToReadableNameFilter.test.js b/test/unit/makeDatasetSlugToReadableNameFilter.test.js index 3193b08fc..b97f164f7 100644 --- a/test/unit/makeDatasetSlugToReadableNameFilter.test.js +++ b/test/unit/makeDatasetSlugToReadableNameFilter.test.js @@ -10,12 +10,18 @@ describe('makeDatasetSlugToReadableNameFilter', () => { const filter = makeDatasetSlugToReadableNameFilter(datasetNameMapping) - it('returns a function that takes a dataset slug and returns its corresponding readable name', () => { - expect(filter('dataset-slug-1')).toBe('Dataset 1') - expect(filter('dataset-slug-2')).toBe('Dataset 2') + it('returns a function that takes a dataset slug and returns its corresponding readable name with lowercase first letter by default', () => { + expect(filter('dataset-slug-1')).toBe('dataset 1') + expect(filter('dataset-slug-2')).toBe('dataset 2') }) - it('returns the original slug if it is not found in the dataset name mapping', () => { - expect(filter('unknown-slug')).toBe('unknown-slug') + it('capitalizes first letter when capitalize param is true', () => { + expect(filter('dataset-slug-1', true)).toBe('Dataset 1') + expect(filter('dataset-slug-2', true)).toBe('Dataset 2') + }) + + it('returns the original slug with lowercase first letter if it is not found in the dataset name mapping', () => { + expect(filter('Unknown-slug')).toBe('unknown-slug') + expect(filter('Unknown-slug', true)).toBe('Unknown-slug') }) }) diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js index 99bafc3de..7ead80739 100644 --- a/test/unit/middleware/common.middleware.test.js +++ b/test/unit/middleware/common.middleware.test.js @@ -5,7 +5,6 @@ import datasette from '../../../src/services/datasette.js' import performanceDbApi from '../../../src/services/performanceDbApi.js' import platformApi from '../../../src/services/platformApi.js' import { isValiError } from 'valibot' -import { MiddlewareError } from '../../../src/utils/errors.js' vi.mock('../../../src/services/performanceDbApi.js') @@ -536,19 +535,18 @@ describe('pullOutDatasetSpecification', () => { expect(next).toHaveBeenCalledWith() }) - it('calls next with an error if dataset specification is an empty array', () => { + it('falls back gracefully if dataset specification is an empty array', () => { const reqWithEmptySpecification = { ...req, specification: { json: JSON.stringify([]) } } - const errors = [] - const next = vi.fn().mockImplementation((val) => errors.push(val)) + const next = vi.fn() pullOutDatasetSpecification(reqWithEmptySpecification, res, next) expect(next).toHaveBeenCalledTimes(1) - expect(next).toHaveBeenCalledWith(expect.any(MiddlewareError)) - expect(errors[0].statusCode).toBe(404) + expect(next).toHaveBeenCalledWith() + expect(reqWithEmptySpecification.specification).toBeNull() }) }) diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js index dabdf6bc3..51181a3c8 100644 --- a/test/unit/views/organisations/dataset-overview.test.js +++ b/test/unit/views/organisations/dataset-overview.test.js @@ -49,19 +49,21 @@ describe('Dataset Overview Page', () => { } } - const nunjucks = setupNunjucks({ datasetNameMapping: new Map() }) + const nunjucks = setupNunjucks({}) const html = stripWhitespace(nunjucks.render('organisations/dataset-overview.html', params)) const dom = new jsdom.JSDOM(html) const document = dom.window.document runGenericPageTests(html, { - pageTitle: 'Mock org - World heritage site buffer zone - Dataset overview - Check and provide planning data' + // Filter returns slug in tests (no mapping loaded, fallback doesn't capitalize) + pageTitle: 'Mock org - world-heritage-site-buffer-zone - Dataset overview - Check and provide planning data' }) it('Renders the correct headings', () => { expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual('Mock org') - expect(document.querySelector('h1').textContent).toContain('World heritage site buffer zone') + // Filter returns slug in tests (no mapping loaded) + expect(document.querySelector('h1').textContent).toContain('world-heritage-site-buffer-zone') }) it('Renders the dataset navigation links correctly', () => { @@ -97,7 +99,8 @@ describe('Dataset Overview Page', () => { expect(breadcrumbs[0].textContent).toContain('Home') expect(breadcrumbs[1].textContent).toContain('Organisations') expect(breadcrumbs[2].textContent).toContain('Mock org') - expect(breadcrumbs[3].textContent).toContain('World heritage site buffer zone') + // Filter returns slug in tests (no mapping loaded) + expect(breadcrumbs[3].textContent).toContain('world-heritage-site-buffer-zone') }) it('Does not render the map section when non mappable dataset is viewed', () => { @@ -184,10 +187,11 @@ describe('Dataset Overview Page', () => { expect(header.textContent.trim()).toEqual('Dataset actions') - expect(links[0].textContent.trim()).toEqual('How to prepare and provide your Article 4 direction area data') + // Filter returns slug in tests (no mapping loaded) + expect(links[0].textContent.trim()).toEqual('How to prepare and provide your article-4-direction-area data') expect(links[0].querySelector('.govuk-link').href).toEqual('/organisations/mock-org/article-4-direction-area/get-started') - expect(links[1].textContent.trim()).toEqual('Check Article 4 direction area dataset') + expect(links[1].textContent.trim()).toEqual('Check article-4-direction-area dataset') expect(links[1].querySelector('.govuk-link').href).toEqual('/check/link?dataset=article-4-direction-area&orgName=Mock%20org&orgId=mock-org') expect(links[2].textContent.trim()).toEqual('Article 4 direction area guidance') diff --git a/test/unit/views/organisations/get-startedPage.test.js b/test/unit/views/organisations/get-startedPage.test.js index 3fc598597..85a976e1b 100644 --- a/test/unit/views/organisations/get-startedPage.test.js +++ b/test/unit/views/organisations/get-startedPage.test.js @@ -13,19 +13,21 @@ const seed = new Date().getTime() describe(`Get Started Page (seed: ${seed})`, () => { const params = mocker(OrgGetStarted, seed) + const html = nunjucks.render('organisations/get-started.html', params) const dom = new jsdom.JSDOM(html) const document = dom.window.document runGenericPageTests(html, { - pageTitle: `${params.organisation.name} - ${params.dataset.name} - Get started - Check and provide planning data` + breadcrumbs: [{ text: 'Home', href: '/' }, { text: 'Organisations', href: '/organisations' }, { text: params.organisation.name }, { text: 'Get started' }] }) it('Renders the correct headings', () => { expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name) - expect(document.querySelector('h1').textContent).toContain(`${params.dataset.name}`) - expect(document.querySelector('#main-content h2').textContent).toContain(`How to prepare and provide your ${params.dataset.name} data`) + // Filter returns slug in tests (no mapping loaded), check slug passes through + expect(document.querySelector('h1').textContent).toContain(params.dataset.dataset) + expect(document.querySelector('#main-content h2').textContent).toContain(`How to prepare and provide your ${params.dataset.dataset} data`) }) it('Renders breadcrumbs correctly', () => { diff --git a/test/unit/views/organisations/issueDetailsPage.test.js b/test/unit/views/organisations/issueDetailsPage.test.js index 8dec92a6b..74febec0e 100644 --- a/test/unit/views/organisations/issueDetailsPage.test.js +++ b/test/unit/views/organisations/issueDetailsPage.test.js @@ -18,19 +18,19 @@ describe(`issueDetails.html(seed: ${seed})`, () => { const document = dom.window.document runGenericPageTests(html, { - pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues - ${config.serviceNames.submit}`, breadcrumbs: [ { text: 'Home', href: '/' }, { text: 'Organisations', href: '/organisations' }, { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` }, - { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` }, + { text: params.dataset.dataset, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` }, { text: 'mock issue' } ] }) it('Renders the correct headings', () => { expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name) - expect(document.querySelector('h1').textContent).toContain(params.dataset.name) + // Filter returns slug in tests (no mapping loaded), check slug passes through + expect(document.querySelector('h1').textContent).toContain(params.dataset.dataset) }) describe('error summary', () => { From 5ed90cc0edea610215e8b3c92f08452eb8a33295 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 10 Feb 2026 12:09:22 +0000 Subject: [PATCH 3/9] submit update name, issue details bug --- src/middleware/entityIssueDetails.middleware.js | 2 +- src/routes/schemas.js | 2 +- src/views/organisations/datasetTaskList.html | 2 +- src/views/organisations/dataview.html | 2 +- src/views/organisations/get-started.html | 2 +- src/views/organisations/http-error.html | 2 +- src/views/organisations/issueDetails.html | 4 ++-- src/views/organisations/issueTable.html | 4 ++-- src/views/submit/dataset-details.html | 2 +- src/views/submit/lpa-details.html | 2 +- test/unit/datasetTaskListPage.test.js | 1 - test/unit/http-errorPage.test.js | 2 +- test/unit/middleware/datasetTaskList.middleware.test.js | 4 ++-- test/unit/views/organisations/issueDetailsPage.test.js | 2 +- 14 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/middleware/entityIssueDetails.middleware.js b/src/middleware/entityIssueDetails.middleware.js index 5a53ac44b..866431cd4 100644 --- a/src/middleware/entityIssueDetails.middleware.js +++ b/src/middleware/entityIssueDetails.middleware.js @@ -60,7 +60,7 @@ export const getIssueField = (text, html, classes) => { }, value: { html: html ? html.toString() : '', - originalValue: html // we don't want any markup here + originalValue: html != null ? html.toString() : undefined }, classes } diff --git a/src/routes/schemas.js b/src/routes/schemas.js index b717c27d8..262bd9e4b 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -211,7 +211,7 @@ export const OrgDatasetTaskList = v.strictObject({ organisation: OrgField, authority: v.string(), dataset: v.strictObject({ - dataset: v.optional(NonEmptyString), + dataset: NonEmptyString, name: NonEmptyString, collection: NonEmptyString }) diff --git a/src/views/organisations/datasetTaskList.html b/src/views/organisations/datasetTaskList.html index 87fcd6fda..0b927638c 100644 --- a/src/views/organisations/datasetTaskList.html +++ b/src/views/organisations/datasetTaskList.html @@ -3,7 +3,7 @@ {% from "../components/dataset-navigation.html" import datasetNavigation %} {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} -{% set pageName %}{{organisation.name}} - {{dataset.name}} - Task list{% endset %} +{% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Task list{% endset %} {% set serviceType = 'Manage' %} {% extends "layouts/main.html" %} diff --git a/src/views/organisations/dataview.html b/src/views/organisations/dataview.html index b89989b4a..881e9086a 100644 --- a/src/views/organisations/dataview.html +++ b/src/views/organisations/dataview.html @@ -6,7 +6,7 @@ {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} {% set serviceType = 'Submit'%} -{% set pageName %}{{organisation.name}} - {{dataset.name}} - Dataset table{% endset %} +{% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Dataset table{% endset %} {% extends "layouts/main.html" %} diff --git a/src/views/organisations/get-started.html b/src/views/organisations/get-started.html index f75d1fc8b..a3702c96b 100644 --- a/src/views/organisations/get-started.html +++ b/src/views/organisations/get-started.html @@ -3,7 +3,7 @@ {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} {% set serviceType = 'Manage' %} {% set pageName %} - {{organisation.name}} - {{dataset.name}} - Get started + {{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Get started {% endset %} {% extends "layouts/main.html" %} diff --git a/src/views/organisations/http-error.html b/src/views/organisations/http-error.html index b07f42b81..58b7e536b 100644 --- a/src/views/organisations/http-error.html +++ b/src/views/organisations/http-error.html @@ -2,7 +2,7 @@ {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %} {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} -{% set pageName %}{{organisation.name}} - {{dataset.name}} - Task list{% endset %} +{% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Task list{% endset %} {% set serviceType = 'manage' %} {% set resubmitUrl %}/organisations/{{ organisation.organisation | urlencode }}/{{ dataset.dataset | urlencode }}/get-started{% endset %} diff --git a/src/views/organisations/issueDetails.html b/src/views/organisations/issueDetails.html index c102c5317..b56775251 100644 --- a/src/views/organisations/issueDetails.html +++ b/src/views/organisations/issueDetails.html @@ -7,9 +7,9 @@ {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} {% set serviceType = 'Submit'%} {% if issueEntitiesCount > 1 %} - {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues (Page {{pageNumber}} of {{issueEntitiesCount}}){% endset %} + {% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Issues (Page {{pageNumber}} of {{issueEntitiesCount}}){% endset %} {% else %} - {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues{% endset %} + {% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Issues{% endset %} {%endif%} {% extends "layouts/main.html" %} diff --git a/src/views/organisations/issueTable.html b/src/views/organisations/issueTable.html index 8d2766535..e9bbfab16 100644 --- a/src/views/organisations/issueTable.html +++ b/src/views/organisations/issueTable.html @@ -7,9 +7,9 @@ {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance']) %} {% set serviceType = 'Submit'%} {% if issueEntitiesCount > 1 %} - {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues (Page {{pageNumber}} of {{issueEntitiesCount}}){% endset %} + {% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Issues (Page {{pageNumber}} of {{issueEntitiesCount}}){% endset %} {% else %} - {% set pageName %}{{organisation.name}} - {{dataset.name}} - Issues{% endset %} + {% set pageName %}{{organisation.name}} - {{dataset.dataset | datasetSlugToReadableName(true)}} - Issues{% endset %} {%endif%} {% extends "layouts/main.html" %} diff --git a/src/views/submit/dataset-details.html b/src/views/submit/dataset-details.html index 6ea88a6e0..3309f0dce 100644 --- a/src/views/submit/dataset-details.html +++ b/src/views/submit/dataset-details.html @@ -27,7 +27,7 @@ {% block content %} {% set pageHeader %} -{{ options.lpa }} — {{ options.datasetName }} +{{ options.lpa }} — {{ options.datasetName | capitalize }}

    {{ pageName }}

    diff --git a/src/views/submit/lpa-details.html b/src/views/submit/lpa-details.html index 3aa82c2d3..ea3dfd6ff 100644 --- a/src/views/submit/lpa-details.html +++ b/src/views/submit/lpa-details.html @@ -16,7 +16,7 @@ {% endblock %} {% set pageHeader %} -{{ options.lpa }} — {{ options.datasetName }} +{{ options.lpa }} — {{ options.datasetName | capitalize }}

    {{ pageName }}

    diff --git a/test/unit/datasetTaskListPage.test.js b/test/unit/datasetTaskListPage.test.js index 2c9d80eee..7c1ea5553 100644 --- a/test/unit/datasetTaskListPage.test.js +++ b/test/unit/datasetTaskListPage.test.js @@ -11,7 +11,6 @@ const seed = new Date().getTime() describe(`Dataset Task List Page (seed: ${seed})`, () => { const params = mocker(OrgDatasetTaskList, seed) - const html = nunjucks.render('organisations/datasetTaskList.html', params) const dom = new jsdom.JSDOM(html) diff --git a/test/unit/http-errorPage.test.js b/test/unit/http-errorPage.test.js index 2c1e1f0d2..d42697e2d 100644 --- a/test/unit/http-errorPage.test.js +++ b/test/unit/http-errorPage.test.js @@ -19,7 +19,7 @@ describe(`http-error.html(seed: ${seed})`, () => { const document = dom.window.document runGenericPageTests(html, { - pageTitle: `${params.organisation.name} - ${params.dataset.name} - Task list - Check and provide planning data` + pageTitle: `${params.organisation.name} - ${params.dataset.dataset} - Task list - Check and provide planning data` }) it('Renders the correct heading', () => { diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index 71dd9b855..e13c28884 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -11,7 +11,7 @@ describe('datasetTaskList.middleware.js', () => { it('sets the correct template params on the request object', async () => { const req = { orgInfo: { name: 'Example Organisation', organisation: 'ORG' }, - dataset: { name: 'Example Dataset', collection: 'collection 1' }, + dataset: { name: 'Example Dataset', collection: 'collection 1', dataset: 'example-dataset' }, authority: 'authoritative', taskList: [ { @@ -29,7 +29,7 @@ describe('datasetTaskList.middleware.js', () => { const templateParams = { taskList: req.taskList, organisation: { name: 'Example Organisation', organisation: 'ORG' }, - dataset: { name: 'Example Dataset', collection: 'collection 1' }, + dataset: { name: 'Example Dataset', collection: 'collection 1', dataset: 'example-dataset' }, authority: 'authoritative' } v.parse(S.OrgDatasetTaskList, templateParams) diff --git a/test/unit/views/organisations/issueDetailsPage.test.js b/test/unit/views/organisations/issueDetailsPage.test.js index 74febec0e..034e7b85e 100644 --- a/test/unit/views/organisations/issueDetailsPage.test.js +++ b/test/unit/views/organisations/issueDetailsPage.test.js @@ -127,7 +127,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => { // const multiPageDocument = multiPageDom.window.document const paginationTitleSection = params.issueEntitiesCount > 1 ? `(Page ${params.pageNumber} of ${params.issueEntitiesCount}) ` : '' runGenericPageTests(multiPageHtml, { - pageTitle: `${params.organisation.name} - ${params.dataset.name} - Issues ${paginationTitleSection}- ${config.serviceNames.submit}`, + pageTitle: `${params.organisation.name} - ${params.dataset.dataset} - Issues ${paginationTitleSection}- ${config.serviceNames.submit}`, breadcrumbs: [ { text: 'Home', href: '/' }, { text: 'Organisations', href: '/organisations' }, From fec93b554bcb6c45024ef72e51aa478c32d5c2d2 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 10 Feb 2026 14:40:38 +0000 Subject: [PATCH 4/9] more capitalize, improve dataset task, overview and view page to get task count from performancedb --- src/middleware/common.middleware.js | 49 ++++++++++--------- ...t-failed-expectation-details.middleware.js | 3 +- src/middleware/datasetOverview.middleware.js | 10 ++-- src/middleware/datasetTaskList.middleware.js | 10 ++-- src/middleware/dataview.middleware.js | 10 ++-- src/middleware/lpa-overview.middleware.js | 18 +------ src/services/performanceDbApi.js | 16 +++--- .../organisations/get-startedPage.test.js | 1 - 8 files changed, 52 insertions(+), 65 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 265189810..44eb02ab5 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -208,7 +208,7 @@ export const prepareAuthority = async (req, res, next) => { if (count !== undefined) { req.entityCount = { entity_count: count } } - console.log(`Authoritative data found with count ${count}, skipping non-authoritative check`) + logger.info(`Authoritative data found with count ${count}, skipping non-authoritative check`) return next() } @@ -800,28 +800,20 @@ export const fetchEntityIssueCounts = fetchMany({ query: ({ req }) => { const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : '' return ` - WITH unique_issues AS ( - SELECT DISTINCT - i.dataset, - i.field, - i.issue_type, - i.entity - FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type - WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') - AND COALESCE(entity, '') <> '' - AND (i.end_date = '' OR i.end_date IS NULL) - AND it.responsibility = 'external' - AND it.severity = 'error' - ${datasetClause} - ) SELECT - dataset, - field, - issue_type, - COUNT(*) AS count - FROM unique_issues - GROUP BY field, issue_type, dataset + i.dataset, + i.field, + i.issue_type, + COUNT(DISTINCT i.entity) AS count + FROM issue i + JOIN issue_type it ON i.issue_type = it.issue_type + WHERE i.resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') + AND i.entity IS NOT NULL AND i.entity <> '' + AND (i.end_date = '' OR i.end_date IS NULL) + AND it.responsibility = 'external' + AND it.severity = 'error' + ${datasetClause} + GROUP BY i.field, i.issue_type, i.dataset ` }, result: 'entityIssueCounts' @@ -1175,3 +1167,16 @@ export const setAvailableDatasets = async (req, res, next) => { req.availableDatasets = await CONSTANTS.availableDatasets() next() } + +/** + * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! + * + * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb + */ +export const fetchEntityIssueCountsPerformanceDb = fetchMany({ + query: ({ params }) => { + return performanceDbApi.fetchEntityIssueCounts(params.lpa, params.dataset) + }, + result: 'entityIssueCounts', + dataset: FetchOptions.performanceDb +}) diff --git a/src/middleware/dataset-failed-expectation-details.middleware.js b/src/middleware/dataset-failed-expectation-details.middleware.js index b7d58281d..297e0ee51 100644 --- a/src/middleware/dataset-failed-expectation-details.middleware.js +++ b/src/middleware/dataset-failed-expectation-details.middleware.js @@ -75,7 +75,7 @@ const preparePaginationInfo = (req, res, next) => { * @returns {undefined} */ const prepareTemplateParams = (req, res, next) => { - const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination } = req + const { orgInfo: organisation, dataset, expectationOutOfBounds, entity, dataRange, pagination, parsedParams } = req const entityAugmented = prepareEntityForTable(entity) @@ -100,6 +100,7 @@ const prepareTemplateParams = (req, res, next) => { } }) }, + pageNumber: parsedParams.pageNumber, dataRange, pagination } diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index 080a7427b..c6c6de415 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' +import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entryIssueCounts, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req + const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req let endpointErrorIssues = 0 const endpoints = sources @@ -219,8 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { if (authority === 'some') { taskCount = 1 } else { - taskCount = (entryIssueCounts ? entryIssueCounts.length : 0) + - (entityIssueCounts ? entityIssueCounts.length : 0) + + taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) + endpointErrorIssues + (expectationOutOfBounds.length > 0 ? 1 : 0) } @@ -264,8 +263,7 @@ export default [ fetchColumnSummary, fetchResources, fetchSources, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, fetchSpecification, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index dcd2d9739..3f2cd4f18 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -19,8 +19,7 @@ import { expectations, fetchDatasetInfo, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, noop, @@ -106,7 +105,7 @@ export function entityOutOfBoundsMessage (dataset, count) { export const prepareTasks = (req, res, next) => { const { lpa, dataset } = req.parsedParams const { entityCount, resources, sources, authority } = req - const { entryIssueCounts, entityIssueCounts, expectationOutOfBounds = [] } = req + const { entityIssueCounts, expectationOutOfBounds = [] } = req // First check, if non authoritative dataset, only one task to show: Provide authoritative data if (authority && authority === 'some') { @@ -119,7 +118,7 @@ export const prepareTasks = (req, res, next) => { }] return next() } - let issues = [...entryIssueCounts, ...entityIssueCounts] + let issues = [...entityIssueCounts] issues = issues.filter( issue => issue.issue_type !== '' && @@ -233,8 +232,7 @@ export default [ isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, addEntityCountsToResources, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, prepareTasks, prepareDatasetTaskListTemplateParams, getDatasetTaskList, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index 49b9b7651..10db4a34e 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -15,8 +15,7 @@ import { logPageError, fetchResources, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, fetchEntitiesPlatformDb } from './common.middleware.js' import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js' @@ -82,10 +81,10 @@ export const constructTableParams = (req, res, next) => { } export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, entryIssueCounts, authority, alternateSources, uniqueDatasetFields } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req // Hard code task count for 'some' authority - const taskCount = authority !== 'some' ? entityIssueCounts.length + entryIssueCounts.length : 1 + const taskCount = authority !== 'some' ? entityIssueCounts.length : 1 // Build the fields query parameter and download url const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0 ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&') @@ -121,8 +120,7 @@ export default [ fetchDatasetInfo, fetchResources, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchEntityIssueCountsPerformanceDb, ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 05bb38cca..922e13e91 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,9 +4,8 @@ * @description Middleware for oragnisation (LPA) overview page */ -import performanceDbApi from '../services/performanceDbApi.js' -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate, parallel } from './middleware.builders.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js' +import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' import logger from '../utils/logger.js' @@ -15,19 +14,6 @@ import platformApi from '../services/platformApi.js' import { types } from '../utils/logging.js' import config from '../../config/index.js' -/** - * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! - * - * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb - */ -const fetchEntityIssueCountsPerformanceDb = fetchMany({ - query: ({ params }) => { - return performanceDbApi.fetchEntityIssueCounts(params.lpa) - }, - result: 'entityIssueCounts', - dataset: FetchOptions.performanceDb -}) - const fetchProvisions = fetchMany({ query: ({ params }) => { return /* sql */ `select dataset, project, provision_reason diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index f21a2bb83..cf903bec7 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -251,18 +251,20 @@ export default { }, // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb - fetchEntityIssueCounts (datasetId) { + fetchEntityIssueCounts (lpa, dataset) { + const datasetClause = dataset ? `AND dataset = '${dataset}'` : '' return /* sql */ ` - SELECT + SELECT dataset, issue_type, field, - COUNT(*) AS count - FROM endpoint_dataset_issue_type_summary - WHERE organisation = '${datasetId}' - AND severity = 'error' - AND responsibility = 'external' + COUNT(*) AS count + FROM endpoint_dataset_issue_type_summary + WHERE organisation = '${lpa}' + AND severity = 'error' + AND responsibility = 'external' AND (resource_end_date = '' OR resource_end_date IS NULL) + ${datasetClause} GROUP BY dataset, issue_type, field` }, diff --git a/test/unit/views/organisations/get-startedPage.test.js b/test/unit/views/organisations/get-startedPage.test.js index 85a976e1b..47d403217 100644 --- a/test/unit/views/organisations/get-startedPage.test.js +++ b/test/unit/views/organisations/get-startedPage.test.js @@ -13,7 +13,6 @@ const seed = new Date().getTime() describe(`Get Started Page (seed: ${seed})`, () => { const params = mocker(OrgGetStarted, seed) - const html = nunjucks.render('organisations/get-started.html', params) const dom = new jsdom.JSDOM(html) From 6e529044ee803a70206906b02fa961c1b5d06620 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 10 Feb 2026 15:25:26 +0000 Subject: [PATCH 5/9] update tests for performance db --- .../datasetOverview.middleware.test.js | 3 +-- .../datasetTaskList.middleware.test.js | 22 ++++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js index 4622e1b72..bf45fc8cc 100644 --- a/test/unit/middleware/datasetOverview.middleware.test.js +++ b/test/unit/middleware/datasetOverview.middleware.test.js @@ -36,7 +36,7 @@ describe('Dataset Overview Middleware', () => { { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: '2024-02-01', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' }, { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: '2023-01-01', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' } ], - entryIssueCounts: [ + entityIssueCounts: [ { issue: 'Example issue 1', issue_type: 'Example issue type 1', @@ -45,7 +45,6 @@ describe('Dataset Overview Middleware', () => { status: 'Error' } ], - entityIssueCounts: [], notice: undefined, authority: '' } diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index e13c28884..fdbef7dd2 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -49,8 +49,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1', 'entity2'], entityCount: { count: 2 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [{ field: 'field2', issue_type: 'issue-type2', count: 1 }] + entityIssueCounts: [ + { field: 'field1', issue_type: 'issue-type1', count: 1 }, + { field: 'field2', issue_type: 'issue-type2', count: 1 } + ] } const res = { @@ -117,8 +119,7 @@ describe('datasetTaskList.middleware.js', () => { entities: [], entityCount: { count: 0 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }], - entityIssueCounts: [] + entityIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }] } const res = { @@ -164,7 +165,6 @@ describe('datasetTaskList.middleware.js', () => { sources: [], entities: ['entity1'], resources: [{ entry_count: 10 }], - entryIssueCounts: [], entityIssueCounts: [] } @@ -193,8 +193,7 @@ describe('datasetTaskList.middleware.js', () => { entityCount: { count: 1 }, sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [] + entityIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }] } const res = { status: vi.fn() } @@ -222,8 +221,7 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }], // Invalid issue type (empty string) - entityIssueCounts: [] + entityIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }] // Invalid issue type (empty string) } const res = { status: vi.fn() } @@ -245,8 +243,7 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [] + entityIssueCounts: [{ issue_type: 'issue-type1', count: 1 }] // Missing field } const res = { status: vi.fn() } @@ -268,8 +265,7 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [], + entityIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field expectationOutOfBounds: [ { dataset: 'some-dataset', passed: 'False', details: { actual: 3, expected: 0 } } ] From aff25aed5a1f965a29f1ffc54faf7fd3b4d2cb44 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 11 Feb 2026 11:22:57 +0000 Subject: [PATCH 6/9] rabbit changes --- src/views/check/error-redirect.html | 4 ++-- test/unit/views/organisations/issueDetailsPage.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index cd655f6f5..2742f7b0f 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -58,8 +58,8 @@ {% elif errorMessage == "URL must be the data layer" %}

    The URL you have provided is an ArcGIS link, which is not the data layer.

    The link to the data layer ends with a forward slash, followed by a number. For example, /8.

    -

    If you believe you have provided the information correctly and the the problem persists, contact us at digitalland@communities.gov.uk

    +

    If you believe you have provided the information correctly and the problem persists, contact us at digitalland@communities.gov.uk

    {% else %}

    Error: {{ errorMessage }} diff --git a/test/unit/views/organisations/issueDetailsPage.test.js b/test/unit/views/organisations/issueDetailsPage.test.js index 034e7b85e..4e62121b3 100644 --- a/test/unit/views/organisations/issueDetailsPage.test.js +++ b/test/unit/views/organisations/issueDetailsPage.test.js @@ -22,7 +22,7 @@ describe(`issueDetails.html(seed: ${seed})`, () => { { text: 'Home', href: '/' }, { text: 'Organisations', href: '/organisations' }, { text: params.organisation.name, href: `/organisations/${params.organisation.organisation}` }, - { text: params.dataset.dataset, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` }, + { text: params.dataset.name, href: `/organisations/${params.organisation.organisation}/${params.dataset.dataset}` }, { text: 'mock issue' } ] }) From 58a68de0a3e855e35d3b83eb232e1115f40909fa Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 11 Feb 2026 21:08:30 +0000 Subject: [PATCH 7/9] revert task list to entry-entity issue merge --- src/middleware/datasetTaskList.middleware.js | 10 ++++--- src/middleware/dataview.middleware.js | 2 +- .../datasetTaskList.middleware.test.js | 28 +++++++++++-------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index 3f2cd4f18..dcd2d9739 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -19,7 +19,8 @@ import { expectations, fetchDatasetInfo, fetchEntityCount, - fetchEntityIssueCountsPerformanceDb, + fetchEntityIssueCounts, + fetchEntryIssueCounts, fetchOrgInfo, fetchResources, fetchSources, logPageError, noop, @@ -105,7 +106,7 @@ export function entityOutOfBoundsMessage (dataset, count) { export const prepareTasks = (req, res, next) => { const { lpa, dataset } = req.parsedParams const { entityCount, resources, sources, authority } = req - const { entityIssueCounts, expectationOutOfBounds = [] } = req + const { entryIssueCounts, entityIssueCounts, expectationOutOfBounds = [] } = req // First check, if non authoritative dataset, only one task to show: Provide authoritative data if (authority && authority === 'some') { @@ -118,7 +119,7 @@ export const prepareTasks = (req, res, next) => { }] return next() } - let issues = [...entityIssueCounts] + let issues = [...entryIssueCounts, ...entityIssueCounts] issues = issues.filter( issue => issue.issue_type !== '' && @@ -232,7 +233,8 @@ export default [ isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, addEntityCountsToResources, fetchEntityCount, - fetchEntityIssueCountsPerformanceDb, + fetchEntityIssueCounts, + fetchEntryIssueCounts, prepareTasks, prepareDatasetTaskListTemplateParams, getDatasetTaskList, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index 10db4a34e..cf63c434c 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -84,7 +84,7 @@ export const prepareTemplateParams = (req, res, next) => { const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req // Hard code task count for 'some' authority - const taskCount = authority !== 'some' ? entityIssueCounts.length : 1 + const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 // Build the fields query parameter and download url const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0 ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&') diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index fdbef7dd2..00562b329 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -11,7 +11,7 @@ describe('datasetTaskList.middleware.js', () => { it('sets the correct template params on the request object', async () => { const req = { orgInfo: { name: 'Example Organisation', organisation: 'ORG' }, - dataset: { name: 'Example Dataset', collection: 'collection 1', dataset: 'example-dataset' }, + dataset: { dataset: 'example-dataset', name: 'Example Dataset', collection: 'collection 1' }, authority: 'authoritative', taskList: [ { @@ -29,7 +29,7 @@ describe('datasetTaskList.middleware.js', () => { const templateParams = { taskList: req.taskList, organisation: { name: 'Example Organisation', organisation: 'ORG' }, - dataset: { name: 'Example Dataset', collection: 'collection 1', dataset: 'example-dataset' }, + dataset: { dataset: 'example-dataset', name: 'Example Dataset', collection: 'collection 1' }, authority: 'authoritative' } v.parse(S.OrgDatasetTaskList, templateParams) @@ -49,10 +49,8 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1', 'entity2'], entityCount: { count: 2 }, resources: [{ entry_count: 10 }], - entityIssueCounts: [ - { field: 'field1', issue_type: 'issue-type1', count: 1 }, - { field: 'field2', issue_type: 'issue-type2', count: 1 } - ] + entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], + entityIssueCounts: [{ field: 'field2', issue_type: 'issue-type2', count: 1 }] } const res = { @@ -119,7 +117,8 @@ describe('datasetTaskList.middleware.js', () => { entities: [], entityCount: { count: 0 }, resources: [{ entry_count: 10 }], - entityIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }] + entryIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }], + entityIssueCounts: [] } const res = { @@ -165,6 +164,7 @@ describe('datasetTaskList.middleware.js', () => { sources: [], entities: ['entity1'], resources: [{ entry_count: 10 }], + entryIssueCounts: [], entityIssueCounts: [] } @@ -193,7 +193,8 @@ describe('datasetTaskList.middleware.js', () => { entityCount: { count: 1 }, sources: [], resources: [{ entry_count: 10 }], - entityIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }] + entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], + entityIssueCounts: [] } const res = { status: vi.fn() } @@ -221,7 +222,8 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], sources: [], resources: [{ entry_count: 10 }], - entityIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }] // Invalid issue type (empty string) + entryIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }], // Invalid issue type (empty string) + entityIssueCounts: [] } const res = { status: vi.fn() } @@ -243,7 +245,8 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entityIssueCounts: [{ issue_type: 'issue-type1', count: 1 }] // Missing field + entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field + entityIssueCounts: [] } const res = { status: vi.fn() } @@ -265,7 +268,8 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entityIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field + entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field + entityIssueCounts: [], expectationOutOfBounds: [ { dataset: 'some-dataset', passed: 'False', details: { actual: 3, expected: 0 } } ] @@ -306,4 +310,4 @@ describe('entityOutOfBoundsMessage()', () => { expect(entityOutOfBoundsMessage('conservation-area', undefined)).toBe('You have conservation areas outside of your boundary') expect(entityOutOfBoundsMessage('new-dataset', undefined)).toBe('You have entities outside of your boundary') }) -}) +}) \ No newline at end of file From 36759fff4524f0fd0380cb4b435040bd66615f8f Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 11 Feb 2026 21:08:51 +0000 Subject: [PATCH 8/9] lint --- test/unit/middleware/datasetTaskList.middleware.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index 00562b329..31876d278 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -310,4 +310,4 @@ describe('entityOutOfBoundsMessage()', () => { expect(entityOutOfBoundsMessage('conservation-area', undefined)).toBe('You have conservation areas outside of your boundary') expect(entityOutOfBoundsMessage('new-dataset', undefined)).toBe('You have entities outside of your boundary') }) -}) \ No newline at end of file +}) From a732cf70f7e7f36a60a96d6161e8938584ef6832 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Thu, 12 Feb 2026 11:36:48 +0000 Subject: [PATCH 9/9] improve fetchEntityIssueCounts sql --- src/middleware/common.middleware.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 44eb02ab5..88ee2fab4 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -806,12 +806,10 @@ export const fetchEntityIssueCounts = fetchMany({ i.issue_type, COUNT(DISTINCT i.entity) AS count FROM issue i - JOIN issue_type it ON i.issue_type = it.issue_type WHERE i.resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') AND i.entity IS NOT NULL AND i.entity <> '' AND (i.end_date = '' OR i.end_date IS NULL) - AND it.responsibility = 'external' - AND it.severity = 'error' + AND i.issue_type IN (SELECT it.issue_type FROM issue_type it WHERE it.responsibility = 'external' AND it.severity = 'error') ${datasetClause} GROUP BY i.field, i.issue_type, i.dataset `