From 3fb13be30a5c6d7b5cd242f3d66f05870d01c711 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 11 Nov 2025 19:48:08 +0000 Subject: [PATCH 1/8] updated overview design and unit tests --- src/middleware/lpa-overview.middleware.js | 7 ++- src/routes/schemas.js | 2 + src/views/organisations/overview.html | 51 ++++++++++++------- .../lpa-overview.middleware.test.js | 18 ++++--- .../organisations/lpaOverviewPage.test.js | 28 ++++++---- 5 files changed, 69 insertions(+), 37 deletions(-) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index d5b4ffcd1..aa3261924 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -28,9 +28,8 @@ const fetchDatasetErrorStatus = fetchMany({ const fetchProvisions = fetchMany({ query: ({ params }) => { - const excludeDatasets = Object.keys(config.datasetsConfig).map(dataset => `'${dataset}'`).join(',') return /* sql */ `select dataset, project, provision_reason - from provision where organisation = '${params.lpa}' and dataset in (${excludeDatasets})` + from provision where organisation = '${params.lpa}'` }, result: 'provisions' }) @@ -285,6 +284,10 @@ export function prepareOverviewTemplateParams (req, res, next) { switch (reason) { case 'statutory': return 'statutory' + case 'expected': + return 'expected' + case 'prospective': + return 'prospective' default: return 'other' } diff --git a/src/routes/schemas.js b/src/routes/schemas.js index 9f7a165b6..a9c4a4d94 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -136,6 +136,8 @@ export const OrgOverviewPage = v.strictObject({ organisation: OrgField, datasets: v.object({ statutory: v.optional(v.array(DatasetItem)), + expected: v.optional(v.array(DatasetItem)), + prospective: v.optional(v.array(DatasetItem)), other: v.optional(v.array(DatasetItem)) }), totalDatasets: v.integer(), diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 3c2f4e682..18daa8506 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -91,11 +91,17 @@

{% endif %} {% endfor %} - {% for dataset in datasets.other %} - {% if dataset.notice %} - {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} - {% endif %} - {% endfor %} + {% for dataset in datasets.expected %} + {% if dataset.notice %} + {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} + {% endif %} + {% endfor %} + + {% for dataset in datasets.prospective %} + {% if dataset.notice %} + {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} + {% endif %} + {% endfor %} @@ -107,17 +113,17 @@

{{datasetsWithEndpoints}}/{{totalDatasets}} - datasets submitted + authoritative dataset{{ "s" if (datasetsWithEndpoints != 1) }} provided
{{datasetsWithErrors}} - dataset{{ "s" if (datasetsWithErrors != 1) }} with endpoint errors + error{{ "s" if (datasetsWithErrors != 1) }} accessing URLs
{{datasetsWithIssues}} - {{ "dataset" | pluralise(datasetsWithIssues) }} need{{ "s" if (datasetsWithIssues == 1) }} fixing + {{ "dataset" | pluralise(datasetsWithIssues) }} can be improved
@@ -126,10 +132,10 @@

- {% if datasets.statutory.length > 0 %}

Datasets {{ organisation.name }} must provide

+

You are required to make these datasets available under legislation. Using the Planning Data Platform to do this can help you to create and maintain authoritative, trustworthy data.

    {% for dataset in datasets.statutory %} {{ datasetItem(dataset) }} @@ -138,22 +144,29 @@

    Datasets {{ organisation.name }} must provide

{% endif %} - {% if datasets.other.length > 0 %} -
-

Datasets organisations in Open Digital Planning programme must provide

- {% if isODPMember %} -

{{ organisation.name}} is a member of the Open Digital Planning programme.

- {% else %} -

{{ organisation.name}} is not a member of the Open Digital Planning programme.

- {% endif %} -
    - {% for dataset in datasets.other %} + {% if datasets.expected.length > 0 and isODPMember %} +
    +

    Datasets {{ organisation.name}} are expected to provide

    +

    We expect you to provide these datasets as a condition of your funding with the Open Digital Planning community.

    +
      + {% for dataset in datasets.expected %} {{ datasetItem(dataset) }} {% endfor %}
    {% endif %} + {% if datasets.prospective.length > 0 %} +
    +

    Datasets {{ organisation.name}} can provide

    +

    You are the authoritative source of these datasets. Providing this data to the platform ensures it is accurate and up to date.

    +
      + {% for dataset in datasets.prospective %} + {{ datasetItem(dataset) }} + {% endfor %} +
    +
    + {% endif %}
diff --git a/test/unit/middleware/lpa-overview.middleware.test.js b/test/unit/middleware/lpa-overview.middleware.test.js index 8514047ea..a0d3309b8 100644 --- a/test/unit/middleware/lpa-overview.middleware.test.js +++ b/test/unit/middleware/lpa-overview.middleware.test.js @@ -96,7 +96,7 @@ describe('lpa-overview.middleware', () => { { endpointCount: 1, status: 'Live', dataset: 'dataset1', error: undefined, issueCount: 0, endpointErrorCount: 0 }, { endpointCount: 1, status: 'Error', dataset: 'dataset3', error: undefined, issueCount: 0, endpointErrorCount: 1 } ]), - other: expect.arrayContaining([ + expected: expect.arrayContaining([ { endpointCount: 0, status: 'Needs fixing', dataset: 'dataset2', error: undefined, issueCount: 1, endpointErrorCount: 0 }, { endpointCount: 0, status: 'Error', dataset: 'dataset4', error: 'There was a 404 error', issueCount: 0, endpointErrorCount: 1 } ]) @@ -114,17 +114,23 @@ describe('lpa-overview.middleware', () => { expect(errorCardNodes[0].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was an error accessing the endpoint URL') expect(errorCardNodes[1].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was a 404 error') - const orgMemebershipInfo = doc.querySelector('.org-membership-info').textContent.trim() - expect(orgMemebershipInfo).toMatch('is a member of the Open Digital Planning programme') + // Check for ODP membership info in the expected datasets section + const expectedSection = doc.querySelector('[data-testid="datasetsExpected"]') + if (expectedSection) { + const orgMembershipInfo = expectedSection.querySelector('.org-membership-info') + expect(orgMembershipInfo.textContent.trim()).toMatch(/Open Digital Planning/) + } - // verify proper label for non-OPD memebers gets rendered + // verify proper label for non-ODP members gets rendered const reqNotMember = structuredClone(reqTemplate) reqNotMember.provisions.forEach((provision) => { provision.project = '' }) prepareOverviewTemplateParams(reqNotMember, res, () => { }) const { doc: docNotMember } = getRenderedErrorCards(reqNotMember.templateParams) - expect(docNotMember.querySelector('.org-membership-info').textContent.trim()).toMatch('is not a member of the Open Digital Planning programme') + // When not an ODP member, expected datasets won't render (requires isODPMember), so datasetsExpected won't be present + const expectedSectionNotMember = docNotMember.querySelector('[data-testid="datasetsExpected"]') + expect(expectedSectionNotMember).toBeNull() }) it('should patch dataset status based on the provision_summary info', () => { @@ -139,7 +145,7 @@ describe('lpa-overview.middleware', () => { expect(ds1.status).toBe('Live') expect(ds1.error).toBeUndefined() - const ds4 = req.templateParams.datasets.other[1] + const ds4 = req.templateParams.datasets.expected[1] expect(ds4.status).toBe('Error') expect(ds4.error).toBe(req.datasets[3].error) // Error message should be left untouched }) diff --git a/test/unit/views/organisations/lpaOverviewPage.test.js b/test/unit/views/organisations/lpaOverviewPage.test.js index 652f768b9..22690eadc 100644 --- a/test/unit/views/organisations/lpaOverviewPage.test.js +++ b/test/unit/views/organisations/lpaOverviewPage.test.js @@ -38,7 +38,7 @@ const datasetGroup = ({ expect }, key, datasets, document) => { describe(`LPA Overview Page (seed: ${seed})`, () => { const params = mocker(OrgOverviewPage, seed) - console.debug(`mocked datasets: statutory = ${params.datasets.statutory?.length ?? 'none'}, other = ${params.datasets.other?.length ?? 'none'}`) + console.debug(`mocked datasets: statutory = ${params.datasets.statutory?.length ?? 'none'}, expected = ${params.datasets.expected?.length ?? 'none'}, prospective = ${params.datasets.prospective?.length ?? 'none'}`) const html = nunjucks.render('organisations/overview.html', params) @@ -53,17 +53,18 @@ describe(`LPA Overview Page (seed: ${seed})`, () => { const statsBoxes = document.querySelector('.dataset-status').children it('Datasets provided gives the correct value', () => { expect(statsBoxes[0].textContent).toContain(`${params.datasetsWithEndpoints}/${params.totalDatasets}`) - expect(statsBoxes[0].textContent).toContain('datasets submitted') + expect(statsBoxes[0].textContent).toContain('authoritative dataset') + expect(statsBoxes[0].textContent).toContain('provided') }) it('Datasets with errors gives the correct value', () => { expect(statsBoxes[1].textContent).toContain(params.datasetsWithErrors) - expect(statsBoxes[1].textContent).toMatch(/datasets? with endpoint errors/) + expect(statsBoxes[1].textContent).toContain('accessing URLs') }) it('Datasets with issues gives the correct value', () => { expect(statsBoxes[2].textContent).toContain(params.datasetsWithIssues) - expect(statsBoxes[2].textContent).toMatch(/datasets? needs? fixing/) + expect(statsBoxes[2].textContent).toContain('can be improved') }) it('The correct number of dataset cards are rendered with the correct titles in group "statutory"', () => { @@ -72,16 +73,23 @@ describe(`LPA Overview Page (seed: ${seed})`, () => { } }) - it('The correct number of dataset cards are rendered with the correct titles in group "other"', () => { - if (params.datasets.other && params.datasets.other.length > 0) { - datasetGroup({ expect }, 'other', params.datasets.other, document) + it('The correct number of dataset cards are rendered with the correct titles in group "expected"', () => { + if (params.datasets.expected && params.datasets.expected.length > 0) { + datasetGroup({ expect }, 'expected', params.datasets.expected, document) + } + }) - const infoText = document.querySelector('.org-membership-info').textContent - expect(infoText).match(new RegExp(`${params.organisation.name} is (a|not a) member of .*`)) + it('The correct number of dataset cards are rendered with the correct titles in group "prospective"', () => { + if (params.datasets.prospective && params.datasets.prospective.length > 0) { + datasetGroup({ expect }, 'prospective', params.datasets.prospective, document) } }) - const allDatasets = [].concat(...Object.values(params.datasets.statutory ?? []), ...Object.values(params.datasets.other ?? [])) + const allDatasets = [ + ...(params.datasets.statutory ?? []), + ...(params.datasets.expected ?? []), + ...(params.datasets.prospective ?? []) + ] allDatasets.forEach((dataset, i) => { it(`dataset cards are rendered with correct hints for dataset='${dataset.dataset}'`, () => { let expectedHint = 'Endpoint URL submitted' From f55c7137e1cd70bad83df6a33f22ff433a80a317 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 09:12:59 +0000 Subject: [PATCH 2/8] testing using the endpoint_dataset_issue_type_summary table - DO NOT USE --- src/middleware/common.middleware.js | 5 +++-- src/middleware/lpa-overview.middleware.js | 13 +++++++++++-- src/services/performanceDbApi.js | 6 ++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index fd736125b..b0a938d44 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -618,8 +618,7 @@ export const fetchEntryIssues = fetchMany({ export const fetchEntityIssueCounts = fetchMany({ query: ({ req }) => { const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : '' - - return ` + const sqlQuery = ` WITH unique_issues AS ( SELECT DISTINCT i.dataset, @@ -643,6 +642,8 @@ export const fetchEntityIssueCounts = fetchMany({ FROM unique_issues GROUP BY field, issue_type, dataset ` + console.log('sqlQuery', sqlQuery) + return sqlQuery }, result: 'entityIssueCounts' }) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index aa3261924..031039d52 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -26,6 +26,14 @@ const fetchDatasetErrorStatus = fetchMany({ dataset: FetchOptions.performanceDb }) +const fetchDatasetIssuePerformanceDb = fetchMany({ + query: ({ params, req }) => { + return performanceDbApi.endpointDatasetIssueQuery(params.lpa, req.resources) + }, + result: 'entityIssueCounts', + dataset: FetchOptions.performanceDb +}) + const fetchProvisions = fetchMany({ query: ({ params }) => { return /* sql */ `select dataset, project, provision_reason @@ -360,9 +368,10 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ export default [ fetchOrgInfo, fetchResources, - fetchDatasetErrorStatus, + //fetchDatasetErrorStatus, fetchEndpointSummary, - fetchEntityIssueCounts, + //fetchEntityIssueCounts, + fetchDatasetIssuePerformanceDb, fetchEntryIssueCounts, fetchEntityCounts, setAvailableDatasets, diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 27b5e496e..1e9914627 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -250,6 +250,12 @@ export default { AND pipeline = '${datasetId}'` }, + endpointDatasetIssueQuery(datasetId, resources) { + return /* sql */ ` + select dataset, field, issue_type, count_issues from endpoint_dataset_issue_type_summary + where organisation = '${datasetId}' and resource IN (${resources.map(resource => `'${resource.resource}'`).join(", ")})` + }, + datasetIssuesQuery, /** From 8811b89572aa7b740d706ee52377d674114fcd9b Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 09:13:13 +0000 Subject: [PATCH 3/8] lint --- src/middleware/lpa-overview.middleware.js | 21 ++++++++++----------- src/services/performanceDbApi.js | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 031039d52..23f62ceab 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -5,10 +5,9 @@ */ import performanceDbApi from '../services/performanceDbApi.js' -import { expectationFetcher, expectations, fetchEndpointSummary, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, logPageError, noop, setAvailableDatasets } from './common.middleware.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, logPageError, noop, setAvailableDatasets } from './common.middleware.js' import { fetchMany, FetchOptions, renderTemplate, fetchOneFromAllDatasets } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' -import config from '../../config/index.js' import _ from 'lodash' import logger from '../utils/logger.js' import { isFeatureEnabled } from '../utils/features.js' @@ -18,13 +17,13 @@ import { isFeatureEnabled } from '../utils/features.js' * * Fetches datasets which have active endpoints in error state. */ -const fetchDatasetErrorStatus = fetchMany({ - query: ({ params }) => { - return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) }) - }, - result: 'datasetErrorStatus', - dataset: FetchOptions.performanceDb -}) +// const fetchDatasetErrorStatus = fetchMany({ +// query: ({ params }) => { +// return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) }) +// }, +// result: 'datasetErrorStatus', +// dataset: FetchOptions.performanceDb +// }) const fetchDatasetIssuePerformanceDb = fetchMany({ query: ({ params, req }) => { @@ -368,9 +367,9 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ export default [ fetchOrgInfo, fetchResources, - //fetchDatasetErrorStatus, + // fetchDatasetErrorStatus, fetchEndpointSummary, - //fetchEntityIssueCounts, + // fetchEntityIssueCounts, fetchDatasetIssuePerformanceDb, fetchEntryIssueCounts, fetchEntityCounts, diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 1e9914627..dcc76c92f 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -250,10 +250,10 @@ export default { AND pipeline = '${datasetId}'` }, - endpointDatasetIssueQuery(datasetId, resources) { + endpointDatasetIssueQuery (datasetId, resources) { return /* sql */ ` select dataset, field, issue_type, count_issues from endpoint_dataset_issue_type_summary - where organisation = '${datasetId}' and resource IN (${resources.map(resource => `'${resource.resource}'`).join(", ")})` + where organisation = '${datasetId}' and resource IN (${resources.map(resource => `'${resource.resource}'`).join(', ')})` }, datasetIssuesQuery, From 2f6f189e30f640e526b14e23dca26e91f77b447d Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 11:36:07 +0000 Subject: [PATCH 4/8] correct use of endpoint_dataset_issue_type_summary and parallel for fast(er) lookup --- src/middleware/lpa-overview.middleware.js | 34 ++++++++++------------- src/services/performanceDbApi.js | 26 +++++++++++------ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 23f62ceab..96e21d3e7 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -6,28 +6,20 @@ import performanceDbApi from '../services/performanceDbApi.js' import { expectationFetcher, expectations, fetchEndpointSummary, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, logPageError, noop, setAvailableDatasets } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate, fetchOneFromAllDatasets } from './middleware.builders.js' +import { fetchMany, FetchOptions, renderTemplate, fetchOneFromAllDatasets, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' import logger from '../utils/logger.js' import { isFeatureEnabled } from '../utils/features.js' /** - * Middleware. Updates req with 'datasetErrorStatus'. + * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! * - * Fetches datasets which have active endpoints in error state. + * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb */ -// const fetchDatasetErrorStatus = fetchMany({ -// query: ({ params }) => { -// return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) }) -// }, -// result: 'datasetErrorStatus', -// dataset: FetchOptions.performanceDb -// }) - -const fetchDatasetIssuePerformanceDb = fetchMany({ - query: ({ params, req }) => { - return performanceDbApi.endpointDatasetIssueQuery(params.lpa, req.resources) +const fetchEntityIssueCountsPerformanceDb = fetchMany({ + query: ({ params }) => { + return performanceDbApi.fetchEntityIssueCounts(params.lpa) }, result: 'entityIssueCounts', dataset: FetchOptions.performanceDb @@ -365,14 +357,17 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ * Organisation (LPA) overview page middleware chain. */ export default [ + parallel([ fetchOrgInfo, fetchResources, - // fetchDatasetErrorStatus, fetchEndpointSummary, - // fetchEntityIssueCounts, - fetchDatasetIssuePerformanceDb, - fetchEntryIssueCounts, - fetchEntityCounts, + fetchEntityIssueCountsPerformanceDb, + fetchProvisions + ]), + parallel([ + fetchEntryIssueCounts, // needs fetchResources to complete + fetchEntityCounts // needs fetchOrgInfo to complete + ]), setAvailableDatasets, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, groupResourcesByDataset, @@ -383,7 +378,6 @@ export default [ // datasetSubmissionDeadlineCheck, // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) // addNoticesToDatasets, // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) - fetchProvisions, prepareOverviewTemplateParams, getOverview, logPageError diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index dcc76c92f..5fc18a773 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -17,7 +17,7 @@ const messages = new Map() /** * Reads messages from CSV files and populates `messages` map. */ -export async function initialiseMessages () { +export async function initialiseMessages() { // the csv parser works streams so we have wrap it with promises const fieldIssuesMessages = new Promise((resolve, reject) => { fs.createReadStream('src/content/fieldIssueMessages.csv').pipe(csv()).on('data', row => { @@ -154,7 +154,7 @@ const entityCountSelectFragment = (dataset, resource, entityCount) => `select '$ * @param {number=} params.entityCounts[].entityCount - Optional entity count * @returns {string} The generated SQL query */ -export function lpaOverviewQuery (lpa, params) { +export function lpaOverviewQuery(lpa, params) { let datasetClause = '' if (params.datasetsFilter) { const datasetString = params.datasetsFilter.map(dataset => `'${dataset}'`).join(',') @@ -250,10 +250,20 @@ export default { AND pipeline = '${datasetId}'` }, - endpointDatasetIssueQuery (datasetId, resources) { + // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb + fetchEntityIssueCounts(datasetId) { return /* sql */ ` - select dataset, field, issue_type, count_issues from endpoint_dataset_issue_type_summary - where organisation = '${datasetId}' and resource IN (${resources.map(resource => `'${resource.resource}'`).join(', ')})` + SELECT + dataset, + issue_type, + field, + COUNT(*) AS count + FROM endpoint_dataset_issue_type_summary + WHERE organisation = '${datasetId}' + AND severity = 'error' + AND responsibility = 'external' + AND (resource_end_date = '' OR resource_end_date IS NULL) + GROUP BY dataset, issue_type, field` }, datasetIssuesQuery, @@ -271,7 +281,7 @@ export default { * * @throws {Error} If the issue type is unknown */ - getTaskMessage ({ + getTaskMessage({ issue_type: issueType, num_issues: numIssues, rowCount, @@ -380,7 +390,7 @@ export default { * @param {string} resources[].dataset - Dataset identifier * @returns {Promise>} Array of objects containing resource, dataset and optional entityCount */ - async getEntityCounts (resources) { + async getEntityCounts(resources) { const requests = resources.map(({ resource, dataset }) => { const q = datasette.runQuery(this.entityCountQuery(resource), dataset) return q @@ -444,7 +454,7 @@ export default { * @param {string} dataset - Dataset identifier * @returns {Promise>} Array of entry objects containing field and value information */ - async getEntry (resourceId, entryNumber, dataset) { + async getEntry(resourceId, entryNumber, dataset) { logger.debug({ message: 'getEntry()', resourceId, entryNumber, dataset, type: types.App }) // TODO: why do we order by rowid? const sql = /* sql */ ` From ff65df4362a412c47e18ec56c8ed41014067d592 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 11:52:19 +0000 Subject: [PATCH 5/8] lint --- src/middleware/lpa-overview.middleware.js | 14 +++++++------- src/services/performanceDbApi.js | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 96e21d3e7..7d151463f 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -358,15 +358,15 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ */ export default [ parallel([ - fetchOrgInfo, - fetchResources, - fetchEndpointSummary, - fetchEntityIssueCountsPerformanceDb, - fetchProvisions + fetchOrgInfo, + fetchResources, + fetchEndpointSummary, + fetchEntityIssueCountsPerformanceDb, + fetchProvisions ]), parallel([ - fetchEntryIssueCounts, // needs fetchResources to complete - fetchEntityCounts // needs fetchOrgInfo to complete + fetchEntryIssueCounts, // needs fetchResources to complete + fetchEntityCounts // needs fetchOrgInfo to complete ]), setAvailableDatasets, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 5fc18a773..d786b7fdd 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -17,7 +17,7 @@ const messages = new Map() /** * Reads messages from CSV files and populates `messages` map. */ -export async function initialiseMessages() { +export async function initialiseMessages () { // the csv parser works streams so we have wrap it with promises const fieldIssuesMessages = new Promise((resolve, reject) => { fs.createReadStream('src/content/fieldIssueMessages.csv').pipe(csv()).on('data', row => { @@ -154,7 +154,7 @@ const entityCountSelectFragment = (dataset, resource, entityCount) => `select '$ * @param {number=} params.entityCounts[].entityCount - Optional entity count * @returns {string} The generated SQL query */ -export function lpaOverviewQuery(lpa, params) { +export function lpaOverviewQuery (lpa, params) { let datasetClause = '' if (params.datasetsFilter) { const datasetString = params.datasetsFilter.map(dataset => `'${dataset}'`).join(',') @@ -251,7 +251,7 @@ export default { }, // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb - fetchEntityIssueCounts(datasetId) { + fetchEntityIssueCounts (datasetId) { return /* sql */ ` SELECT dataset, @@ -281,7 +281,7 @@ export default { * * @throws {Error} If the issue type is unknown */ - getTaskMessage({ + getTaskMessage ({ issue_type: issueType, num_issues: numIssues, rowCount, @@ -390,7 +390,7 @@ export default { * @param {string} resources[].dataset - Dataset identifier * @returns {Promise>} Array of objects containing resource, dataset and optional entityCount */ - async getEntityCounts(resources) { + async getEntityCounts (resources) { const requests = resources.map(({ resource, dataset }) => { const q = datasette.runQuery(this.entityCountQuery(resource), dataset) return q @@ -454,7 +454,7 @@ export default { * @param {string} dataset - Dataset identifier * @returns {Promise>} Array of entry objects containing field and value information */ - async getEntry(resourceId, entryNumber, dataset) { + async getEntry (resourceId, entryNumber, dataset) { logger.debug({ message: 'getEntry()', resourceId, entryNumber, dataset, type: types.App }) // TODO: why do we order by rowid? const sql = /* sql */ ` From dda71b519adf9e2aa3afa0326739e4d8a1d667ca Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 15:34:15 +0000 Subject: [PATCH 6/8] change spacing and text for bottome prospective section on dashboard --- src/views/organisations/overview.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 18daa8506..6971c6e4e 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -148,7 +148,7 @@

Datasets {{ organisation.name }} must provide

Datasets {{ organisation.name}} are expected to provide

We expect you to provide these datasets as a condition of your funding with the Open Digital Planning community.

-
    +
      {% for dataset in datasets.expected %} {{ datasetItem(dataset) }} {% endfor %} @@ -159,7 +159,7 @@

      Datasets {{ organisation.name}} are expected to prov {% if datasets.prospective.length > 0 %}

      Datasets {{ organisation.name}} can provide

      -

      You are the authoritative source of these datasets. Providing this data to the platform ensures it is accurate and up to date.

      +

      Your organisation is the authoritative source of this data. Providing this data to the platform helps ensure it is accurate and up to date.

        {% for dataset in datasets.prospective %} {{ datasetItem(dataset) }} From d4390616de395dedd520df632737d1843828442a Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 16:19:37 +0000 Subject: [PATCH 7/8] acceptance test update --- package.json | 2 +- test/acceptance/dataset_overview.test.js | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 7c1cfd9d9..425a2433d 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "test:integration": "NODE_ENV=test playwright test --config ./test/integration/playwright.config.js", "test:acceptance": "NODE_ENV=development playwright test --config ./test/acceptance/playwright.config.js", "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js", - "test:acceptance:local": "NODE_ENV=local playwright test --config ./test/acceptance/playwright.config.js", + "test:acceptance:local": "NODE_ENV=local playwright test --config ./test/acceptance/playwright.config.js --ui", "playwright-codegen": "playwright codegen http://localhost:5000", "lint": "standard", "lint:fix": "standard --fix", diff --git a/test/acceptance/dataset_overview.test.js b/test/acceptance/dataset_overview.test.js index 0d80d4942..bdb402367 100644 --- a/test/acceptance/dataset_overview.test.js +++ b/test/acceptance/dataset_overview.test.js @@ -19,10 +19,10 @@ test.describe('Dataset overview', () => { expect(await page.locator('h1').innerText()).toEqual('London Borough of Lambeth overview') expect(await page.locator('[data-testid=datasetsMandatory] h2.govuk-heading-m').innerText()) .toEqual('Datasets London Borough of Lambeth must provide') - expect(await page.locator('[data-testid=datasetsMandatory] .govuk-task-list li').count()).toBeGreaterThanOrEqual(1) - expect(await page.locator('[data-testid=datasetsOdpMandatory] h2.govuk-heading-m').innerText()) - .toEqual('Datasets organisations in Open Digital Planning programme must provide') - expect(await page.locator('[data-testid=datasetsOdpMandatory] .govuk-task-list li').count()).toBeGreaterThanOrEqual(8) + expect(await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').count()).toBeGreaterThanOrEqual(1) + expect(await page.locator('[data-testid=datasetsExpected] h2.govuk-heading-m').innerText()) + .toEqual('Datasets London Borough of Lambeth are expected to provide') + expect(await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').count()).toBeGreaterThanOrEqual(8) }) test('Can view dataset overview', async ({ page, context }) => { @@ -99,7 +99,7 @@ test.describe('Dataset overview', () => { await organisationOverviewPage.navigateHere() expect(await page.locator('h1').innerText()).toEqual('London Borough of Lambeth overview') - const datasetElements = await page.locator('[data-testid=datasetsOdpMandatory] .govuk-task-list li').allInnerTexts() + const datasetElements = await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').allInnerTexts() const displayedNames = datasetElements .map(text => text.split('\n')[0].trim()) From 38f71e7570a6026e3709e07812cd145dde8fcc57 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 12 Nov 2025 16:49:10 +0000 Subject: [PATCH 8/8] remove console log --- src/middleware/common.middleware.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index b0a938d44..0ef111ee3 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -618,7 +618,7 @@ export const fetchEntryIssues = fetchMany({ export const fetchEntityIssueCounts = fetchMany({ query: ({ req }) => { const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : '' - const sqlQuery = ` + return ` WITH unique_issues AS ( SELECT DISTINCT i.dataset, @@ -642,8 +642,6 @@ export const fetchEntityIssueCounts = fetchMany({ FROM unique_issues GROUP BY field, issue_type, dataset ` - console.log('sqlQuery', sqlQuery) - return sqlQuery }, result: 'entityIssueCounts' })