diff --git a/package.json b/package.json index 7c1cfd9d9..425a2433d 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "test:integration": "NODE_ENV=test playwright test --config ./test/integration/playwright.config.js", "test:acceptance": "NODE_ENV=development playwright test --config ./test/acceptance/playwright.config.js", "test:acceptance:ci": "NODE_ENV=ci playwright test --config ./test/acceptance/playwright.config.js", - "test:acceptance:local": "NODE_ENV=local playwright test --config ./test/acceptance/playwright.config.js", + "test:acceptance:local": "NODE_ENV=local playwright test --config ./test/acceptance/playwright.config.js --ui", "playwright-codegen": "playwright codegen http://localhost:5000", "lint": "standard", "lint:fix": "standard --fix", diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index fd736125b..0ef111ee3 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -618,7 +618,6 @@ export const fetchEntryIssues = fetchMany({ export const fetchEntityIssueCounts = fetchMany({ query: ({ req }) => { const datasetClause = req.params.dataset ? `AND i.dataset = '${req.params.dataset}'` : '' - return ` WITH unique_issues AS ( SELECT DISTINCT diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index d5b4ffcd1..7d151463f 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -5,32 +5,30 @@ */ import performanceDbApi from '../services/performanceDbApi.js' -import { expectationFetcher, expectations, fetchEndpointSummary, fetchEntityIssueCounts, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, logPageError, noop, setAvailableDatasets } from './common.middleware.js' -import { fetchMany, FetchOptions, renderTemplate, fetchOneFromAllDatasets } from './middleware.builders.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchEntryIssueCounts, fetchOrgInfo, fetchResources, logPageError, noop, setAvailableDatasets } from './common.middleware.js' +import { fetchMany, FetchOptions, renderTemplate, fetchOneFromAllDatasets, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' -import config from '../../config/index.js' import _ from 'lodash' import logger from '../utils/logger.js' import { isFeatureEnabled } from '../utils/features.js' /** - * Middleware. Updates req with 'datasetErrorStatus'. + * Middleware. Updates req with 'entityIssueCounts' same as fetchEntityIssueCounts so not to be used together! * - * Fetches datasets which have active endpoints in error state. + * Functionally equivalent (for the utilization of the LPA Dashboard) to fetchEntityIssueCounts but using performanceDb */ -const fetchDatasetErrorStatus = fetchMany({ +const fetchEntityIssueCountsPerformanceDb = fetchMany({ query: ({ params }) => { - return performanceDbApi.datasetErrorStatusQuery(params.lpa, { datasetsFilter: Object.keys(config.datasetsConfig) }) + return performanceDbApi.fetchEntityIssueCounts(params.lpa) }, - result: 'datasetErrorStatus', + result: 'entityIssueCounts', dataset: FetchOptions.performanceDb }) const fetchProvisions = fetchMany({ query: ({ params }) => { - const excludeDatasets = Object.keys(config.datasetsConfig).map(dataset => `'${dataset}'`).join(',') return /* sql */ `select dataset, project, provision_reason - from provision where organisation = '${params.lpa}' and dataset in (${excludeDatasets})` + from provision where organisation = '${params.lpa}'` }, result: 'provisions' }) @@ -285,6 +283,10 @@ export function prepareOverviewTemplateParams (req, res, next) { switch (reason) { case 'statutory': return 'statutory' + case 'expected': + return 'expected' + case 'prospective': + return 'prospective' default: return 'other' } @@ -355,13 +357,17 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ * Organisation (LPA) overview page middleware chain. */ export default [ - fetchOrgInfo, - fetchResources, - fetchDatasetErrorStatus, - fetchEndpointSummary, - fetchEntityIssueCounts, - fetchEntryIssueCounts, - fetchEntityCounts, + parallel([ + fetchOrgInfo, + fetchResources, + fetchEndpointSummary, + fetchEntityIssueCountsPerformanceDb, + fetchProvisions + ]), + parallel([ + fetchEntryIssueCounts, // needs fetchResources to complete + fetchEntityCounts // needs fetchOrgInfo to complete + ]), setAvailableDatasets, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, groupResourcesByDataset, @@ -372,7 +378,6 @@ export default [ // datasetSubmissionDeadlineCheck, // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) // addNoticesToDatasets, // commented out as the logic is currently incorrect (https://github.com/digital-land/submit/issues/824) - fetchProvisions, prepareOverviewTemplateParams, getOverview, logPageError diff --git a/src/routes/schemas.js b/src/routes/schemas.js index 9f7a165b6..a9c4a4d94 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -136,6 +136,8 @@ export const OrgOverviewPage = v.strictObject({ organisation: OrgField, datasets: v.object({ statutory: v.optional(v.array(DatasetItem)), + expected: v.optional(v.array(DatasetItem)), + prospective: v.optional(v.array(DatasetItem)), other: v.optional(v.array(DatasetItem)) }), totalDatasets: v.integer(), diff --git a/src/services/performanceDbApi.js b/src/services/performanceDbApi.js index 27b5e496e..d786b7fdd 100644 --- a/src/services/performanceDbApi.js +++ b/src/services/performanceDbApi.js @@ -250,6 +250,22 @@ export default { AND pipeline = '${datasetId}'` }, + // Query to simulate fetchEntityIssueCounts in common.middleware.js but against performanceDb + fetchEntityIssueCounts (datasetId) { + return /* sql */ ` + SELECT + dataset, + issue_type, + field, + COUNT(*) AS count + FROM endpoint_dataset_issue_type_summary + WHERE organisation = '${datasetId}' + AND severity = 'error' + AND responsibility = 'external' + AND (resource_end_date = '' OR resource_end_date IS NULL) + GROUP BY dataset, issue_type, field` + }, + datasetIssuesQuery, /** diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 3c2f4e682..6971c6e4e 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -91,11 +91,17 @@

{% endif %} {% endfor %} - {% for dataset in datasets.other %} - {% if dataset.notice %} - {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} - {% endif %} - {% endfor %} + {% for dataset in datasets.expected %} + {% if dataset.notice %} + {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} + {% endif %} + {% endfor %} + + {% for dataset in datasets.prospective %} + {% if dataset.notice %} + {{ deadlineNotice(dataset.dataset, dataset.notice, organisation) }} + {% endif %} + {% endfor %} @@ -107,17 +113,17 @@

{{datasetsWithEndpoints}}/{{totalDatasets}} - datasets submitted + authoritative dataset{{ "s" if (datasetsWithEndpoints != 1) }} provided
{{datasetsWithErrors}} - dataset{{ "s" if (datasetsWithErrors != 1) }} with endpoint errors + error{{ "s" if (datasetsWithErrors != 1) }} accessing URLs
{{datasetsWithIssues}} - {{ "dataset" | pluralise(datasetsWithIssues) }} need{{ "s" if (datasetsWithIssues == 1) }} fixing + {{ "dataset" | pluralise(datasetsWithIssues) }} can be improved
@@ -126,10 +132,10 @@

- {% if datasets.statutory.length > 0 %}

Datasets {{ organisation.name }} must provide

+

You are required to make these datasets available under legislation. Using the Planning Data Platform to do this can help you to create and maintain authoritative, trustworthy data.

    {% for dataset in datasets.statutory %} {{ datasetItem(dataset) }} @@ -138,22 +144,29 @@

    Datasets {{ organisation.name }} must provide

{% endif %} - {% if datasets.other.length > 0 %} -
-

Datasets organisations in Open Digital Planning programme must provide

- {% if isODPMember %} -

{{ organisation.name}} is a member of the Open Digital Planning programme.

- {% else %} -

{{ organisation.name}} is not a member of the Open Digital Planning programme.

- {% endif %} -
    - {% for dataset in datasets.other %} + {% if datasets.expected.length > 0 and isODPMember %} +
    +

    Datasets {{ organisation.name}} are expected to provide

    +

    We expect you to provide these datasets as a condition of your funding with the Open Digital Planning community.

    +
      + {% for dataset in datasets.expected %} {{ datasetItem(dataset) }} {% endfor %}
    {% endif %} + {% if datasets.prospective.length > 0 %} +
    +

    Datasets {{ organisation.name}} can provide

    +

    Your organisation is the authoritative source of this data. Providing this data to the platform helps ensure it is accurate and up to date.

    +
      + {% for dataset in datasets.prospective %} + {{ datasetItem(dataset) }} + {% endfor %} +
    +
    + {% endif %}
diff --git a/test/acceptance/dataset_overview.test.js b/test/acceptance/dataset_overview.test.js index 0d80d4942..bdb402367 100644 --- a/test/acceptance/dataset_overview.test.js +++ b/test/acceptance/dataset_overview.test.js @@ -19,10 +19,10 @@ test.describe('Dataset overview', () => { expect(await page.locator('h1').innerText()).toEqual('London Borough of Lambeth overview') expect(await page.locator('[data-testid=datasetsMandatory] h2.govuk-heading-m').innerText()) .toEqual('Datasets London Borough of Lambeth must provide') - expect(await page.locator('[data-testid=datasetsMandatory] .govuk-task-list li').count()).toBeGreaterThanOrEqual(1) - expect(await page.locator('[data-testid=datasetsOdpMandatory] h2.govuk-heading-m').innerText()) - .toEqual('Datasets organisations in Open Digital Planning programme must provide') - expect(await page.locator('[data-testid=datasetsOdpMandatory] .govuk-task-list li').count()).toBeGreaterThanOrEqual(8) + expect(await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').count()).toBeGreaterThanOrEqual(1) + expect(await page.locator('[data-testid=datasetsExpected] h2.govuk-heading-m').innerText()) + .toEqual('Datasets London Borough of Lambeth are expected to provide') + expect(await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').count()).toBeGreaterThanOrEqual(8) }) test('Can view dataset overview', async ({ page, context }) => { @@ -99,7 +99,7 @@ test.describe('Dataset overview', () => { await organisationOverviewPage.navigateHere() expect(await page.locator('h1').innerText()).toEqual('London Borough of Lambeth overview') - const datasetElements = await page.locator('[data-testid=datasetsOdpMandatory] .govuk-task-list li').allInnerTexts() + const datasetElements = await page.locator('[data-testid=datasetsExpected] .govuk-task-list li').allInnerTexts() const displayedNames = datasetElements .map(text => text.split('\n')[0].trim()) diff --git a/test/unit/middleware/lpa-overview.middleware.test.js b/test/unit/middleware/lpa-overview.middleware.test.js index 8514047ea..a0d3309b8 100644 --- a/test/unit/middleware/lpa-overview.middleware.test.js +++ b/test/unit/middleware/lpa-overview.middleware.test.js @@ -96,7 +96,7 @@ describe('lpa-overview.middleware', () => { { endpointCount: 1, status: 'Live', dataset: 'dataset1', error: undefined, issueCount: 0, endpointErrorCount: 0 }, { endpointCount: 1, status: 'Error', dataset: 'dataset3', error: undefined, issueCount: 0, endpointErrorCount: 1 } ]), - other: expect.arrayContaining([ + expected: expect.arrayContaining([ { endpointCount: 0, status: 'Needs fixing', dataset: 'dataset2', error: undefined, issueCount: 1, endpointErrorCount: 0 }, { endpointCount: 0, status: 'Error', dataset: 'dataset4', error: 'There was a 404 error', issueCount: 0, endpointErrorCount: 1 } ]) @@ -114,17 +114,23 @@ describe('lpa-overview.middleware', () => { expect(errorCardNodes[0].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was an error accessing the endpoint URL') expect(errorCardNodes[1].querySelector('.govuk-task-list__hint').textContent.trim()).toBe('There was a 404 error') - const orgMemebershipInfo = doc.querySelector('.org-membership-info').textContent.trim() - expect(orgMemebershipInfo).toMatch('is a member of the Open Digital Planning programme') + // Check for ODP membership info in the expected datasets section + const expectedSection = doc.querySelector('[data-testid="datasetsExpected"]') + if (expectedSection) { + const orgMembershipInfo = expectedSection.querySelector('.org-membership-info') + expect(orgMembershipInfo.textContent.trim()).toMatch(/Open Digital Planning/) + } - // verify proper label for non-OPD memebers gets rendered + // verify proper label for non-ODP members gets rendered const reqNotMember = structuredClone(reqTemplate) reqNotMember.provisions.forEach((provision) => { provision.project = '' }) prepareOverviewTemplateParams(reqNotMember, res, () => { }) const { doc: docNotMember } = getRenderedErrorCards(reqNotMember.templateParams) - expect(docNotMember.querySelector('.org-membership-info').textContent.trim()).toMatch('is not a member of the Open Digital Planning programme') + // When not an ODP member, expected datasets won't render (requires isODPMember), so datasetsExpected won't be present + const expectedSectionNotMember = docNotMember.querySelector('[data-testid="datasetsExpected"]') + expect(expectedSectionNotMember).toBeNull() }) it('should patch dataset status based on the provision_summary info', () => { @@ -139,7 +145,7 @@ describe('lpa-overview.middleware', () => { expect(ds1.status).toBe('Live') expect(ds1.error).toBeUndefined() - const ds4 = req.templateParams.datasets.other[1] + const ds4 = req.templateParams.datasets.expected[1] expect(ds4.status).toBe('Error') expect(ds4.error).toBe(req.datasets[3].error) // Error message should be left untouched }) diff --git a/test/unit/views/organisations/lpaOverviewPage.test.js b/test/unit/views/organisations/lpaOverviewPage.test.js index 652f768b9..22690eadc 100644 --- a/test/unit/views/organisations/lpaOverviewPage.test.js +++ b/test/unit/views/organisations/lpaOverviewPage.test.js @@ -38,7 +38,7 @@ const datasetGroup = ({ expect }, key, datasets, document) => { describe(`LPA Overview Page (seed: ${seed})`, () => { const params = mocker(OrgOverviewPage, seed) - console.debug(`mocked datasets: statutory = ${params.datasets.statutory?.length ?? 'none'}, other = ${params.datasets.other?.length ?? 'none'}`) + console.debug(`mocked datasets: statutory = ${params.datasets.statutory?.length ?? 'none'}, expected = ${params.datasets.expected?.length ?? 'none'}, prospective = ${params.datasets.prospective?.length ?? 'none'}`) const html = nunjucks.render('organisations/overview.html', params) @@ -53,17 +53,18 @@ describe(`LPA Overview Page (seed: ${seed})`, () => { const statsBoxes = document.querySelector('.dataset-status').children it('Datasets provided gives the correct value', () => { expect(statsBoxes[0].textContent).toContain(`${params.datasetsWithEndpoints}/${params.totalDatasets}`) - expect(statsBoxes[0].textContent).toContain('datasets submitted') + expect(statsBoxes[0].textContent).toContain('authoritative dataset') + expect(statsBoxes[0].textContent).toContain('provided') }) it('Datasets with errors gives the correct value', () => { expect(statsBoxes[1].textContent).toContain(params.datasetsWithErrors) - expect(statsBoxes[1].textContent).toMatch(/datasets? with endpoint errors/) + expect(statsBoxes[1].textContent).toContain('accessing URLs') }) it('Datasets with issues gives the correct value', () => { expect(statsBoxes[2].textContent).toContain(params.datasetsWithIssues) - expect(statsBoxes[2].textContent).toMatch(/datasets? needs? fixing/) + expect(statsBoxes[2].textContent).toContain('can be improved') }) it('The correct number of dataset cards are rendered with the correct titles in group "statutory"', () => { @@ -72,16 +73,23 @@ describe(`LPA Overview Page (seed: ${seed})`, () => { } }) - it('The correct number of dataset cards are rendered with the correct titles in group "other"', () => { - if (params.datasets.other && params.datasets.other.length > 0) { - datasetGroup({ expect }, 'other', params.datasets.other, document) + it('The correct number of dataset cards are rendered with the correct titles in group "expected"', () => { + if (params.datasets.expected && params.datasets.expected.length > 0) { + datasetGroup({ expect }, 'expected', params.datasets.expected, document) + } + }) - const infoText = document.querySelector('.org-membership-info').textContent - expect(infoText).match(new RegExp(`${params.organisation.name} is (a|not a) member of .*`)) + it('The correct number of dataset cards are rendered with the correct titles in group "prospective"', () => { + if (params.datasets.prospective && params.datasets.prospective.length > 0) { + datasetGroup({ expect }, 'prospective', params.datasets.prospective, document) } }) - const allDatasets = [].concat(...Object.values(params.datasets.statutory ?? []), ...Object.values(params.datasets.other ?? [])) + const allDatasets = [ + ...(params.datasets.statutory ?? []), + ...(params.datasets.expected ?? []), + ...(params.datasets.prospective ?? []) + ] allDatasets.forEach((dataset, i) => { it(`dataset cards are rendered with correct hints for dataset='${dataset.dataset}'`, () => { let expectedHint = 'Endpoint URL submitted'