From 8b6b4cf8e4c65b6e96a29527df29bd5de464a83d Mon Sep 17 00:00:00 2001 From: Feba Date: Thu, 26 Jun 2025 15:31:29 +0100 Subject: [PATCH 1/7] update endpoint status check to only show error if all endpoints fail --- src/middleware/lpa-overview.middleware.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 78c4cf662..fda89dca7 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -205,7 +205,6 @@ export const addNoticesToDatasets = (req, res, next) => { */ export function prepareDatasetObjects (req, res, next) { const { issues, endpoints, expectationOutOfBounds, availableDatasets } = req - const outOfBoundsViolations = new Set((expectationOutOfBounds ?? []).map(o => o.dataset)) req.datasets = availableDatasets.map((dataset) => { const datasetEndpoints = endpoints[dataset] @@ -216,7 +215,10 @@ export function prepareDatasetObjects (req, res, next) { } const endpointCount = datasetEndpoints.length - const httpStatus = datasetEndpoints.find(endpoint => endpoint.latest_status !== '200')?.latest_status + // const httpStatus = datasetEndpoints.find(endpoint => endpoint.latest_status !== '200')?.latest_status + // const error = httpStatus !== undefined ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined + const allError = datasetEndpoints.every(endpoint => endpoint.latest_status !== '200') + const httpStatus = allError ? datasetEndpoints[0]?.latest_status : undefined const error = httpStatus !== undefined ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined const expectationFailed = outOfBoundsViolations.has(dataset) const issueCount = (datasetIssues?.length || 0) + (expectationFailed ? 1 : 0) @@ -351,7 +353,6 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ expectation: expectations.entitiesOutOfBounds, result: 'expectationOutOfBounds' }) - /** * Organisation (LPA) overview page middleware chain. */ From 7d0da94d8bf1e3ccab6459ae1ffddf34f35f52dd Mon Sep 17 00:00:00 2001 From: Feba Date: Mon, 30 Jun 2025 15:11:18 +0100 Subject: [PATCH 2/7] added endpoint_entry_date in dataset details page and updated endpoint status logic for dashboard display --- src/middleware/datasetOverview.middleware.js | 49 ++++++++-------- src/middleware/lpa-overview.middleware.js | 10 ++-- src/routes/schemas.js | 1 + src/views/organisations/dataset-overview.html | 10 ++++ .../datasetOverview.middleware.test.js | 4 +- .../lpa-overview.middleware.test.js | 57 +++++++++++++++++-- .../organisations/dataset-overview.test.js | 14 +++-- 7 files changed, 105 insertions(+), 40 deletions(-) diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index 3e5811c60..e3ca95a5a 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -201,31 +201,34 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { const numberOfExpectedFields = specFields.length let endpointErrorIssues = 0 - const endpoints = sources.sort((a, b) => { - if (a.status && a.status >= 200 && a.status < 300) return -1 - if (b.status && b.status >= 200 && b.status < 300) return 1 - return 0 - }).map((source, index) => { - let error - - if (!source.status || source.status < 200 || source.status >= 300) { - error = { - code: source.status, - exception: source.exception + // const endpoints = sources.sort((a, b) => { + // if (a.status && a.status >= 200 && a.status < 300) return -1 + // if (b.status && b.status >= 200 && b.status < 300) return 1 + // return 0 + const endpoints = sources.slice() + .sort((a, b) => new Date(b.endpoint_entry_date) - new Date(a.endpoint_entry_date)) + .map((source, index) => { + let error + + if (!source.status || source.status < 200 || source.status >= 300) { + error = { + code: source.status, + exception: source.exception + } + endpointErrorIssues += 1 } - endpointErrorIssues += 1 - } - return { - name: `Endpoint URL ${index}`, - endpoint: source.endpoint, - endpoint_url: source.endpoint_url, - documentation_url: source.documentation_url, - lastAccessed: source.latest_log_entry_date, - lastUpdated: source.resource_start_date, // as in: when was the _resource_ updated, not data under that resource - error - } - }) + return { + name: `Endpoint URL ${index}`, + endpoint: source.endpoint, + endpoint_url: source.endpoint_url, + documentation_url: source.documentation_url, + lastAccessed: source.latest_log_entry_date, + lastUpdated: source.resource_start_date, // as in: when was the _resource_ updated, not data under that resource + entryDate: source.endpoint_entry_date, + error + } + }) const taskCount = (entryIssueCounts ? entryIssueCounts.length : 0) + (entityIssueCounts ? entityIssueCounts.length : 0) + diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index fda89dca7..489e545ea 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -209,24 +209,22 @@ export function prepareDatasetObjects (req, res, next) { req.datasets = availableDatasets.map((dataset) => { const datasetEndpoints = endpoints[dataset] const datasetIssues = issues[dataset] - if (!datasetEndpoints) { return { status: 'Not submitted', endpointCount: 0, dataset } } const endpointCount = datasetEndpoints.length - // const httpStatus = datasetEndpoints.find(endpoint => endpoint.latest_status !== '200')?.latest_status - // const error = httpStatus !== undefined ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined const allError = datasetEndpoints.every(endpoint => endpoint.latest_status !== '200') + const someError = datasetEndpoints.some(endpoint => endpoint.latest_status !== '200') const httpStatus = allError ? datasetEndpoints[0]?.latest_status : undefined - const error = httpStatus !== undefined ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined + const error = allError ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined const expectationFailed = outOfBoundsViolations.has(dataset) const issueCount = (datasetIssues?.length || 0) + (expectationFailed ? 1 : 0) let status - if (error) { + if (allError) { status = 'Error' - } else if (issueCount > 0) { + } else if (someError || issueCount > 0) { status = 'Needs fixing' } else { status = 'Live' diff --git a/src/routes/schemas.js b/src/routes/schemas.js index 54feb61ca..e274dcb07 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -169,6 +169,7 @@ export const OrgDatasetOverview = v.strictObject({ endpoint: NonEmptyString, lastAccessed: v.string(), lastUpdated: v.nullable(v.string()), + entryDate: v.optional(v.nullable(v.string())), error: v.optional(v.strictObject({ code: v.integer(), exception: v.string() diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index c3de02c49..a47b79954 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -165,6 +165,16 @@ } %} {{ rows.push(lastUpdatedRow) }} + {% set entryDateRow = { + key: { + text: 'Endpoint entry date' + }, + value: { + text: endpoint.entryDate and ( endpoint.entryDate | govukDateTime ) + } + } %} + {{ rows.push(entryDateRow) }} + {% endfor %} {% block content %} diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js index e6569c036..4d0d6500f 100644 --- a/test/unit/middleware/datasetOverview.middleware.test.js +++ b/test/unit/middleware/datasetOverview.middleware.test.js @@ -53,8 +53,8 @@ describe('Dataset Overview Middleware', () => { numberOfExpectedFields: 2, numberOfRecords: 10, endpoints: [ - { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01' }, - { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02' } + { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01', entryDate: 'LU1' }, + { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02', entryDate: 'LU2' } ] }, notice: undefined diff --git a/test/unit/middleware/lpa-overview.middleware.test.js b/test/unit/middleware/lpa-overview.middleware.test.js index dcdfb52e1..a0617a891 100644 --- a/test/unit/middleware/lpa-overview.middleware.test.js +++ b/test/unit/middleware/lpa-overview.middleware.test.js @@ -82,7 +82,7 @@ describe('lpa-overview.middleware', () => { const req = structuredClone(reqTemplate) const res = { render: vi.fn() } - prepareOverviewTemplateParams(req, res, () => {}) + prepareOverviewTemplateParams(req, res, () => { }) const expectedTemplateParams = { organisation: { name: 'Example LPA', organisation: 'LPA' }, @@ -117,7 +117,7 @@ describe('lpa-overview.middleware', () => { reqNotMember.provisions.forEach((provision) => { provision.project = '' }) - prepareOverviewTemplateParams(reqNotMember, res, () => {}) + prepareOverviewTemplateParams(reqNotMember, res, () => { }) const { doc: docNotMember } = getRenderedErrorCards(reqNotMember.templateParams) expect(docNotMember.querySelector('.org-membership-info').textContent.trim()).toMatch('is not a member of the Open Digital Planning programme') }) @@ -128,7 +128,7 @@ describe('lpa-overview.middleware', () => { req.datasetErrorStatus = [{ dataset: 'dataset1' }] const res = { render: vi.fn() } - prepareOverviewTemplateParams(req, res, () => {}) + prepareOverviewTemplateParams(req, res, () => { }) const ds1 = req.templateParams.datasets.statutory[0] expect(ds1.status).toBe('Live') @@ -152,12 +152,61 @@ describe('lpa-overview.middleware', () => { req.expectationOutOfBounds = [{ passed: false, dataset: 'datasetA' }] const res = { render: vi.fn() } - prepareDatasetObjects(req, res, () => {}) + prepareDatasetObjects(req, res, () => { }) expect(req.datasets[0].error).toBeUndefined() expect(req.datasets[0].issueCount).toBe(1) expect(req.datasets[0].status).toBe('Needs fixing') }) + it('should not show an error if atleast one endpoint is 200', () => { + const req = { + expectationOutOfBounds: [], + issues: {}, + endpoints: { datasetA: [{ latest_status: '200' }, { latest_status: '504' }, { latest_status: '504' }] }, + availableDatasets: ['datasetA'], + datasets: undefined + } + const res = { render: vi.fn() } + + prepareDatasetObjects(req, res, () => { }) + + expect(req.datasets[0].error).toBeUndefined() + expect(req.datasets[0].status).toBe('Needs fixing') + }) + it('should show an error all endpoints have status !== 200', () => { + const req = { + expectationOutOfBounds: [], + issues: {}, + endpoints: { datasetA: [{ latest_status: '504' }, { latest_status: '504' }] }, + availableDatasets: ['datasetA'], + datasets: undefined + } + const res = { render: vi.fn() } + + prepareDatasetObjects(req, res, () => { }) + + expect(req.datasets[0].error).toContain('504') + expect(req.datasets[0].status).toBe('Error') + }) + it('should show live if all endpoint status == 200', () => { + const req = { + expectationOutOfBounds: [], + issues: {}, + endpoints: { + datasetA: [{ latest_status: '200' }, + { latest_status: '200' }, + { latest_status: '200' }] + }, + availableDatasets: ['datasetA'], + datasets: undefined + } + const res = { render: vi.fn() } + + prepareDatasetObjects(req, res, () => { }) + + expect(req.datasets[0].error).toBeUndefined() + expect(req.datasets[0].status).toBe('Live') + }) }) describe('getOverview', () => { diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js index 1066cf6f8..983ad556f 100644 --- a/test/unit/views/organisations/dataset-overview.test.js +++ b/test/unit/views/organisations/dataset-overview.test.js @@ -30,7 +30,8 @@ describe('Dataset Overview Page', () => { endpoint_url: 'http://endpoint1.co.uk', documentation_url: 'http://endpoint1-docs.co.uk', lastAccessed: '2024-09-09', - lastUpdated: '2024-09-09' + lastUpdated: '2024-09-09', + entryDate: '2025-03-08' }, { name: 'endpoint 2', @@ -38,6 +39,7 @@ describe('Dataset Overview Page', () => { endpoint_url: 'http://endpoint2.co.uk', lastAccessed: '2024-19-19', lastUpdated: '2024-19-19', + entryDate: '2024-12-12', error: { code: 404, exception: '' @@ -82,10 +84,12 @@ describe('Dataset Overview Page', () => { expect(summaryListValues[5].textContent).toContain(params.stats.endpoints[0].documentation_url) expect(summaryListValues[6].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastAccessed)) expect(summaryListValues[7].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastUpdated)) - expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].endpoint_url) - expect(summaryListValues[9].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastAccessed)) - expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].error.code) - expect(summaryListValues[10].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated)) + expect(summaryListValues[8].textContent).toContain(govukDateTime(params.stats.endpoints[0].entryDate)) + expect(summaryListValues[9].textContent).toContain(params.stats.endpoints[1].endpoint_url) + expect(summaryListValues[10].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastAccessed)) + expect(summaryListValues[10].textContent).toContain(params.stats.endpoints[1].error.code) + expect(summaryListValues[11].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated)) + expect(summaryListValues[12].textContent).toContain(govukDateTime(params.stats.endpoints[1].entryDate)) }) it('Renders breadcrumbs correctly', () => { From 4c22e6ab7babaa5f1edf33a35c18f2e875a25bc2 Mon Sep 17 00:00:00 2001 From: Feba Date: Mon, 30 Jun 2025 16:28:19 +0100 Subject: [PATCH 3/7] Use console.log in template to test --- src/views/organisations/overview.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 7c120fbe9..3ccc6804f 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -11,6 +11,14 @@ {% macro datasetItem(dataset) %} +

  • From 8932d2c3affbdd5a8d01c5d9b9e98322fd691b86 Mon Sep 17 00:00:00 2001 From: Feba Date: Mon, 30 Jun 2025 16:46:22 +0100 Subject: [PATCH 4/7] removed console.log from template --- src/views/organisations/overview.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index 3ccc6804f..7c120fbe9 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -11,14 +11,6 @@ {% macro datasetItem(dataset) %} -

  • From 26c77b111e714b5f3c3aca98f0e940ea412eccc4 Mon Sep 17 00:00:00 2001 From: Feba Date: Mon, 30 Jun 2025 17:16:37 +0100 Subject: [PATCH 5/7] remove commented code --- src/middleware/datasetOverview.middleware.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index e3ca95a5a..950074abe 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -201,10 +201,6 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { const numberOfExpectedFields = specFields.length let endpointErrorIssues = 0 - // const endpoints = sources.sort((a, b) => { - // if (a.status && a.status >= 200 && a.status < 300) return -1 - // if (b.status && b.status >= 200 && b.status < 300) return 1 - // return 0 const endpoints = sources.slice() .sort((a, b) => new Date(b.endpoint_entry_date) - new Date(a.endpoint_entry_date)) .map((source, index) => { From 5cb7979935af6ef368d747ed4674a3b4acbb4fb3 Mon Sep 17 00:00:00 2001 From: Feba Date: Tue, 1 Jul 2025 11:02:36 +0100 Subject: [PATCH 6/7] lint fix --- test/unit/views/organisations/dataset-overview.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/views/organisations/dataset-overview.test.js b/test/unit/views/organisations/dataset-overview.test.js index c31d9b581..955ebe1e5 100644 --- a/test/unit/views/organisations/dataset-overview.test.js +++ b/test/unit/views/organisations/dataset-overview.test.js @@ -86,7 +86,6 @@ describe('Dataset Overview Page', () => { expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].error.code) expect(summaryListValues[9].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated)) expect(summaryListValues[10].textContent).toContain(govukDateTime(params.stats.endpoints[1].entryDate)) - }) it('Renders breadcrumbs correctly', () => { From 1ca1c9efe4610ca94bf902157f64007cf6c16222 Mon Sep 17 00:00:00 2001 From: Feba Date: Wed, 2 Jul 2025 14:00:56 +0100 Subject: [PATCH 7/7] fix/valid entryDate format --- src/middleware/datasetOverview.middleware.js | 2 +- test/unit/middleware/datasetOverview.middleware.test.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index 0eeb29ea5..60e7b2205 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -187,7 +187,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { const { orgInfo, entityCount, sources, dataset, entryIssueCounts, entityIssueCounts, notice, expectationOutOfBounds = [] } = req let endpointErrorIssues = 0 - const endpoints = sources.slice() + const endpoints = sources .sort((a, b) => new Date(b.endpoint_entry_date) - new Date(a.endpoint_entry_date)) .map((source, index) => { let error diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js index fc4ac56fa..cd0011828 100644 --- a/test/unit/middleware/datasetOverview.middleware.test.js +++ b/test/unit/middleware/datasetOverview.middleware.test.js @@ -27,8 +27,8 @@ describe('Dataset Overview Middleware', () => { entityCount: { entity_count: 10 }, expectationOutOfBounds: [{ dataset: req.params.dataset, passed: 'False' }], sources: [ - { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: 'LU1', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' }, - { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: 'LU2', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' } + { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: '2024-02-01', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' }, + { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: '2023-01-01', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' } ], entryIssueCounts: [ { @@ -50,8 +50,8 @@ describe('Dataset Overview Middleware', () => { stats: { numberOfRecords: 10, endpoints: [ - { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01', entryDate: 'LU1' }, - { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02', entryDate: 'LU2' } + { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01', entryDate: '2024-02-01' }, + { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02', entryDate: '2023-01-01' } ] }, notice: undefined