update endpoint status check to only show error if all endpoints fail

src/middleware/datasetOverview.middleware.js

@@ -187,31 +187,30 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
   const { orgInfo, entityCount, sources, dataset, entryIssueCounts, entityIssueCounts, notice, expectationOutOfBounds = [] } = req
 
   let endpointErrorIssues = 0
-  const endpoints = sources.sort((a, b) => {
-    if (a.status && a.status >= 200 && a.status < 300) return -1
-    if (b.status && b.status >= 200 && b.status < 300) return 1
-    return 0
-  }).map((source, index) => {
-    let error
-
-    if (!source.status || source.status < 200 || source.status >= 300) {
-      error = {
-        code: source.status,
-        exception: source.exception
+  const endpoints = sources
+    .sort((a, b) => new Date(b.endpoint_entry_date) - new Date(a.endpoint_entry_date))
+    .map((source, index) => {
+      let error
+
+      if (!source.status || source.status < 200 || source.status >= 300) {
+        error = {
+          code: source.status,
+          exception: source.exception
+        }
+        endpointErrorIssues += 1
       }
-      endpointErrorIssues += 1
-    }
 
-    return {
-      name: `Endpoint URL ${index}`,
-      endpoint: source.endpoint,
-      endpoint_url: source.endpoint_url,
-      documentation_url: source.documentation_url,
-      lastAccessed: source.latest_log_entry_date,
-      lastUpdated: source.resource_start_date, // as in: when was the _resource_ updated, not data under that resource
-      error
-    }
-  })
+      return {
+        name: `Endpoint URL ${index}`,
+        endpoint: source.endpoint,
+        endpoint_url: source.endpoint_url,
+        documentation_url: source.documentation_url,
+        lastAccessed: source.latest_log_entry_date,
+        lastUpdated: source.resource_start_date, // as in: when was the _resource_ updated, not data under that resource
+        entryDate: source.endpoint_entry_date,
+        error
+      }
+    })
 
   const taskCount = (entryIssueCounts ? entryIssueCounts.length : 0) +
     (entityIssueCounts ? entityIssueCounts.length : 0) +

src/middleware/lpa-overview.middleware.js

@@ -205,26 +205,26 @@ export const addNoticesToDatasets = (req, res, next) => {
  */
 export function prepareDatasetObjects (req, res, next) {
   const { issues, endpoints, expectationOutOfBounds, availableDatasets } = req
-
   const outOfBoundsViolations = new Set((expectationOutOfBounds ?? []).map(o => o.dataset))
   req.datasets = availableDatasets.map((dataset) => {
     const datasetEndpoints = endpoints[dataset]
     const datasetIssues = issues[dataset]
-
     if (!datasetEndpoints) {
       return { status: 'Not submitted', endpointCount: 0, dataset }
     }
 
     const endpointCount = datasetEndpoints.length
-    const httpStatus = datasetEndpoints.find(endpoint => endpoint.latest_status !== '200')?.latest_status
-    const error = httpStatus !== undefined ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined
+    const allError = datasetEndpoints.every(endpoint => endpoint.latest_status !== '200')
+    const someError = datasetEndpoints.some(endpoint => endpoint.latest_status !== '200')
+    const httpStatus = allError ? datasetEndpoints[0]?.latest_status : undefined
+    const error = allError ? `There was a ${httpStatus} error accessing the endpoint URL` : undefined
     const expectationFailed = outOfBoundsViolations.has(dataset)
     const issueCount = (datasetIssues?.length || 0) + (expectationFailed ? 1 : 0)
 
     let status
-    if (error) {
+    if (allError) {
       status = 'Error'
-    } else if (issueCount > 0) {
+    } else if (someError || issueCount > 0) {
       status = 'Needs fixing'
     } else {
       status = 'Live'
@@ -351,7 +351,6 @@ const fetchOutOfBoundsExpectations = expectationFetcher({
   expectation: expectations.entitiesOutOfBounds,
   result: 'expectationOutOfBounds'
 })
-
 /**
  * Organisation (LPA) overview page middleware chain.
  */

src/routes/schemas.js

@@ -166,6 +166,7 @@ export const OrgDatasetOverview = v.strictObject({
       endpoint: NonEmptyString,
       lastAccessed: v.string(),
       lastUpdated: v.nullable(v.string()),
+      entryDate: v.optional(v.nullable(v.string())),
       error: v.optional(v.strictObject({
         code: v.integer(),
         exception: v.string()

src/views/organisations/dataset-overview.html

@@ -149,6 +149,16 @@
   } %}
   {{ rows.push(lastUpdatedRow) }}
 
+  {% set entryDateRow = {
+    key: {
+      text: 'Endpoint entry date'
+    },
+    value: {
+      text: endpoint.entryDate and ( endpoint.entryDate | govukDateTime )
+    }
+  } %}
+  {{ rows.push(entryDateRow) }}
+
 {% endfor %}
 
 {% block content %}

test/unit/middleware/datasetOverview.middleware.test.js

@@ -27,8 +27,8 @@ describe('Dataset Overview Middleware', () => {
         entityCount: { entity_count: 10 },
         expectationOutOfBounds: [{ dataset: req.params.dataset, passed: 'False' }],
         sources: [
-          { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: 'LU1', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' },
-          { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: 'LU2', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' }
+          { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: '2024-02-01', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' },
+          { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: '2023-01-01', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' }
         ],
         entryIssueCounts: [
           {
@@ -50,8 +50,8 @@ describe('Dataset Overview Middleware', () => {
         stats: {
           numberOfRecords: 10,
           endpoints: [
-            { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01' },
-            { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02' }
+            { name: 'Endpoint URL 0', endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', error: undefined, lastAccessed: 'LA1', lastUpdated: '2023-01-01', entryDate: '2024-02-01' },
+            { name: 'Endpoint URL 1', endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', error: { code: 404, exception: 'exception' }, lastAccessed: 'LA2', lastUpdated: '2023-01-02', entryDate: '2023-01-01' }
           ]
         },
         notice: undefined

test/unit/middleware/lpa-overview.middleware.test.js

@@ -82,7 +82,7 @@ describe('lpa-overview.middleware', () => {
       const req = structuredClone(reqTemplate)
       const res = { render: vi.fn() }
 
-      prepareOverviewTemplateParams(req, res, () => {})
+      prepareOverviewTemplateParams(req, res, () => { })
 
       const expectedTemplateParams = {
         organisation: { name: 'Example LPA', organisation: 'LPA' },
@@ -117,7 +117,7 @@ describe('lpa-overview.middleware', () => {
       reqNotMember.provisions.forEach((provision) => {
         provision.project = ''
       })
-      prepareOverviewTemplateParams(reqNotMember, res, () => {})
+      prepareOverviewTemplateParams(reqNotMember, res, () => { })
       const { doc: docNotMember } = getRenderedErrorCards(reqNotMember.templateParams)
       expect(docNotMember.querySelector('.org-membership-info').textContent.trim()).toMatch('is not a member of the Open Digital Planning programme')
     })
@@ -128,7 +128,7 @@ describe('lpa-overview.middleware', () => {
       req.datasetErrorStatus = [{ dataset: 'dataset1' }]
       const res = { render: vi.fn() }
 
-      prepareOverviewTemplateParams(req, res, () => {})
+      prepareOverviewTemplateParams(req, res, () => { })
 
       const ds1 = req.templateParams.datasets.statutory[0]
       expect(ds1.status).toBe('Live')
@@ -152,12 +152,61 @@ describe('lpa-overview.middleware', () => {
       req.expectationOutOfBounds = [{ passed: false, dataset: 'datasetA' }]
       const res = { render: vi.fn() }
 
-      prepareDatasetObjects(req, res, () => {})
+      prepareDatasetObjects(req, res, () => { })
 
       expect(req.datasets[0].error).toBeUndefined()
       expect(req.datasets[0].issueCount).toBe(1)
       expect(req.datasets[0].status).toBe('Needs fixing')
     })
+    it('should not show an error if atleast one endpoint is 200', () => {
+      const req = {
+        expectationOutOfBounds: [],
+        issues: {},
+        endpoints: { datasetA: [{ latest_status: '200' }, { latest_status: '504' }, { latest_status: '504' }] },
+        availableDatasets: ['datasetA'],
+        datasets: undefined
+      }
+      const res = { render: vi.fn() }
+
+      prepareDatasetObjects(req, res, () => { })
+
+      expect(req.datasets[0].error).toBeUndefined()
+      expect(req.datasets[0].status).toBe('Needs fixing')
+    })
+    it('should show an error all endpoints have status !== 200', () => {
+      const req = {
+        expectationOutOfBounds: [],
+        issues: {},
+        endpoints: { datasetA: [{ latest_status: '504' }, { latest_status: '504' }] },
+        availableDatasets: ['datasetA'],
+        datasets: undefined
+      }
+      const res = { render: vi.fn() }
+
+      prepareDatasetObjects(req, res, () => { })
+
+      expect(req.datasets[0].error).toContain('504')
+      expect(req.datasets[0].status).toBe('Error')
+    })
+    it('should show live if all endpoint status == 200', () => {
+      const req = {
+        expectationOutOfBounds: [],
+        issues: {},
+        endpoints: {
+          datasetA: [{ latest_status: '200' },
+            { latest_status: '200' },
+            { latest_status: '200' }]
+        },
+        availableDatasets: ['datasetA'],
+        datasets: undefined
+      }
+      const res = { render: vi.fn() }
+
+      prepareDatasetObjects(req, res, () => { })
+
+      expect(req.datasets[0].error).toBeUndefined()
+      expect(req.datasets[0].status).toBe('Live')
+    })
   })
 
   describe('getOverview', () => {

test/unit/views/organisations/dataset-overview.test.js

@@ -27,14 +27,16 @@ describe('Dataset Overview Page', () => {
           endpoint_url: 'http://endpoint1.co.uk',
           documentation_url: 'http://endpoint1-docs.co.uk',
           lastAccessed: '2024-09-09',
-          lastUpdated: '2024-09-09'
+          lastUpdated: '2024-09-09',
+          entryDate: '2025-03-08'
         },
         {
           name: 'endpoint 2',
           endpoint: 'BAR',
           endpoint_url: 'http://endpoint2.co.uk',
           lastAccessed: '2024-19-19',
           lastUpdated: '2024-19-19',
+          entryDate: '2024-12-12',
           error: {
             code: 404,
             exception: ''
@@ -78,10 +80,12 @@ describe('Dataset Overview Page', () => {
     expect(summaryListValues[3].textContent).toContain(params.stats.endpoints[0].documentation_url)
     expect(summaryListValues[4].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastAccessed))
     expect(summaryListValues[5].textContent).toContain(govukDateTime(params.stats.endpoints[0].lastUpdated))
-    expect(summaryListValues[6].textContent).toContain(params.stats.endpoints[1].endpoint_url)
-    expect(summaryListValues[7].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastAccessed))
-    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[1].error.code)
-    expect(summaryListValues[8].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated))
+    expect(summaryListValues[6].textContent).toContain(govukDateTime(params.stats.endpoints[0].entryDate))
+    expect(summaryListValues[7].textContent).toContain(params.stats.endpoints[1].endpoint_url)
+    expect(summaryListValues[8].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastAccessed))
+    expect(summaryListValues[8].textContent).toContain(params.stats.endpoints[1].error.code)
+    expect(summaryListValues[9].textContent).toContain(govukDateTime(params.stats.endpoints[1].lastUpdated))
+    expect(summaryListValues[10].textContent).toContain(govukDateTime(params.stats.endpoints[1].entryDate))
   })
 
   it('Renders breadcrumbs correctly', () => {