From 1b4dfc3a519bde5eded3d531da2b5236182f0a79 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 23 Jun 2026 13:27:56 +0100 Subject: [PATCH 1/6] Migration from issue based tasks to platform api generated tasks --- config/default.yaml | 2 +- config/development.yaml | 3 +- config/production.yaml | 1 + config/staging.yaml | 1 + src/middleware/common.middleware.js | 35 ++++++++ src/middleware/datasetOverview.middleware.js | 8 +- src/middleware/datasetTaskList.middleware.js | 34 +++----- src/middleware/dataview.middleware.js | 20 +++-- src/middleware/lpa-overview.middleware.js | 8 +- src/services/platformApi.js | 19 +++++ .../unit/middleware/common.middleware.test.js | 82 ++++++++++++++++++- .../datasetOverview.middleware.test.js | 10 +-- .../datasetTaskList.middleware.test.js | 42 ++++++---- .../middleware/dataview.middleware.test.js | 3 +- .../lpa-overview.middleware.test.js | 44 +++++++++- test/unit/services/platformApi.test.js | 55 +++++++++++++ 16 files changed, 301 insertions(+), 66 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 278cd98f6..a88e40de1 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -16,7 +16,7 @@ aws: { } # ToDo: url and name might need updating url: 'https://provide.planning.data.gov.uk' -mainWebsiteUrl: https://www.planning.data.gov.uk +mainWebsiteUrl: https://www.development.planning.data.gov.uk dataDesignUrl: https://design.planning.data.gov.uk downloadUrl: 'https://download.planning.data.gov.uk' # NOTE: this property is deprecated, use serviceNames instead diff --git a/config/development.yaml b/config/development.yaml index 34d59afcd..2dea5db3c 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -6,8 +6,9 @@ aws: { bucket: 'development-pub-async-request-files', } manageServiceUrl: 'https://manage.development.planning.data.gov.uk' +mainWebsiteUrl: 'https://www.development.planning.data.gov.uk' datasetteUrl: https://datasette.development.planning.data.gov.uk -downloadUrl: 'https://download.development.planning.data.gov.uk' +downloadUrl: 'https://download.development.planning.data.gov.uk' redis: { secure: true, host: 'development-pub-async-redis-3dqynz.serverless.euw2.cache.amazonaws.com', diff --git a/config/production.yaml b/config/production.yaml index 0035a4a79..b98dc8998 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -11,6 +11,7 @@ aws: { bucket: 'production-pub-async-request-files', } manageServiceUrl: 'https://manage.planning.data.gov.uk' +mainWebsiteUrl: 'https://www.planning.data.gov.uk' redis: { secure: true, host: 'production-pub-async-redis-eihmmv.serverless.euw2.cache.amazonaws.com', diff --git a/config/staging.yaml b/config/staging.yaml index 29469261d..19d5b60a1 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -11,6 +11,7 @@ aws: { bucket: 'staging-pub-async-request-files', } manageServiceUrl: 'https://manage.staging.planning.data.gov.uk/' +mainWebsiteUrl: 'https://www.staging.planning.data.gov.uk' redis: { secure: true, host: 'staging-pub-async-redis-kh1elu.serverless.euw2.cache.amazonaws.com', diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 33526b4b2..cf3e160c9 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -1224,6 +1224,41 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({ dataset: FetchOptions.performanceDb }) +/** + * Fetches error-severity issue tasks from the platform API for the current organisation, + * then deduplicates by (dataset, issue_type, field) keeping the highest count per group. + * When `req.params.dataset` is set (e.g. dataset task list page), filters to that + * dataset and uses a limit of 100. Without a dataset (e.g. LPA overview page), + * fetches across all datasets with a limit of 500. + * Result is stored on `req.tasks` as `{ tasks: [...], count: N }`. + */ +export const fetchTasksFromPlatformApi = async (req, res, next) => { + const dataset = req.params.dataset + try { + const { formattedData } = await platformApi.fetchTasks({ + organisation: req.orgInfo?.organisation, + ...(dataset && { dataset }), + severity: 'error', + task_source: 'issue', + limit: dataset ? 100 : 500 + }) + const deduplicated = Object.values( + (formattedData.tasks ?? []).reduce((acc, task) => { + const key = `${task.dataset}::${task.details?.issue_type}::${task.details?.field}` + if (!acc[key] || task.details?.count > acc[key].details?.count) acc[key] = task + return acc + }, {}) + ) + req.tasks = { tasks: deduplicated, count: deduplicated.length } + next() + } catch (err) { + logger.warn('fetchTasksFromPlatformApi: failed to fetch tasks', { err }) + req.tasks = { tasks: [], count: 0 } + next() + } +} + + /** * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs: *. - takes org code and: diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index d9881cbb8..0797a0464 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js' +import { fetchDatasetPlatformInfo, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchTasksFromPlatformApi } from './common.middleware.js' import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req + const { orgInfo, entityCount, sources, dataset, tasks, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req let endpointErrorIssues = 0 const endpoints = sources @@ -219,7 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { if (authority === 'some') { taskCount = 1 } else { - taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) + + taskCount = (tasks?.count ?? 0) + endpointErrorIssues + (expectationOutOfBounds.length > 0 ? 1 : 0) } @@ -271,7 +271,7 @@ export default [ fetchColumnSummary, fetchResources, fetchSources, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchSpecification, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index c0c05f629..8521db6ec 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -19,8 +19,7 @@ import { expectations, fetchDatasetInfo, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchTasksFromPlatformApi, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, fetchResources, fetchSources, @@ -108,7 +107,7 @@ export function entityOutOfBoundsMessage (dataset, count) { export const prepareTasks = (req, res, next) => { const { lpa, dataset } = req.parsedParams const { entityCount, resources, sources, authority } = req - const { entryIssueCounts, entityIssueCounts, expectationOutOfBounds = [] } = req + const { tasks: tasksData = { tasks: [] }, expectationOutOfBounds = [] } = req // First check, if non authoritative dataset, only one task to show: Provide authoritative data if (authority && authority === 'some') { @@ -121,25 +120,17 @@ export const prepareTasks = (req, res, next) => { }] return next() } - let issues = [...entryIssueCounts, ...entityIssueCounts] - issues = issues.filter( - issue => issue.issue_type !== '' && - issue.issue_type !== undefined && - issue.field !== '' && - issue.field !== undefined - ) - - const taskList = Object.values(issues).map(({ field, issue_type: type, count }) => { - // if the issue doesn't have an entity, or is one of the special case issue types then we should use the resource_row_count + // Tasks are pre-deduplicated by fetchTasksFromPlatformApi (highest count per issue_type+field) + // TODO: show tasks broken down by endpoint + const taskList = (tasksData.tasks || []) + .filter(task => task.details?.issue_type && task.details?.field !== undefined) + .map(task => { + const { issue_type: type, field, count } = task.details let rowCount = entityCount.count if (SPECIAL_ISSUE_TYPES.includes(type)) { - if (resources.length > 0) { - rowCount = resources[0].entry_count - } else { - rowCount = 0 - } + rowCount = resources.length > 0 ? resources[0].entry_count : 0 } let title @@ -156,9 +147,7 @@ export const prepareTasks = (req, res, next) => { } return { - title: { - text: title - }, + title: { text: title }, href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`, status: getStatusTag('Needs improving') } @@ -241,8 +230,7 @@ export default [ isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, addEntityCountsToResources, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchTasksFromPlatformApi, prepareTasks, prepareDatasetTaskListTemplateParams, getDatasetTaskList, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index afb0d9c34..71f424eda 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -1,11 +1,14 @@ import config from '../../config/index.js' import { createPaginationTemplateParams, + expectationFetcher, + expectations, extractJsonFieldFromEntities, fetchDatasetInfo, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, + noop, processAuthoritativeMiddlewares, processSpecificationMiddlewares, replaceUnderscoreInEntities, @@ -17,10 +20,11 @@ import { logPageError, fetchResources, fetchEntityCount, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchEntitiesPlatformDb } from './common.middleware.js' import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js' +import { isFeatureEnabled } from '../utils/features.js' import * as v from 'valibot' import { splitByLeading } from '../utils/table.js' @@ -82,11 +86,16 @@ export const constructTableParams = (req, res, next) => { next() } +const fetchOutOfBoundsExpectations = expectationFetcher({ + expectation: expectations.entitiesOutOfBounds, + result: 'expectationOutOfBounds' +}) + export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, tasks, authority, alternateSources, uniqueDatasetFields, provisions, expectationOutOfBounds } = req - // Hard code task count for 'some' authority - const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 + const outOfBoundsCount = (expectationOutOfBounds?.length ?? 0) > 0 ? 1 : 0 + const taskCount = authority !== 'some' ? (tasks?.count ?? 0) + outOfBoundsCount : 1 // Build the fields query parameter and download url const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0 ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&') @@ -129,7 +138,8 @@ export default [ fetchDatasetInfo, fetchResources, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, + isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 43842e1f0..d0400a7a8 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for oragnisation (LPA) overview page */ -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchTasksFromPlatformApi, fetchLocalPlanningGroups } from './common.middleware.js' import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' @@ -417,9 +417,9 @@ export const getOverview = renderTemplate({ }) export function groupIssuesCountsByDataset (req, res, next) { - const { entityIssueCounts = [] } = req + const tasks = req.tasks?.tasks ?? [] - req.issues = entityIssueCounts.reduce((acc, current) => { + req.issues = tasks.reduce((acc, current) => { if (!acc[current.dataset]) { acc[current.dataset] = [] } @@ -457,7 +457,7 @@ export default [ parallel([ fetchLocalPlanningGroups, fetchEndpointSummary, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchProvisions ]), diff --git a/src/services/platformApi.js b/src/services/platformApi.js index 1c22ef68c..cd729f7a6 100644 --- a/src/services/platformApi.js +++ b/src/services/platformApi.js @@ -121,6 +121,25 @@ export default { data, formattedData: datasets } + }, + + fetchTasks: async (params) => { + const queryParams = new URLSearchParams() + + if (params.organisation) queryParams.append('organisation', params.organisation) + if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.severity) queryParams.append('severity', params.severity) + if (params.responsibility) queryParams.append('responsibility', params.responsibility) + if (params.task_source) queryParams.append('task_source', params.task_source) + if (params.limit) queryParams.append('limit', params.limit) + + const url = `${config.mainWebsiteUrl}/task.json?${queryParams.toString()}` + const data = await queryPlatformAPI(url, params) + + return { + data, + formattedData: { tasks: data?.tasks ?? [], count: data?.count ?? 0 } + } } } diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js index cbfc15a17..28658b4e2 100644 --- a/test/unit/middleware/common.middleware.test.js +++ b/test/unit/middleware/common.middleware.test.js @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { filterOutEntitiesWithoutIssues, createPaginationTemplateParams, addDatabaseFieldToSpecification, replaceUnderscoreInSpecification, pullOutDatasetSpecification, extractJsonFieldFromEntities, replaceUnderscoreInEntities, setDefaultParams, getUniqueDatasetFieldsFromSpecification, show404IfPageNumberNotInRange, removeIssuesThatHaveBeenFixed, addFieldMappingsToIssue, getSetDataRange, getErrorSummaryItems, getSetBaseSubPath, prepareIssueDetailsTemplateParams, preventIndexing, getIssueSpecification, fetchEntities, prepareAuthority } from '../../../src/middleware/common.middleware' +import { filterOutEntitiesWithoutIssues, createPaginationTemplateParams, addDatabaseFieldToSpecification, replaceUnderscoreInSpecification, pullOutDatasetSpecification, extractJsonFieldFromEntities, replaceUnderscoreInEntities, setDefaultParams, getUniqueDatasetFieldsFromSpecification, show404IfPageNumberNotInRange, removeIssuesThatHaveBeenFixed, addFieldMappingsToIssue, getSetDataRange, getErrorSummaryItems, getSetBaseSubPath, prepareIssueDetailsTemplateParams, preventIndexing, getIssueSpecification, fetchEntities, prepareAuthority, fetchTasksFromPlatformApi } from '../../../src/middleware/common.middleware' import logger from '../../../src/utils/logger' import datasette from '../../../src/services/datasette.js' import performanceDbApi from '../../../src/services/performanceDbApi.js' @@ -18,7 +18,8 @@ vi.mock('../../../src/services/performanceDbApi.js') vi.mock('../../../src/services/platformApi.js', () => ({ default: { fetchEntities: vi.fn(), - fetchAllEntities: vi.fn() + fetchAllEntities: vi.fn(), + fetchTasks: vi.fn() } })) @@ -1654,3 +1655,80 @@ describe('preventIndexing middleware', () => { }) }) }) + +describe('fetchTasksFromPlatformApi', () => { + const next = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('calls the API with dataset filter and limit 100 when dataset param is set', async () => { + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks: [], count: 0 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: { dataset: 'brownfield-land' } } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(platformApi.fetchTasks).toHaveBeenCalledWith(expect.objectContaining({ + organisation: 'local-authority:TST', + dataset: 'brownfield-land', + severity: 'error', + task_source: 'issue', + limit: 100 + })) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('calls the API without dataset and with limit 500 when no dataset param', async () => { + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks: [], count: 0 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(platformApi.fetchTasks).toHaveBeenCalledWith(expect.objectContaining({ + limit: 500 + })) + expect(platformApi.fetchTasks.mock.calls[0][0]).not.toHaveProperty('dataset') + }) + + it('deduplicates tasks keeping the highest count per (dataset, issue_type, field)', async () => { + const tasks = [ + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 6 } }, + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 9 } }, + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } } + ] + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks, count: 3 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks.tasks).toHaveLength(2) + const uriTask = req.tasks.tasks.find(t => t.details.field === 'SiteplanURL') + expect(uriTask.details.count).toBe(9) + expect(req.tasks.count).toBe(2) + }) + + it('does not merge tasks with the same issue_type+field across different datasets', async () => { + const tasks = [ + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 2 } }, + { dataset: 'conservation-area', details: { issue_type: 'missing value', field: 'name', count: 3 } } + ] + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks, count: 2 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks.tasks).toHaveLength(2) + expect(req.tasks.count).toBe(2) + }) + + it('falls back to empty tasks on API error', async () => { + platformApi.fetchTasks.mockRejectedValueOnce(new Error('API down')) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: { dataset: 'brownfield-land' } } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks).toEqual({ tasks: [], count: 0 }) + expect(next).toHaveBeenCalledTimes(1) + }) +}) diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js index bf45fc8cc..24a090abe 100644 --- a/test/unit/middleware/datasetOverview.middleware.test.js +++ b/test/unit/middleware/datasetOverview.middleware.test.js @@ -36,15 +36,7 @@ describe('Dataset Overview Middleware', () => { { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: '2024-02-01', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' }, { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: '2023-01-01', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' } ], - entityIssueCounts: [ - { - issue: 'Example issue 1', - issue_type: 'Example issue type 1', - field: 'Example issue field 1', - num_issues: 1, - status: 'Error' - } - ], + tasks: { tasks: [{ reference: 'task-1', severity: 'error' }], count: 1 }, notice: undefined, authority: '' } diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index 4c2510488..e4730536c 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -49,8 +49,13 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1', 'entity2'], entityCount: { count: 2 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [{ field: 'field2', issue_type: 'issue-type2', count: 1 }] + tasks: { + tasks: [ + { details: { field: 'field1', issue_type: 'issue-type1', count: 1 }, severity: 'error' }, + { details: { field: 'field2', issue_type: 'issue-type2', count: 1 }, severity: 'error' } + ], + count: 2 + } } const res = { @@ -117,8 +122,10 @@ describe('datasetTaskList.middleware.js', () => { entities: [], entityCount: { count: 0 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }], - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: 'reference values are not unique', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { @@ -164,8 +171,7 @@ describe('datasetTaskList.middleware.js', () => { sources: [], entities: ['entity1'], resources: [{ entry_count: 10 }], - entryIssueCounts: [], - entityIssueCounts: [] + tasks: { tasks: [], count: 0 } } const res = { status: vi.fn() } @@ -193,8 +199,10 @@ describe('datasetTaskList.middleware.js', () => { entityCount: { count: 1 }, sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -222,8 +230,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }], // Invalid issue type (empty string) - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: '', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -245,8 +255,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [] + tasks: { + tasks: [{ details: { issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -268,8 +280,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [], + tasks: { + tasks: [{ details: { issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + }, expectationOutOfBounds: [ { dataset: 'some-dataset', passed: 'False', details: { actual: 3, expected: 0 } } ] diff --git a/test/unit/middleware/dataview.middleware.test.js b/test/unit/middleware/dataview.middleware.test.js index 1e246d27d..1bd515852 100644 --- a/test/unit/middleware/dataview.middleware.test.js +++ b/test/unit/middleware/dataview.middleware.test.js @@ -184,8 +184,7 @@ describe('dataview.middleware.test.js', () => { orgInfo: { name: 'Mock Org', entity: 'mock-entity' }, dataset: { name: 'Mock Dataset', dataset: 'mock-dataset' }, tableParams: { columns: ['foo'], fields: ['foo'] }, - entityIssueCounts: [], - entryIssueCounts: [], + tasks: { tasks: [], count: 0 }, pagination: {}, entityCount: { count: 1 }, offset: 0, diff --git a/test/unit/middleware/lpa-overview.middleware.test.js b/test/unit/middleware/lpa-overview.middleware.test.js index eb5c9fcb1..287faf964 100644 --- a/test/unit/middleware/lpa-overview.middleware.test.js +++ b/test/unit/middleware/lpa-overview.middleware.test.js @@ -1,5 +1,5 @@ import { describe, it, vi, expect, beforeEach, afterEach } from 'vitest' -import { addNoticesToDatasets, datasetSubmissionDeadlineCheck, getOverview, prepareDatasetObjects, prepareOverviewTemplateParams, prepareAuthorityBatch } from '../../../src/middleware/lpa-overview.middleware.js' +import { addNoticesToDatasets, datasetSubmissionDeadlineCheck, getOverview, prepareDatasetObjects, prepareOverviewTemplateParams, prepareAuthorityBatch, groupIssuesCountsByDataset } from '../../../src/middleware/lpa-overview.middleware.js' import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js' import jsdom from 'jsdom' import platformApi from '../../../src/services/platformApi.js' @@ -500,4 +500,46 @@ describe('lpa-overview.middleware', () => { expect(next).toHaveBeenCalled() }) }) + + describe('groupIssuesCountsByDataset', () => { + it('groups tasks by dataset into req.issues', () => { + const req = { + tasks: { + tasks: [ + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } }, + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 3 } }, + { dataset: 'conservation-area', details: { issue_type: 'missing value', field: 'name', count: 2 } } + ], + count: 3 + } + } + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues['brownfield-land']).toHaveLength(2) + expect(req.issues['conservation-area']).toHaveLength(1) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('produces an empty object when tasks is empty', () => { + const req = { tasks: { tasks: [], count: 0 } } + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues).toEqual({}) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('handles missing tasks gracefully', () => { + const req = {} + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues).toEqual({}) + expect(next).toHaveBeenCalledTimes(1) + }) + }) }) diff --git a/test/unit/services/platformApi.test.js b/test/unit/services/platformApi.test.js index fac586777..a3830037c 100644 --- a/test/unit/services/platformApi.test.js +++ b/test/unit/services/platformApi.test.js @@ -176,3 +176,58 @@ describe('platformApi.fetchEntities', () => { await expect(platformApi.fetchEntities(params)).rejects.toThrow('Network Error') }) }) + +describe('platformApi.fetchTasks', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('builds the correct URL with dataset filter', async () => { + axios.get.mockResolvedValueOnce({ data: { tasks: [], count: 0 } }) + + await platformApi.fetchTasks({ + organisation: 'local-authority:TST', + dataset: 'brownfield-land', + severity: 'error', + task_source: 'issue', + limit: 100 + }) + + expect(axios.get).toHaveBeenCalledWith( + 'https://www.planning.data.gov.uk/task.json?organisation=local-authority%3ATST&dataset=brownfield-land&severity=error&task_source=issue&limit=100', + expect.any(Object) + ) + }) + + it('omits dataset param when not provided', async () => { + axios.get.mockResolvedValueOnce({ data: { tasks: [], count: 0 } }) + + await platformApi.fetchTasks({ + organisation: 'local-authority:TST', + severity: 'error', + task_source: 'issue', + limit: 500 + }) + + const calledUrl = axios.get.mock.calls[0][0] + expect(calledUrl).not.toContain('dataset=') + expect(calledUrl).toContain('limit=500') + }) + + it('returns formattedData with tasks and count', async () => { + const tasks = [{ reference: 'abc', dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } }] + axios.get.mockResolvedValueOnce({ data: { tasks, count: 1 } }) + + const result = await platformApi.fetchTasks({ organisation: 'local-authority:TST' }) + + expect(result.formattedData).toEqual({ tasks, count: 1 }) + }) + + it('returns empty formattedData when response is missing fields', async () => { + axios.get.mockResolvedValueOnce({ data: {} }) + + const result = await platformApi.fetchTasks({ organisation: 'local-authority:TST' }) + + expect(result.formattedData).toEqual({ tasks: [], count: 0 }) + }) +}) From 3cf0d645c0e920b47f4a383d617e187d1399b776 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Tue, 23 Jun 2026 13:28:42 +0100 Subject: [PATCH 2/6] lint --- src/middleware/common.middleware.js | 1 - src/middleware/datasetTaskList.middleware.js | 52 ++++++++++---------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index cf3e160c9..b756956c8 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -1258,7 +1258,6 @@ export const fetchTasksFromPlatformApi = async (req, res, next) => { } } - /** * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs: *. - takes org code and: diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index 8521db6ec..d1b77aa8a 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -126,32 +126,32 @@ export const prepareTasks = (req, res, next) => { const taskList = (tasksData.tasks || []) .filter(task => task.details?.issue_type && task.details?.field !== undefined) .map(task => { - const { issue_type: type, field, count } = task.details - - let rowCount = entityCount.count - if (SPECIAL_ISSUE_TYPES.includes(type)) { - rowCount = resources.length > 0 ? resources[0].entry_count : 0 - } - - let title - try { - title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset }) - } catch (e) { - logger.warn('Failed to generate task title', { - type: types.App, - errorMessage: e.message, - errorStack: e.stack, - params: { num_issues: count, rowCount, field, issue_type: type } - }) - title = `${count} issue${count > 1 ? 's' : ''} of type ${type}` - } - - return { - title: { text: title }, - href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`, - status: getStatusTag('Needs improving') - } - }) + const { issue_type: type, field, count } = task.details + + let rowCount = entityCount.count + if (SPECIAL_ISSUE_TYPES.includes(type)) { + rowCount = resources.length > 0 ? resources[0].entry_count : 0 + } + + let title + try { + title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset }) + } catch (e) { + logger.warn('Failed to generate task title', { + type: types.App, + errorMessage: e.message, + errorStack: e.stack, + params: { num_issues: count, rowCount, field, issue_type: type } + }) + title = `${count} issue${count > 1 ? 's' : ''} of type ${type}` + } + + return { + title: { text: title }, + href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`, + status: getStatusTag('Needs improving') + } + }) // include sources which couldn't be accessed for (const source of sources) { From dc569a11a96768e777edfef5c847a7705f6a5eff Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 24 Jun 2026 11:41:36 +0100 Subject: [PATCH 3/6] change default to prod api --- config/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.yaml b/config/default.yaml index a88e40de1..278cd98f6 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -16,7 +16,7 @@ aws: { } # ToDo: url and name might need updating url: 'https://provide.planning.data.gov.uk' -mainWebsiteUrl: https://www.development.planning.data.gov.uk +mainWebsiteUrl: https://www.planning.data.gov.uk dataDesignUrl: https://design.planning.data.gov.uk downloadUrl: 'https://download.planning.data.gov.uk' # NOTE: this property is deprecated, use serviceNames instead From 2e57b7881d391bad6691ce077bc1edf8baa238f7 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 24 Jun 2026 13:32:32 +0100 Subject: [PATCH 4/6] add dropdown for arcgis layer details in get started --- src/views/organisations/get-started.html | 29 +++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/views/organisations/get-started.html b/src/views/organisations/get-started.html index c9f7b1395..69cc82e8c 100644 --- a/src/views/organisations/get-started.html +++ b/src/views/organisations/get-started.html @@ -1,4 +1,5 @@ {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %} +{% from "govuk/components/details/macro.njk" import govukDetails %} {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance', 'community', 'extract']) %} {% set serviceType = 'Manage' %} @@ -115,19 +116,21 @@

Hosting your data

  • file hosted on your web server - like URLs that end in .json or .csv
  • live data feed from an API - hosted by your Geographic Information System (GIS) software or open data platform
    • -
    -

    Help with providing data using an ArcGIS data layer

    -

    An ArcGIS data layer URL usually looks like this:

    -

    https://maps.example.gov.uk/arcgis/rest/services/Planning/LocalPlans/FeatureServer/0

    -

    This URL is made up of:

    -
      -
    • the organisation's website (maps.example.gov.uk)
      • -
    • the ArcGIS REST services path (/arcgis/rest/services)
      • -
    • the name of the service (Planning/LocalPlans)
      • -
    • the type of service (for example FeatureServer or MapServer)
      • -
    • a number that identifies the layer within the service (/0)
      • -
    -
    + {{ govukDetails({ + summaryText: "Help with providing data using an ArcGIS data layer", + html: ' +

    An ArcGIS data layer URL usually looks like this:

    +

    https://maps.example.gov.uk/arcgis/rest/services/Planning/LocalPlans/FeatureServer/0

    +

    This URL is made up of:

    +
      +
    • the organisation\'s website (maps.example.gov.uk)
      • +
    • the ArcGIS REST services path (/arcgis/rest/services)
      • +
    • the name of the service (Planning/LocalPlans)
      • +
    • the type of service (for example FeatureServer or MapServer)
      • +
    • a number that identifies the layer within the service (/0)
      • +
    + ' + }) }}

    You only need to make changes to your data at your endpoint URL. Do not change your endpoint URL when you make updates.

    Create your webpage

    From 06d55f056ff67549d5145f0a07020cc6aae45d8a Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 24 Jun 2026 14:39:24 +0100 Subject: [PATCH 5/6] make issues source from database overide --- src/middleware/common.middleware.js | 12 +++--------- src/middleware/entryIssueDetails.middleware.js | 3 +-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index b756956c8..37244cb7f 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -668,6 +668,7 @@ export const filterOutEntitiesWithoutIssues = (req, res, next) => { // entity issues const fetchEntityIssuesForFieldAndType = fetchMany({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : '' const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : '' @@ -679,18 +680,14 @@ const fetchEntityIssuesForFieldAndType = fetchMany({ field, entity, message, - severity, value, ROW_NUMBER() OVER ( PARTITION BY i.issue_type, entity ORDER BY i.rowid ) AS rn FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') ${issueTypeClause} - AND it.responsibility = 'external' - AND it.severity = 'error' ${issueFieldClause} AND i.dataset = '${req.params.dataset}' AND entity != '' @@ -704,7 +701,6 @@ const fetchEntityIssuesForFieldAndType = fetchMany({ field, entity, message, - severity, value FROM ranked WHERE rn = 1 @@ -805,17 +801,15 @@ export const addFieldMappingsToIssue = (req, res, next) => { // We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed? export const fetchEntryIssues = fetchMany({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : '' const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : '' return ` - select i.issue_type, field, entity, message, severity, value, line_number + select i.issue_type, field, entity, message, value, line_number from issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource = '${req.resources[0].resource}' ${issueTypeClause} - AND it.responsibility = 'external' - AND it.severity = 'error' AND i.dataset = '${req.params.dataset}' ${issueFieldClause} AND (entity = '' OR entity is NULL OR i.issue_type in ('${entryIssueGroups.map(issue => issue.type).join("', '")}')) diff --git a/src/middleware/entryIssueDetails.middleware.js b/src/middleware/entryIssueDetails.middleware.js index 24c73dd2b..e3e90b8e2 100644 --- a/src/middleware/entryIssueDetails.middleware.js +++ b/src/middleware/entryIssueDetails.middleware.js @@ -42,6 +42,7 @@ export const addResourceMetaDataToResources = (req, res, next) => { } const fetchIssueCount = fetchOne({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { if (!req.resources[0]) { return 'SELECT 0 AS count' @@ -49,10 +50,8 @@ const fetchIssueCount = fetchOne({ return ` SELECT count(*) AS count FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource = '${req.resources[0].resource}' AND i.issue_type = '${params.issue_type}' - AND it.responsibility = 'external' AND field = '${params.issue_field}' ` }, From f35f4408893639c3afe574b4709ad9d790c713e7 Mon Sep 17 00:00:00 2001 From: Matt Poole Date: Wed, 24 Jun 2026 16:50:37 +0100 Subject: [PATCH 6/6] query guard for sql no resource --- src/middleware/common.middleware.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 37244cb7f..842bca238 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -803,6 +803,7 @@ export const addFieldMappingsToIssue = (req, res, next) => { export const fetchEntryIssues = fetchMany({ dataset: FetchOptions.fromParams, query: ({ req, params }) => { + if (!req.resources[0]) return 'SELECT 1 WHERE 0' const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : '' const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : '' return `