diff --git a/config/development.yaml b/config/development.yaml index 34d59afc..2dea5db3 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -6,8 +6,9 @@ aws: { bucket: 'development-pub-async-request-files', } manageServiceUrl: 'https://manage.development.planning.data.gov.uk' +mainWebsiteUrl: 'https://www.development.planning.data.gov.uk' datasetteUrl: https://datasette.development.planning.data.gov.uk -downloadUrl: 'https://download.development.planning.data.gov.uk' +downloadUrl: 'https://download.development.planning.data.gov.uk' redis: { secure: true, host: 'development-pub-async-redis-3dqynz.serverless.euw2.cache.amazonaws.com', diff --git a/config/production.yaml b/config/production.yaml index 0035a4a7..b98dc899 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -11,6 +11,7 @@ aws: { bucket: 'production-pub-async-request-files', } manageServiceUrl: 'https://manage.planning.data.gov.uk' +mainWebsiteUrl: 'https://www.planning.data.gov.uk' redis: { secure: true, host: 'production-pub-async-redis-eihmmv.serverless.euw2.cache.amazonaws.com', diff --git a/config/staging.yaml b/config/staging.yaml index 29469261..19d5b60a 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -11,6 +11,7 @@ aws: { bucket: 'staging-pub-async-request-files', } manageServiceUrl: 'https://manage.staging.planning.data.gov.uk/' +mainWebsiteUrl: 'https://www.staging.planning.data.gov.uk' redis: { secure: true, host: 'staging-pub-async-redis-kh1elu.serverless.euw2.cache.amazonaws.com', diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index 33526b4b..842bca23 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -668,6 +668,7 @@ export const filterOutEntitiesWithoutIssues = (req, res, next) => { // entity issues const fetchEntityIssuesForFieldAndType = fetchMany({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : '' const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : '' @@ -679,18 +680,14 @@ const fetchEntityIssuesForFieldAndType = fetchMany({ field, entity, message, - severity, value, ROW_NUMBER() OVER ( PARTITION BY i.issue_type, entity ORDER BY i.rowid ) AS rn FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}') ${issueTypeClause} - AND it.responsibility = 'external' - AND it.severity = 'error' ${issueFieldClause} AND i.dataset = '${req.params.dataset}' AND entity != '' @@ -704,7 +701,6 @@ const fetchEntityIssuesForFieldAndType = fetchMany({ field, entity, message, - severity, value FROM ranked WHERE rn = 1 @@ -805,17 +801,16 @@ export const addFieldMappingsToIssue = (req, res, next) => { // We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed? export const fetchEntryIssues = fetchMany({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { + if (!req.resources[0]) return 'SELECT 1 WHERE 0' const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : '' const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : '' return ` - select i.issue_type, field, entity, message, severity, value, line_number + select i.issue_type, field, entity, message, value, line_number from issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource = '${req.resources[0].resource}' ${issueTypeClause} - AND it.responsibility = 'external' - AND it.severity = 'error' AND i.dataset = '${req.params.dataset}' ${issueFieldClause} AND (entity = '' OR entity is NULL OR i.issue_type in ('${entryIssueGroups.map(issue => issue.type).join("', '")}')) @@ -1224,6 +1219,40 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({ dataset: FetchOptions.performanceDb }) +/** + * Fetches error-severity issue tasks from the platform API for the current organisation, + * then deduplicates by (dataset, issue_type, field) keeping the highest count per group. + * When `req.params.dataset` is set (e.g. dataset task list page), filters to that + * dataset and uses a limit of 100. Without a dataset (e.g. LPA overview page), + * fetches across all datasets with a limit of 500. + * Result is stored on `req.tasks` as `{ tasks: [...], count: N }`. + */ +export const fetchTasksFromPlatformApi = async (req, res, next) => { + const dataset = req.params.dataset + try { + const { formattedData } = await platformApi.fetchTasks({ + organisation: req.orgInfo?.organisation, + ...(dataset && { dataset }), + severity: 'error', + task_source: 'issue', + limit: dataset ? 100 : 500 + }) + const deduplicated = Object.values( + (formattedData.tasks ?? []).reduce((acc, task) => { + const key = `${task.dataset}::${task.details?.issue_type}::${task.details?.field}` + if (!acc[key] || task.details?.count > acc[key].details?.count) acc[key] = task + return acc + }, {}) + ) + req.tasks = { tasks: deduplicated, count: deduplicated.length } + next() + } catch (err) { + logger.warn('fetchTasksFromPlatformApi: failed to fetch tasks', { err }) + req.tasks = { tasks: [], count: 0 } + next() + } +} + /** * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs: *. - takes org code and: diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index d9881cbb..0797a046 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js' +import { fetchDatasetPlatformInfo, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchTasksFromPlatformApi } from './common.middleware.js' import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req + const { orgInfo, entityCount, sources, dataset, tasks, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req let endpointErrorIssues = 0 const endpoints = sources @@ -219,7 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { if (authority === 'some') { taskCount = 1 } else { - taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) + + taskCount = (tasks?.count ?? 0) + endpointErrorIssues + (expectationOutOfBounds.length > 0 ? 1 : 0) } @@ -271,7 +271,7 @@ export default [ fetchColumnSummary, fetchResources, fetchSources, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchSpecification, isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index c0c05f62..d1b77aa8 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -19,8 +19,7 @@ import { expectations, fetchDatasetInfo, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchTasksFromPlatformApi, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, fetchResources, fetchSources, @@ -108,7 +107,7 @@ export function entityOutOfBoundsMessage (dataset, count) { export const prepareTasks = (req, res, next) => { const { lpa, dataset } = req.parsedParams const { entityCount, resources, sources, authority } = req - const { entryIssueCounts, entityIssueCounts, expectationOutOfBounds = [] } = req + const { tasks: tasksData = { tasks: [] }, expectationOutOfBounds = [] } = req // First check, if non authoritative dataset, only one task to show: Provide authoritative data if (authority && authority === 'some') { @@ -121,48 +120,38 @@ export const prepareTasks = (req, res, next) => { }] return next() } - let issues = [...entryIssueCounts, ...entityIssueCounts] - - issues = issues.filter( - issue => issue.issue_type !== '' && - issue.issue_type !== undefined && - issue.field !== '' && - issue.field !== undefined - ) - - const taskList = Object.values(issues).map(({ field, issue_type: type, count }) => { - // if the issue doesn't have an entity, or is one of the special case issue types then we should use the resource_row_count - - let rowCount = entityCount.count - if (SPECIAL_ISSUE_TYPES.includes(type)) { - if (resources.length > 0) { - rowCount = resources[0].entry_count - } else { - rowCount = 0 + + // Tasks are pre-deduplicated by fetchTasksFromPlatformApi (highest count per issue_type+field) + // TODO: show tasks broken down by endpoint + const taskList = (tasksData.tasks || []) + .filter(task => task.details?.issue_type && task.details?.field !== undefined) + .map(task => { + const { issue_type: type, field, count } = task.details + + let rowCount = entityCount.count + if (SPECIAL_ISSUE_TYPES.includes(type)) { + rowCount = resources.length > 0 ? resources[0].entry_count : 0 } - } - let title - try { - title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset }) - } catch (e) { - logger.warn('Failed to generate task title', { - type: types.App, - errorMessage: e.message, - errorStack: e.stack, - params: { num_issues: count, rowCount, field, issue_type: type } - }) - title = `${count} issue${count > 1 ? 's' : ''} of type ${type}` - } + let title + try { + title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset }) + } catch (e) { + logger.warn('Failed to generate task title', { + type: types.App, + errorMessage: e.message, + errorStack: e.stack, + params: { num_issues: count, rowCount, field, issue_type: type } + }) + title = `${count} issue${count > 1 ? 's' : ''} of type ${type}` + } - return { - title: { - text: title - }, - href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`, - status: getStatusTag('Needs improving') - } - }) + return { + title: { text: title }, + href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`, + status: getStatusTag('Needs improving') + } + }) // include sources which couldn't be accessed for (const source of sources) { @@ -241,8 +230,7 @@ export default [ isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, addEntityCountsToResources, fetchEntityCount, - fetchEntityIssueCounts, - fetchEntryIssueCounts, + fetchTasksFromPlatformApi, prepareTasks, prepareDatasetTaskListTemplateParams, getDatasetTaskList, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index afb0d9c3..71f424ed 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -1,11 +1,14 @@ import config from '../../config/index.js' import { createPaginationTemplateParams, + expectationFetcher, + expectations, extractJsonFieldFromEntities, fetchDatasetInfo, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, + noop, processAuthoritativeMiddlewares, processSpecificationMiddlewares, replaceUnderscoreInEntities, @@ -17,10 +20,11 @@ import { logPageError, fetchResources, fetchEntityCount, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchEntitiesPlatformDb } from './common.middleware.js' import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js' +import { isFeatureEnabled } from '../utils/features.js' import * as v from 'valibot' import { splitByLeading } from '../utils/table.js' @@ -82,11 +86,16 @@ export const constructTableParams = (req, res, next) => { next() } +const fetchOutOfBoundsExpectations = expectationFetcher({ + expectation: expectations.entitiesOutOfBounds, + result: 'expectationOutOfBounds' +}) + export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, tasks, authority, alternateSources, uniqueDatasetFields, provisions, expectationOutOfBounds } = req - // Hard code task count for 'some' authority - const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 + const outOfBoundsCount = (expectationOutOfBounds?.length ?? 0) > 0 ? 1 : 0 + const taskCount = authority !== 'some' ? (tasks?.count ?? 0) + outOfBoundsCount : 1 // Build the fields query parameter and download url const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0 ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&') @@ -129,7 +138,8 @@ export default [ fetchDatasetInfo, fetchResources, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, + isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop, ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API diff --git a/src/middleware/entryIssueDetails.middleware.js b/src/middleware/entryIssueDetails.middleware.js index 24c73dd2..e3e90b8e 100644 --- a/src/middleware/entryIssueDetails.middleware.js +++ b/src/middleware/entryIssueDetails.middleware.js @@ -42,6 +42,7 @@ export const addResourceMetaDataToResources = (req, res, next) => { } const fetchIssueCount = fetchOne({ + dataset: FetchOptions.fromParams, query: ({ req, params }) => { if (!req.resources[0]) { return 'SELECT 0 AS count' @@ -49,10 +50,8 @@ const fetchIssueCount = fetchOne({ return ` SELECT count(*) AS count FROM issue i - LEFT JOIN issue_type it ON i.issue_type = it.issue_type WHERE resource = '${req.resources[0].resource}' AND i.issue_type = '${params.issue_type}' - AND it.responsibility = 'external' AND field = '${params.issue_field}' ` }, diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 43842e1f..d0400a7a 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for oragnisation (LPA) overview page */ -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchTasksFromPlatformApi, fetchLocalPlanningGroups } from './common.middleware.js' import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' @@ -417,9 +417,9 @@ export const getOverview = renderTemplate({ }) export function groupIssuesCountsByDataset (req, res, next) { - const { entityIssueCounts = [] } = req + const tasks = req.tasks?.tasks ?? [] - req.issues = entityIssueCounts.reduce((acc, current) => { + req.issues = tasks.reduce((acc, current) => { if (!acc[current.dataset]) { acc[current.dataset] = [] } @@ -457,7 +457,7 @@ export default [ parallel([ fetchLocalPlanningGroups, fetchEndpointSummary, - fetchEntityIssueCountsPerformanceDb, + fetchTasksFromPlatformApi, fetchProvisions ]), diff --git a/src/services/platformApi.js b/src/services/platformApi.js index 1c22ef68..cd729f7a 100644 --- a/src/services/platformApi.js +++ b/src/services/platformApi.js @@ -121,6 +121,25 @@ export default { data, formattedData: datasets } + }, + + fetchTasks: async (params) => { + const queryParams = new URLSearchParams() + + if (params.organisation) queryParams.append('organisation', params.organisation) + if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.severity) queryParams.append('severity', params.severity) + if (params.responsibility) queryParams.append('responsibility', params.responsibility) + if (params.task_source) queryParams.append('task_source', params.task_source) + if (params.limit) queryParams.append('limit', params.limit) + + const url = `${config.mainWebsiteUrl}/task.json?${queryParams.toString()}` + const data = await queryPlatformAPI(url, params) + + return { + data, + formattedData: { tasks: data?.tasks ?? [], count: data?.count ?? 0 } + } } } diff --git a/src/views/organisations/get-started.html b/src/views/organisations/get-started.html index c9f7b139..69cc82e8 100644 --- a/src/views/organisations/get-started.html +++ b/src/views/organisations/get-started.html @@ -1,4 +1,5 @@ {% from "govuk/components/breadcrumbs/macro.njk" import govukBreadcrumbs %} +{% from "govuk/components/details/macro.njk" import govukDetails %} {% set navigationItems = currentPath | getNavigationLinks(['organisations', 'guidance', 'community', 'extract']) %} {% set serviceType = 'Manage' %} @@ -115,19 +116,21 @@

Hosting your data

  • file hosted on your web server - like URLs that end in .json or .csv
  • live data feed from an API - hosted by your Geographic Information System (GIS) software or open data platform
    • -
    -

    Help with providing data using an ArcGIS data layer

    -

    An ArcGIS data layer URL usually looks like this:

    -

    https://maps.example.gov.uk/arcgis/rest/services/Planning/LocalPlans/FeatureServer/0

    -

    This URL is made up of:

    - -
    + {{ govukDetails({ + summaryText: "Help with providing data using an ArcGIS data layer", + html: ' +

    An ArcGIS data layer URL usually looks like this:

    +

    https://maps.example.gov.uk/arcgis/rest/services/Planning/LocalPlans/FeatureServer/0

    +

    This URL is made up of:

    + + ' + }) }}

    You only need to make changes to your data at your endpoint URL. Do not change your endpoint URL when you make updates.

    Create your webpage

    diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js index cbfc15a1..28658b4e 100644 --- a/test/unit/middleware/common.middleware.test.js +++ b/test/unit/middleware/common.middleware.test.js @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { filterOutEntitiesWithoutIssues, createPaginationTemplateParams, addDatabaseFieldToSpecification, replaceUnderscoreInSpecification, pullOutDatasetSpecification, extractJsonFieldFromEntities, replaceUnderscoreInEntities, setDefaultParams, getUniqueDatasetFieldsFromSpecification, show404IfPageNumberNotInRange, removeIssuesThatHaveBeenFixed, addFieldMappingsToIssue, getSetDataRange, getErrorSummaryItems, getSetBaseSubPath, prepareIssueDetailsTemplateParams, preventIndexing, getIssueSpecification, fetchEntities, prepareAuthority } from '../../../src/middleware/common.middleware' +import { filterOutEntitiesWithoutIssues, createPaginationTemplateParams, addDatabaseFieldToSpecification, replaceUnderscoreInSpecification, pullOutDatasetSpecification, extractJsonFieldFromEntities, replaceUnderscoreInEntities, setDefaultParams, getUniqueDatasetFieldsFromSpecification, show404IfPageNumberNotInRange, removeIssuesThatHaveBeenFixed, addFieldMappingsToIssue, getSetDataRange, getErrorSummaryItems, getSetBaseSubPath, prepareIssueDetailsTemplateParams, preventIndexing, getIssueSpecification, fetchEntities, prepareAuthority, fetchTasksFromPlatformApi } from '../../../src/middleware/common.middleware' import logger from '../../../src/utils/logger' import datasette from '../../../src/services/datasette.js' import performanceDbApi from '../../../src/services/performanceDbApi.js' @@ -18,7 +18,8 @@ vi.mock('../../../src/services/performanceDbApi.js') vi.mock('../../../src/services/platformApi.js', () => ({ default: { fetchEntities: vi.fn(), - fetchAllEntities: vi.fn() + fetchAllEntities: vi.fn(), + fetchTasks: vi.fn() } })) @@ -1654,3 +1655,80 @@ describe('preventIndexing middleware', () => { }) }) }) + +describe('fetchTasksFromPlatformApi', () => { + const next = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('calls the API with dataset filter and limit 100 when dataset param is set', async () => { + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks: [], count: 0 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: { dataset: 'brownfield-land' } } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(platformApi.fetchTasks).toHaveBeenCalledWith(expect.objectContaining({ + organisation: 'local-authority:TST', + dataset: 'brownfield-land', + severity: 'error', + task_source: 'issue', + limit: 100 + })) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('calls the API without dataset and with limit 500 when no dataset param', async () => { + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks: [], count: 0 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(platformApi.fetchTasks).toHaveBeenCalledWith(expect.objectContaining({ + limit: 500 + })) + expect(platformApi.fetchTasks.mock.calls[0][0]).not.toHaveProperty('dataset') + }) + + it('deduplicates tasks keeping the highest count per (dataset, issue_type, field)', async () => { + const tasks = [ + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 6 } }, + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 9 } }, + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } } + ] + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks, count: 3 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks.tasks).toHaveLength(2) + const uriTask = req.tasks.tasks.find(t => t.details.field === 'SiteplanURL') + expect(uriTask.details.count).toBe(9) + expect(req.tasks.count).toBe(2) + }) + + it('does not merge tasks with the same issue_type+field across different datasets', async () => { + const tasks = [ + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 2 } }, + { dataset: 'conservation-area', details: { issue_type: 'missing value', field: 'name', count: 3 } } + ] + platformApi.fetchTasks.mockResolvedValueOnce({ formattedData: { tasks, count: 2 } }) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: {} } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks.tasks).toHaveLength(2) + expect(req.tasks.count).toBe(2) + }) + + it('falls back to empty tasks on API error', async () => { + platformApi.fetchTasks.mockRejectedValueOnce(new Error('API down')) + + const req = { orgInfo: { organisation: 'local-authority:TST' }, params: { dataset: 'brownfield-land' } } + await fetchTasksFromPlatformApi(req, {}, next) + + expect(req.tasks).toEqual({ tasks: [], count: 0 }) + expect(next).toHaveBeenCalledTimes(1) + }) +}) diff --git a/test/unit/middleware/datasetOverview.middleware.test.js b/test/unit/middleware/datasetOverview.middleware.test.js index bf45fc8c..24a090ab 100644 --- a/test/unit/middleware/datasetOverview.middleware.test.js +++ b/test/unit/middleware/datasetOverview.middleware.test.js @@ -36,15 +36,7 @@ describe('Dataset Overview Middleware', () => { { endpoint: 'endpoint1', endpoint_url: 'endpoint1', documentation_url: 'doc-url1', status: 200, endpoint_entry_date: '2024-02-01', latest_log_entry_date: 'LA1', resource_start_date: '2023-01-01' }, { endpoint: 'endpoint2', endpoint_url: 'endpoint2', documentation_url: 'doc-url2', status: 404, exception: 'exception', endpoint_entry_date: '2023-01-01', latest_log_entry_date: 'LA2', resource_start_date: '2023-01-02' } ], - entityIssueCounts: [ - { - issue: 'Example issue 1', - issue_type: 'Example issue type 1', - field: 'Example issue field 1', - num_issues: 1, - status: 'Error' - } - ], + tasks: { tasks: [{ reference: 'task-1', severity: 'error' }], count: 1 }, notice: undefined, authority: '' } diff --git a/test/unit/middleware/datasetTaskList.middleware.test.js b/test/unit/middleware/datasetTaskList.middleware.test.js index 4c251048..e4730536 100644 --- a/test/unit/middleware/datasetTaskList.middleware.test.js +++ b/test/unit/middleware/datasetTaskList.middleware.test.js @@ -49,8 +49,13 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1', 'entity2'], entityCount: { count: 2 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [{ field: 'field2', issue_type: 'issue-type2', count: 1 }] + tasks: { + tasks: [ + { details: { field: 'field1', issue_type: 'issue-type1', count: 1 }, severity: 'error' }, + { details: { field: 'field2', issue_type: 'issue-type2', count: 1 }, severity: 'error' } + ], + count: 2 + } } const res = { @@ -117,8 +122,10 @@ describe('datasetTaskList.middleware.js', () => { entities: [], entityCount: { count: 0 }, resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'reference values are not unique', count: 1 }], - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: 'reference values are not unique', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { @@ -164,8 +171,7 @@ describe('datasetTaskList.middleware.js', () => { sources: [], entities: ['entity1'], resources: [{ entry_count: 10 }], - entryIssueCounts: [], - entityIssueCounts: [] + tasks: { tasks: [], count: 0 } } const res = { status: vi.fn() } @@ -193,8 +199,10 @@ describe('datasetTaskList.middleware.js', () => { entityCount: { count: 1 }, sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: 'issue-type1', count: 1 }], - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -222,8 +230,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], sources: [], resources: [{ entry_count: 10 }], - entryIssueCounts: [{ field: 'field1', issue_type: '', count: 1 }], // Invalid issue type (empty string) - entityIssueCounts: [] + tasks: { + tasks: [{ details: { field: 'field1', issue_type: '', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -245,8 +255,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [] + tasks: { + tasks: [{ details: { issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + } } const res = { status: vi.fn() } @@ -268,8 +280,10 @@ describe('datasetTaskList.middleware.js', () => { entities: ['entity1'], resources: [{ entry_count: 10 }], sources: [], - entryIssueCounts: [{ issue_type: 'issue-type1', count: 1 }], // Missing field - entityIssueCounts: [], + tasks: { + tasks: [{ details: { issue_type: 'issue-type1', count: 1 }, severity: 'error' }], + count: 1 + }, expectationOutOfBounds: [ { dataset: 'some-dataset', passed: 'False', details: { actual: 3, expected: 0 } } ] diff --git a/test/unit/middleware/dataview.middleware.test.js b/test/unit/middleware/dataview.middleware.test.js index 1e246d27..1bd51585 100644 --- a/test/unit/middleware/dataview.middleware.test.js +++ b/test/unit/middleware/dataview.middleware.test.js @@ -184,8 +184,7 @@ describe('dataview.middleware.test.js', () => { orgInfo: { name: 'Mock Org', entity: 'mock-entity' }, dataset: { name: 'Mock Dataset', dataset: 'mock-dataset' }, tableParams: { columns: ['foo'], fields: ['foo'] }, - entityIssueCounts: [], - entryIssueCounts: [], + tasks: { tasks: [], count: 0 }, pagination: {}, entityCount: { count: 1 }, offset: 0, diff --git a/test/unit/middleware/lpa-overview.middleware.test.js b/test/unit/middleware/lpa-overview.middleware.test.js index eb5c9fcb..287faf96 100644 --- a/test/unit/middleware/lpa-overview.middleware.test.js +++ b/test/unit/middleware/lpa-overview.middleware.test.js @@ -1,5 +1,5 @@ import { describe, it, vi, expect, beforeEach, afterEach } from 'vitest' -import { addNoticesToDatasets, datasetSubmissionDeadlineCheck, getOverview, prepareDatasetObjects, prepareOverviewTemplateParams, prepareAuthorityBatch } from '../../../src/middleware/lpa-overview.middleware.js' +import { addNoticesToDatasets, datasetSubmissionDeadlineCheck, getOverview, prepareDatasetObjects, prepareOverviewTemplateParams, prepareAuthorityBatch, groupIssuesCountsByDataset } from '../../../src/middleware/lpa-overview.middleware.js' import { setupNunjucks } from '../../../src/serverSetup/nunjucks.js' import jsdom from 'jsdom' import platformApi from '../../../src/services/platformApi.js' @@ -500,4 +500,46 @@ describe('lpa-overview.middleware', () => { expect(next).toHaveBeenCalled() }) }) + + describe('groupIssuesCountsByDataset', () => { + it('groups tasks by dataset into req.issues', () => { + const req = { + tasks: { + tasks: [ + { dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } }, + { dataset: 'brownfield-land', details: { issue_type: 'invalid URI', field: 'SiteplanURL', count: 3 } }, + { dataset: 'conservation-area', details: { issue_type: 'missing value', field: 'name', count: 2 } } + ], + count: 3 + } + } + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues['brownfield-land']).toHaveLength(2) + expect(req.issues['conservation-area']).toHaveLength(1) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('produces an empty object when tasks is empty', () => { + const req = { tasks: { tasks: [], count: 0 } } + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues).toEqual({}) + expect(next).toHaveBeenCalledTimes(1) + }) + + it('handles missing tasks gracefully', () => { + const req = {} + const next = vi.fn() + + groupIssuesCountsByDataset(req, {}, next) + + expect(req.issues).toEqual({}) + expect(next).toHaveBeenCalledTimes(1) + }) + }) }) diff --git a/test/unit/services/platformApi.test.js b/test/unit/services/platformApi.test.js index fac58677..a3830037 100644 --- a/test/unit/services/platformApi.test.js +++ b/test/unit/services/platformApi.test.js @@ -176,3 +176,58 @@ describe('platformApi.fetchEntities', () => { await expect(platformApi.fetchEntities(params)).rejects.toThrow('Network Error') }) }) + +describe('platformApi.fetchTasks', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('builds the correct URL with dataset filter', async () => { + axios.get.mockResolvedValueOnce({ data: { tasks: [], count: 0 } }) + + await platformApi.fetchTasks({ + organisation: 'local-authority:TST', + dataset: 'brownfield-land', + severity: 'error', + task_source: 'issue', + limit: 100 + }) + + expect(axios.get).toHaveBeenCalledWith( + 'https://www.planning.data.gov.uk/task.json?organisation=local-authority%3ATST&dataset=brownfield-land&severity=error&task_source=issue&limit=100', + expect.any(Object) + ) + }) + + it('omits dataset param when not provided', async () => { + axios.get.mockResolvedValueOnce({ data: { tasks: [], count: 0 } }) + + await platformApi.fetchTasks({ + organisation: 'local-authority:TST', + severity: 'error', + task_source: 'issue', + limit: 500 + }) + + const calledUrl = axios.get.mock.calls[0][0] + expect(calledUrl).not.toContain('dataset=') + expect(calledUrl).toContain('limit=500') + }) + + it('returns formattedData with tasks and count', async () => { + const tasks = [{ reference: 'abc', dataset: 'brownfield-land', details: { issue_type: 'missing value', field: 'name', count: 1 } }] + axios.get.mockResolvedValueOnce({ data: { tasks, count: 1 } }) + + const result = await platformApi.fetchTasks({ organisation: 'local-authority:TST' }) + + expect(result.formattedData).toEqual({ tasks, count: 1 }) + }) + + it('returns empty formattedData when response is missing fields', async () => { + axios.get.mockResolvedValueOnce({ data: {} }) + + const result = await platformApi.fetchTasks({ organisation: 'local-authority:TST' }) + + expect(result.formattedData).toEqual({ tasks: [], count: 0 }) + }) +})