diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37f26d8fd..ff60c6639 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,7 @@ jobs: environment: development env: DOCKER_REPO: ${{ secrets.DEPLOY_DOCKER_REPOSITORY }} + TESTCONTAINERS_REUSE_ENABLE: true steps: - uses: actions/checkout@v4 - name: Install AWS CLI diff --git a/config/default.yaml b/config/default.yaml index 7ddd0f94c..22ae91755 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -116,9 +116,14 @@ datasetsConfig: base: infrastructure funding statement variable: statement # Any dataset that has one of these three previous rules set, will be displayed in the LPA Dashboard to be checked. +organisationTypes: + - local-authority + - national-park-authority + - development-corporation + - local-planning-group provisionReasons: - statutory - - prospective + - prospective - expected - encouraged tablePageLength: 50 diff --git a/config/production.yaml b/config/production.yaml index 0a27a30ee..bb60eb0c9 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -1,3 +1,7 @@ +organisationTypes: + - local-authority + - national-park-authority + - development-corporation asyncRequestApi: { url: http://production-pub-async-api-lb-636110663.eu-west-2.elb.amazonaws.com } diff --git a/config/staging.yaml b/config/staging.yaml index 833f0bbb1..dde3d2d27 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -1,3 +1,7 @@ +organisationTypes: + - local-authority + - national-park-authority + - development-corporation asyncRequestApi: { url: http://staging-pub-async-api-lb-12493311.eu-west-2.elb.amazonaws.com } diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss index 71c196ccc..58ee8e60e 100644 --- a/src/assets/scss/index.scss +++ b/src/assets/scss/index.scss @@ -93,7 +93,7 @@ $govuk-global-styles: true; } .big-number { - @include govuk-font($size: 80, $weight: bold); + @include govuk-font($size: 48, $weight: bold); display: block; } diff --git a/src/filters/filters.js b/src/filters/filters.js index 655025d74..f303e19c7 100644 --- a/src/filters/filters.js +++ b/src/filters/filters.js @@ -32,7 +32,8 @@ export function statusToTagClass (status) { const { govukMarkdown, govukDateTime } = xGovFilters const addFilters = (nunjucksEnv) => { - nunjucksEnv.addFilter('datasetSlugToReadableName', datasetSlugToReadableName) + // Wrapper function for datasetSlugToReadableName filter to allow working with async loading of the mapping data + nunjucksEnv.addFilter('datasetSlugToReadableName', (...args) => datasetSlugToReadableName(...args)) nunjucksEnv.addFilter('govukMarkdown', govukMarkdown) nunjucksEnv.addFilter('govukDateTime', govukDateTime) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index f6d7ee810..b25ac2059 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -110,7 +110,7 @@ export const takeResourceIdFromParams = (req) => { export const fetchOrgInfo = fetchOne({ query: ({ params }) => { - return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'` + return `SELECT name, organisation, entity, statistical_geography, dataset FROM organisation WHERE organisation = '${params.lpa}'` }, result: 'orgInfo' }) @@ -212,8 +212,8 @@ export const prepareAuthority = async (req, res, next) => { return next() } - // Second query: Check for 'some' quality - const someResult = await platformApi.fetchEntities({ + // Second query: Check for 'some' quality, need all, if stroing alternateEntityList + const someResult = await platformApi.fetchAllEntities({ organisation_entity: orgInfo.entity, dataset: params.dataset, quality: 'some' @@ -377,7 +377,7 @@ export const fetchSpecification = fetchOne({ // Fall back dataset fields if no specification found export const fetchDatasetFields = fetchMany({ - query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.collection}'`, + query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.dataset}'`, result: 'datasetFields' }) @@ -1179,3 +1179,75 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({ result: 'entityIssueCounts', dataset: FetchOptions.performanceDb }) + +/** + * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs: + *. - takes org code and: + * - req.parentGroup {Object[]|null} - If this org is a member of any planning group(s), returns an array of those + * groups with { entity, name, organisation }. Null if this org belongs to no planning groups. + * + * - req.planningGroupMembers {Object[]|null} - If this org IS a planning group, returns an array of its member + * organisations with { organisation, name }, where name is resolved via a parallel Platform API lookup. + * Falls back to the org code as name if the lookup fails. Null if this org is not a planning group. + */ +export const fetchLocalPlanningGroups = async (req, res, next) => { + try { + const { formattedData: allGroups } = await platformApi.fetchAllEntities({ prefix: 'local-planning-group' }) + // Remove all end-dated planning groups + const today = new Date().toISOString().slice(0, 10) + const groups = allGroups.filter(g => !g['end-date'] || g['end-date'] > today) + const orgCode = req.orgInfo.organisation + + // Find any groups this org is within the organisations field. + const parentMatches = groups.filter(g => (g.organisations || '').split(';').includes(orgCode)) + req.parentGroup = parentMatches.length > 0 + ? parentMatches.map(g => ({ entity: g['organisation-entity'], name: g.name, organisation: `${g.prefix}:${g.reference}` })) + : null + + // Look to see if this org is a local-planning group + const ownGroup = groups.find(g => String(g['organisation-entity']) === String(req.orgInfo.entity)) + + // if group get all members, and resolve their names in a single call to the Platform API, then map back to the org codes + if (ownGroup) { + const orgCodes = (ownGroup.organisations || '').split(';').filter(Boolean) + const { flat: allOrgs } = await platformApi.fetchOrganisations() + const nameMap = new Map(allOrgs.map(o => [o.organisation, o.name])) + req.planningGroupMembers = orgCodes.map(organisation => ({ + organisation, + name: nameMap.get(organisation) ?? organisation + })) + } else { + req.planningGroupMembers = null + } + } catch (error) { + logger.warn({ message: `fetchLocalPlanningGroups(): ${error.message}`, type: types.App }) + req.parentGroup = null + req.planningGroupMembers = null + } + next() +} + +/** + * Fetches provision records for the current organisation and any planning groups it belongs to, + * filtered to the current dataset. Includes the organisation name via a JOIN on the organisation table. + * + * Requires: req.params.lpa, req.params.dataset, req.parentGroup (set by fetchLocalPlanningGroups) + * Sets: req.provisions — array of { dataset, project, provision_reason, organisation, name } + * + * TODO: Does it need fetchMany any more, would allow an append of Org Name to fetchLocalPlanningGroups result + */ +export const fetchProvisionsByOrgsAndDatasets = fetchMany({ + query: ({ params, req }) => { + const orgs = [params.lpa] + if (req.parentGroup) { + orgs.push(...req.parentGroup.map(g => g.organisation)) + } + const inClause = orgs.map(o => `'${o}'`).join(', ') + return /* sql */ `select p.dataset, p.project, p.provision_reason, p.organisation, o.name + from provision p + left join organisation o on o.organisation = p.organisation + where p.organisation IN (${inClause}) + AND p.dataset = '${params.dataset}'` + }, + result: 'provisions' +}) diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index c6c6de415..d9881cbb8 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' +import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js' import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req + const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req let endpointErrorIssues = 0 const endpoints = sources @@ -231,6 +231,10 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { : '' const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}` + const planningGroupProvisions = provisions.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + req.templateParams = { downloadUrl, authority, @@ -239,6 +243,8 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { dataset, taskCount, alternateSources, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined, + parentGroup, stats: { numberOfRecords: entityCount.entity_count, endpoints @@ -259,6 +265,8 @@ const getDatasetOverview = renderTemplate( export default [ fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetPlatformInfo, fetchColumnSummary, fetchResources, diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index dcd2d9739..c0c05f629 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -21,6 +21,8 @@ import { fetchEntityCount, fetchEntityIssueCounts, fetchEntryIssueCounts, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, fetchResources, fetchSources, logPageError, noop, @@ -206,13 +208,17 @@ export const prepareTasks = (req, res, next) => { * @param {*} next */ export const prepareDatasetTaskListTemplateParams = (req, res, next) => { - const { taskList, dataset, orgInfo: organisation, authority } = req + const { taskList, dataset, orgInfo: organisation, authority, provisions } = req + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] req.templateParams = { taskList, organisation, authority, - dataset + dataset, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined } next() } @@ -226,6 +232,8 @@ const getDatasetTaskList = renderTemplate({ export default [ validateOrgAndDatasetQueryParams, fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchSources, fetchDatasetInfo, fetchResources, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index cf63c434c..afb0d9c34 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -3,6 +3,8 @@ import { createPaginationTemplateParams, extractJsonFieldFromEntities, fetchDatasetInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, processAuthoritativeMiddlewares, processSpecificationMiddlewares, @@ -81,7 +83,7 @@ export const constructTableParams = (req, res, next) => { } export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req // Hard code task count for 'some' authority const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 @@ -91,6 +93,10 @@ export const prepareTemplateParams = (req, res, next) => { : '' const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}` + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + req.templateParams = { downloadUrl, organisation: orgInfo, @@ -100,7 +106,8 @@ export const prepareTemplateParams = (req, res, next) => { tableParams, pagination, dataRange, - alternateSources + alternateSources, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined } next() } @@ -117,6 +124,8 @@ export default [ getSetBaseSubPath(['data']), fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetInfo, fetchResources, diff --git a/src/middleware/getStarted.middleware.js b/src/middleware/getStarted.middleware.js index 6cebe94da..3d9629e07 100644 --- a/src/middleware/getStarted.middleware.js +++ b/src/middleware/getStarted.middleware.js @@ -1,10 +1,18 @@ -import { fetchDatasetInfo, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js' +import { fetchDatasetInfo, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js' import { renderTemplate } from './middleware.builders.js' export const getGetStarted = renderTemplate({ templateParams (req) { - const { orgInfo: organisation, dataset, authority } = req - return { organisation, dataset, authority } + const { orgInfo: organisation, dataset, authority, provisions } = req + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + return { + organisation, + dataset, + authority, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined + } }, template: 'organisations/get-started.html', handlerName: 'getStarted' @@ -12,6 +20,8 @@ export const getGetStarted = renderTemplate({ export default [ fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetInfo, prepareAuthority, getGetStarted, diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 922e13e91..43842e1f0 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for oragnisation (LPA) overview page */ -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js' import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' @@ -246,7 +246,7 @@ export function prepareDatasetObjects (req, res, next) { * @returns {void} */ export function prepareOverviewTemplateParams (req, res, next) { - const { orgInfo: organisation, provisions, datasets, availableDatasets } = req + const { orgInfo: organisation, provisions, datasets, availableDatasets, parentGroup, planningGroupMembers } = req const provisionData = new Map() for (const provision of provisions ?? []) { @@ -268,7 +268,6 @@ export function prepareOverviewTemplateParams (req, res, next) { }) const isODPMember = provisions.findIndex((p) => p.project === 'open-digital-planning') >= 0 - const totalDatasets = datasets.length const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce(orgStatsReducer, [0, 0, 0]) const datasetsByReason = _.groupBy(datasets, (ds) => { const reason = provisionData.get(ds.dataset)?.provision_reason @@ -285,6 +284,7 @@ export function prepareOverviewTemplateParams (req, res, next) { return 'other' } }) + const totalDatasets = (datasetsByReason.statutory?.length ?? 0) + (datasetsByReason.expected?.length ?? 0) + (datasetsByReason.prospective?.length ?? 0) for (const coll of Object.values(datasetsByReason)) { coll.sort((a, b) => a.dataset.localeCompare(b.dataset)) @@ -297,7 +297,9 @@ export function prepareOverviewTemplateParams (req, res, next) { datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors, - isODPMember + isODPMember, + parentGroup, + planningGroupMembers } next() @@ -453,6 +455,7 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ export default [ fetchOrgInfo, parallel([ + fetchLocalPlanningGroups, fetchEndpointSummary, fetchEntityIssueCountsPerformanceDb, fetchProvisions diff --git a/src/middleware/organisations.middleware.js b/src/middleware/organisations.middleware.js index 0d21ec356..5ad2c7dab 100644 --- a/src/middleware/organisations.middleware.js +++ b/src/middleware/organisations.middleware.js @@ -8,15 +8,14 @@ const fetchOrganisations = fetchMany({ query: ({ req, params }) => ` SELECT DISTINCT p.organisation, - o.name + o.name, + o.dataset FROM provision p LEFT JOIN organisation o ON p.organisation = o.organisation WHERE ( - p.organisation LIKE 'local-authority:%' - OR p.organisation LIKE 'national-park-authority:%' - OR p.organisation LIKE 'development-corporation:%' + ${config.organisationTypes.map(type => `p.organisation LIKE '${type}:%'`).join('\n OR ')} ) AND ( p.end_date IS NULL @@ -69,14 +68,14 @@ export const prepareGetOrganisationsTemplateParams = (req, res, next) => { return a.name.localeCompare(b.name) }) - const alphabetisedOrgs = sortedResults.reduce((acc, current) => { - const firstLetter = current.name.charAt(0).toUpperCase() - acc[firstLetter] = acc[firstLetter] || [] - acc[firstLetter].push(current) + const orgsByDataset = sortedResults.reduce((acc, current) => { + const datasetKey = current.dataset || 'other' + acc[datasetKey] = acc[datasetKey] || [] + acc[datasetKey].push(current) return acc }, {}) - req.templateParams = { alphabetisedOrgs } + req.templateParams = { orgsByDataset } next() } diff --git a/src/routes/schemas.js b/src/routes/schemas.js index b9658d07f..7193cc74a 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -115,8 +115,15 @@ const IssueSpecification = v.optional(v.looseObject({ guidance: v.optional(NonEmptyString) })) -const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) }) +const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()), dataset: v.optional(v.string()) }) const DatasetNameField = v.looseObject({ name: NonEmptyString, dataset: NonEmptyString, collection: v.string() }) +const PlanningGroupProvisionsField = v.optional(v.array(v.strictObject({ + organisation: NonEmptyString, + name: v.nullable(v.string()), + dataset: NonEmptyString, + project: v.nullable(v.string()), + provision_reason: v.nullable(v.string()) +}))) const DatasetItem = v.strictObject({ endpointCount: v.optional(v.number()), status: v.enum(datasetStatusEnum), @@ -145,17 +152,27 @@ export const OrgOverviewPage = v.strictObject({ datasetsWithEndpoints: v.integer(), datasetsWithIssues: v.integer(), datasetsWithErrors: v.integer(), - isODPMember: v.boolean() + isODPMember: v.boolean(), + parentGroup: v.optional(v.nullable(v.array(v.strictObject({ + entity: v.integer(), + name: NonEmptyString, + organisation: NonEmptyString + })))), + planningGroupMembers: v.optional(v.nullable(v.array(v.strictObject({ + organisation: NonEmptyString, + name: NonEmptyString + })))) }) export const OrgFindPage = v.strictObject({ - alphabetisedOrgs: v.record(NonEmptyString, v.array(OrgField)) + orgsByDataset: v.record(NonEmptyString, v.array(OrgField)) }) export const OrgGetStarted = v.strictObject({ organisation: OrgField, dataset: DatasetNameField, - authority: v.string() + authority: v.string(), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgDatasetOverview = v.strictObject({ @@ -182,6 +199,12 @@ export const OrgDatasetOverview = v.strictObject({ })) })) }), + planningGroupProvisions: PlanningGroupProvisionsField, + parentGroup: v.optional(v.nullable(v.array(v.strictObject({ + entity: v.integer(), + name: NonEmptyString, + organisation: NonEmptyString + })))), notice: v.optional(DeadlineNoticeField) }) @@ -194,7 +217,8 @@ export const OrgDataView = v.strictObject({ tableParams, pagination: PaginationParams, dataRange: dataRangeParams, - alternateSources: v.optional(v.array(v.strictObject({ name: NonEmptyString }))) + alternateSources: v.optional(v.array(v.strictObject({ name: NonEmptyString }))), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgDatasetTaskList = v.strictObject({ @@ -214,7 +238,8 @@ export const OrgDatasetTaskList = v.strictObject({ dataset: NonEmptyString, name: NonEmptyString, collection: NonEmptyString - }) + }), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgEndpointError = v.strictObject({ diff --git a/src/services/platformApi.js b/src/services/platformApi.js index 429e751ef..1f6f1dea7 100644 --- a/src/services/platformApi.js +++ b/src/services/platformApi.js @@ -12,8 +12,9 @@ export default { * Fetches entities from the Platform API /entity.json endpoint * * @param {Object} params - Query parameters - * @param {string} params.organisation_entity - The organisation entity ID - * @param {string} params.dataset - The dataset name + * @param {string} [params.organisation_entity] - The organisation entity ID + * @param {string} [params.dataset] - The dataset name + * @param {string} [params.prefix] - Entity prefix filter (e.g., 'local-planning-group') * @param {number} [params.limit] - Maximum number of results * @param {number} [params.offset] - Number of results to skip * @param {string} [params.quality] - The quality level (e.g., 'authoritative', 'some') @@ -21,13 +22,15 @@ export default { * @throws {Error} If the query fails or there is an error communicating with the Platform API */ fetchEntities: async (params) => { - if (!params.organisation_entity && !params.dataset) { - throw new Error('organisation_entity or dataset are required parameters') + if (!params.organisation_entity && !params.dataset && !params.prefix && !params.organisation) { + throw new Error('organisation_entity, dataset, prefix, or organisation are required parameters') } const queryParams = new URLSearchParams() if (params.organisation_entity) queryParams.append('organisation_entity', params.organisation_entity) if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.prefix) queryParams.append('prefix', params.prefix) + if (params.organisation) queryParams.append('organisation', params.organisation) if (params.limit) queryParams.append('limit', params.limit) if (params.offset) queryParams.append('offset', params.offset) if (params.quality) queryParams.append('quality', params.quality) @@ -53,6 +56,54 @@ export default { * @returns {Promise<{data: object, formattedData: object[]}>} - A promise that resolves to formatted dataset data * @throws {Error} If the query fails or there is an error communicating with the Platform API */ + /** + * Fetches all entities from the Platform API /entity.json endpoint, paginating through all results. + * Accepts the same params as fetchEntities (except limit/offset which are managed internally). + */ + fetchAllEntities: async (params) => { + const pageSize = 100 + let offset = 0 + let allEntities = [] + + while (true) { + const queryParams = new URLSearchParams() + if (params.organisation_entity) queryParams.append('organisation_entity', params.organisation_entity) + if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.prefix) queryParams.append('prefix', params.prefix) + if (params.organisation) queryParams.append('organisation', params.organisation) + if (params.quality) queryParams.append('quality', params.quality) + queryParams.append('limit', pageSize) + queryParams.append('offset', offset) + const data = await queryPlatformAPI(`${config.mainWebsiteUrl}/entity.json?${queryParams.toString()}`, params) + const entities = data?.entities || [] + allEntities = allEntities.concat(entities) + if (!data?.links?.next || entities.length < pageSize) break + offset += pageSize + } + + return { formattedData: allEntities } + }, + + /** + * Fetches organisations from /organisation.json. + * + * @param {Object} [params] + * @param {string} [params.organisations] - Optional dataset type to filter by (e.g. 'local-authority') + * @returns {Promise<{data: object, grouped: object, flat: object[]}>} + * - data: raw response + * - grouped: organisations keyed by dataset type + * - flat: all organisations as a flat array across all types + */ + fetchOrganisations: async (params = {}) => { + const queryParams = new URLSearchParams() + if (params.organisations) queryParams.append('organisations', params.organisations) + const url = `${config.mainWebsiteUrl}/organisation.json${queryParams.toString() ? '?' + queryParams.toString() : ''}` + const data = await queryPlatformAPI(url, params) + const grouped = data?.organisations || {} + const flat = Object.values(grouped).flat() + return { data, grouped, flat } + }, + fetchDatasets: async (params) => { const queryParams = new URLSearchParams() diff --git a/src/utils/datasetSlugToReadableName.js b/src/utils/datasetSlugToReadableName.js index 94c6e16d6..b6a3ff5b8 100644 --- a/src/utils/datasetSlugToReadableName.js +++ b/src/utils/datasetSlugToReadableName.js @@ -1,16 +1,26 @@ import { makeDatasetSlugToReadableNameFilter } from '../filters/makeDatasetSlugToReadableNameFilter.js' import { getDatasetSlugNameMapping } from './datasetteQueries/getDatasetSlugNameMapping.js' +import logger from './logger.js' + +const RETRY_INTERVAL_MS = 30_000 let datasetSlugToReadableName = (slug) => slug -const initDatasetSlugToReadableNameFilter = async () => { - try { - const mapping = await getDatasetSlugNameMapping() - datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(mapping) - } catch (error) { - console.error('Failed to load dataset mapping:', error) - datasetSlugToReadableName = (slug) => slug // Fallback to using the slug as-is +const retryUntilLoaded = async () => { + while (true) { + try { + const mapping = await getDatasetSlugNameMapping() + datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(mapping) + return + } catch (error) { + logger.warn(`Failed to load dataset mapping, retrying in ${RETRY_INTERVAL_MS / 1000}s:`, error) + await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL_MS)) + } } +} + +const initDatasetSlugToReadableNameFilter = async () => { + retryUntilLoaded() // keeps trying to update mapping until success return datasetSlugToReadableName } diff --git a/src/views/includes/_planning-group-notice.html b/src/views/includes/_planning-group-notice.html new file mode 100644 index 000000000..40023e26e --- /dev/null +++ b/src/views/includes/_planning-group-notice.html @@ -0,0 +1,17 @@ +{% if planningGroupProvisions and planningGroupProvisions | length > 0 %} +
+
+

+ Important +

+
+
+

This organisation is attached to a planning group which is also provisioned to provide this data:

+ +
+
+{% endif %} diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index f4af56cf4..db44d5f9f 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -193,6 +193,8 @@ {% include "includes/_dataset-page-header.html" %} +{% include "includes/_planning-group-notice.html" %} +
{{ datasetNavigation({ diff --git a/src/views/organisations/datasetTaskList.html b/src/views/organisations/datasetTaskList.html index 0b927638c..87a8a937b 100644 --- a/src/views/organisations/datasetTaskList.html +++ b/src/views/organisations/datasetTaskList.html @@ -41,6 +41,8 @@ {% include "includes/_dataset-page-header.html" %}
+{% include "includes/_planning-group-notice.html" %} +
{{ datasetNavigation({ diff --git a/src/views/organisations/dataview.html b/src/views/organisations/dataview.html index 881e9086a..8cb38a7a0 100644 --- a/src/views/organisations/dataview.html +++ b/src/views/organisations/dataview.html @@ -43,6 +43,8 @@ {% include "includes/_dataset-page-header.html" %}
+{% include "includes/_planning-group-notice.html" %} +
{{ datasetNavigation({ diff --git a/src/views/organisations/find.html b/src/views/organisations/find.html index 2a4637059..685ef8931 100644 --- a/src/views/organisations/find.html +++ b/src/views/organisations/find.html @@ -46,19 +46,19 @@

{{ pageName }}

- {% for letter, orgs in alphabetisedOrgs %} + {% for datasetType, orgs in orgsByDataset %}
-

{{ letter }}

+

{{ datasetType | datasetSlugToReadableName(true) }}

+ {{ orgs | length }}
- + {% for org in orgs %} + + {% endfor %}

diff --git a/src/views/organisations/get-started.html b/src/views/organisations/get-started.html index a3702c96b..005d0392c 100644 --- a/src/views/organisations/get-started.html +++ b/src/views/organisations/get-started.html @@ -40,6 +40,8 @@ {% include "includes/_dataset-page-header.html" %}
+{% include "includes/_planning-group-notice.html" %} +
{% if authority and authority === "some" %} diff --git a/src/views/organisations/overview.html b/src/views/organisations/overview.html index b4e5b4cd7..616f99f04 100644 --- a/src/views/organisations/overview.html +++ b/src/views/organisations/overview.html @@ -81,11 +81,16 @@

{% block content %}
-
+
+ {% if organisation.dataset %} + {{ organisation.dataset | datasetSlugToReadableName(true) }} + {% endif %} +

{{ pageName }}

+
+
-

- {{ pageName }} -

+
+
{% for dataset in datasets.statutory %} {% if dataset.notice %} @@ -171,6 +176,37 @@

Datasets {{ organisation.name}} can provide

{% endif %}
+ +
+ {% if planningGroupMembers %} +
+
+

Membership

+
+
+ +
+
+ {% endif %} + {% if parentGroup %} +
+
+

Group {{ "Membership" | pluralise(parentGroup | length) }}

+
+
+ +
+
+ {% endif %} +
{% endblock %} diff --git a/test/integration/authoritative_data.playwright.test.js b/test/integration/authoritative_data.playwright.test.js index 273d2810a..4220c4743 100644 --- a/test/integration/authoritative_data.playwright.test.js +++ b/test/integration/authoritative_data.playwright.test.js @@ -1,9 +1,12 @@ import { test, expect } from '@playwright/test' const resetWiremock = async () => { - await fetch('http://localhost:8001/__admin/mappings/reset', { + const response = await fetch('http://localhost:8001/__admin/mappings/reset', { method: 'POST' }) + if (!response.ok) { + throw new Error(`Failed to reset WireMock mappings: ${response.status} ${await response.text()}`) + } } test.beforeEach(async () => { diff --git a/test/integration/global-setup.js b/test/integration/global-setup.js index 8e8d11cc0..d1224f34c 100644 --- a/test/integration/global-setup.js +++ b/test/integration/global-setup.js @@ -12,6 +12,10 @@ setup('Global setup', async () => { await new Wiremock().start() - localstack = await new Localstack().start() - await localstack.createBucket(config.aws.bucket) + // In CI, LocalStack is provided by docker-compose with bucket already created via bootstrap script. + // Only start the testcontainer locally where docker-compose is not running. + if (!process.env.CI) { + localstack = await new Localstack().start() + await localstack.createBucket(config.aws.bucket) + } }) diff --git a/test/integration/global-teardown.js b/test/integration/global-teardown.js index 19cc8040e..de5a7b6af 100644 --- a/test/integration/global-teardown.js +++ b/test/integration/global-teardown.js @@ -10,5 +10,8 @@ teardown('Global teardown', async () => { await new Wiremock().stop() - await new Localstack().stop() + // Only stop the testcontainer locally — in CI it was never started (docker-compose handles LocalStack). + if (!process.env.CI) { + await new Localstack().stop() + } }) diff --git a/test/integration/planning_group.playwright.test.js b/test/integration/planning_group.playwright.test.js new file mode 100644 index 000000000..fcbdc4a09 --- /dev/null +++ b/test/integration/planning_group.playwright.test.js @@ -0,0 +1,66 @@ +import { test, expect } from '@playwright/test' + +const resetWiremock = async () => { + const response = await fetch('http://localhost:8001/__admin/mappings/reset', { + method: 'POST' + }) + if (!response.ok) { + throw new Error(`Failed to reset WireMock mappings: ${response.status} ${await response.text()}`) + } +} + +const addWiremockStub = async (stub) => { + const response = await fetch('http://localhost:8001/__admin/mappings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(stub) + }) + if (!response.ok) { + throw new Error(`Failed to add WireMock stub: ${response.status} ${await response.text()}`) + } +} + +test.describe('Planning group', () => { + test.describe.configure({ mode: 'serial' }) + + test.beforeEach(async () => { + await resetWiremock() + }) + + test('An LPA that is a member of a planning group shows the group membership banner on the overview page', async ({ page }) => { + // Stub the local-planning-group entity endpoint to return a group containing local-authority:SLF + await addWiremockStub({ + priority: 1, + request: { + method: 'GET', + urlPattern: '/entity.json.*prefix=local-planning-group.*' + }, + response: { + status: 200, + jsonBody: { + entities: [ + { + entity: 9000001, + name: 'South London Joint Planning Group', + prefix: 'local-planning-group', + reference: 'south-london-joint', + 'organisation-entity': 9000001, + organisations: 'local-authority:SLF;local-authority:LBH', + 'end-date': '' + } + ], + count: 1 + } + } + }) + + // Navigate to the LPA overview for SLF + await page.goto('/organisations/local-authority:SLF') + + // The Group Membership banner should be visible + await expect(page.getByText('Group Membership')).toBeVisible() + + // The planning group name should appear as a link + await expect(page.getByRole('link', { name: 'South London Joint Planning Group' })).toBeVisible() + }) +}) diff --git a/test/testContainers/localstack.js b/test/testContainers/localstack.js index 36290f3c3..d6bcee579 100644 --- a/test/testContainers/localstack.js +++ b/test/testContainers/localstack.js @@ -3,7 +3,7 @@ import { S3Client, CreateBucketCommand } from '@aws-sdk/client-s3' export default class Localstack { constructor () { - this.image = 'localstack/localstack:latest' + this.image = 'localstack/localstack:3.1' this.buckets = [] } @@ -16,7 +16,6 @@ export default class Localstack { async stop () { console.log('Stopping LocalstackContainer') this.container = await new LocalstackContainer(this.image).withReuse(true).start() - this.container.stop() await this.container.stop() } diff --git a/test/testContainers/wiremock.js b/test/testContainers/wiremock.js index 99b74163b..bf905b035 100644 --- a/test/testContainers/wiremock.js +++ b/test/testContainers/wiremock.js @@ -43,6 +43,6 @@ export default class Wiremock { host: config.asyncRequestApi.port }) .withReuse(true).start() - this.container.stop() + await this.container.stop() } } diff --git a/test/unit/findPage.test.js b/test/unit/findPage.test.js index 29cb15ecb..9d47dc582 100644 --- a/test/unit/findPage.test.js +++ b/test/unit/findPage.test.js @@ -40,13 +40,12 @@ describe(`Organisations Find Page (seed: ${seed})`, () => { it('Renders the correct organisation list with appropriate attributes', () => { const organisationList = document.querySelector('#search_results') - expect(organisationList.children.length).toBe(Object.keys(params.alphabetisedOrgs).length) + expect(organisationList.children.length).toBe(Object.keys(params.orgsByDataset).length) - Object.keys(params.alphabetisedOrgs).forEach((letter, i) => { + Object.keys(params.orgsByDataset).forEach((datasetType, i) => { const organisationSection = organisationList.children[i] - expect(organisationSection.querySelector('.blockHeading').textContent).toBe(letter) - const organisationListItems = organisationSection.querySelector('.govuk-list').children - params.alphabetisedOrgs[letter].forEach((organisation, j) => { + const organisationListItems = organisationSection.querySelectorAll('[data-filter="item"]') + params.orgsByDataset[datasetType].forEach((organisation, j) => { expect(organisationListItems[j].textContent).toContain(organisation.name) expect(organisationListItems[j].getAttribute('data-filter')).toEqual('item') expect(organisationListItems[j].getAttribute('data-filter-terms')).toEqual(organisation.name) diff --git a/test/unit/middleware/common.middleware.test.js b/test/unit/middleware/common.middleware.test.js index 7ead80739..0f93afeca 100644 --- a/test/unit/middleware/common.middleware.test.js +++ b/test/unit/middleware/common.middleware.test.js @@ -10,7 +10,8 @@ vi.mock('../../../src/services/performanceDbApi.js') vi.mock('../../../src/services/platformApi.js', () => ({ default: { - fetchEntities: vi.fn() + fetchEntities: vi.fn(), + fetchAllEntities: vi.fn() } })) @@ -1580,20 +1581,20 @@ describe('preventIndexing middleware', () => { }) it('sets req.authority to "some" when no authoritative but some quality entities exist', async () => { - platformApi.fetchEntities - .mockResolvedValueOnce({ formattedData: [] }) // authoritative query returns empty - .mockResolvedValueOnce({ formattedData: [{ entity: '2', name: 'Some Quality Entity' }] }) // some query returns data + platformApi.fetchEntities.mockResolvedValueOnce({ formattedData: [] }) // authoritative query returns empty + platformApi.fetchAllEntities.mockResolvedValueOnce({ formattedData: [{ entity: '2', name: 'Some Quality Entity' }] }) // some query returns data await prepareAuthority(req, res, next) - expect(platformApi.fetchEntities).toHaveBeenCalledTimes(2) - expect(platformApi.fetchEntities).toHaveBeenNthCalledWith(1, { + expect(platformApi.fetchEntities).toHaveBeenCalledTimes(1) + expect(platformApi.fetchEntities).toHaveBeenCalledWith({ organisation_entity: '12345', dataset: 'local-plan-boundary', quality: 'authoritative', limit: 1 }) - expect(platformApi.fetchEntities).toHaveBeenNthCalledWith(2, { + expect(platformApi.fetchAllEntities).toHaveBeenCalledTimes(1) + expect(platformApi.fetchAllEntities).toHaveBeenCalledWith({ organisation_entity: '12345', dataset: 'local-plan-boundary', quality: 'some' @@ -1603,13 +1604,13 @@ describe('preventIndexing middleware', () => { }) it('sets req.authority to empty string when neither authoritative nor some quality entities exist', async () => { - platformApi.fetchEntities - .mockResolvedValueOnce({ formattedData: [] }) // authoritative query returns empty - .mockResolvedValueOnce({ formattedData: [] }) // some query returns empty + platformApi.fetchEntities.mockResolvedValueOnce({ formattedData: [] }) // authoritative query returns empty + platformApi.fetchAllEntities.mockResolvedValueOnce({ formattedData: [] }) // some query returns empty await prepareAuthority(req, res, next) - expect(platformApi.fetchEntities).toHaveBeenCalledTimes(2) + expect(platformApi.fetchEntities).toHaveBeenCalledTimes(1) + expect(platformApi.fetchAllEntities).toHaveBeenCalledTimes(1) expect(req.authority).toBe('') expect(next).toHaveBeenCalledTimes(1) }) @@ -1626,9 +1627,8 @@ describe('preventIndexing middleware', () => { }) it('handles null formattedData gracefully', async () => { - platformApi.fetchEntities - .mockResolvedValueOnce({ formattedData: null }) // authoritative query returns null - .mockResolvedValueOnce({ formattedData: null }) // some query returns null + platformApi.fetchEntities.mockResolvedValueOnce({ formattedData: null }) // authoritative query returns null + platformApi.fetchAllEntities.mockResolvedValueOnce({ formattedData: null }) // some query returns null await prepareAuthority(req, res, next) @@ -1637,9 +1637,8 @@ describe('preventIndexing middleware', () => { }) it('handles undefined formattedData gracefully', async () => { - platformApi.fetchEntities - .mockResolvedValueOnce({}) // authoritative query returns no formattedData - .mockResolvedValueOnce({}) // some query returns no formattedData + platformApi.fetchEntities.mockResolvedValueOnce({}) // authoritative query returns no formattedData + platformApi.fetchAllEntities.mockResolvedValueOnce({}) // some query returns no formattedData await prepareAuthority(req, res, next) diff --git a/test/unit/middleware/organisations.middleware.test.js b/test/unit/middleware/organisations.middleware.test.js index 9650e533a..4ce7ecf28 100644 --- a/test/unit/middleware/organisations.middleware.test.js +++ b/test/unit/middleware/organisations.middleware.test.js @@ -21,21 +21,13 @@ describe('organisations.middleware.js', () => { prepareGetOrganisationsTemplateParams(req, res, next) const expectedTemplatePrams = { - alphabetisedOrgs: { - A: [ - { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' } - ], - B: [ + orgsByDataset: { + other: [ + { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' }, { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' }, - { name: 'Bristol Hospital', organisation: 'Bristol Hospital' } - ], - C: [ - { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' } - ], - D: [ - { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' } - ], - E: [ + { name: 'Bristol Hospital', organisation: 'Bristol Hospital' }, + { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' }, + { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' }, { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' } ] } @@ -52,22 +44,10 @@ describe('organisations.middleware.js', () => { const next = vi.fn() req.templateParams = { - alphabetisedOrgs: { - A: [ - { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' } - ], - B: [ - { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' }, - { name: 'Bristol Hospital', organisation: 'Bristol Hospital' } - ], - C: [ - { name: 'Cardiff Health Board', organisation: 'Cardiff Health Board' } - ], - D: [ - { name: 'Derbyshire Healthcare', organisation: 'Derbyshire Healthcare' } - ], - E: [ - { name: 'East Sussex NHS Trust', organisation: 'East Sussex NHS Trust' } + orgsByDataset: { + other: [ + { name: 'Aardvark Healthcare', organisation: 'Aardvark Healthcare' }, + { name: 'Bath NHS Trust', organisation: 'Bath NHS Trust' } ] } } diff --git a/test/unit/views/organisations/get-startedPage.test.js b/test/unit/views/organisations/get-startedPage.test.js index 47d403217..441b6a7ac 100644 --- a/test/unit/views/organisations/get-startedPage.test.js +++ b/test/unit/views/organisations/get-startedPage.test.js @@ -26,7 +26,7 @@ describe(`Get Started Page (seed: ${seed})`, () => { expect(document.querySelector('span.govuk-caption-xl').textContent).toEqual(params.organisation.name) // Filter returns slug in tests (no mapping loaded), check slug passes through expect(document.querySelector('h1').textContent).toContain(params.dataset.dataset) - expect(document.querySelector('#main-content h2').textContent).toContain(`How to prepare and provide your ${params.dataset.dataset} data`) + expect(document.querySelector('#main-content h2.govuk-heading-l').textContent).toContain(`How to prepare and provide your ${params.dataset.dataset} data`) }) it('Renders breadcrumbs correctly', () => {