diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37f26d8fd..ff60c6639 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,7 @@ jobs: environment: development env: DOCKER_REPO: ${{ secrets.DEPLOY_DOCKER_REPOSITORY }} + TESTCONTAINERS_REUSE_ENABLE: true steps: - uses: actions/checkout@v4 - name: Install AWS CLI diff --git a/config/default.yaml b/config/default.yaml index 7ddd0f94c..22ae91755 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -116,9 +116,14 @@ datasetsConfig: base: infrastructure funding statement variable: statement # Any dataset that has one of these three previous rules set, will be displayed in the LPA Dashboard to be checked. +organisationTypes: + - local-authority + - national-park-authority + - development-corporation + - local-planning-group provisionReasons: - statutory - - prospective + - prospective - expected - encouraged tablePageLength: 50 diff --git a/config/production.yaml b/config/production.yaml index 0a27a30ee..bb60eb0c9 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -1,3 +1,7 @@ +organisationTypes: + - local-authority + - national-park-authority + - development-corporation asyncRequestApi: { url: http://production-pub-async-api-lb-636110663.eu-west-2.elb.amazonaws.com } diff --git a/config/staging.yaml b/config/staging.yaml index 833f0bbb1..dde3d2d27 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -1,3 +1,7 @@ +organisationTypes: + - local-authority + - national-park-authority + - development-corporation asyncRequestApi: { url: http://staging-pub-async-api-lb-12493311.eu-west-2.elb.amazonaws.com } diff --git a/src/assets/scss/index.scss b/src/assets/scss/index.scss index 71c196ccc..58ee8e60e 100644 --- a/src/assets/scss/index.scss +++ b/src/assets/scss/index.scss @@ -93,7 +93,7 @@ $govuk-global-styles: true; } .big-number { - @include govuk-font($size: 80, $weight: bold); + @include govuk-font($size: 48, $weight: bold); display: block; } diff --git a/src/filters/filters.js b/src/filters/filters.js index 655025d74..f303e19c7 100644 --- a/src/filters/filters.js +++ b/src/filters/filters.js @@ -32,7 +32,8 @@ export function statusToTagClass (status) { const { govukMarkdown, govukDateTime } = xGovFilters const addFilters = (nunjucksEnv) => { - nunjucksEnv.addFilter('datasetSlugToReadableName', datasetSlugToReadableName) + // Wrapper function for datasetSlugToReadableName filter to allow working with async loading of the mapping data + nunjucksEnv.addFilter('datasetSlugToReadableName', (...args) => datasetSlugToReadableName(...args)) nunjucksEnv.addFilter('govukMarkdown', govukMarkdown) nunjucksEnv.addFilter('govukDateTime', govukDateTime) diff --git a/src/middleware/common.middleware.js b/src/middleware/common.middleware.js index f6d7ee810..b25ac2059 100644 --- a/src/middleware/common.middleware.js +++ b/src/middleware/common.middleware.js @@ -110,7 +110,7 @@ export const takeResourceIdFromParams = (req) => { export const fetchOrgInfo = fetchOne({ query: ({ params }) => { - return `SELECT name, organisation, entity, statistical_geography FROM organisation WHERE organisation = '${params.lpa}'` + return `SELECT name, organisation, entity, statistical_geography, dataset FROM organisation WHERE organisation = '${params.lpa}'` }, result: 'orgInfo' }) @@ -212,8 +212,8 @@ export const prepareAuthority = async (req, res, next) => { return next() } - // Second query: Check for 'some' quality - const someResult = await platformApi.fetchEntities({ + // Second query: Check for 'some' quality, need all, if stroing alternateEntityList + const someResult = await platformApi.fetchAllEntities({ organisation_entity: orgInfo.entity, dataset: params.dataset, quality: 'some' @@ -377,7 +377,7 @@ export const fetchSpecification = fetchOne({ // Fall back dataset fields if no specification found export const fetchDatasetFields = fetchMany({ - query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.collection}'`, + query: ({ req }) => `select field from dataset_field where dataset = '${req.dataset.dataset}'`, result: 'datasetFields' }) @@ -1179,3 +1179,75 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({ result: 'entityIssueCounts', dataset: FetchOptions.performanceDb }) + +/** + * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs: + *. - takes org code and: + * - req.parentGroup {Object[]|null} - If this org is a member of any planning group(s), returns an array of those + * groups with { entity, name, organisation }. Null if this org belongs to no planning groups. + * + * - req.planningGroupMembers {Object[]|null} - If this org IS a planning group, returns an array of its member + * organisations with { organisation, name }, where name is resolved via a parallel Platform API lookup. + * Falls back to the org code as name if the lookup fails. Null if this org is not a planning group. + */ +export const fetchLocalPlanningGroups = async (req, res, next) => { + try { + const { formattedData: allGroups } = await platformApi.fetchAllEntities({ prefix: 'local-planning-group' }) + // Remove all end-dated planning groups + const today = new Date().toISOString().slice(0, 10) + const groups = allGroups.filter(g => !g['end-date'] || g['end-date'] > today) + const orgCode = req.orgInfo.organisation + + // Find any groups this org is within the organisations field. + const parentMatches = groups.filter(g => (g.organisations || '').split(';').includes(orgCode)) + req.parentGroup = parentMatches.length > 0 + ? parentMatches.map(g => ({ entity: g['organisation-entity'], name: g.name, organisation: `${g.prefix}:${g.reference}` })) + : null + + // Look to see if this org is a local-planning group + const ownGroup = groups.find(g => String(g['organisation-entity']) === String(req.orgInfo.entity)) + + // if group get all members, and resolve their names in a single call to the Platform API, then map back to the org codes + if (ownGroup) { + const orgCodes = (ownGroup.organisations || '').split(';').filter(Boolean) + const { flat: allOrgs } = await platformApi.fetchOrganisations() + const nameMap = new Map(allOrgs.map(o => [o.organisation, o.name])) + req.planningGroupMembers = orgCodes.map(organisation => ({ + organisation, + name: nameMap.get(organisation) ?? organisation + })) + } else { + req.planningGroupMembers = null + } + } catch (error) { + logger.warn({ message: `fetchLocalPlanningGroups(): ${error.message}`, type: types.App }) + req.parentGroup = null + req.planningGroupMembers = null + } + next() +} + +/** + * Fetches provision records for the current organisation and any planning groups it belongs to, + * filtered to the current dataset. Includes the organisation name via a JOIN on the organisation table. + * + * Requires: req.params.lpa, req.params.dataset, req.parentGroup (set by fetchLocalPlanningGroups) + * Sets: req.provisions — array of { dataset, project, provision_reason, organisation, name } + * + * TODO: Does it need fetchMany any more, would allow an append of Org Name to fetchLocalPlanningGroups result + */ +export const fetchProvisionsByOrgsAndDatasets = fetchMany({ + query: ({ params, req }) => { + const orgs = [params.lpa] + if (req.parentGroup) { + orgs.push(...req.parentGroup.map(g => g.organisation)) + } + const inClause = orgs.map(o => `'${o}'`).join(', ') + return /* sql */ `select p.dataset, p.project, p.provision_reason, p.organisation, o.name + from provision p + left join organisation o on o.organisation = p.organisation + where p.organisation IN (${inClause}) + AND p.dataset = '${params.dataset}'` + }, + result: 'provisions' +}) diff --git a/src/middleware/datasetOverview.middleware.js b/src/middleware/datasetOverview.middleware.js index c6c6de415..d9881cbb8 100644 --- a/src/middleware/datasetOverview.middleware.js +++ b/src/middleware/datasetOverview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview) */ -import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares } from './common.middleware.js' +import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js' import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import logger from '../utils/logger.js' @@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({ * @param {Function} next - Express next middleware function */ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { - const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [] } = req + const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req let endpointErrorIssues = 0 const endpoints = sources @@ -231,6 +231,10 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { : '' const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}` + const planningGroupProvisions = provisions.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + req.templateParams = { downloadUrl, authority, @@ -239,6 +243,8 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => { dataset, taskCount, alternateSources, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined, + parentGroup, stats: { numberOfRecords: entityCount.entity_count, endpoints @@ -259,6 +265,8 @@ const getDatasetOverview = renderTemplate( export default [ fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetPlatformInfo, fetchColumnSummary, fetchResources, diff --git a/src/middleware/datasetTaskList.middleware.js b/src/middleware/datasetTaskList.middleware.js index dcd2d9739..c0c05f629 100644 --- a/src/middleware/datasetTaskList.middleware.js +++ b/src/middleware/datasetTaskList.middleware.js @@ -21,6 +21,8 @@ import { fetchEntityCount, fetchEntityIssueCounts, fetchEntryIssueCounts, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, fetchResources, fetchSources, logPageError, noop, @@ -206,13 +208,17 @@ export const prepareTasks = (req, res, next) => { * @param {*} next */ export const prepareDatasetTaskListTemplateParams = (req, res, next) => { - const { taskList, dataset, orgInfo: organisation, authority } = req + const { taskList, dataset, orgInfo: organisation, authority, provisions } = req + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] req.templateParams = { taskList, organisation, authority, - dataset + dataset, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined } next() } @@ -226,6 +232,8 @@ const getDatasetTaskList = renderTemplate({ export default [ validateOrgAndDatasetQueryParams, fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchSources, fetchDatasetInfo, fetchResources, diff --git a/src/middleware/dataview.middleware.js b/src/middleware/dataview.middleware.js index cf63c434c..afb0d9c34 100644 --- a/src/middleware/dataview.middleware.js +++ b/src/middleware/dataview.middleware.js @@ -3,6 +3,8 @@ import { createPaginationTemplateParams, extractJsonFieldFromEntities, fetchDatasetInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, processAuthoritativeMiddlewares, processSpecificationMiddlewares, @@ -81,7 +83,7 @@ export const constructTableParams = (req, res, next) => { } export const prepareTemplateParams = (req, res, next) => { - const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields } = req + const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req // Hard code task count for 'some' authority const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1 @@ -91,6 +93,10 @@ export const prepareTemplateParams = (req, res, next) => { : '' const downloadUrl = config.downloadUrl + `/${encodeURIComponent(dataset.dataset)}.csv?organisation-entity=${encodeURIComponent(orgInfo.entity)}&quality=${encodeURIComponent(authority)}${fieldsParams ? '&' + fieldsParams : ''}` + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + req.templateParams = { downloadUrl, organisation: orgInfo, @@ -100,7 +106,8 @@ export const prepareTemplateParams = (req, res, next) => { tableParams, pagination, dataRange, - alternateSources + alternateSources, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined } next() } @@ -117,6 +124,8 @@ export default [ getSetBaseSubPath(['data']), fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetInfo, fetchResources, diff --git a/src/middleware/getStarted.middleware.js b/src/middleware/getStarted.middleware.js index 6cebe94da..3d9629e07 100644 --- a/src/middleware/getStarted.middleware.js +++ b/src/middleware/getStarted.middleware.js @@ -1,10 +1,18 @@ -import { fetchDatasetInfo, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js' +import { fetchDatasetInfo, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchOrgInfo, logPageError, prepareAuthority } from './common.middleware.js' import { renderTemplate } from './middleware.builders.js' export const getGetStarted = renderTemplate({ templateParams (req) { - const { orgInfo: organisation, dataset, authority } = req - return { organisation, dataset, authority } + const { orgInfo: organisation, dataset, authority, provisions } = req + const planningGroupProvisions = provisions?.length > 1 + ? provisions.filter(p => p.organisation !== req.params.lpa) + : [] + return { + organisation, + dataset, + authority, + planningGroupProvisions: planningGroupProvisions.length > 0 ? planningGroupProvisions : undefined + } }, template: 'organisations/get-started.html', handlerName: 'getStarted' @@ -12,6 +20,8 @@ export const getGetStarted = renderTemplate({ export default [ fetchOrgInfo, + fetchLocalPlanningGroups, + fetchProvisionsByOrgsAndDatasets, fetchDatasetInfo, prepareAuthority, getGetStarted, diff --git a/src/middleware/lpa-overview.middleware.js b/src/middleware/lpa-overview.middleware.js index 922e13e91..43842e1f0 100644 --- a/src/middleware/lpa-overview.middleware.js +++ b/src/middleware/lpa-overview.middleware.js @@ -4,7 +4,7 @@ * @description Middleware for oragnisation (LPA) overview page */ -import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb } from './common.middleware.js' +import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js' import { fetchMany, renderTemplate, parallel } from './middleware.builders.js' import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js' import _ from 'lodash' @@ -246,7 +246,7 @@ export function prepareDatasetObjects (req, res, next) { * @returns {void} */ export function prepareOverviewTemplateParams (req, res, next) { - const { orgInfo: organisation, provisions, datasets, availableDatasets } = req + const { orgInfo: organisation, provisions, datasets, availableDatasets, parentGroup, planningGroupMembers } = req const provisionData = new Map() for (const provision of provisions ?? []) { @@ -268,7 +268,6 @@ export function prepareOverviewTemplateParams (req, res, next) { }) const isODPMember = provisions.findIndex((p) => p.project === 'open-digital-planning') >= 0 - const totalDatasets = datasets.length const [datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors] = datasets.reduce(orgStatsReducer, [0, 0, 0]) const datasetsByReason = _.groupBy(datasets, (ds) => { const reason = provisionData.get(ds.dataset)?.provision_reason @@ -285,6 +284,7 @@ export function prepareOverviewTemplateParams (req, res, next) { return 'other' } }) + const totalDatasets = (datasetsByReason.statutory?.length ?? 0) + (datasetsByReason.expected?.length ?? 0) + (datasetsByReason.prospective?.length ?? 0) for (const coll of Object.values(datasetsByReason)) { coll.sort((a, b) => a.dataset.localeCompare(b.dataset)) @@ -297,7 +297,9 @@ export function prepareOverviewTemplateParams (req, res, next) { datasetsWithEndpoints, datasetsWithIssues, datasetsWithErrors, - isODPMember + isODPMember, + parentGroup, + planningGroupMembers } next() @@ -453,6 +455,7 @@ const fetchOutOfBoundsExpectations = expectationFetcher({ export default [ fetchOrgInfo, parallel([ + fetchLocalPlanningGroups, fetchEndpointSummary, fetchEntityIssueCountsPerformanceDb, fetchProvisions diff --git a/src/middleware/organisations.middleware.js b/src/middleware/organisations.middleware.js index 0d21ec356..5ad2c7dab 100644 --- a/src/middleware/organisations.middleware.js +++ b/src/middleware/organisations.middleware.js @@ -8,15 +8,14 @@ const fetchOrganisations = fetchMany({ query: ({ req, params }) => ` SELECT DISTINCT p.organisation, - o.name + o.name, + o.dataset FROM provision p LEFT JOIN organisation o ON p.organisation = o.organisation WHERE ( - p.organisation LIKE 'local-authority:%' - OR p.organisation LIKE 'national-park-authority:%' - OR p.organisation LIKE 'development-corporation:%' + ${config.organisationTypes.map(type => `p.organisation LIKE '${type}:%'`).join('\n OR ')} ) AND ( p.end_date IS NULL @@ -69,14 +68,14 @@ export const prepareGetOrganisationsTemplateParams = (req, res, next) => { return a.name.localeCompare(b.name) }) - const alphabetisedOrgs = sortedResults.reduce((acc, current) => { - const firstLetter = current.name.charAt(0).toUpperCase() - acc[firstLetter] = acc[firstLetter] || [] - acc[firstLetter].push(current) + const orgsByDataset = sortedResults.reduce((acc, current) => { + const datasetKey = current.dataset || 'other' + acc[datasetKey] = acc[datasetKey] || [] + acc[datasetKey].push(current) return acc }, {}) - req.templateParams = { alphabetisedOrgs } + req.templateParams = { orgsByDataset } next() } diff --git a/src/routes/schemas.js b/src/routes/schemas.js index b9658d07f..7193cc74a 100644 --- a/src/routes/schemas.js +++ b/src/routes/schemas.js @@ -115,8 +115,15 @@ const IssueSpecification = v.optional(v.looseObject({ guidance: v.optional(NonEmptyString) })) -const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()) }) +const OrgField = v.strictObject({ name: NonEmptyString, organisation: NonEmptyString, statistical_geography: v.optional(v.string()), entity: v.optional(v.integer()), dataset: v.optional(v.string()) }) const DatasetNameField = v.looseObject({ name: NonEmptyString, dataset: NonEmptyString, collection: v.string() }) +const PlanningGroupProvisionsField = v.optional(v.array(v.strictObject({ + organisation: NonEmptyString, + name: v.nullable(v.string()), + dataset: NonEmptyString, + project: v.nullable(v.string()), + provision_reason: v.nullable(v.string()) +}))) const DatasetItem = v.strictObject({ endpointCount: v.optional(v.number()), status: v.enum(datasetStatusEnum), @@ -145,17 +152,27 @@ export const OrgOverviewPage = v.strictObject({ datasetsWithEndpoints: v.integer(), datasetsWithIssues: v.integer(), datasetsWithErrors: v.integer(), - isODPMember: v.boolean() + isODPMember: v.boolean(), + parentGroup: v.optional(v.nullable(v.array(v.strictObject({ + entity: v.integer(), + name: NonEmptyString, + organisation: NonEmptyString + })))), + planningGroupMembers: v.optional(v.nullable(v.array(v.strictObject({ + organisation: NonEmptyString, + name: NonEmptyString + })))) }) export const OrgFindPage = v.strictObject({ - alphabetisedOrgs: v.record(NonEmptyString, v.array(OrgField)) + orgsByDataset: v.record(NonEmptyString, v.array(OrgField)) }) export const OrgGetStarted = v.strictObject({ organisation: OrgField, dataset: DatasetNameField, - authority: v.string() + authority: v.string(), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgDatasetOverview = v.strictObject({ @@ -182,6 +199,12 @@ export const OrgDatasetOverview = v.strictObject({ })) })) }), + planningGroupProvisions: PlanningGroupProvisionsField, + parentGroup: v.optional(v.nullable(v.array(v.strictObject({ + entity: v.integer(), + name: NonEmptyString, + organisation: NonEmptyString + })))), notice: v.optional(DeadlineNoticeField) }) @@ -194,7 +217,8 @@ export const OrgDataView = v.strictObject({ tableParams, pagination: PaginationParams, dataRange: dataRangeParams, - alternateSources: v.optional(v.array(v.strictObject({ name: NonEmptyString }))) + alternateSources: v.optional(v.array(v.strictObject({ name: NonEmptyString }))), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgDatasetTaskList = v.strictObject({ @@ -214,7 +238,8 @@ export const OrgDatasetTaskList = v.strictObject({ dataset: NonEmptyString, name: NonEmptyString, collection: NonEmptyString - }) + }), + planningGroupProvisions: PlanningGroupProvisionsField }) export const OrgEndpointError = v.strictObject({ diff --git a/src/services/platformApi.js b/src/services/platformApi.js index 429e751ef..1f6f1dea7 100644 --- a/src/services/platformApi.js +++ b/src/services/platformApi.js @@ -12,8 +12,9 @@ export default { * Fetches entities from the Platform API /entity.json endpoint * * @param {Object} params - Query parameters - * @param {string} params.organisation_entity - The organisation entity ID - * @param {string} params.dataset - The dataset name + * @param {string} [params.organisation_entity] - The organisation entity ID + * @param {string} [params.dataset] - The dataset name + * @param {string} [params.prefix] - Entity prefix filter (e.g., 'local-planning-group') * @param {number} [params.limit] - Maximum number of results * @param {number} [params.offset] - Number of results to skip * @param {string} [params.quality] - The quality level (e.g., 'authoritative', 'some') @@ -21,13 +22,15 @@ export default { * @throws {Error} If the query fails or there is an error communicating with the Platform API */ fetchEntities: async (params) => { - if (!params.organisation_entity && !params.dataset) { - throw new Error('organisation_entity or dataset are required parameters') + if (!params.organisation_entity && !params.dataset && !params.prefix && !params.organisation) { + throw new Error('organisation_entity, dataset, prefix, or organisation are required parameters') } const queryParams = new URLSearchParams() if (params.organisation_entity) queryParams.append('organisation_entity', params.organisation_entity) if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.prefix) queryParams.append('prefix', params.prefix) + if (params.organisation) queryParams.append('organisation', params.organisation) if (params.limit) queryParams.append('limit', params.limit) if (params.offset) queryParams.append('offset', params.offset) if (params.quality) queryParams.append('quality', params.quality) @@ -53,6 +56,54 @@ export default { * @returns {Promise<{data: object, formattedData: object[]}>} - A promise that resolves to formatted dataset data * @throws {Error} If the query fails or there is an error communicating with the Platform API */ + /** + * Fetches all entities from the Platform API /entity.json endpoint, paginating through all results. + * Accepts the same params as fetchEntities (except limit/offset which are managed internally). + */ + fetchAllEntities: async (params) => { + const pageSize = 100 + let offset = 0 + let allEntities = [] + + while (true) { + const queryParams = new URLSearchParams() + if (params.organisation_entity) queryParams.append('organisation_entity', params.organisation_entity) + if (params.dataset) queryParams.append('dataset', params.dataset) + if (params.prefix) queryParams.append('prefix', params.prefix) + if (params.organisation) queryParams.append('organisation', params.organisation) + if (params.quality) queryParams.append('quality', params.quality) + queryParams.append('limit', pageSize) + queryParams.append('offset', offset) + const data = await queryPlatformAPI(`${config.mainWebsiteUrl}/entity.json?${queryParams.toString()}`, params) + const entities = data?.entities || [] + allEntities = allEntities.concat(entities) + if (!data?.links?.next || entities.length < pageSize) break + offset += pageSize + } + + return { formattedData: allEntities } + }, + + /** + * Fetches organisations from /organisation.json. + * + * @param {Object} [params] + * @param {string} [params.organisations] - Optional dataset type to filter by (e.g. 'local-authority') + * @returns {Promise<{data: object, grouped: object, flat: object[]}>} + * - data: raw response + * - grouped: organisations keyed by dataset type + * - flat: all organisations as a flat array across all types + */ + fetchOrganisations: async (params = {}) => { + const queryParams = new URLSearchParams() + if (params.organisations) queryParams.append('organisations', params.organisations) + const url = `${config.mainWebsiteUrl}/organisation.json${queryParams.toString() ? '?' + queryParams.toString() : ''}` + const data = await queryPlatformAPI(url, params) + const grouped = data?.organisations || {} + const flat = Object.values(grouped).flat() + return { data, grouped, flat } + }, + fetchDatasets: async (params) => { const queryParams = new URLSearchParams() diff --git a/src/utils/datasetSlugToReadableName.js b/src/utils/datasetSlugToReadableName.js index 94c6e16d6..b6a3ff5b8 100644 --- a/src/utils/datasetSlugToReadableName.js +++ b/src/utils/datasetSlugToReadableName.js @@ -1,16 +1,26 @@ import { makeDatasetSlugToReadableNameFilter } from '../filters/makeDatasetSlugToReadableNameFilter.js' import { getDatasetSlugNameMapping } from './datasetteQueries/getDatasetSlugNameMapping.js' +import logger from './logger.js' + +const RETRY_INTERVAL_MS = 30_000 let datasetSlugToReadableName = (slug) => slug -const initDatasetSlugToReadableNameFilter = async () => { - try { - const mapping = await getDatasetSlugNameMapping() - datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(mapping) - } catch (error) { - console.error('Failed to load dataset mapping:', error) - datasetSlugToReadableName = (slug) => slug // Fallback to using the slug as-is +const retryUntilLoaded = async () => { + while (true) { + try { + const mapping = await getDatasetSlugNameMapping() + datasetSlugToReadableName = makeDatasetSlugToReadableNameFilter(mapping) + return + } catch (error) { + logger.warn(`Failed to load dataset mapping, retrying in ${RETRY_INTERVAL_MS / 1000}s:`, error) + await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL_MS)) + } } +} + +const initDatasetSlugToReadableNameFilter = async () => { + retryUntilLoaded() // keeps trying to update mapping until success return datasetSlugToReadableName } diff --git a/src/views/includes/_planning-group-notice.html b/src/views/includes/_planning-group-notice.html new file mode 100644 index 000000000..40023e26e --- /dev/null +++ b/src/views/includes/_planning-group-notice.html @@ -0,0 +1,17 @@ +{% if planningGroupProvisions and planningGroupProvisions | length > 0 %} +
+{% endif %} diff --git a/src/views/organisations/dataset-overview.html b/src/views/organisations/dataset-overview.html index f4af56cf4..db44d5f9f 100644 --- a/src/views/organisations/dataset-overview.html +++ b/src/views/organisations/dataset-overview.html @@ -193,6 +193,8 @@ {% include "includes/_dataset-page-header.html" %} +{% include "includes/_planning-group-notice.html" %} +