1180 display tasks in lpa dashboard

config/development.yaml

@@ -6,8 +6,9 @@ aws: {
     bucket: 'development-pub-async-request-files',
 }
 manageServiceUrl: 'https://manage.development.planning.data.gov.uk'
+mainWebsiteUrl: 'https://www.development.planning.data.gov.uk'
 datasetteUrl: https://datasette.development.planning.data.gov.uk
-downloadUrl: 'https://download.development.planning.data.gov.uk' 
+downloadUrl: 'https://download.development.planning.data.gov.uk'
 redis: {
     secure: true,
     host: 'development-pub-async-redis-3dqynz.serverless.euw2.cache.amazonaws.com',

config/production.yaml

@@ -11,6 +11,7 @@ aws: {
     bucket: 'production-pub-async-request-files',
 }
 manageServiceUrl: 'https://manage.planning.data.gov.uk'
+mainWebsiteUrl: 'https://www.planning.data.gov.uk'
 redis: {
     secure: true,
     host: 'production-pub-async-redis-eihmmv.serverless.euw2.cache.amazonaws.com',

config/staging.yaml

@@ -11,6 +11,7 @@ aws: {
     bucket: 'staging-pub-async-request-files',
 }
 manageServiceUrl: 'https://manage.staging.planning.data.gov.uk/'
+mainWebsiteUrl: 'https://www.staging.planning.data.gov.uk'
 redis: {
     secure: true,
     host: 'staging-pub-async-redis-kh1elu.serverless.euw2.cache.amazonaws.com',

src/middleware/common.middleware.js

@@ -668,6 +668,7 @@ export const filterOutEntitiesWithoutIssues = (req, res, next) => {
 // entity issues
 
 const fetchEntityIssuesForFieldAndType = fetchMany({
+  dataset: FetchOptions.fromParams,
   query: ({ req, params }) => {
     const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
     const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
@@ -679,18 +680,14 @@ const fetchEntityIssuesForFieldAndType = fetchMany({
             field,
             entity,
             message,
-            severity,
             value,
             ROW_NUMBER() OVER (
               PARTITION BY i.issue_type, entity
               ORDER BY i.rowid
             ) AS rn
           FROM issue i
-          LEFT JOIN issue_type it ON i.issue_type = it.issue_type
           WHERE resource IN ('${req.resources.map(resource => resource.resource).join("', '")}')
             ${issueTypeClause}
-            AND it.responsibility = 'external'
-            AND it.severity = 'error'
             ${issueFieldClause}
             AND i.dataset = '${req.params.dataset}'
             AND entity != ''
@@ -704,7 +701,6 @@ const fetchEntityIssuesForFieldAndType = fetchMany({
           field,
           entity,
           message,
-          severity,
           value
         FROM ranked
         WHERE rn = 1
@@ -805,17 +801,16 @@ export const addFieldMappingsToIssue = (req, res, next) => {
 
 // We can only get the issues without entity from the latest resource as we have no way of knowing if those in previous resources have been fixed?
 export const fetchEntryIssues = fetchMany({
+  dataset: FetchOptions.fromParams,
   query: ({ req, params }) => {
+    if (!req.resources[0]) return 'SELECT 1 WHERE 0'
     const issueTypeClause = params.issue_type ? `AND i.issue_type = '${params.issue_type}'` : ''
     const issueFieldClause = params.issue_field ? `AND field = '${params.issue_field}'` : ''
     return `
-      select i.issue_type, field, entity, message, severity, value, line_number
+      select i.issue_type, field, entity, message, value, line_number
       from issue i
-      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
       WHERE resource = '${req.resources[0].resource}'
       ${issueTypeClause}
-      AND it.responsibility = 'external'
-      AND it.severity = 'error'
       AND i.dataset = '${req.params.dataset}'
       ${issueFieldClause}
       AND (entity = '' OR entity is NULL OR i.issue_type in ('${entryIssueGroups.map(issue => issue.type).join("', '")}'))
@@ -1224,6 +1219,40 @@ export const fetchEntityIssueCountsPerformanceDb = fetchMany({
   dataset: FetchOptions.performanceDb
 })
 
+/**
+ * Fetches error-severity issue tasks from the platform API for the current organisation,
+ * then deduplicates by (dataset, issue_type, field) keeping the highest count per group.
+ * When `req.params.dataset` is set (e.g. dataset task list page), filters to that
+ * dataset and uses a limit of 100. Without a dataset (e.g. LPA overview page),
+ * fetches across all datasets with a limit of 500.
+ * Result is stored on `req.tasks` as `{ tasks: [...], count: N }`.
+ */
+export const fetchTasksFromPlatformApi = async (req, res, next) => {
+  const dataset = req.params.dataset
+  try {
+    const { formattedData } = await platformApi.fetchTasks({
+      organisation: req.orgInfo?.organisation,
+      ...(dataset && { dataset }),
+      severity: 'error',
+      task_source: 'issue',
+      limit: dataset ? 100 : 500
+    })
+    const deduplicated = Object.values(
+      (formattedData.tasks ?? []).reduce((acc, task) => {
+        const key = `${task.dataset}::${task.details?.issue_type}::${task.details?.field}`
+        if (!acc[key] || task.details?.count > acc[key].details?.count) acc[key] = task
+        return acc
+      }, {})
+    )
+    req.tasks = { tasks: deduplicated, count: deduplicated.length }
+    next()
+  } catch (err) {
+    logger.warn('fetchTasksFromPlatformApi: failed to fetch tasks', { err })
+    req.tasks = { tasks: [], count: 0 }
+    next()
+  }
+}
+
 /**
  * Middleware. Fetches all local-planning-group entities from the Platform API in a single call and derives two outputs:
  *. - takes org code and:

src/middleware/datasetOverview.middleware.js

@@ -4,7 +4,7 @@
  * @description Middleware for dataset overview page (under /oranisations/:lpa/:dataset/overview)
  */
 
-import { fetchDatasetPlatformInfo, fetchEntityIssueCountsPerformanceDb, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets } from './common.middleware.js'
+import { fetchDatasetPlatformInfo, fetchOrgInfo, fetchResources, fetchSources, logPageError, processSpecificationMiddlewares, expectationFetcher, expectations, noop, processAuthoritativeMiddlewares, fetchLocalPlanningGroups, fetchProvisionsByOrgsAndDatasets, fetchTasksFromPlatformApi } from './common.middleware.js'
 import { fetchOne, fetchMany, onlyIf, renderTemplate, FetchOptions, FetchOneFallbackPolicy } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import logger from '../utils/logger.js'
@@ -186,7 +186,7 @@ export const fetchEntityCount = fetchOne({
  * @param {Function} next - Express next middleware function
  */
 export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
-  const { orgInfo, entityCount, sources, dataset, entityIssueCounts, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req
+  const { orgInfo, entityCount, sources, dataset, tasks, notice, authority, alternateSources, uniqueDatasetFields, expectationOutOfBounds = [], provisions = [], parentGroup } = req
 
   let endpointErrorIssues = 0
   const endpoints = sources
@@ -219,7 +219,7 @@ export const prepareDatasetOverviewTemplateParams = (req, res, next) => {
   if (authority === 'some') {
     taskCount = 1
   } else {
-    taskCount = (entityIssueCounts ? entityIssueCounts.length : 0) +
+    taskCount = (tasks?.count ?? 0) +
     endpointErrorIssues +
     (expectationOutOfBounds.length > 0 ? 1 : 0)
   }
@@ -271,7 +271,7 @@ export default [
   fetchColumnSummary,
   fetchResources,
   fetchSources,
-  fetchEntityIssueCountsPerformanceDb,
+  fetchTasksFromPlatformApi,
   fetchSpecification,
   isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop,
   ...processAuthoritativeMiddlewares,

src/middleware/datasetTaskList.middleware.js

@@ -19,8 +19,7 @@ import {
   expectations,
   fetchDatasetInfo,
   fetchEntityCount,
-  fetchEntityIssueCounts,
-  fetchEntryIssueCounts,
+  fetchTasksFromPlatformApi,
   fetchLocalPlanningGroups,
   fetchProvisionsByOrgsAndDatasets,
   fetchOrgInfo, fetchResources, fetchSources,
@@ -108,7 +107,7 @@ export function entityOutOfBoundsMessage (dataset, count) {
 export const prepareTasks = (req, res, next) => {
   const { lpa, dataset } = req.parsedParams
   const { entityCount, resources, sources, authority } = req
-  const { entryIssueCounts, entityIssueCounts, expectationOutOfBounds = [] } = req
+  const { tasks: tasksData = { tasks: [] }, expectationOutOfBounds = [] } = req
 
   // First check, if non authoritative dataset, only one task to show: Provide authoritative data
   if (authority && authority === 'some') {
@@ -121,48 +120,38 @@ export const prepareTasks = (req, res, next) => {
     }]
     return next()
   }
-  let issues = [...entryIssueCounts, ...entityIssueCounts]
-
-  issues = issues.filter(
-    issue => issue.issue_type !== '' &&
-    issue.issue_type !== undefined &&
-    issue.field !== '' &&
-    issue.field !== undefined
-  )
-
-  const taskList = Object.values(issues).map(({ field, issue_type: type, count }) => {
-    // if the issue doesn't have an entity, or is one of the special case issue types then we should use the resource_row_count
-
-    let rowCount = entityCount.count
-    if (SPECIAL_ISSUE_TYPES.includes(type)) {
-      if (resources.length > 0) {
-        rowCount = resources[0].entry_count
-      } else {
-        rowCount = 0
+
+  // Tasks are pre-deduplicated by fetchTasksFromPlatformApi (highest count per issue_type+field)
+  // TODO: show tasks broken down by endpoint
+  const taskList = (tasksData.tasks || [])
+    .filter(task => task.details?.issue_type && task.details?.field !== undefined)
+    .map(task => {
+      const { issue_type: type, field, count } = task.details
+
+      let rowCount = entityCount.count
+      if (SPECIAL_ISSUE_TYPES.includes(type)) {
+        rowCount = resources.length > 0 ? resources[0].entry_count : 0
       }
-    }
 
-    let title
-    try {
-      title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset })
-    } catch (e) {
-      logger.warn('Failed to generate task title', {
-        type: types.App,
-        errorMessage: e.message,
-        errorStack: e.stack,
-        params: { num_issues: count, rowCount, field, issue_type: type }
-      })
-      title = `${count} issue${count > 1 ? 's' : ''} of type ${type}`
-    }
+      let title
+      try {
+        title = performanceDbApi.getTaskMessage({ num_issues: count, rowCount, field, issue_type: type, dataset })
+      } catch (e) {
+        logger.warn('Failed to generate task title', {
+          type: types.App,
+          errorMessage: e.message,
+          errorStack: e.stack,
+          params: { num_issues: count, rowCount, field, issue_type: type }
+        })
+        title = `${count} issue${count > 1 ? 's' : ''} of type ${type}`
+      }
 
-    return {
-      title: {
-        text: title
-      },
-      href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`,
-      status: getStatusTag('Needs improving')
-    }
-  })
+      return {
+        title: { text: title },
+        href: `/organisations/${encodeURIComponent(lpa)}/${encodeURIComponent(dataset)}/${encodeURIComponent(type)}/${encodeURIComponent(field)}`,
+        status: getStatusTag('Needs improving')
+      }
+    })
 
   // include sources which couldn't be accessed
   for (const source of sources) {
@@ -241,8 +230,7 @@ export default [
   isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop,
   addEntityCountsToResources,
   fetchEntityCount,
-  fetchEntityIssueCounts,
-  fetchEntryIssueCounts,
+  fetchTasksFromPlatformApi,
   prepareTasks,
   prepareDatasetTaskListTemplateParams,
   getDatasetTaskList,

src/middleware/dataview.middleware.js

@@ -1,11 +1,14 @@
 import config from '../../config/index.js'
 import {
   createPaginationTemplateParams,
+  expectationFetcher,
+  expectations,
   extractJsonFieldFromEntities,
   fetchDatasetInfo,
   fetchLocalPlanningGroups,
   fetchProvisionsByOrgsAndDatasets,
   fetchOrgInfo,
+  noop,
   processAuthoritativeMiddlewares,
   processSpecificationMiddlewares,
   replaceUnderscoreInEntities,
@@ -17,10 +20,11 @@ import {
   logPageError,
   fetchResources,
   fetchEntityCount,
-  fetchEntityIssueCountsPerformanceDb,
+  fetchTasksFromPlatformApi,
   fetchEntitiesPlatformDb
 } from './common.middleware.js'
 import { fetchMany, FetchOptions, onlyIf, renderTemplate } from './middleware.builders.js'
+import { isFeatureEnabled } from '../utils/features.js'
 import * as v from 'valibot'
 import { splitByLeading } from '../utils/table.js'
 
@@ -82,11 +86,16 @@ export const constructTableParams = (req, res, next) => {
   next()
 }
 
+const fetchOutOfBoundsExpectations = expectationFetcher({
+  expectation: expectations.entitiesOutOfBounds,
+  result: 'expectationOutOfBounds'
+})
+
 export const prepareTemplateParams = (req, res, next) => {
-  const { orgInfo, dataset, tableParams, pagination, dataRange, entityIssueCounts, authority, alternateSources, uniqueDatasetFields, provisions } = req
+  const { orgInfo, dataset, tableParams, pagination, dataRange, tasks, authority, alternateSources, uniqueDatasetFields, provisions, expectationOutOfBounds } = req
 
-  // Hard code task count for 'some' authority
-  const taskCount = authority !== 'some' ? (entityIssueCounts ? entityIssueCounts.length : 0) : 1
+  const outOfBoundsCount = (expectationOutOfBounds?.length ?? 0) > 0 ? 1 : 0
+  const taskCount = authority !== 'some' ? (tasks?.count ?? 0) + outOfBoundsCount : 1
   // Build the fields query parameter and download url
   const fieldsParams = uniqueDatasetFields && uniqueDatasetFields.length > 0
     ? uniqueDatasetFields.map(field => `field=${encodeURIComponent(field)}`).join('&')
@@ -129,7 +138,8 @@ export default [
   fetchDatasetInfo,
 
   fetchResources,
-  fetchEntityIssueCountsPerformanceDb,
+  fetchTasksFromPlatformApi,
+  isFeatureEnabled('expectationOutOfBoundsTask') ? fetchOutOfBoundsExpectations : noop,
 
   ...processAuthoritativeMiddlewares, // Sets authority and entityCount from Platform API
 

src/middleware/entryIssueDetails.middleware.js

@@ -42,17 +42,16 @@ export const addResourceMetaDataToResources = (req, res, next) => {
 }
 
 const fetchIssueCount = fetchOne({
+  dataset: FetchOptions.fromParams,
   query: ({ req, params }) => {
     if (!req.resources[0]) {
       return 'SELECT 0 AS count'
     }
     return `
       SELECT count(*) AS count
       FROM issue i
-      LEFT JOIN issue_type it ON i.issue_type = it.issue_type
       WHERE resource = '${req.resources[0].resource}'
       AND i.issue_type = '${params.issue_type}'
-      AND it.responsibility = 'external'
       AND field = '${params.issue_field}'
     `
   },

src/middleware/lpa-overview.middleware.js

@@ -4,7 +4,7 @@
  * @description Middleware for oragnisation (LPA) overview page
  */
 
-import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchEntityIssueCountsPerformanceDb, fetchLocalPlanningGroups } from './common.middleware.js'
+import { expectationFetcher, expectations, fetchEndpointSummary, fetchOrgInfo, logPageError, noop, setAvailableDatasets, fetchTasksFromPlatformApi, fetchLocalPlanningGroups } from './common.middleware.js'
 import { fetchMany, renderTemplate, parallel } from './middleware.builders.js'
 import { getDeadlineHistory, requiredDatasets } from '../utils/utils.js'
 import _ from 'lodash'
@@ -417,9 +417,9 @@ export const getOverview = renderTemplate({
 })
 
 export function groupIssuesCountsByDataset (req, res, next) {
-  const { entityIssueCounts = [] } = req
+  const tasks = req.tasks?.tasks ?? []
 
-  req.issues = entityIssueCounts.reduce((acc, current) => {
+  req.issues = tasks.reduce((acc, current) => {
     if (!acc[current.dataset]) {
       acc[current.dataset] = []
     }
@@ -457,7 +457,7 @@ export default [
   parallel([
     fetchLocalPlanningGroups,
     fetchEndpointSummary,
-    fetchEntityIssueCountsPerformanceDb,
+    fetchTasksFromPlatformApi,
     fetchProvisions
   ]),
 

src/services/platformApi.js

@@ -121,6 +121,25 @@ export default {
       data,
       formattedData: datasets
     }
+  },
+
+  fetchTasks: async (params) => {
+    const queryParams = new URLSearchParams()
+
+    if (params.organisation) queryParams.append('organisation', params.organisation)
+    if (params.dataset) queryParams.append('dataset', params.dataset)
+    if (params.severity) queryParams.append('severity', params.severity)
+    if (params.responsibility) queryParams.append('responsibility', params.responsibility)
+    if (params.task_source) queryParams.append('task_source', params.task_source)
+    if (params.limit) queryParams.append('limit', params.limit)
+
+    const url = `${config.mainWebsiteUrl}/task.json?${queryParams.toString()}`
+    const data = await queryPlatformAPI(url, params)
+
+    return {
+      data,
+      formattedData: { tasks: data?.tasks ?? [], count: data?.count ?? 0 }
+    }
   }
 
 }