feat: Implement endpoint duplication check in CheckConfirmationContro…

src/assets/js/statusPage.js

@@ -8,7 +8,7 @@ const finishedProcessingStatuses = [
 
 export default class StatusPage {
   constructor (pollingInterval, maxPollAttempts) {
-    this.pollingInterval = pollingInterval || 1000
+    this.pollingInterval = pollingInterval || 3000
     this.maxPollAttempts = maxPollAttempts || 30
     this.pollingOffset = 400
     this.pollAttempts = 0

src/controllers/checkConfirmationController.js

@@ -1,12 +1,42 @@
 import PageController from './pageController.js'
+import { getRequestData } from '../services/asyncRequestApi.js'
+import { endpointAlreadyCollectedForDataset } from '../utils/datasetteQueries/endpointAlreadyCollected.js'
+import logger from '../utils/logger.js'
+import { types } from '../utils/logging.js'
 
 class CheckConfirmationController extends PageController {
-  locals (req, res, next) {
+  async locals (req, res, next) {
     const isUrlCheck = req.sessionModel.get('upload-method') === 'url'
     if (isUrlCheck) {
       const requestId = req.sessionModel.get('request_id')
       req.form.options.requestId = requestId
-      req.session.checkRequestId = requestId
+      try {
+        const requestData = await getRequestData(requestId)
+        const params = requestData.getParams() ?? {}
+        if (params.dataset) {
+          req.sessionModel.set('dataset', params.dataset)
+        }
+        if (params.organisationName) {
+          req.sessionModel.set('orgId', params.organisationName)
+        }
+        req.form.options.alreadyCollectingEndpoint = await endpointAlreadyCollectedForDataset({
+          endpointUrl: params.url,
+          dataset: params.dataset,
+          organisation: params.organisationName
+        })
+      } catch (error) {
+        logger.warn('CheckConfirmationController: could not check whether endpoint is already collected', {
+          type: types.App,
+          requestId,
+          errorMessage: error.message
+        })
+      }
+
+      if (req.form.options.alreadyCollectingEndpoint) {
+        delete req.session.checkRequestId
+      } else {
+        req.session.checkRequestId = requestId
+      }
     }
     super.locals(req, res, next)
   }

src/controllers/statusController.js

@@ -81,7 +81,11 @@ class StatusController extends PageController {
       req.form.options.messageTexts = messageTexts
       req.form.options.buttonTexts = buttonTexts
       req.form.options.buttonAriaLabels = buttonAriaLabels
-      req.form.options.pollingEndpoint = `/api/status/${req.form.options.data.id}`
+      const pollingParams = new URLSearchParams()
+      const uniqueDatasetFields = req.uniqueDatasetFields || []
+      uniqueDatasetFields.forEach(field => pollingParams.append('field', field))
+      const pollingQuery = pollingParams.toString()
+      req.form.options.pollingEndpoint = `/api/status/${req.form.options.data.id}${pollingQuery ? `?${pollingQuery}` : ''}`
       const now = new Date()
       req.form.options.lastUpdated = now.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit', hour12: true, timeZone: 'Europe/London' }).replace(' ', '').toLowerCase() +
         ' on ' +

src/routes/api.js

@@ -15,7 +15,7 @@ const router = express.Router()
  * @returns {Promise<object>} Returns a JSON object with the request data if successful.
  * If an error occurs, returns a JSON object with an error message and sets the HTTP status code to 500.
  */
-router.get('/status/:result_id', async (req, res) => {
+export async function getStatus (req, res) {
   res.set('Cache-Control', 'no-store')
   try {
     const resultData = await getRequestData(req.params.result_id)
@@ -24,7 +24,8 @@ router.get('/status/:result_id', async (req, res) => {
     // compute whether we should show column mapping when the request finished
     if (typeof resultData.isComplete === 'function' && resultData.isComplete() && !resultData.isFailed?.()) {
       try {
-        const show = await shouldShowColumnMapping(resultData, [])
+        const uniqueDatasetFields = getUniqueDatasetFieldsFromQuery(req.query)
+        const show = await shouldShowColumnMapping(resultData, uniqueDatasetFields)
         payload.showColumnMapping = show
         if (show) payload.columnMappingUrl = `/check/column-mapping/${resultData.id}`
       } catch (e) {
@@ -35,7 +36,17 @@ router.get('/status/:result_id', async (req, res) => {
   } catch (error) {
     return res.status(500).json({ error })
   }
-})
+}
+
+router.get('/status/:result_id', getStatus)
+
+export function getUniqueDatasetFieldsFromQuery (query = {}) {
+  if (!query.field) return []
+
+  return Array.isArray(query.field)
+    ? query.field
+    : [query.field]
+}
 
 /**
  * Retrieves the boundary data for a local planning authority (LPA) by boundary ID.

src/utils/datasetteQueries/endpointAlreadyCollected.js

@@ -0,0 +1,26 @@
+import datasette from '../../services/datasette.js'
+
+function sqlString (value) {
+  return String(value).replaceAll("'", "''")
+}
+
+export async function endpointAlreadyCollectedForDataset ({ endpointUrl, dataset, organisation }) {
+  if (!endpointUrl || !dataset || !organisation) return false
+
+  const sql = /* sql */ `
+    SELECT 1
+    FROM endpoint e
+    JOIN resource_endpoint re ON e.endpoint = re.endpoint
+    JOIN resource r ON re.resource = r.resource
+    JOIN resource_dataset rd ON r.resource = rd.resource
+    JOIN resource_organisation ro ON r.resource = ro.resource
+    WHERE e.endpoint_url = '${sqlString(endpointUrl)}'
+      AND rd.dataset = '${sqlString(dataset)}'
+      AND ro.organisation = '${sqlString(organisation)}'
+      AND (e.end_date IS NULL OR e.end_date = '')
+      AND (r.end_date IS NULL OR r.end_date = '')
+    LIMIT 1`
+
+  const response = await datasette.runQuery(sql)
+  return response.formattedData.length > 0
+}

src/views/check/confirmation.html

@@ -6,12 +6,20 @@
 
 {% set serviceType = 'Check' %}
 
-{% if options.requestId %}
+{% if options.alreadyCollectingEndpoint %}
+  {% set pageName = "Data checked" %}
+{% elif options.requestId %}
   {% set pageName = "Provide your data" %}
 {% else %}
   {% set pageName = "Publish your data" %}
 {% endif %}
 
+{% if options.deepLink %}
+  {% set datasetOverviewHref = options.deepLink.referrer %}
+{% elif options.orgId and options.dataset %}
+  {% set datasetOverviewHref = "/organisations/" + options.orgId + "/" + options.dataset %}
+{% endif %}
+
 {% block beforeContent %}
   {{ govukBackLink({
     text: "Back",
@@ -22,15 +30,29 @@
 
 {% set content %}
 
-  # What happens next
+  ## What happens next
+
+  {% if options.alreadyCollectingEndpoint %}
+
+  We are already collecting data from this endpoint URL, so we will process any changes you make automatically.
+
+  You do not need to submit this endpoint URL again.
+
+  {% if datasetOverviewHref and options.dataset %}
+<p><a class="govuk-link" href="{{ datasetOverviewHref }}">Return to {{ options.dataset | datasetSlugToReadableName | replace("-", " ") }} overview</a></p>
+  {% else %}
+<p><a class="govuk-link" href="/">Return to Home</a></p>
+  {% endif %}
+
+  {% else %}
 
   {% if options.requestId %}
 
-  ## 1. Make sure that your data is published on your website
+  ### 1. Make sure that your data is published on your website
 
   {% else %}
 
-  ## 1. Publish data to your website
+  ### 1. Publish data to your website
 
   {% endif %}
 
@@ -41,7 +63,7 @@
 
   {% if options.requestId %}
 
-  ## 2. Provide your data
+  ### 2. Provide your data
 
   You need to submit:
 
@@ -54,28 +76,30 @@
   <div class="govuk-button-group">
     <a class="govuk-button submit-link" href="/submit/lpa-details">Provide your data</a>
     {% if options.deepLink %}
-      <a class="govuk-link" href="{{ options.deepLink.referrer }}">Return to {{ options.deepLink.dataset | datasetSlugToReadableName }} overview</a>
+      <a class="govuk-link" href="{{ options.deepLink.referrer }}">Return to {{ options.deepLink.dataset | datasetSlugToReadableName  | replace("-", " ") }} overview</a>
     {% else %}
       <a class="govuk-link" href="/">Return to Home</a>
     {% endif %}
   </div>
 
   {% else %}
 
-  ## 2. Provide your data
+  ### 2. Provide your data
 
   After you publish, you should [provide your data to the Planning Data Platform](/check/url).
 
   [Find out more about how to check and provide your data](/guidance)
 
   {% if options.deepLink %}
-    <p><a class="govuk-link" href="{{ options.deepLink.referrer }}">Return to {{ options.deepLink.dataset | datasetSlugToReadableName }} overview</a></p>
+    <p><a class="govuk-link" href="{{ options.deepLink.referrer }}">Return to {{ options.deepLink.dataset | datasetSlugToReadableName | replace("-", " ") }} overview</a></p>
   {% else %}
     <p><a class="govuk-link" href="/">Return to Home</a></p>
   {% endif %}
 
   {% endif %}
 
+  {% endif %}
+
   ## Give feedback
 
   [Give feedback about this service]({{feedbackLink}}) (takes 30 seconds).

src/views/check/statusPage/status.html

@@ -59,7 +59,7 @@
   {{ super() }}
   <script>
     window.serverContext = {
-      pollingEndpoint: "{{ options.pollingEndpoint }}"
+      pollingEndpoint: {{ options.pollingEndpoint | dump | safe }}
     }
   </script>
   <script src="/public/js/statusPage.bundle.js"></script>

test/unit/check/confirmationPage.test.js

@@ -32,6 +32,15 @@ describe('Check confirmation View', () => {
     it('should not render the submit link when requestId is absent', () => {
       expect(doc.querySelector('a.submit-link')).toBeNull()
     })
+
+    it('should render the standard confirmation heading structure', () => {
+      expect(doc.querySelector('main h1')?.textContent.trim()).toBe('Publish your data')
+      expect(doc.querySelector('main h2')?.textContent.trim()).toBe('What happens next')
+      expect([...doc.querySelectorAll('main h3')].map(heading => heading.textContent.trim())).toEqual([
+        '1. Publish data to your website',
+        '2. Provide your data'
+      ])
+    })
   })
 
   describe('with requestId', () => {
@@ -50,5 +59,78 @@ describe('Check confirmation View', () => {
       expect(submitLink).not.toBeNull()
       expect(submitLink.getAttribute('href')).toBe('/submit/lpa-details')
     })
+
+    it('should render the standard confirmation heading structure', () => {
+      expect(doc.querySelector('main h1')?.textContent.trim()).toBe('Provide your data')
+      expect(doc.querySelector('main h2')?.textContent.trim()).toBe('What happens next')
+      expect([...doc.querySelectorAll('main h3')].map(heading => heading.textContent.trim())).toEqual([
+        '1. Make sure that your data is published on your website',
+        '2. Provide your data'
+      ])
+    })
+  })
+
+  describe('when the endpoint is already being collected for the dataset', () => {
+    const templateParams = {
+      options: {
+        ...baseOptions,
+        requestId: 'abc-123',
+        alreadyCollectingEndpoint: true,
+        orgId: 'local-authority:ABC',
+        dataset: 'brownfield-land'
+      }
+    }
+    const html = stripWhitespace(nunjucks.render('check/confirmation.html', templateParams))
+    const dom = new JSDOM(html)
+    const doc = dom.window.document
+
+    runGenericPageTests(html, {
+      pageTitle: 'Data checked - Check your planning data'
+    })
+
+    it('should render the data checked panel', () => {
+      const regex = new RegExp('<h1 class="govuk-panel__title".*Data checked.*</h1>', 'g')
+      expect(html).toMatch(regex)
+    })
+
+    it('should render the already collecting content', () => {
+      expect(doc.body.textContent).toContain('We are already collecting data from this endpoint URL, so we will process any changes you make automatically.')
+      expect(doc.body.textContent).toContain('You do not need to submit this endpoint URL again.')
+    })
+
+    it('should not render the submit link', () => {
+      expect(doc.querySelector('a.submit-link')).toBeNull()
+    })
+
+    it('should render a dataset overview link', () => {
+      const overviewLink = doc.querySelector('a[href="/organisations/local-authority:ABC/brownfield-land"]')
+      expect(overviewLink).not.toBeNull()
+      expect(overviewLink.textContent.trim()).toBe('Return to brownfield land overview')
+    })
+
+    it('should render What happens next as a second-level heading', () => {
+      expect(doc.querySelector('main h1')?.textContent.trim()).toBe('Data checked')
+      expect(doc.querySelector('main h2')?.textContent.trim()).toBe('What happens next')
+      expect([...doc.querySelectorAll('main h1')]).toHaveLength(1)
+    })
+
+    it('should render a home link when an overview URL cannot be built', () => {
+      const html = stripWhitespace(nunjucks.render('check/confirmation.html', {
+        options: {
+          ...baseOptions,
+          requestId: 'abc-123',
+          alreadyCollectingEndpoint: true,
+          orgId: undefined,
+          dataset: 'brownfield-land'
+        }
+      }))
+      const doc = new JSDOM(html).window.document
+      const homeLink = doc.querySelector('a[href="/"]')
+      const overviewLink = doc.querySelector('a[href^="/organisations/"]')
+
+      expect(homeLink).not.toBeNull()
+      expect(homeLink.textContent.trim()).toBe('Return to Home')
+      expect(overviewLink).toBeNull()
+    })
   })
 })