From 5385679c69500e67486a6e6914ffc2f3ed882737 Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Thu, 9 Jul 2026 18:00:14 +0100 Subject: [PATCH] fix: Exclude ended resources from duplicate endpoint check in query --- .../datasetteQueries/endpointAlreadyCollected.js | 1 - test/unit/utils/endpointAlreadyCollected.test.js | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/utils/datasetteQueries/endpointAlreadyCollected.js b/src/utils/datasetteQueries/endpointAlreadyCollected.js index 107ce31f..44638bff 100644 --- a/src/utils/datasetteQueries/endpointAlreadyCollected.js +++ b/src/utils/datasetteQueries/endpointAlreadyCollected.js @@ -18,7 +18,6 @@ export async function endpointAlreadyCollectedForDataset ({ endpointUrl, dataset AND rd.dataset = '${sqlString(dataset)}' AND ro.organisation = '${sqlString(organisation)}' AND (e.end_date IS NULL OR e.end_date = '') - AND (r.end_date IS NULL OR r.end_date = '') LIMIT 1` const response = await datasette.runQuery(sql) diff --git a/test/unit/utils/endpointAlreadyCollected.test.js b/test/unit/utils/endpointAlreadyCollected.test.js index c4825f7f..8234a8a1 100644 --- a/test/unit/utils/endpointAlreadyCollected.test.js +++ b/test/unit/utils/endpointAlreadyCollected.test.js @@ -86,6 +86,21 @@ describe('endpointAlreadyCollectedForDataset', () => { expect(datasette.runQuery.mock.calls[0][0]).not.toContain('REPLACE') }) + it('does not exclude ended resources from the duplicate endpoint check', async () => { + datasette.runQuery.mockResolvedValue({ formattedData: [] }) + + await endpointAlreadyCollectedForDataset({ + endpointUrl: 'https://example.com/data.csv', + dataset: 'brownfield-land', + organisation: 'local-authority:ABC' + }) + + const query = datasette.runQuery.mock.calls[0][0] + + expect(query).toContain("(e.end_date IS NULL OR e.end_date = '')") + expect(query).not.toContain('r.end_date') + }) + it('does not query Datasette when endpoint URL or dataset is missing', async () => { await expect(endpointAlreadyCollectedForDataset({ endpointUrl: '',