diff --git a/.github/workflows/ci-feed-validation.yml b/.github/workflows/ci-feed-validation.yml index 49207e21..3306eac0 100644 --- a/.github/workflows/ci-feed-validation.yml +++ b/.github/workflows/ci-feed-validation.yml @@ -49,8 +49,7 @@ jobs: run: npm ci --no-audit --no-fund - name: Install Playwright Chromium - if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }} - run: npx playwright install chromium --with-deps + run: npx playwright install --with-deps chromium - name: Validate feed data files run: npm run validate:feed-data @@ -135,8 +134,7 @@ jobs: run: npm ci --no-audit --no-fund - name: Install Playwright Chromium - if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }} - run: npx playwright install chromium --with-deps + run: npx playwright install --with-deps chromium - name: Seed quarantine from production health run: node ./scripts/ci/seed-quarantine-from-health.mjs diff --git a/.github/workflows/update-live-feed.yml b/.github/workflows/update-live-feed.yml index e96d550c..27e21d0d 100644 --- a/.github/workflows/update-live-feed.yml +++ b/.github/workflows/update-live-feed.yml @@ -36,8 +36,7 @@ jobs: run: npm ci --no-audit --no-fund - name: Install Playwright Chromium - if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }} - run: npx playwright install chromium --with-deps + run: npx playwright install --with-deps chromium - name: Validate feed data files run: npm run validate:feed-data diff --git a/scripts/build-live-feed.mjs b/scripts/build-live-feed.mjs index ace9af58..5d19f491 100644 --- a/scripts/build-live-feed.mjs +++ b/scripts/build-live-feed.mjs @@ -8,6 +8,7 @@ import { AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD, AUTO_QUARANTINE_DEAD_URL_THRESHOLD, AUTO_QUARANTINE_RECHECK_HOURS, + AUTO_QUARANTINE_TIMEOUT_THRESHOLD, AUTO_SKIP_EMPTY_THRESHOLD, BLOCKED_NON_CONTENT_COOLDOWN_HOURS, BLOCKED_NON_CONTENT_FAIL_THRESHOLD, @@ -163,6 +164,17 @@ function isNotFoundFailureCategory(category) { return category === 'not-found-404'; } +function isTimeoutFailureCategory(category) { + return category === 'timeout'; +} + +function shouldAutoQuarantineHealthEntry(entry = {}) { + const consecutiveFailures = Number(entry.consecutiveFailures || 0); + const consecutiveTimeoutFailures = Number(entry.consecutiveTimeoutFailures || 0); + return consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD + || consecutiveTimeoutFailures >= AUTO_QUARANTINE_TIMEOUT_THRESHOLD; +} + function sourceMayAutoCooldown(source, previousEntry, buildDate) { if (!previousEntry) return null; @@ -265,6 +277,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { const prior = previousEntry && typeof previousEntry === 'object' ? previousEntry : {}; const priorBlockedFailures = Number(prior.consecutiveBlockedFailures || 0); const priorDeadUrlFailures = Number(prior.consecutiveDeadUrlFailures || 0); + const priorTimeoutFailures = Number(prior.consecutiveTimeoutFailures || 0); const priorHealthScore = Number.isFinite(Number(prior.healthScore)) ? Number(prior.healthScore) : HEALTH_SCORE_INITIAL; const priorRecentErrors = Array.isArray(prior.recentErrors) ? prior.recentErrors : []; const generatedAtMs = Date.parse(generatedAt); @@ -283,6 +296,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { consecutiveEmptyRuns: Number(prior.consecutiveEmptyRuns || 0), consecutiveBlockedFailures: priorBlockedFailures, consecutiveDeadUrlFailures: priorDeadUrlFailures, + consecutiveTimeoutFailures: priorTimeoutFailures, lastSuccessfulAt: prior.lastSuccessfulAt || null, lastFailureAt: prior.lastFailureAt || null, lastEmptyAt: prior.lastEmptyAt || null, @@ -304,6 +318,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.consecutiveEmptyRuns = 0; next.consecutiveBlockedFailures = 0; next.consecutiveDeadUrlFailures = 0; + next.consecutiveTimeoutFailures = 0; next.lastErrorCategory = null; next.lastErrorMessage = null; next.recentErrors = []; @@ -321,11 +336,13 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { const blockedNonContent = stat?.statusKind === 'blocked_non_content' || blockedFailure; const deadUrlFailure = isDeadUrlFailureCategory(stat?.lastErrorCategory); const notFoundFailure = isNotFoundFailureCategory(stat?.lastErrorCategory); + const timeoutFailure = isTimeoutFailureCategory(stat?.lastErrorCategory); next.failedRuns += 1; next.consecutiveFailures += 1; next.consecutiveEmptyRuns = 0; next.consecutiveBlockedFailures = blockedNonContent ? priorBlockedFailures + 1 : 0; next.consecutiveDeadUrlFailures = deadUrlFailure ? priorDeadUrlFailures + 1 : 0; + next.consecutiveTimeoutFailures = timeoutFailure ? priorTimeoutFailures + 1 : 0; next.lastFailureAt = generatedAt; next.lastErrorCategory = stat?.lastErrorCategory || null; next.lastErrorMessage = stat?.lastErrorMessage || null; @@ -342,20 +359,21 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { const isCritical = notFoundFailure || deadUrlFailure || (source?.kind === 'html' && next.consecutiveBlockedFailures >= AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD) || next.consecutiveDeadUrlFailures >= AUTO_QUARANTINE_DEAD_URL_THRESHOLD + || next.consecutiveTimeoutFailures >= AUTO_QUARANTINE_TIMEOUT_THRESHOLD || next.consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD; const penalty = isCritical ? HEALTH_SCORE_CRITICAL_FAILURE_PENALTY : HEALTH_SCORE_FAILURE_PENALTY; next.healthScore = Math.max(0, priorHealthScore - penalty); // Auto-quarantine only after repeated consecutive failures. - const overFailureThreshold = next.consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD; - if (overFailureThreshold && !next.quarantined) { + const shouldAutoQuarantine = shouldAutoQuarantineHealthEntry(next); + if (shouldAutoQuarantine && !next.quarantined) { next.quarantined = true; next.quarantinedAt = generatedAt; next.quarantineReason = analyseErrorPattern(next.recentErrors, { healthScore: next.healthScore, kind: source?.kind }); - } else if (!overFailureThreshold) { + } else if (!shouldAutoQuarantine) { next.quarantined = false; next.quarantinedAt = null; next.quarantineReason = null; @@ -381,6 +399,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.consecutiveFailures = 0; next.consecutiveBlockedFailures = 0; next.consecutiveDeadUrlFailures = 0; + next.consecutiveTimeoutFailures = 0; next.quarantined = false; next.quarantinedAt = null; next.quarantineReason = null; @@ -1027,8 +1046,9 @@ function buildQuarantinedSourceEntries(sources, sourceHealth) { const manuallyQuarantined = Boolean(source?.quarantined); const healthScore = Number.isFinite(Number(health?.healthScore)) ? Number(health.healthScore) : HEALTH_SCORE_INITIAL; const consecutiveFailures = Number(health?.consecutiveFailures || 0); - const healthQuarantined = Boolean(health?.quarantined) && consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD; - const needsReview = manuallyQuarantined || healthQuarantined || consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD; + const consecutiveTimeoutFailures = Number(health?.consecutiveTimeoutFailures || 0); + const healthQuarantined = Boolean(health?.quarantined) && shouldAutoQuarantineHealthEntry(health); + const needsReview = manuallyQuarantined || healthQuarantined || shouldAutoQuarantineHealthEntry(health); if (!needsReview) return null; return { id: clean(source?.id), @@ -1046,6 +1066,7 @@ function buildQuarantinedSourceEntries(sources, sourceHealth) { lastErrorCategory: clean(health?.lastErrorCategory), lastErrorMessage: clean(health?.lastErrorMessage), consecutiveFailures, + consecutiveTimeoutFailures, consecutiveBlockedFailures: Number(health?.consecutiveBlockedFailures || 0), consecutiveDeadUrlFailures: Number(health?.consecutiveDeadUrlFailures || 0), recentErrors: Array.isArray(health?.recentErrors) ? health.recentErrors : [], @@ -3076,9 +3097,8 @@ async function main() { const deferred = autoDeferredSources.find((entry) => entry.id === source.id); if (deferred) { const priorHealthScore = Number.isFinite(Number(priorEntry?.healthScore)) ? Number(priorEntry.healthScore) : HEALTH_SCORE_INITIAL; - const priorConsecutiveFailures = Number(priorEntry?.consecutiveFailures || 0); const isQuarantined = Boolean(priorEntry?.quarantined) - && priorConsecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD; + && shouldAutoQuarantineHealthEntry(priorEntry); const deferredNextFetchAt = deferred.until || (isQuarantined ? priorEntry?.nextFetchAt : null) || sourceScheduleNextFetchAt(source, buildDate, priorEntry, true); diff --git a/scripts/build-live-feed/config.mjs b/scripts/build-live-feed/config.mjs index cce8e18b..2b454cf4 100644 --- a/scripts/build-live-feed/config.mjs +++ b/scripts/build-live-feed/config.mjs @@ -285,6 +285,7 @@ export const AUTO_QUARANTINE_RECHECK_HOURS = 7 * 24; export const AUTO_SKIP_EMPTY_THRESHOLD = 10; export const AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD', 4, 1); export const AUTO_QUARANTINE_DEAD_URL_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_DEAD_URL_THRESHOLD', 2, 1); +export const AUTO_QUARANTINE_TIMEOUT_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_TIMEOUT_THRESHOLD', 4, 1); export const AUTO_QUARANTINE_FAILURE_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_FAILURE_THRESHOLD', 6, 1); export const HEALTH_SCORE_INITIAL = envInt('ALBERTALERT_HEALTH_SCORE_INITIAL', 80, 0); export const HEALTH_SCORE_DEPRIORITISE_THRESHOLD = envInt('ALBERTALERT_HEALTH_SCORE_DEPRIORITISE_THRESHOLD', 40, 0); @@ -427,7 +428,7 @@ export function sourceRefreshEveryHours(source) { const byLane = DEFAULT_SOURCE_REFRESH_HOURS_BY_LANE[source?.lane] || DEFAULT_SOURCE_REFRESH_HOURS_BY_LANE.default; if (source?.lane === 'incidents') return 0.25; - if (source?.kind === 'html') return Math.max(byLane, 1); + if (source?.kind === 'html') return Math.max(byLane * 2, 1); return byLane; } diff --git a/scripts/ci/seed-quarantine-from-health.mjs b/scripts/ci/seed-quarantine-from-health.mjs index 76646875..1162ad01 100644 --- a/scripts/ci/seed-quarantine-from-health.mjs +++ b/scripts/ci/seed-quarantine-from-health.mjs @@ -18,6 +18,9 @@ import path from 'node:path'; const AUTO_QUARANTINE_FAILURE_THRESHOLD = Number( process.env.ALBERTALERT_AUTO_QUARANTINE_FAILURE_THRESHOLD || 6 ); +const AUTO_QUARANTINE_TIMEOUT_THRESHOLD = Number( + process.env.ALBERTALERT_AUTO_QUARANTINE_TIMEOUT_THRESHOLD || 4 +); // ── Paths ──────────────────────────────────────────────────────────── const repoRoot = path.resolve( @@ -36,6 +39,7 @@ const sourcesPath = process.argv[3] || path.join(repoRoot, 'data', 'sources.json */ export function deriveQuarantineIds(sourceHealth, options = {}) { const failureThreshold = options.failureThreshold ?? AUTO_QUARANTINE_FAILURE_THRESHOLD; + const timeoutThreshold = options.timeoutThreshold ?? AUTO_QUARANTINE_TIMEOUT_THRESHOLD; const ids = new Set(); if (!sourceHealth || typeof sourceHealth !== 'object') return ids; @@ -45,10 +49,12 @@ export function deriveQuarantineIds(sourceHealth, options = {}) { const alreadyQuarantined = Boolean(entry.quarantined); const consecutiveFailures = Number(entry.consecutiveFailures || 0); + const consecutiveTimeoutFailures = Number(entry.consecutiveTimeoutFailures || 0); if ( alreadyQuarantined || - consecutiveFailures >= failureThreshold + consecutiveFailures >= failureThreshold || + consecutiveTimeoutFailures >= timeoutThreshold ) { ids.add(id); } diff --git a/tests/decision-logic.test.mjs b/tests/decision-logic.test.mjs index f670f9ae..0734ce7e 100644 --- a/tests/decision-logic.test.mjs +++ b/tests/decision-logic.test.mjs @@ -61,9 +61,11 @@ import { DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT_MS, FEED_SOURCE_CONCURRENCY, + AUTO_QUARANTINE_TIMEOUT_THRESHOLD, MAX_FEED_PREFETCH_ITEMS, MAX_HTML_PREFETCH_ITEMS, MAX_HTML_SOURCES_PER_RUN, + sourceRefreshEveryHours, shouldRefreshSourceThisRun, sourceScheduleIntervalMinutes, sourceScheduleOffsetMinutes, @@ -1392,6 +1394,13 @@ test('html source run cap keeps control scheduler budget at legacy value', () => assert.equal(CONTROL_MAX_HTML_SOURCES_PER_RUN, 30); }); +test('non-incident HTML sources refresh less often than machine-readable equivalents', () => { + assert.equal(sourceRefreshEveryHours({ lane: 'context', kind: 'rss' }), 0.5); + assert.equal(sourceRefreshEveryHours({ lane: 'context', kind: 'html' }), 1); + assert.equal(sourceRefreshEveryHours({ lane: 'oversight', kind: 'rss' }), 1); + assert.equal(sourceRefreshEveryHours({ lane: 'oversight', kind: 'html' }), 2); +}); + test('default fetch/runtime tuning constants remain stable', () => { assert.equal(DEFAULT_TIMEOUT_MS, 12000); assert.equal(DEFAULT_MAX_RETRIES, 2); @@ -1399,6 +1408,7 @@ test('default fetch/runtime tuning constants remain stable', () => { assert.equal(MAX_HTML_PREFETCH_ITEMS, 12); assert.equal(MAX_FEED_PREFETCH_ITEMS, 8); assert.equal(AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD, 4); + assert.equal(AUTO_QUARANTINE_TIMEOUT_THRESHOLD, 4); }); test('source error summary classifies HTTP 404 separately for direct quarantine routing', async () => { diff --git a/tests/error-pattern-analysis.test.mjs b/tests/error-pattern-analysis.test.mjs index 264a9aca..aad2fec5 100644 --- a/tests/error-pattern-analysis.test.mjs +++ b/tests/error-pattern-analysis.test.mjs @@ -136,6 +136,7 @@ test('nextSourceHealthEntry – failure appends to empty recentErrors', () => { assert.equal(result.recentErrors.length, 1); assert.equal(result.recentErrors[0].category, 'not-found-404'); assert.equal(result.recentErrors[0].at, '2026-01-01T00:00:00Z'); + assert.equal(result.consecutiveTimeoutFailures, 0); }); test('nextSourceHealthEntry – failures accumulate in rolling window', () => { @@ -147,6 +148,7 @@ test('nextSourceHealthEntry – failures accumulate in rolling window', () => { assert.equal(prior.recentErrors.length, 3); assert.equal(prior.recentErrors[0].category, 'timeout'); assert.equal(prior.recentErrors[2].at, '2026-01-03T00:00:00Z'); + assert.equal(prior.consecutiveTimeoutFailures, 3); }); test('nextSourceHealthEntry – rolling window caps at configured size', () => { @@ -175,6 +177,7 @@ test('nextSourceHealthEntry – success clears recentErrors', () => { const result = nextSourceHealthEntry(source, successStat(), prior, '2026-01-03T00:00:00Z'); assert.deepEqual(result.recentErrors, []); assert.equal(result.lastErrorCategory, null); + assert.equal(result.consecutiveTimeoutFailures, 0); }); test('nextSourceHealthEntry – empty run preserves existing recentErrors', () => { @@ -186,6 +189,7 @@ test('nextSourceHealthEntry – empty run preserves existing recentErrors', () = const result = nextSourceHealthEntry(source, emptyStat(), prior, '2026-01-02T00:00:00Z'); assert.equal(result.recentErrors.length, 1); assert.equal(result.recentErrors[0].category, 'timeout'); + assert.equal(result.consecutiveTimeoutFailures, 0); }); test('nextSourceHealthEntry – quarantine reason uses pattern analysis', () => { @@ -254,6 +258,29 @@ test('nextSourceHealthEntry – never-verified source with non-definitive failur const source = makeSource(); const result = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), {}, '2026-01-01T00:00:00Z'); assert.equal(result.quarantined, false, 'timeout alone should not immediately quarantine a new source'); + assert.equal(result.consecutiveTimeoutFailures, 1); +}); + +test('nextSourceHealthEntry – four consecutive timeouts trigger auto-quarantine', () => { + const source = makeSource(); + let prior = {}; + for (let i = 0; i < 4; i++) { + prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, `2026-01-0${i + 1}T00:00:00Z`); + } + assert.equal(prior.consecutiveFailures, 4); + assert.equal(prior.consecutiveTimeoutFailures, 4); + assert.equal(prior.quarantined, true); + assert.match(prior.quarantineReason, /persistent timeouts/i); +}); + +test('nextSourceHealthEntry – non-timeout failure resets timeout streak', () => { + const source = makeSource(); + let prior = {}; + prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, '2026-01-01T00:00:00Z'); + prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, '2026-01-02T00:00:00Z'); + prior = nextSourceHealthEntry(source, failStat('network-failure', 'ECONNRESET'), prior, '2026-01-03T00:00:00Z'); + assert.equal(prior.consecutiveTimeoutFailures, 0); + assert.equal(prior.quarantined, false); }); test('nextSourceHealthEntry – previously-verified source with 404 is NOT immediately quarantined', () => { diff --git a/tests/seed-quarantine.test.mjs b/tests/seed-quarantine.test.mjs index 35fa1513..66977a94 100644 --- a/tests/seed-quarantine.test.mjs +++ b/tests/seed-quarantine.test.mjs @@ -42,6 +42,14 @@ test('deriveQuarantineIds flags source at or above failure threshold', () => { assert.ok(ids.has('src-c')); }); +test('deriveQuarantineIds flags source at or above timeout threshold', () => { + const ids = deriveQuarantineIds( + { 'src-timeout': { healthScore: 50, quarantined: false, consecutiveFailures: 4, consecutiveTimeoutFailures: 4 } }, + { failureThreshold: 6, timeoutThreshold: 4 } + ); + assert.ok(ids.has('src-timeout')); +}); + test('deriveQuarantineIds does not flag healthy source', () => { const ids = deriveQuarantineIds( { 'src-d': { healthScore: 80, quarantined: false, consecutiveFailures: 1 } },