Install Playwright browsers in CI and push non-incident HTML sources behind feeds

.github/workflows/ci-feed-validation.yml

@@ -49,8 +49,7 @@ jobs:
         run: npm ci --no-audit --no-fund
 
       - name: Install Playwright Chromium
-        if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }}
-        run: npx playwright install chromium --with-deps
+        run: npx playwright install --with-deps chromium
 
       - name: Validate feed data files
         run: npm run validate:feed-data
@@ -135,8 +134,7 @@ jobs:
         run: npm ci --no-audit --no-fund
 
       - name: Install Playwright Chromium
-        if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }}
-        run: npx playwright install chromium --with-deps
+        run: npx playwright install --with-deps chromium
 
       - name: Seed quarantine from production health
         run: node ./scripts/ci/seed-quarantine-from-health.mjs

.github/workflows/update-live-feed.yml

@@ -36,8 +36,7 @@ jobs:
         run: npm ci --no-audit --no-fund
 
       - name: Install Playwright Chromium
-        if: ${{ env.ALBERTALERT_ENABLE_PLAYWRIGHT_FALLBACK == 'true' }}
-        run: npx playwright install chromium --with-deps
+        run: npx playwright install --with-deps chromium
 
       - name: Validate feed data files
         run: npm run validate:feed-data

scripts/build-live-feed.mjs

@@ -8,6 +8,7 @@ import {
   AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD,
   AUTO_QUARANTINE_DEAD_URL_THRESHOLD,
   AUTO_QUARANTINE_RECHECK_HOURS,
+  AUTO_QUARANTINE_TIMEOUT_THRESHOLD,
   AUTO_SKIP_EMPTY_THRESHOLD,
   BLOCKED_NON_CONTENT_COOLDOWN_HOURS,
   BLOCKED_NON_CONTENT_FAIL_THRESHOLD,
@@ -163,6 +164,17 @@ function isNotFoundFailureCategory(category) {
   return category === 'not-found-404';
 }
 
+function isTimeoutFailureCategory(category) {
+  return category === 'timeout';
+}
+
+function shouldAutoQuarantineHealthEntry(entry = {}) {
+  const consecutiveFailures = Number(entry.consecutiveFailures || 0);
+  const consecutiveTimeoutFailures = Number(entry.consecutiveTimeoutFailures || 0);
+  return consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD
+    || consecutiveTimeoutFailures >= AUTO_QUARANTINE_TIMEOUT_THRESHOLD;
+}
+
 function sourceMayAutoCooldown(source, previousEntry, buildDate) {
   if (!previousEntry) return null;
 
@@ -265,6 +277,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
   const prior = previousEntry && typeof previousEntry === 'object' ? previousEntry : {};
   const priorBlockedFailures = Number(prior.consecutiveBlockedFailures || 0);
   const priorDeadUrlFailures = Number(prior.consecutiveDeadUrlFailures || 0);
+  const priorTimeoutFailures = Number(prior.consecutiveTimeoutFailures || 0);
   const priorHealthScore = Number.isFinite(Number(prior.healthScore)) ? Number(prior.healthScore) : HEALTH_SCORE_INITIAL;
   const priorRecentErrors = Array.isArray(prior.recentErrors) ? prior.recentErrors : [];
   const generatedAtMs = Date.parse(generatedAt);
@@ -283,6 +296,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
     consecutiveEmptyRuns: Number(prior.consecutiveEmptyRuns || 0),
     consecutiveBlockedFailures: priorBlockedFailures,
     consecutiveDeadUrlFailures: priorDeadUrlFailures,
+    consecutiveTimeoutFailures: priorTimeoutFailures,
     lastSuccessfulAt: prior.lastSuccessfulAt || null,
     lastFailureAt: prior.lastFailureAt || null,
     lastEmptyAt: prior.lastEmptyAt || null,
@@ -304,6 +318,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
     next.consecutiveEmptyRuns = 0;
     next.consecutiveBlockedFailures = 0;
     next.consecutiveDeadUrlFailures = 0;
+    next.consecutiveTimeoutFailures = 0;
     next.lastErrorCategory = null;
     next.lastErrorMessage = null;
     next.recentErrors = [];
@@ -321,11 +336,13 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
     const blockedNonContent = stat?.statusKind === 'blocked_non_content' || blockedFailure;
     const deadUrlFailure = isDeadUrlFailureCategory(stat?.lastErrorCategory);
     const notFoundFailure = isNotFoundFailureCategory(stat?.lastErrorCategory);
+    const timeoutFailure = isTimeoutFailureCategory(stat?.lastErrorCategory);
     next.failedRuns += 1;
     next.consecutiveFailures += 1;
     next.consecutiveEmptyRuns = 0;
     next.consecutiveBlockedFailures = blockedNonContent ? priorBlockedFailures + 1 : 0;
     next.consecutiveDeadUrlFailures = deadUrlFailure ? priorDeadUrlFailures + 1 : 0;
+    next.consecutiveTimeoutFailures = timeoutFailure ? priorTimeoutFailures + 1 : 0;
     next.lastFailureAt = generatedAt;
     next.lastErrorCategory = stat?.lastErrorCategory || null;
     next.lastErrorMessage = stat?.lastErrorMessage || null;
@@ -342,20 +359,21 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
     const isCritical = notFoundFailure || deadUrlFailure
       || (source?.kind === 'html' && next.consecutiveBlockedFailures >= AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD)
       || next.consecutiveDeadUrlFailures >= AUTO_QUARANTINE_DEAD_URL_THRESHOLD
+      || next.consecutiveTimeoutFailures >= AUTO_QUARANTINE_TIMEOUT_THRESHOLD
       || next.consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD;
     const penalty = isCritical ? HEALTH_SCORE_CRITICAL_FAILURE_PENALTY : HEALTH_SCORE_FAILURE_PENALTY;
     next.healthScore = Math.max(0, priorHealthScore - penalty);
 
     // Auto-quarantine only after repeated consecutive failures.
-    const overFailureThreshold = next.consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD;
-    if (overFailureThreshold && !next.quarantined) {
+    const shouldAutoQuarantine = shouldAutoQuarantineHealthEntry(next);
+    if (shouldAutoQuarantine && !next.quarantined) {
       next.quarantined = true;
       next.quarantinedAt = generatedAt;
       next.quarantineReason = analyseErrorPattern(next.recentErrors, {
         healthScore: next.healthScore,
         kind: source?.kind
       });
-    } else if (!overFailureThreshold) {
+    } else if (!shouldAutoQuarantine) {
       next.quarantined = false;
       next.quarantinedAt = null;
       next.quarantineReason = null;
@@ -381,6 +399,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) {
   next.consecutiveFailures = 0;
   next.consecutiveBlockedFailures = 0;
   next.consecutiveDeadUrlFailures = 0;
+  next.consecutiveTimeoutFailures = 0;
   next.quarantined = false;
   next.quarantinedAt = null;
   next.quarantineReason = null;
@@ -1027,8 +1046,9 @@ function buildQuarantinedSourceEntries(sources, sourceHealth) {
       const manuallyQuarantined = Boolean(source?.quarantined);
       const healthScore = Number.isFinite(Number(health?.healthScore)) ? Number(health.healthScore) : HEALTH_SCORE_INITIAL;
       const consecutiveFailures = Number(health?.consecutiveFailures || 0);
-      const healthQuarantined = Boolean(health?.quarantined) && consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD;
-      const needsReview = manuallyQuarantined || healthQuarantined || consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD;
+      const consecutiveTimeoutFailures = Number(health?.consecutiveTimeoutFailures || 0);
+      const healthQuarantined = Boolean(health?.quarantined) && shouldAutoQuarantineHealthEntry(health);
+      const needsReview = manuallyQuarantined || healthQuarantined || shouldAutoQuarantineHealthEntry(health);
       if (!needsReview) return null;
       return {
         id: clean(source?.id),
@@ -1046,6 +1066,7 @@ function buildQuarantinedSourceEntries(sources, sourceHealth) {
         lastErrorCategory: clean(health?.lastErrorCategory),
         lastErrorMessage: clean(health?.lastErrorMessage),
         consecutiveFailures,
+        consecutiveTimeoutFailures,
         consecutiveBlockedFailures: Number(health?.consecutiveBlockedFailures || 0),
         consecutiveDeadUrlFailures: Number(health?.consecutiveDeadUrlFailures || 0),
         recentErrors: Array.isArray(health?.recentErrors) ? health.recentErrors : [],
@@ -3076,9 +3097,8 @@ async function main() {
     const deferred = autoDeferredSources.find((entry) => entry.id === source.id);
     if (deferred) {
       const priorHealthScore = Number.isFinite(Number(priorEntry?.healthScore)) ? Number(priorEntry.healthScore) : HEALTH_SCORE_INITIAL;
-      const priorConsecutiveFailures = Number(priorEntry?.consecutiveFailures || 0);
       const isQuarantined = Boolean(priorEntry?.quarantined)
-        && priorConsecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD;
+        && shouldAutoQuarantineHealthEntry(priorEntry);
       const deferredNextFetchAt = deferred.until
         || (isQuarantined ? priorEntry?.nextFetchAt : null)
         || sourceScheduleNextFetchAt(source, buildDate, priorEntry, true);

scripts/build-live-feed/config.mjs

@@ -285,6 +285,7 @@ export const AUTO_QUARANTINE_RECHECK_HOURS = 7 * 24;
 export const AUTO_SKIP_EMPTY_THRESHOLD = 10;
 export const AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD', 4, 1);
 export const AUTO_QUARANTINE_DEAD_URL_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_DEAD_URL_THRESHOLD', 2, 1);
+export const AUTO_QUARANTINE_TIMEOUT_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_TIMEOUT_THRESHOLD', 4, 1);
 export const AUTO_QUARANTINE_FAILURE_THRESHOLD = envInt('ALBERTALERT_AUTO_QUARANTINE_FAILURE_THRESHOLD', 6, 1);
 export const HEALTH_SCORE_INITIAL = envInt('ALBERTALERT_HEALTH_SCORE_INITIAL', 80, 0);
 export const HEALTH_SCORE_DEPRIORITISE_THRESHOLD = envInt('ALBERTALERT_HEALTH_SCORE_DEPRIORITISE_THRESHOLD', 40, 0);
@@ -427,7 +428,7 @@ export function sourceRefreshEveryHours(source) {
 
   const byLane = DEFAULT_SOURCE_REFRESH_HOURS_BY_LANE[source?.lane] || DEFAULT_SOURCE_REFRESH_HOURS_BY_LANE.default;
   if (source?.lane === 'incidents') return 0.25;
-  if (source?.kind === 'html') return Math.max(byLane, 1);
+  if (source?.kind === 'html') return Math.max(byLane * 2, 1);
   return byLane;
 }
 

scripts/ci/seed-quarantine-from-health.mjs

@@ -18,6 +18,9 @@ import path from 'node:path';
 const AUTO_QUARANTINE_FAILURE_THRESHOLD = Number(
   process.env.ALBERTALERT_AUTO_QUARANTINE_FAILURE_THRESHOLD || 6
 );
+const AUTO_QUARANTINE_TIMEOUT_THRESHOLD = Number(
+  process.env.ALBERTALERT_AUTO_QUARANTINE_TIMEOUT_THRESHOLD || 4
+);
 
 // ── Paths ────────────────────────────────────────────────────────────
 const repoRoot = path.resolve(
@@ -36,6 +39,7 @@ const sourcesPath = process.argv[3] || path.join(repoRoot, 'data', 'sources.json
  */
 export function deriveQuarantineIds(sourceHealth, options = {}) {
   const failureThreshold = options.failureThreshold ?? AUTO_QUARANTINE_FAILURE_THRESHOLD;
+  const timeoutThreshold = options.timeoutThreshold ?? AUTO_QUARANTINE_TIMEOUT_THRESHOLD;
 
   const ids = new Set();
   if (!sourceHealth || typeof sourceHealth !== 'object') return ids;
@@ -45,10 +49,12 @@ export function deriveQuarantineIds(sourceHealth, options = {}) {
 
     const alreadyQuarantined = Boolean(entry.quarantined);
     const consecutiveFailures = Number(entry.consecutiveFailures || 0);
+    const consecutiveTimeoutFailures = Number(entry.consecutiveTimeoutFailures || 0);
 
     if (
       alreadyQuarantined ||
-      consecutiveFailures >= failureThreshold
+      consecutiveFailures >= failureThreshold ||
+      consecutiveTimeoutFailures >= timeoutThreshold
     ) {
       ids.add(id);
     }

tests/decision-logic.test.mjs

@@ -61,9 +61,11 @@ import {
   DEFAULT_MAX_RETRIES,
   DEFAULT_TIMEOUT_MS,
   FEED_SOURCE_CONCURRENCY,
+  AUTO_QUARANTINE_TIMEOUT_THRESHOLD,
   MAX_FEED_PREFETCH_ITEMS,
   MAX_HTML_PREFETCH_ITEMS,
   MAX_HTML_SOURCES_PER_RUN,
+  sourceRefreshEveryHours,
   shouldRefreshSourceThisRun,
   sourceScheduleIntervalMinutes,
   sourceScheduleOffsetMinutes,
@@ -1392,13 +1394,21 @@ test('html source run cap keeps control scheduler budget at legacy value', () =>
   assert.equal(CONTROL_MAX_HTML_SOURCES_PER_RUN, 30);
 });
 
+test('non-incident HTML sources refresh less often than machine-readable equivalents', () => {
+  assert.equal(sourceRefreshEveryHours({ lane: 'context', kind: 'rss' }), 0.5);
+  assert.equal(sourceRefreshEveryHours({ lane: 'context', kind: 'html' }), 1);
+  assert.equal(sourceRefreshEveryHours({ lane: 'oversight', kind: 'rss' }), 1);
+  assert.equal(sourceRefreshEveryHours({ lane: 'oversight', kind: 'html' }), 2);
+});
+
 test('default fetch/runtime tuning constants remain stable', () => {
   assert.equal(DEFAULT_TIMEOUT_MS, 12000);
   assert.equal(DEFAULT_MAX_RETRIES, 2);
   assert.equal(FEED_SOURCE_CONCURRENCY, 4);
   assert.equal(MAX_HTML_PREFETCH_ITEMS, 12);
   assert.equal(MAX_FEED_PREFETCH_ITEMS, 8);
   assert.equal(AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD, 4);
+  assert.equal(AUTO_QUARANTINE_TIMEOUT_THRESHOLD, 4);
 });
 
 test('source error summary classifies HTTP 404 separately for direct quarantine routing', async () => {

tests/error-pattern-analysis.test.mjs

@@ -136,6 +136,7 @@ test('nextSourceHealthEntry – failure appends to empty recentErrors', () => {
   assert.equal(result.recentErrors.length, 1);
   assert.equal(result.recentErrors[0].category, 'not-found-404');
   assert.equal(result.recentErrors[0].at, '2026-01-01T00:00:00Z');
+  assert.equal(result.consecutiveTimeoutFailures, 0);
 });
 
 test('nextSourceHealthEntry – failures accumulate in rolling window', () => {
@@ -147,6 +148,7 @@ test('nextSourceHealthEntry – failures accumulate in rolling window', () => {
   assert.equal(prior.recentErrors.length, 3);
   assert.equal(prior.recentErrors[0].category, 'timeout');
   assert.equal(prior.recentErrors[2].at, '2026-01-03T00:00:00Z');
+  assert.equal(prior.consecutiveTimeoutFailures, 3);
 });
 
 test('nextSourceHealthEntry – rolling window caps at configured size', () => {
@@ -175,6 +177,7 @@ test('nextSourceHealthEntry – success clears recentErrors', () => {
   const result = nextSourceHealthEntry(source, successStat(), prior, '2026-01-03T00:00:00Z');
   assert.deepEqual(result.recentErrors, []);
   assert.equal(result.lastErrorCategory, null);
+  assert.equal(result.consecutiveTimeoutFailures, 0);
 });
 
 test('nextSourceHealthEntry – empty run preserves existing recentErrors', () => {
@@ -186,6 +189,7 @@ test('nextSourceHealthEntry – empty run preserves existing recentErrors', () =
   const result = nextSourceHealthEntry(source, emptyStat(), prior, '2026-01-02T00:00:00Z');
   assert.equal(result.recentErrors.length, 1);
   assert.equal(result.recentErrors[0].category, 'timeout');
+  assert.equal(result.consecutiveTimeoutFailures, 0);
 });
 
 test('nextSourceHealthEntry – quarantine reason uses pattern analysis', () => {
@@ -254,6 +258,29 @@ test('nextSourceHealthEntry – never-verified source with non-definitive failur
   const source = makeSource();
   const result = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), {}, '2026-01-01T00:00:00Z');
   assert.equal(result.quarantined, false, 'timeout alone should not immediately quarantine a new source');
+  assert.equal(result.consecutiveTimeoutFailures, 1);
+});
+
+test('nextSourceHealthEntry – four consecutive timeouts trigger auto-quarantine', () => {
+  const source = makeSource();
+  let prior = {};
+  for (let i = 0; i < 4; i++) {
+    prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, `2026-01-0${i + 1}T00:00:00Z`);
+  }
+  assert.equal(prior.consecutiveFailures, 4);
+  assert.equal(prior.consecutiveTimeoutFailures, 4);
+  assert.equal(prior.quarantined, true);
+  assert.match(prior.quarantineReason, /persistent timeouts/i);
+});
+
+test('nextSourceHealthEntry – non-timeout failure resets timeout streak', () => {
+  const source = makeSource();
+  let prior = {};
+  prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, '2026-01-01T00:00:00Z');
+  prior = nextSourceHealthEntry(source, failStat('timeout', 'Timed out'), prior, '2026-01-02T00:00:00Z');
+  prior = nextSourceHealthEntry(source, failStat('network-failure', 'ECONNRESET'), prior, '2026-01-03T00:00:00Z');
+  assert.equal(prior.consecutiveTimeoutFailures, 0);
+  assert.equal(prior.quarantined, false);
 });
 
 test('nextSourceHealthEntry – previously-verified source with 404 is NOT immediately quarantined', () => {

tests/seed-quarantine.test.mjs

@@ -42,6 +42,14 @@ test('deriveQuarantineIds flags source at or above failure threshold', () => {
   assert.ok(ids.has('src-c'));
 });
 
+test('deriveQuarantineIds flags source at or above timeout threshold', () => {
+  const ids = deriveQuarantineIds(
+    { 'src-timeout': { healthScore: 50, quarantined: false, consecutiveFailures: 4, consecutiveTimeoutFailures: 4 } },
+    { failureThreshold: 6, timeoutThreshold: 4 }
+  );
+  assert.ok(ids.has('src-timeout'));
+});
+
 test('deriveQuarantineIds does not flag healthy source', () => {
   const ids = deriveQuarantineIds(
     { 'src-d': { healthScore: 80, quarantined: false, consecutiveFailures: 1 } },