From 60f0a87507556349e1ca6cd12b2ed445242b283f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:30:06 +0000 Subject: [PATCH 1/9] Randomize source selection order within priority tiers on each feed refresh - Randomize rankedScheduledSources tie-breaking with Math.random() jitter instead of stable array index, so same-priority sources are shuffled each run - Randomize selectHtmlSourcesForRun rotateTier with jitter-based tie-breaking instead of deterministic offset rotation within same weight buckets - Randomize continuationCandidates tie-breaking with jitter so backup sources are also picked in varying order Priority ordering is preserved (incidents > trusted > general), but sources at the same priority level are shuffled randomly each run to keep the feed fresh. Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/0e50b7ca-6e9d-4b18-9f43-44f8eaeaac44 Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- scripts/build-live-feed.mjs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/build-live-feed.mjs b/scripts/build-live-feed.mjs index 097567c8..f8df91de 100644 --- a/scripts/build-live-feed.mjs +++ b/scripts/build-live-feed.mjs @@ -429,9 +429,10 @@ function selectHtmlSourcesForRun(rankedHtmlEntries, buildDate, maxSources) { } const rotateTier = (entries, weighted = false) => { - const sorted = [...entries].sort((left, right) => { - const leftSource = left?.source || {}; - const rightSource = right?.source || {}; + const withJitter = entries.map((entry) => ({ entry, jitter: Math.random() })); + const sorted = withJitter.sort((left, right) => { + const leftSource = left.entry?.source || {}; + const rightSource = right.entry?.source || {}; const leftWeight = weighted ? sourceDeterministicHash(`${leftSource.id || leftSource.endpoint}|${runSeed}`) % ROTATION_WEIGHT_BUCKETS : 0; @@ -439,11 +440,9 @@ function selectHtmlSourcesForRun(rankedHtmlEntries, buildDate, maxSources) { ? sourceDeterministicHash(`${rightSource.id || rightSource.endpoint}|${runSeed}`) % ROTATION_WEIGHT_BUCKETS : 0; if (rightWeight !== leftWeight) return rightWeight - leftWeight; - return left.index - right.index; + return left.jitter - right.jitter; }); - if (!sorted.length) return []; - const offset = runSeed % sorted.length; - return [...sorted.slice(offset), ...sorted.slice(0, offset)]; + return sorted.map((item) => item.entry); }; const ordered = [ @@ -2116,11 +2115,12 @@ async function main() { .map((source, index) => ({ source, index, - priority: sourceSchedulingPriority(source) + priority: sourceSchedulingPriority(source), + jitter: Math.random() })) .sort((left, right) => { if (right.priority !== left.priority) return right.priority - left.priority; - return left.index - right.index; + return left.jitter - right.jitter; }); const machineReadableScheduled = rankedScheduledSources .filter((entry) => isMachineReadableSourceKind(entry.source?.kind)) @@ -2151,10 +2151,11 @@ async function main() { } } const continuationCandidates = [...continuationCandidatesById.values()] + .map((entry) => ({ ...entry, jitter: Math.random() })) .sort((left, right) => { const priorityDelta = sourceSchedulingPriority(right.source) - sourceSchedulingPriority(left.source); if (priorityDelta !== 0) return priorityDelta; - return left.index - right.index; + return left.jitter - right.jitter; }) .map((entry) => entry.source); From 14abd928f7efac693606227ed338c1ca6fa9f8b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:56:05 +0000 Subject: [PATCH 2/9] Fix XSS, error path leaks, code duplication, missing Playwright exports, confidence bounds - Escape entry.source and entry.sourceUrl in HTML templates (alert-view-model.mjs) - Remove file paths from parseJsonFile error messages (github-persistence.js) - Sanitize workflowMessage in approve-source.js and restore-source.js - Remove raw GitHub API error forwarding in trigger-live-feed.js - Deduplicate source-processing logic: main loop now calls attemptSourceBuild() - Add missing Playwright constant exports to config.mjs - Add 0-1 bounds check on confidenceScore (alert-view-model.mjs) Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/125f5bed-88b1-4f52-9efa-960df9dc4e41 Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- api/_lib/github-persistence.js | 8 +- api/approve-source.js | 4 +- api/restore-source.js | 4 +- api/trigger-live-feed.js | 11 +- scripts/build-live-feed.mjs | 224 +++++++++-------------------- scripts/build-live-feed/config.mjs | 5 + shared/alert-view-model.mjs | 7 +- 7 files changed, 89 insertions(+), 174 deletions(-) diff --git a/api/_lib/github-persistence.js b/api/_lib/github-persistence.js index 4b135aba..2623154a 100644 --- a/api/_lib/github-persistence.js +++ b/api/_lib/github-persistence.js @@ -66,13 +66,13 @@ function encodeBase64(plainText) { return Buffer.from(plainText, 'utf8').toString('base64'); } -function parseJsonFile(raw, filePath) { +function parseJsonFile(raw) { try { return JSON.parse(raw); - } catch (error) { + } catch { throw new ApiError( 'persistence-failure', - `Invalid JSON in ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + 'Repository data file contains invalid JSON.', 500 ); } @@ -120,7 +120,7 @@ export async function loadJsonFile(pathName) { config, path: pathName, sha: payload.sha, - data: parseJsonFile(content, pathName) + data: parseJsonFile(content) }; } diff --git a/api/approve-source.js b/api/approve-source.js index 3d5c9141..807bba88 100644 --- a/api/approve-source.js +++ b/api/approve-source.js @@ -251,8 +251,8 @@ export default async function handler(request, response) { try { await dispatchWorkflow(requestsFile.config, FEED_WORKFLOW_FILENAME); workflowTriggered = true; - } catch (error) { - workflowMessage = error instanceof Error ? error.message : String(error); + } catch { + workflowMessage = 'Feed refresh workflow could not be triggered.'; } } diff --git a/api/restore-source.js b/api/restore-source.js index 8ae89803..5106b6fc 100644 --- a/api/restore-source.js +++ b/api/restore-source.js @@ -274,8 +274,8 @@ export default async function handler(request, response) { try { await dispatchWorkflow(quarantinedFile.config, FEED_WORKFLOW_FILENAME); workflowTriggered = true; - } catch (error) { - workflowMessage = error instanceof Error ? error.message : String(error); + } catch { + workflowMessage = 'Feed refresh workflow could not be triggered.'; } } diff --git a/api/trigger-live-feed.js b/api/trigger-live-feed.js index ee133a65..e78b3a3f 100644 --- a/api/trigger-live-feed.js +++ b/api/trigger-live-feed.js @@ -79,20 +79,13 @@ export default async function handler(request, response) { }); if (!dispatchResponse.ok) { - const errorText = await dispatchResponse.text().catch(() => ''); let errorMessage = 'Failed to trigger workflow dispatch.'; - try { - const errorPayload = JSON.parse(errorText); - errorMessage = errorPayload.message || errorMessage; - } catch { - // Use default error message - } if (dispatchResponse.status === 401 || dispatchResponse.status === 403) { - throw new ApiError('unauthorized', errorMessage, 503); + throw new ApiError('unauthorized', 'GitHub API authentication failed.', 503); } if (dispatchResponse.status === 404) { - throw new ApiError('workflow-not-found', `Workflow ${WORKFLOW_FILENAME} not found.`, 404); + throw new ApiError('workflow-not-found', 'Workflow not found.', 404); } throw new ApiError('trigger-failed', errorMessage, 500); } diff --git a/scripts/build-live-feed.mjs b/scripts/build-live-feed.mjs index f8df91de..d78b67f9 100644 --- a/scripts/build-live-feed.mjs +++ b/scripts/build-live-feed.mjs @@ -556,6 +556,23 @@ function shouldTryPlaywrightForThinHtml(source, body, playwrightBudget) { async function attemptSourceBuild(source, requestState, playwrightBudget, priorHealthEntry) { const localErrors = []; const builtAlerts = []; + const failureReasonCounts = { + success: 0, + unchanged: 0, + 'stale-endpoint': 0, + 'blocked-or-anti-bot': 0, + 'timeout-or-aborted': 0, + 'parser-failure': 0, + 'empty-or-no-items': 0, + unknown: 0 + }; + const discardReasons = { + parseNoItems: 0, + droppedByFilter: 0, + droppedByMissingOrInvalidDate: 0, + droppedByItemCap: 0, + buildFailures: 0 + }; let body; let usedPlaywrightFallback = false; let finalUrl = clean(source?.endpoint); @@ -574,6 +591,10 @@ async function attemptSourceBuild(source, requestState, playwrightBudget, priorH } } catch (error) { const summary = summariseSourceError(source, error); + const reason = classifyFetchFailure(summary); + if (reason === 'stale-endpoint') failureReasonCounts['stale-endpoint'] += 1; + else if (reason === 'bot-block') failureReasonCounts['blocked-or-anti-bot'] += 1; + else if (reason === 'timeout') failureReasonCounts['timeout-or-aborted'] += 1; if (shouldTryPlaywrightFallback(source, summary, playwrightBudget)) { playwrightBudget.attempts += 1; body = await fetchTextWithPlaywright(source.endpoint, { @@ -603,12 +624,16 @@ async function attemptSourceBuild(source, requestState, playwrightBudget, priorH usedPlaywrightFallback = true; playwrightBudget.successes += 1; parsed = parseHtmlItems(source, body); - } catch (error) { - // ignore, handled below + } catch { + // fall through to normal empty parse handling } } if (!parsed.length) { + discardReasons.parseNoItems += 1; + if (fetchOutcome !== 'unchanged') { + failureReasonCounts['empty-or-no-items'] += 1; + } localErrors.push(summariseSourceError( source, buildFetchError('No items parsed from source payload', 'brittle-selectors-or-js-rendering') @@ -628,7 +653,11 @@ async function attemptSourceBuild(source, requestState, playwrightBudget, priorH const reliabilityProfile = inferReliabilityProfile(source, inferSourceTier(source)); const filtered = hydrated.filter((item) => { try { - return discardReasonForItem(source, item) === null; + const discardReason = discardReasonForItem(source, item); + if (discardReason === 'missing-or-invalid-date') { + discardReasons.droppedByMissingOrInvalidDate += 1; + } + return discardReason === null; } catch (error) { localErrors.push(summariseSourceError(source, error)); return false; @@ -641,15 +670,28 @@ async function attemptSourceBuild(source, requestState, playwrightBudget, priorH filteredCount: filtered.length }); const kept = filtered.slice(0, itemLimit); + discardReasons.droppedByFilter += Math.max(0, hydrated.length - filtered.length); + discardReasons.droppedByItemCap += Math.max(0, filtered.length - kept.length); kept.forEach((item, idx) => { try { builtAlerts.push(buildAlert(source, item, idx)); } catch (error) { localErrors.push(summariseSourceError(source, error)); + discardReasons.buildFailures += 1; } }); + const parserFailures = localErrors + .map((errorSummary) => classifyFetchFailure(errorSummary)) + .filter((reason) => reason === 'parser-failure').length; + if (parserFailures > 0) failureReasonCounts['parser-failure'] += parserFailures; + if (fetchOutcome === 'unchanged') { + failureReasonCounts.unchanged += 1; + } else if (builtAlerts.length > 0) { + failureReasonCounts.success += 1; + } + return { alerts: builtAlerts, sourceErrors: localErrors, @@ -670,7 +712,9 @@ async function attemptSourceBuild(source, requestState, playwrightBudget, priorH usedPlaywrightFallback, finalUrl, status: responseStatus, - fetchOutcome + fetchOutcome, + failureReasonCounts, + discardReasons } }; } @@ -2176,167 +2220,39 @@ async function main() { batch, FEED_SOURCE_CONCURRENCY, async (source, sourceIndex) => { - const localErrors = []; - const builtAlerts = []; - const failureReasonCounts = { - success: 0, - unchanged: 0, - 'stale-endpoint': 0, - 'blocked-or-anti-bot': 0, - 'timeout-or-aborted': 0, - 'parser-failure': 0, - 'empty-or-no-items': 0, - unknown: 0 - }; - const discardReasons = { - parseNoItems: 0, - droppedByFilter: 0, - droppedByMissingOrInvalidDate: 0, - droppedByItemCap: 0, - buildFailures: 0 - }; - try { const baseDelay = (sourceAttemptOffset + sourceIndex) * DEFAULT_FETCH_STAGGER_MS; const jitter = MAX_FETCH_STAGGER_JITTER_MS > 0 ? Math.floor(Math.random() * (MAX_FETCH_STAGGER_JITTER_MS + 1)) : 0; await sleep(baseDelay + jitter); - let body; - let usedPlaywrightFallback = false; - let finalUrl = clean(source?.endpoint); - let responseStatus = null; - let fetchOutcome = 'unknown'; - try { - const fetched = await fetchText(source.endpoint, 1, { source, requestState, includeMeta: true }); - if (typeof fetched === 'string') { - body = fetched; - fetchOutcome = 'success'; - } else { - body = fetched.text; - finalUrl = clean(fetched.finalUrl || source.endpoint); - responseStatus = Number.isFinite(Number(fetched.status)) ? Number(fetched.status) : null; - fetchOutcome = (responseStatus === 304 || fetched.unchanged304) ? 'unchanged' : 'success'; - } - } catch (error) { - const summary = summariseSourceError(source, error); - const reason = classifyFetchFailure(summary); - if (reason === 'stale-endpoint') failureReasonCounts['stale-endpoint'] += 1; - else if (reason === 'bot-block') failureReasonCounts['blocked-or-anti-bot'] += 1; - else if (reason === 'timeout') failureReasonCounts['timeout-or-aborted'] += 1; - if (shouldTryPlaywrightFallback(source, summary, playwrightBudget)) { - playwrightBudget.attempts += 1; - body = await fetchTextWithPlaywright(source.endpoint, { - source, - timeoutMs: PLAYWRIGHT_FALLBACK_TIMEOUT_MS - }); - usedPlaywrightFallback = true; - playwrightBudget.successes += 1; - fetchOutcome = 'success'; - } else { - throw error; - } - } - let parsed = source.kind === 'rss' || source.kind === 'atom' || source.kind === 'json' - ? parseFeedItems(source, body) - : parseHtmlItems(source, body); - if (!parsed.length && source.kind === 'html' && !usedPlaywrightFallback && shouldTryPlaywrightForThinHtml(source, body, playwrightBudget)) { - try { - playwrightBudget.attempts += 1; - body = await fetchTextWithPlaywright(source.endpoint, { - source, - timeoutMs: PLAYWRIGHT_FALLBACK_TIMEOUT_MS, - contentSelectors: source?.playwright?.contentSelectors - }); - usedPlaywrightFallback = true; - playwrightBudget.successes += 1; - parsed = parseHtmlItems(source, body); - } catch (error) { - // fall through to normal empty parse handling - } - } - if (!parsed.length) { - discardReasons.parseNoItems += 1; - if (fetchOutcome !== 'unchanged') { - failureReasonCounts['empty-or-no-items'] += 1; - } - localErrors.push(summariseSourceError( - source, - buildFetchError('No items parsed from source payload', 'brittle-selectors-or-js-rendering') - )); - } - const preLimit = source.kind === 'html' ? MAX_HTML_PREFETCH_ITEMS : MAX_FEED_PREFETCH_ITEMS; - const preLimited = parsed.slice(0, preLimit); - const hydrated = source.kind === 'html' ? await enrichHtmlItems(source, preLimited) : preLimited; - const reliabilityProfile = inferReliabilityProfile(source, inferSourceTier(source)); - const filtered = hydrated.filter((item) => { - try { - const discardReason = discardReasonForItem(source, item); - if (discardReason === 'missing-or-invalid-date') { - discardReasons.droppedByMissingOrInvalidDate += 1; - } - return discardReason === null; - } catch (error) { - localErrors.push(summariseSourceError(source, error)); - console.error(`Source item filter failed: ${source.id} - ${error instanceof Error ? error.message : String(error)}`); - return false; - } - }); - const itemLimit = computeDynamicItemLimit({ - reliabilityProfile, - lane: source.lane, - sourceHealth: sourceHealthEntry(previousHealth, source.id), - filteredCount: filtered.length - }); - const kept = filtered.slice(0, itemLimit); - discardReasons.droppedByFilter += Math.max(0, hydrated.length - filtered.length); - discardReasons.droppedByItemCap += Math.max(0, filtered.length - kept.length); - - kept.forEach((item, idx) => { - try { - builtAlerts.push(buildAlert(source, item, idx)); - } catch (error) { - localErrors.push(summariseSourceError(source, error)); - discardReasons.buildFailures += 1; - console.error(`Alert build failed: ${source.id} - ${error instanceof Error ? error.message : String(error)}`); - } - }); - const parserFailures = localErrors - .map((errorSummary) => classifyFetchFailure(errorSummary)) - .filter((reason) => reason === 'parser-failure').length; - if (parserFailures > 0) failureReasonCounts['parser-failure'] += parserFailures; - if (fetchOutcome === 'unchanged') { - failureReasonCounts.unchanged += 1; - } else if (builtAlerts.length > 0) { - failureReasonCounts.success += 1; - } + const result = await attemptSourceBuild(source, requestState, playwrightBudget, sourceHealthEntry(previousHealth, source.id)); return { checked: 1, - alerts: builtAlerts, - sourceErrors: localErrors, - sourceStat: { - id: source.id, - provider: source.provider, - lane: source.lane, - kind: source.kind, - parsed: parsed.length, - hydrated: hydrated.length, - filtered: filtered.length, - kept: kept.length, - built: builtAlerts.length, - errors: localErrors.length, - lastErrorCategory: localErrors[0]?.category || null, - lastErrorMessage: localErrors[0]?.message || null, - usedPlaywrightFallback, - finalUrl, - status: responseStatus, - fetchOutcome, - failureReasonCounts, - discardReasons - } + alerts: result.alerts, + sourceErrors: result.sourceErrors, + sourceStat: result.sourceStat }; } catch (error) { + const localErrors = []; + const failureReasonCounts = { + success: 0, + unchanged: 0, + 'stale-endpoint': 0, + 'blocked-or-anti-bot': 0, + 'timeout-or-aborted': 0, + 'parser-failure': 0, + 'empty-or-no-items': 0, + unknown: 0 + }; + const discardReasons = { + parseNoItems: 0, + droppedByFilter: 0, + droppedByMissingOrInvalidDate: 0, + droppedByItemCap: 0, + buildFailures: 0 + }; const summary = summariseSourceError(source, error); localErrors.push(summary); const reason = classifyFetchFailure(summary); diff --git a/scripts/build-live-feed/config.mjs b/scripts/build-live-feed/config.mjs index 17ee6d65..367481b1 100644 --- a/scripts/build-live-feed/config.mjs +++ b/scripts/build-live-feed/config.mjs @@ -94,6 +94,11 @@ export const PLAYWRIGHT_FALLBACK_TIMEOUT_MS = Math.max( ? Math.floor(Number(process.env.BRIALERT_PLAYWRIGHT_TIMEOUT_MS)) : 12000 ); +export const DEFAULT_PLAYWRIGHT_TIMEOUT_MS = envInt('BRIALERT_DEFAULT_PLAYWRIGHT_TIMEOUT_MS', 15000, 3000); +export const DEFAULT_PLAYWRIGHT_PAGE_SETTLE_MS = envInt('BRIALERT_DEFAULT_PLAYWRIGHT_PAGE_SETTLE_MS', 2000, 500); +export const MAX_PLAYWRIGHT_ITEM_SUMMARY_CHARS = 400; +export const MAX_PLAYWRIGHT_RAW_CANDIDATES = 60; +export const PLAYWRIGHT_SCRAPER_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'; export const GUARDRAIL_MAX_RUNTIME_MS = Math.max( 60_000, Number.isFinite(Number(process.env.BRIALERT_GUARDRAIL_MAX_RUNTIME_MS)) diff --git a/shared/alert-view-model.mjs b/shared/alert-view-model.mjs index e66b81f3..9d77d659 100644 --- a/shared/alert-view-model.mjs +++ b/shared/alert-view-model.mjs @@ -8,6 +8,7 @@ import { import { laneLabels } from './ui-data.mjs'; import { formatAgeFromDate } from './time-format.mjs'; import { DEFAULT_LANE, LANE_KEYS, STATUS_LABELS } from './ui-constants.mjs'; +import { escapeHtml } from '../app/utils/text.mjs'; export function formatAgeFrom(dateLike) { return formatAgeFromDate(dateLike); @@ -438,7 +439,7 @@ export function renderSceneClock(alert) { return `
${items.map(({ label, entry, fallback }) => `
${label} -

${entry ? `${clockDisplay(entry.publishedAt)} | ${sceneClockStamp(entry.publishedAt)}${entry.source ? ` | ${entry.source}` : ''}` : fallback}

+

${entry ? `${clockDisplay(entry.publishedAt)} | ${sceneClockStamp(entry.publishedAt)}${entry.source ? ` | ${escapeHtml(entry.source)}` : ''}` : fallback}

`).join('')}
`; } @@ -466,7 +467,7 @@ export function renderCorroboratingSources(alert) { } return `
${sources.map((entry) => `
- ${entry.source} + ${escapeHtml(entry.source)}

${reliabilityLabel(normaliseReliabilityProfile(entry.reliabilityProfile))} | ${clean(entry.sourceTier) || 'source tier unknown'} | ${clean(entry.publishedAt) ? formatAgeFrom(entry.publishedAt) : 'age unknown'}

`).join('')}
`; } @@ -554,7 +555,7 @@ export function normaliseAlert(alert, index, geoLookup = []) { freshUntil: clean(alert.freshUntil), needsHumanReview: !!alert.needsHumanReview, priorityScore: Number.isFinite(alert.priorityScore) ? alert.priorityScore : null, - confidenceScore: Number.isFinite(alert.confidenceScore) ? alert.confidenceScore : null, + confidenceScore: Number.isFinite(alert.confidenceScore) && alert.confidenceScore >= 0 && alert.confidenceScore <= 1 ? alert.confidenceScore : null, publishedAt: clean(alert.publishedAt), freshnessBucket: Number.isFinite(alert.freshnessBucket) ? alert.freshnessBucket : null, keywordHits: Array.isArray(alert.keywordHits) ? alert.keywordHits.filter(Boolean) : [], From 011fd54bd039aee65135981f0c020a3a50bf965f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:01:01 +0000 Subject: [PATCH 3/9] Cache Date.parse(generatedAt) in nextSourceHealthEntry to avoid 7 redundant parses Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/42b1a3c3-155f-47e7-aa93-203dba553b5b Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- scripts/build-live-feed.mjs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build-live-feed.mjs b/scripts/build-live-feed.mjs index d78b67f9..4a782d28 100644 --- a/scripts/build-live-feed.mjs +++ b/scripts/build-live-feed.mjs @@ -261,7 +261,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.lastErrorCategory = stat?.lastErrorCategory || null; next.lastErrorMessage = stat?.lastErrorMessage || null; if (blockedNonContent && next.consecutiveBlockedFailures >= BLOCKED_NON_CONTENT_FAIL_THRESHOLD) { - next.cooldownUntil = new Date(Date.parse(generatedAt) + BLOCKED_NON_CONTENT_COOLDOWN_HOURS * 3600000).toISOString(); + next.cooldownUntil = new Date(generatedAtMs + BLOCKED_NON_CONTENT_COOLDOWN_HOURS * 3600000).toISOString(); next.autoSkipReason = 'blocked-cooldown'; next.nextFetchAt = next.cooldownUntil; return next; @@ -272,7 +272,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.quarantineReason = 'HTTP 404 not found; needs manual source URL review'; next.autoSkipReason = 'review-quarantine'; next.cooldownUntil = null; - next.nextFetchAt = new Date(Date.parse(generatedAt) + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); + next.nextFetchAt = new Date(generatedAtMs + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); return next; } if (!next.quarantined && source?.kind === 'html' && next.consecutiveBlockedFailures >= AUTO_QUARANTINE_BLOCKED_HTML_THRESHOLD) { @@ -281,7 +281,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.quarantineReason = 'Repeated blocked-or-auth failures on html source'; next.autoSkipReason = 'review-quarantine'; next.cooldownUntil = null; - next.nextFetchAt = new Date(Date.parse(generatedAt) + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); + next.nextFetchAt = new Date(generatedAtMs + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); return next; } if (!next.quarantined && next.consecutiveDeadUrlFailures >= AUTO_QUARANTINE_DEAD_URL_THRESHOLD) { @@ -290,7 +290,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.quarantineReason = 'Repeated dead-or-moved-url failures'; next.autoSkipReason = 'review-quarantine'; next.cooldownUntil = null; - next.nextFetchAt = new Date(Date.parse(generatedAt) + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); + next.nextFetchAt = new Date(generatedAtMs + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); return next; } if (!next.quarantined && next.consecutiveFailures >= AUTO_QUARANTINE_FAILURE_THRESHOLD) { @@ -299,12 +299,12 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.quarantineReason = `Repeated failures (${next.consecutiveFailures}) need manual review`; next.autoSkipReason = 'review-quarantine'; next.cooldownUntil = null; - next.nextFetchAt = new Date(Date.parse(generatedAt) + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); + next.nextFetchAt = new Date(generatedAtMs + AUTO_QUARANTINE_RECHECK_HOURS * 3600000).toISOString(); return next; } if (next.consecutiveFailures >= AUTO_SKIP_FAILURE_THRESHOLD) { const cooldownHours = sourceFailureCooldownHours(source, stat?.lastErrorCategory || ''); - next.cooldownUntil = new Date(Date.parse(generatedAt) + cooldownHours * 3600000).toISOString(); + next.cooldownUntil = new Date(generatedAtMs + cooldownHours * 3600000).toISOString(); next.autoSkipReason = 'failure-cooldown'; next.nextFetchAt = next.cooldownUntil; } @@ -320,7 +320,7 @@ function nextSourceHealthEntry(source, stat, previousEntry, generatedAt) { next.lastErrorMessage = null; next.lastEmptyAt = generatedAt; if (!source.isTrustedOfficial && source.lane !== 'incidents' && next.consecutiveEmptyRuns >= AUTO_SKIP_EMPTY_THRESHOLD) { - next.cooldownUntil = new Date(Date.parse(generatedAt) + SOURCE_EMPTY_COOLDOWN_HOURS * 3600000).toISOString(); + next.cooldownUntil = new Date(generatedAtMs + SOURCE_EMPTY_COOLDOWN_HOURS * 3600000).toISOString(); next.autoSkipReason = 'empty-cooldown'; next.nextFetchAt = next.cooldownUntil; } From 077ca894264a079df0d166ba95abf38f68a09b95 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:03:13 +0000 Subject: [PATCH 4/9] Add SSRF protection: block private/internal IP ranges in URL validation Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/42b1a3c3-155f-47e7-aa93-203dba553b5b Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- api/_lib/github-persistence.js | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/api/_lib/github-persistence.js b/api/_lib/github-persistence.js index 2623154a..86a08a84 100644 --- a/api/_lib/github-persistence.js +++ b/api/_lib/github-persistence.js @@ -78,6 +78,50 @@ function parseJsonFile(raw) { } } +/** + * Returns true when a hostname looks like a private, loopback, link-local, or + * otherwise non-routable address that must be rejected to prevent SSRF. + * Checks the raw hostname string — no DNS resolution is performed, so only + * literal IP addresses and well-known internal hostnames are caught. + */ +function isPrivateOrReservedHost(hostname) { + const host = String(hostname || '').toLowerCase(); + if (!host) return true; + + // Loopback / localhost + if (host === 'localhost' || host === '[::1]') return true; + + // Strip IPv6 brackets for numeric checks + const bare = host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host; + + // IPv6 loopback & link-local + if (bare === '::1' || bare.startsWith('fe80:') || bare.startsWith('fc00:') || bare.startsWith('fd00:')) return true; + + // IPv4 dotted-quad checks + const ipv4Parts = bare.split('.'); + if (ipv4Parts.length === 4 && ipv4Parts.every((p) => /^\d{1,3}$/.test(p))) { + const octets = ipv4Parts.map(Number); + if (octets.some((o) => o > 255)) return true; // malformed — block as suspicious + const [a, b] = octets; + if (a === 127) return true; // 127.0.0.0/8 loopback + if (a === 10) return true; // 10.0.0.0/8 private + if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 private + if (a === 192 && b === 168) return true; // 192.168.0.0/16 private + if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local + if (a === 0) return true; // 0.0.0.0/8 + if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGN shared + if (a === 198 && (b === 18 || b === 19)) return true; // 198.18.0.0/15 benchmarking + if (a >= 224) return true; // 224+ multicast / reserved + } + + // Block common internal / metadata hostnames + if (host.endsWith('.local') || host.endsWith('.internal') || host.endsWith('.localhost')) return true; + // Cloud provider metadata endpoints + if (host === '169.254.169.254' || host === 'metadata.google.internal') return true; + + return false; +} + export function validateAbsoluteHttpUrl(value, code = 'invalid-url') { const trimmed = String(value || '').trim(); if (!trimmed) { @@ -92,6 +136,9 @@ export function validateAbsoluteHttpUrl(value, code = 'invalid-url') { if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { throw new ApiError(code, 'Replacement URL must use http or https.', 400); } + if (isPrivateOrReservedHost(parsed.hostname)) { + throw new ApiError(code, 'URL must not point to a private or internal network address.', 400); + } return parsed.toString(); } From 46de2c0fe6fab2f23551203ae2f5a83c07f59d86 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:10:16 +0000 Subject: [PATCH 5/9] Cache geo regex, extract constants to config, add Leaflet defer, remove !important, rename duplicate clean() Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/6ba44832-573e-445c-b4f5-c0e7c05c9c0b Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- app/feed/source-requests.mjs | 13 +++++++------ index.html | 2 +- scripts/build-live-feed.mjs | 4 ++-- scripts/build-live-feed/config.mjs | 2 ++ scripts/build-live-feed/geo.mjs | 11 +++++++++-- styles.css | 4 ++-- 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/app/feed/source-requests.mjs b/app/feed/source-requests.mjs index 58a40395..925b756d 100644 --- a/app/feed/source-requests.mjs +++ b/app/feed/source-requests.mjs @@ -11,7 +11,8 @@ const sourceRequestRateState = { lastAttemptAtMs: 0 }; -function clean(value) { +/** Trim-only string coercion — intentionally simpler than taxonomy.clean() which also splits camelCase. */ +function trimString(value) { return String(value || '').trim(); } @@ -22,7 +23,7 @@ function currentOriginBase() { } function resolveApiUrls(apiUrl) { - const trimmed = clean(apiUrl); + const trimmed = trimString(apiUrl); if (!trimmed) return []; if (/^https?:\/\//i.test(trimmed)) return [trimmed]; const bases = [SOURCE_REQUEST_BACKEND_BASE, currentOriginBase()].filter(Boolean); @@ -32,7 +33,7 @@ function resolveApiUrls(apiUrl) { function normaliseRequestUrl(value) { let parsed; try { - parsed = new URL(clean(value)); + parsed = new URL(trimString(value)); } catch { throw new Error('Enter a valid http(s) source link.'); } @@ -65,11 +66,11 @@ function enforceClientRateLimit(nowMs = Date.now()) { function normalisedEndpointKey(endpoint) { try { - const url = new URL(clean(endpoint)); + const url = new URL(trimString(endpoint)); url.hash = ''; return url.toString().replace(/\/$/, ''); } catch { - return clean(endpoint).replace(/\/$/, ''); + return trimString(endpoint).replace(/\/$/, ''); } } @@ -150,7 +151,7 @@ export async function submitSourceRequest(state, { apiUrl, url, regionHint }) { return {}; }); if (!response.ok) { - throw new Error(clean(payload?.detail) || `HTTP ${response.status}`); + throw new Error(trimString(payload?.detail) || `HTTP ${response.status}`); } const requests = Array.isArray(payload?.requests) diff --git a/index.html b/index.html index c406de77..add157ee 100644 --- a/index.html +++ b/index.html @@ -261,7 +261,7 @@ - + diff --git a/scripts/build-live-feed.mjs b/scripts/build-live-feed.mjs index 4a782d28..2a341ce1 100644 --- a/scripts/build-live-feed.mjs +++ b/scripts/build-live-feed.mjs @@ -40,6 +40,8 @@ import { SOURCE_BLOCKED_FAILURE_COOLDOWN_HOURS, SOURCE_FAILURE_COOLDOWN_HOURS, TARGET_SUCCESSFUL_SOURCES_PER_RUN, + CONTINUATION_RUNTIME_HEADROOM_MS, + ROTATION_WEIGHT_BUCKETS, computeDynamicItemLimit, AUTO_QUARANTINE_FAILURE_THRESHOLD, isMachineReadableSourceKind, @@ -412,7 +414,6 @@ function sourceDomain(source) { function selectHtmlSourcesForRun(rankedHtmlEntries, buildDate, maxSources) { const safeEntries = Array.isArray(rankedHtmlEntries) ? rankedHtmlEntries : []; const runSeed = Math.floor(buildDate.getTime() / 3600000); - const ROTATION_WEIGHT_BUCKETS = 7; const runCap = Math.max(0, Number(maxSources || 0)); const domainUse = new Map(); const domainCappedSourceIds = new Set(); @@ -2063,7 +2064,6 @@ async function syncBuilderSQLite(snapshot) { async function main() { const runStartedAtMs = Date.now(); - const CONTINUATION_RUNTIME_HEADROOM_MS = 20_000; const buildDate = new Date(); const existing = await readExisting(); const geoLookupFallbackNote = await safeLoadGeoLookup(existing); diff --git a/scripts/build-live-feed/config.mjs b/scripts/build-live-feed/config.mjs index 367481b1..5f7d9a6f 100644 --- a/scripts/build-live-feed/config.mjs +++ b/scripts/build-live-feed/config.mjs @@ -126,6 +126,8 @@ export const TARGET_SUCCESSFUL_SOURCES_PER_RUN = Math.max( ? Math.floor(Number(process.env.BRIALERT_TARGET_SUCCESSFUL_SOURCES_PER_RUN)) : 60 ); +export const CONTINUATION_RUNTIME_HEADROOM_MS = envInt('BRIALERT_CONTINUATION_RUNTIME_HEADROOM_MS', 20_000, 1000); +export const ROTATION_WEIGHT_BUCKETS = envInt('BRIALERT_ROTATION_WEIGHT_BUCKETS', 7, 2); /** Base lane caps – used as the starting point before dynamic adjustments. */ const BASE_LANE_LIMITS = Object.freeze({ incidents: 6, diff --git a/scripts/build-live-feed/geo.mjs b/scripts/build-live-feed/geo.mjs index b5987687..35ffd2a4 100644 --- a/scripts/build-live-feed/geo.mjs +++ b/scripts/build-live-feed/geo.mjs @@ -18,9 +18,16 @@ function normaliseGeoText(value) { .trim(); } +const geoTermRegexCache = new Map(); + function geoTermRegex(term) { - const escaped = escapeRegex(normaliseGeoText(term)); - return new RegExp(`(^|\\W)${escaped}(?=$|\\W)`, 'i'); + const normalised = normaliseGeoText(term); + let cached = geoTermRegexCache.get(normalised); + if (cached) return cached; + const escaped = escapeRegex(normalised); + cached = new RegExp(`(^|\\W)${escaped}(?=$|\\W)`, 'i'); + geoTermRegexCache.set(normalised, cached); + return cached; } function scoreGeoEntryMatch(entry, haystack) { diff --git a/styles.css b/styles.css index c370531f..7bb6fac3 100644 --- a/styles.css +++ b/styles.css @@ -393,11 +393,11 @@ h1 { } .screen.briefing-mode .briefing-mode-hidden { - display: none !important; + display: none; } .screen.briefing-mode .briefing-mode-panel.hidden { - display: block !important; + display: block; } .screen.briefing-mode .tab-panel.active { From bb8d55c178030896ed5e5f2611c1ef42e903bf9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Apr 2026 23:15:01 +0000 Subject: [PATCH 6/9] Add form labels for accessibility, fix rate limiter clock skew, extract 404 inline JS Agent-Logs-Url: https://github.com/potemkin666/Brialert/sessions/192e4b0f-d0f3-4fea-890d-f58fbf3a9011 Co-authored-by: potemkin666 <183807833+potemkin666@users.noreply.github.com> --- 404.html | 11 +---------- app/feed/source-requests.mjs | 9 ++++++++- app/redirect.js | 10 ++++++++++ index.html | 3 +++ 4 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 app/redirect.js diff --git a/404.html b/404.html index b0025d94..43e16ebc 100644 --- a/404.html +++ b/404.html @@ -5,16 +5,7 @@ Brialert redirect - +