Skip to content

10 scraping reliability improvements for source ingestion pipeline #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Copilot wants to merge 3 commits into
base: main
Choose a base branch
from
Open

10 scraping reliability improvements for source ingestion pipeline #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d4ee0f1
10 scraping reliability improvements for Brialert source ingestion
Copilot Apr 12, 2026
ce3c03b
Implement 10 scraping reliability improvements
Copilot Apr 12, 2026
610968f
Fix review feedback: robust LD+JSON URL extraction, order-independent…
Copilot Apr 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified data/brialert.sqlite
View file
Open in desktop
Binary file not shown.
107 changes: 72 additions & 35 deletions data/sources.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,13 @@
{
"id": "frontex-news-releases",
"provider": "Frontex a News Releases",
"endpoint": "https://www.frontex.europa.eu/media-centre/news/news-release/feed",
"endpoint": "https://www.frontex.europa.eu/media-centre/news/news-release/",
"kind": "html",
"lane": "border",
"region": "eu",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "No RSS endpoint consistently published; using the News Release listing and relying on keyword filtering.",
"quarantined": false,
"restoredAt": "2026-04-12T08:47:26.742Z"
"comment": "No RSS endpoint consistently published; using the News Release listing and relying on keyword filtering."
},
{
"id": "austria-interior-ministry-news",
Expand Down Expand Up @@ -71,15 +69,13 @@
{
"id": "belgium-crisis-centre-news",
"provider": "Belgian National Crisis Centre - News",
"endpoint": "https://crisiscenter.be/en/newsroom",
"endpoint": "https://crisiscenter.be/en/news",
"kind": "html",
"lane": "context",
"region": "eu",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "Belgian crisis centre updates for threat level, emergency posture and civil contingency context.",
"quarantined": false,
"restoredAt": "2026-04-12T08:47:52.261Z"
"comment": "Belgian crisis centre updates for threat level, emergency posture and civil contingency context."
},
{
"id": "denmark-pet-news",
Expand Down Expand Up @@ -313,15 +309,13 @@
{
"id": "czech-police-news",
"provider": "Police of the Czech Republic - News",
"endpoint": "https://english.radio.cz/news",
"endpoint": "https://www.policie.cz/clanek/news.aspx",
"kind": "html",
"lane": "incidents",
"region": "eu",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "Czech Police updates for arrests, security incidents and public-order operations relevant to regional threat context.",
"quarantined": false,
"restoredAt": "2026-04-12T08:50:52.579Z"
"comment": "Czech Police updates for arrests, security incidents and public-order operations relevant to regional threat context."
},
{
"id": "dutch-police-rss-index",
Expand Down Expand Up @@ -566,15 +560,13 @@
{
"id": "eu-council-rss-index",
"provider": "Council of the EU - RSS Index",
"endpoint": "https://securitymattersmagazine.com/news",
"endpoint": "https://www.consilium.europa.eu/en/about-site/rss/",
"kind": "html",
"lane": "oversight",
"region": "eu",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "Council of the EU RSS index page for official feed discovery and source maintenance.",
"quarantined": false,
"restoredAt": "2026-04-12T08:51:52.686Z"
"comment": "Council of the EU RSS index page for official feed discovery and source maintenance."
},
{
"id": "eu-ct-coordinator",
Expand Down Expand Up @@ -927,15 +919,13 @@
{
"id": "politico-europe",
"provider": "Politico Europe",
"endpoint": "https://www.politico.eu/section/defense/",
"endpoint": "https://www.politico.eu/",
"kind": "html",
"lane": "context",
"region": "europe",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Politico Europe used as corroborating political-security context rather than live trigger input.",
"quarantined": false,
"restoredAt": "2026-04-12T08:52:42.850Z"
"comment": "Politico Europe used as corroborating political-security context rather than live trigger input."
},
{
"id": "rte-ireland-news-rss",
Expand Down Expand Up @@ -1160,15 +1150,13 @@
{
"id": "nctv-nl-english-news",
"provider": "NCTV Netherlands latest news (English)",
"endpoint": "https://english.nctv.nl/api/rss?query=%7B%22filters%22%3A%5B%7B%22field%22%3A%22content_type%22%2C%22values%22%3A%5B%22pro%3AnewsDocument%22%5D%2C%22type%22%3A%22all%22%7D%5D%2C%22resultSearchTerm%22%3A%22%22%7D",
"endpoint": "https://english.nctv.nl/latest/news",
"kind": "html",
"lane": "prevention",
"region": "europe",
"isTrustedOfficial": true,
"requiresKeywordMatch": false,
"comment": "Latest news page of the Netherlands' National Coordinator for Security and Counterterrorism (NCTV), in English, listing news and press releases.",
"quarantined": false,
"restoredAt": "2026-04-12T08:53:02.544Z"
"comment": "Latest news page of the Netherlands' National Coordinator for Security and Counterterrorism (NCTV), in English, listing news and press releases."
},
{
"id": "ocam-be-english",
Expand Down Expand Up @@ -1456,14 +1444,12 @@
{
"id": "nato-news-rss",
"provider": "NATO a News (RSS)",
"endpoint": "https://www.nato.int/en/news-and-events/articles/news",
"endpoint": "https://www.nato.int/cps/en/natohq/rss/news.rss",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"quarantined": false,
"restoredAt": "2026-04-12T08:53:42.867Z"
"requiresKeywordMatch": true
},
{
"id": "occrp-news",
Expand Down Expand Up @@ -1575,6 +1561,61 @@
"requiresKeywordMatch": true,
"comment": "War on the Rocks publishes analysis on national security, military affairs and counter-terrorism; keyword filtering applied."
},
{
"id": "google-news-terrorism-uk",
"provider": "Google News - Terrorism UK",
"endpoint": "https://news.google.com/rss/search?q=terrorism+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK terrorism coverage. Acts as a safety net when individual institutional sources fail."
},
{
"id": "google-news-counter-terrorism",
"provider": "Google News - Counter Terrorism",
"endpoint": "https://news.google.com/rss/search?q=%22counter+terrorism%22&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for counter-terrorism policy and operations coverage."
},
{
"id": "google-news-terror-attack",
"provider": "Google News - Terror Attack",
"endpoint": "https://news.google.com/rss/search?q=%22terror+attack%22&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for terror attack incident coverage."
},
{
"id": "google-news-extremism",
"provider": "Google News - Extremism",
"endpoint": "https://news.google.com/rss/search?q=extremism+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK extremism reporting and prevention coverage."
},
{
"id": "google-alerts-uk-security-threat",
"provider": "Google News - UK Security Threat",
"endpoint": "https://news.google.com/rss/search?q=%22security+threat%22+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK security threat level changes and alerts."
},
{
"id": "gctf-news",
"provider": "Global Counterterrorism Forum (GCTF) a News",
Expand Down Expand Up @@ -1713,9 +1754,7 @@
"region": "london",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "Barking and Dagenham borough news used for East London incident corroboration and civic response.",
"quarantined": false,
"restoredAt": "2026-04-12T08:54:18.092Z"
"comment": "Barking and Dagenham borough news used for East London incident corroboration and civic response."
},
{
"id": "barnet-council-news",
Expand Down Expand Up @@ -1973,15 +2012,13 @@
{
"id": "homerton-healthcare-news",
"provider": "Homerton Healthcare NHS Foundation Trust a News",
"endpoint": "https://hospitaltimes.co.uk/",
"endpoint": "https://www.homerton.nhs.uk/latest-news/",
"kind": "html",
"lane": "context",
"region": "london",
"isTrustedOfficial": true,
"requiresKeywordMatch": true,
"comment": "Homerton Healthcare news used for East London hospital and emergency-response context.",
"quarantined": false,
"restoredAt": "2026-04-12T08:55:08.895Z"
"comment": "Homerton Healthcare news used for East London hospital and emergency-response context."
},
{
"id": "hounslow-council-news",
Expand Down
55 changes: 55 additions & 0 deletions data/sources/international/context.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,61 @@
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "War on the Rocks publishes analysis on national security, military affairs and counter-terrorism; keyword filtering applied."
},
{
"id": "google-news-terrorism-uk",
"provider": "Google News - Terrorism UK",
"endpoint": "https://news.google.com/rss/search?q=terrorism+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK terrorism coverage. Acts as a safety net when individual institutional sources fail."
},
{
"id": "google-news-counter-terrorism",
"provider": "Google News - Counter Terrorism",
"endpoint": "https://news.google.com/rss/search?q=%22counter+terrorism%22&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for counter-terrorism policy and operations coverage."
},
{
"id": "google-news-terror-attack",
"provider": "Google News - Terror Attack",
"endpoint": "https://news.google.com/rss/search?q=%22terror+attack%22&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for terror attack incident coverage."
},
{
"id": "google-news-extremism",
"provider": "Google News - Extremism",
"endpoint": "https://news.google.com/rss/search?q=extremism+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK extremism reporting and prevention coverage."
},
{
"id": "google-alerts-uk-security-threat",
"provider": "Google News - UK Security Threat",
"endpoint": "https://news.google.com/rss/search?q=%22security+threat%22+UK&hl=en-GB&gl=GB&ceid=GB:en",
"kind": "rss",
"lane": "context",
"region": "international",
"isTrustedOfficial": false,
"requiresKeywordMatch": true,
"comment": "Google News RSS aggregator for UK security threat level changes and alerts."
}
]
}
Loading