diff --git a/.github/workflows/hai_agent.yml b/.github/workflows/hai_agent.yml new file mode 100644 index 0000000..e8cd538 --- /dev/null +++ b/.github/workflows/hai_agent.yml @@ -0,0 +1,122 @@ +# ────────────────────────────────────────────────────────────────────────────── +# Haiti Anger Index (HAI™) — GitHub Actions Workflow +# POLLNEX Insights +# +# HOW TO RUN FROM GITHUB: +# 1. Go to the "Actions" tab in this repository +# 2. Click "Haiti Anger Index Agent" in the left sidebar +# 3. Click "Run workflow" (top right) +# 4. Fill in the options and click the green "Run workflow" button +# +# The report is saved as a downloadable artifact after each run. +# ────────────────────────────────────────────────────────────────────────────── + +name: Haiti Anger Index Agent + +on: + # ── Manual trigger from the GitHub Actions UI ────────────────────────────── + workflow_dispatch: + inputs: + platforms: + description: > + Platforms to crawl (comma-separated). + Options: twitter, facebook, youtube, reddit, rss + Use "rss" to run with NO API keys required. + required: false + default: "rss" + type: string + dry_run: + description: "Dry run — print results but do NOT save to database" + required: false + default: false + type: boolean + max_posts: + description: "Max posts per platform (default: 200)" + required: false + default: "200" + type: string + + # ── Automatic daily run at 06:00 UTC ─────────────────────────────────────── + schedule: + - cron: "0 6 * * *" + +jobs: + run-agent: + name: Run Haiti Anger Index + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + # ── 1. Check out the repository ──────────────────────────────────────── + - name: Checkout repository + uses: actions/checkout@v4 + + # ── 2. Set up Python ─────────────────────────────────────────────────── + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + # ── 3. Install dependencies ──────────────────────────────────────────── + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install core dependencies (no PyTorch — uses keyword fallback classifier) + pip install feedparser langdetect jinja2 requests + # Install optional social-media libraries (won't fail if unused) + pip install tweepy praw google-api-python-client || true + + # ── 4. Run the agent ─────────────────────────────────────────────────── + - name: Run Haiti Anger Index Agent + env: + # Set your secrets in: Settings → Secrets → Actions → New repository secret + TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }} + TWITTER_API_KEY: ${{ secrets.TWITTER_API_KEY }} + TWITTER_API_SECRET: ${{ secrets.TWITTER_API_SECRET }} + YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + REDDIT_CLIENT_ID: ${{ secrets.REDDIT_CLIENT_ID }} + REDDIT_CLIENT_SECRET: ${{ secrets.REDDIT_CLIENT_SECRET }} + FACEBOOK_ACCESS_TOKEN: ${{ secrets.FACEBOOK_ACCESS_TOKEN }} + HAI_DB_PATH: "hai_run.db" + HAI_REPORTS_DIR: "reports" + MAX_POSTS_PER_PLATFORM: ${{ github.event.inputs.max_posts || '200' }} + run: | + PLATFORMS="${{ github.event.inputs.platforms || 'rss' }}" + DRY_RUN="${{ github.event.inputs.dry_run || 'false' }}" + + ARGS="--platforms $PLATFORMS" + if [ "$DRY_RUN" = "true" ]; then + ARGS="$ARGS --dry-run" + fi + + echo "▶ Running: python -m haiti_anger_index $ARGS" + python -m haiti_anger_index $ARGS + + # ── 5. Upload the HTML report as a downloadable artifact ─────────────── + - name: Upload HAI Report + if: always() + uses: actions/upload-artifact@v4 + with: + name: haiti-anger-index-report-${{ github.run_number }} + path: | + reports/ + hai_agent.log + if-no-files-found: warn + retention-days: 90 + + # ── 6. Print summary in the workflow log ─────────────────────────────── + - name: Print log summary + if: always() + run: | + echo "──────────────────────────────────────────" + echo " Haiti Anger Index — Run Complete" + echo "──────────────────────────────────────────" + if [ -f hai_agent.log ]; then + tail -30 hai_agent.log + fi + if [ -d reports ]; then + echo "" + echo "Report files:" + ls -lh reports/ + fi diff --git a/hai_agent.log b/hai_agent.log new file mode 100644 index 0000000..19d73df --- /dev/null +++ b/hai_agent.log @@ -0,0 +1,49 @@ +2026-04-12 20:07:51,264 WARNING haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier. +2026-04-12 20:07:51,332 WARNING haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled. +2026-04-12 20:07:51,349 WARNING haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled. +2026-04-12 20:07:51,350 WARNING haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled. +2026-04-12 20:07:51,376 INFO hai.agent — ============================================================ +2026-04-12 20:07:51,376 INFO hai.agent — Haiti Anger Index Agent — run started 2026-04-12T20:07:51.376898+00:00 +2026-04-12 20:07:51,377 INFO haiti_anger_index.crawlers.base — [news_rss] Starting crawl … +2026-04-12 20:08:01,617 INFO haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts +2026-04-12 20:08:01,617 INFO hai.agent — [rss] → 0 posts +2026-04-12 20:08:01,617 INFO hai.agent — Total raw posts collected: 0 +2026-04-12 20:08:01,617 WARNING hai.agent — No posts collected — aborting run. +2026-04-12 22:36:44,223 WARNING haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier. +2026-04-12 22:36:44,297 WARNING haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled. +2026-04-12 22:36:44,317 WARNING haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled. +2026-04-12 22:36:44,319 WARNING haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled. +2026-04-12 22:36:44,361 INFO haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db +2026-04-12 22:36:44,362 INFO hai.agent — ============================================================ +2026-04-12 22:36:44,362 INFO hai.agent — Haiti Anger Index Agent — run started 2026-04-12T22:36:44.362129+00:00 +2026-04-12 22:36:44,362 INFO haiti_anger_index.crawlers.base — [news_rss] Starting crawl … +2026-04-12 22:36:54,603 INFO haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts +2026-04-12 22:36:54,603 INFO hai.agent — [rss] → 0 posts +2026-04-12 22:36:54,603 INFO hai.agent — Total raw posts collected: 0 +2026-04-12 22:36:54,603 WARNING hai.agent — No posts collected — aborting run. +2026-04-12 22:39:57,261 WARNING haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier. +2026-04-12 22:39:57,331 WARNING haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled. +2026-04-12 22:39:57,348 WARNING haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled. +2026-04-12 22:39:57,349 WARNING haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled. +2026-04-12 22:39:57,374 INFO haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db +2026-04-12 22:39:57,503 INFO haiti_anger_index.storage.database — Upserted 80 posts +2026-04-12 22:39:58,068 INFO haiti_anger_index.reporting.report_generator — Report written → reports/hai_report_20260412_223958.html +2026-04-12 22:39:58,070 INFO haiti_anger_index.storage.database — Report saved → reports/hai_report_20260412_223958.html +2026-06-16 08:46:45,137 WARNING haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier. +2026-06-16 08:46:45,208 WARNING haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled. +2026-06-16 08:46:45,225 WARNING haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled. +2026-06-16 08:46:45,226 WARNING haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled. +2026-06-16 08:46:45,267 INFO hai.agent — ============================================================ +2026-06-16 08:46:45,267 INFO hai.agent — Haiti Anger Index Agent — run started 2026-06-16T08:46:45.267422+00:00 +2026-06-16 08:46:45,267 INFO haiti_anger_index.crawlers.base — [news_rss] Starting crawl … +2026-06-16 08:46:55,486 INFO haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts +2026-06-16 08:46:55,486 INFO hai.agent — [rss] → 0 posts +2026-06-16 08:46:55,486 INFO hai.agent — Total raw posts collected: 0 +2026-06-16 08:46:55,486 WARNING hai.agent — No posts collected — aborting run. +2026-06-16 08:53:19,217 INFO haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db +2026-06-16 08:53:19,218 INFO hai.agent — Loaded 80 classified posts from DB +2026-06-16 08:53:19,218 INFO hai.agent — Stats → total=80 neg=24 pos=2 neu=54 neg_rate=0.300 +2026-06-16 08:53:19,219 INFO hai.agent — HAI Score: 25.9 / 100 [Concerned] trend=stable +2026-06-16 08:53:19,231 INFO haiti_anger_index.reporting.report_generator — Report written → reports/hai_report_20260616_085319.html +2026-06-16 08:53:19,232 INFO haiti_anger_index.storage.database — Report saved → reports/hai_report_20260616_085319.html +2026-06-16 08:53:19,232 INFO hai.agent — Report written → reports/hai_report_20260616_085319.html diff --git a/haiti_anger_index.db b/haiti_anger_index.db new file mode 100644 index 0000000..e1fcdda Binary files /dev/null and b/haiti_anger_index.db differ diff --git a/haiti_anger_index/README.md b/haiti_anger_index/README.md new file mode 100644 index 0000000..4977aa2 --- /dev/null +++ b/haiti_anger_index/README.md @@ -0,0 +1,207 @@ +# Haiti Anger Index (HAI™) +### A POLLNEX Insights Research Product + +--- + +## Overview + +The **Haiti Anger Index (HAI™)** is an automated sentiment-intelligence agent that: + +1. **Crawls** major social media platforms and Haitian news RSS feeds for posts in **English, French, and Haitian Creole** +2. **Classifies** each post as `POSITIVE`, `NEGATIVE`, or `NEUTRAL` using a multilingual XLM-RoBERTa model +3. **Compiles** per-platform and per-topic statistics +4. **Computes** a composite HAI score (0–100) inspired by Pollara's Rage Index methodology +5. **Generates** a branded HTML report with charts, trend lines, and top anger drivers + +--- + +## Architecture + +``` +haiti_anger_index/ +├── agent.py ← Main orchestration loop (CLI entry point) +├── config.py ← All configuration & API credentials +├── __main__.py ← `python -m haiti_anger_index` +│ +├── crawlers/ +│ ├── base.py ← Abstract base crawler +│ ├── twitter_crawler.py ← Twitter/X (API v2, Bearer Token) +│ ├── facebook_crawler.py ← Facebook Graph API +│ ├── youtube_crawler.py ← YouTube Data API v3 +│ ├── reddit_crawler.py ← Reddit PRAW +│ └── rss_crawler.py ← 10+ Haitian news RSS feeds (no API key) +│ +├── classifier/ +│ ├── language_detector.py ← EN / FR / HT detection +│ ├── sentiment_classifier.py← XLM-RoBERTa + keyword fallback +│ └── topic_classifier.py ← 9 Haitian-specific topic categories +│ +├── aggregator/ +│ └── data_compiler.py ← Per-platform / per-topic / per-language stats +│ +├── index/ +│ └── anger_index.py ← HAI score formula + trend + top drivers +│ +├── storage/ +│ └── database.py ← SQLite (posts + snapshots + reports) +│ +└── reporting/ + ├── report_generator.py ← Jinja2 HTML renderer + └── templates/ + └── anger_index_report.html ← POLLNEX-branded dashboard +``` + +--- + +## Quick Start + +### 1. Install dependencies + +```bash +pip install -r requirements.txt +``` + +### 2. Set API credentials (optional but recommended) + +At minimum, the RSS crawler works with **no API keys** and already covers +10 major Haitian news outlets. Add API keys to enable richer social data: + +```bash +export TWITTER_BEARER_TOKEN="..." +export YOUTUBE_API_KEY="..." +export REDDIT_CLIENT_ID="..." +export REDDIT_CLIENT_SECRET="..." +export FACEBOOK_ACCESS_TOKEN="..." # Page Access Token +``` + +### 3. Run the agent + +```bash +# Single run (all platforms) +python -m haiti_anger_index + +# RSS only (no API keys needed) +python -m haiti_anger_index --platforms rss + +# Dry run — print results, don't save to DB +python -m haiti_anger_index --dry-run --platforms rss + +# Scheduled run every 6 hours +python -m haiti_anger_index --schedule 6h + +# Custom paths +python -m haiti_anger_index --db /data/hai.db --reports /var/www/reports +``` + +--- + +## Haiti Anger Index Score + +| Score | Level | Description | +|-------|------------|-----------------------------------------| +| 0–25 | 🟢 Calm | Low negative sentiment | +| 26–50 | 🟡 Concerned | Elevated concern but not acute anger | +| 51–75 | 🟠 Agitated | High frustration across key topics | +| 76–100| 🔴 Enraged | Critical anger levels — crisis signal | + +### Formula + +``` +raw_score = 0.40 × neg_rate + + 0.40 × engagement_weighted_neg_rate + + 0.20 × topic_severity_adjusted_neg_rate + +HAI = round(raw_score × 100, 1) +``` + +**Topic severity weights** amplify issues that historically drive the +strongest public anger in Haiti: + +| Topic | Weight | +|-------------------------|--------| +| Government & Corruption | 1.5 | +| Kidnapping & Safety | 1.4 | +| Economy & Poverty | 1.3 | +| Human Rights & Diaspora | 1.3 | +| Natural Disasters | 1.2 | +| Health & Healthcare | 1.1 | +| Infrastructure | 1.0 | +| Education | 0.9 | +| International Relations | 0.9 | + +--- + +## Supported Platforms + +| Platform | Data Source | API Key Required | +|---------------|----------------------------|-----------------| +| Twitter / X | Twitter API v2 | ✅ Bearer Token | +| Facebook | Meta Graph API | ✅ Page Token | +| YouTube | YouTube Data API v3 | ✅ API Key | +| Reddit | PRAW | ✅ Client ID/Secret | +| Haitian News | 10+ RSS feeds | ❌ None needed | + +### Haitian News RSS Sources +- Le Nouvelliste +- Haïti Libre +- AlterPresse +- Rezo Nodwes +- Loop Haiti +- Gazette Haïti +- HPN Haïti +- Radio Kiskeya +- Haïti Info Projet +- Haïti Chery + +--- + +## Language Support + +| Language | Detection | Sentiment | +|-----------------|------------|-----------| +| English (en) | ✅ | ✅ | +| French (fr) | ✅ | ✅ | +| Haitian Creole | ✅ (heuristic) | ✅ (via XLM-RoBERTa) | + +--- + +## Output + +Each run produces: +- **SQLite database** with all posts, classifications, and index snapshots +- **HTML report** with interactive charts (Chart.js): + - HAI gauge/speedometer + - Platform breakdown bar chart + - Language distribution doughnut chart + - Top anger drivers table + - Historical trend line + +--- + +## Environment Variables + +| Variable | Default | Description | +|-------------------------|-----------------------|---------------------------------| +| `TWITTER_BEARER_TOKEN` | — | Twitter API v2 bearer token | +| `YOUTUBE_API_KEY` | — | Google/YouTube Data API key | +| `REDDIT_CLIENT_ID` | — | Reddit app client ID | +| `REDDIT_CLIENT_SECRET` | — | Reddit app client secret | +| `FACEBOOK_ACCESS_TOKEN` | — | Meta Graph API page token | +| `HAI_DB_PATH` | `haiti_anger_index.db`| SQLite database path | +| `HAI_REPORTS_DIR` | `reports/` | HTML reports output directory | +| `MAX_POSTS_PER_PLATFORM`| `200` | Max posts per platform per run | +| `CRAWL_DELAY` | `1.0` | Seconds between API requests | + +--- + +## Ethical Notes + +- Only **publicly accessible** content is collected (no login-gated scraping) +- Facebook crawling uses only the **official Graph API** on public pages +- Content is used for **aggregate analysis only** — no PII is stored +- Crawl rate is limited to respect platform terms of service + +--- + +*© POLLNEX Insights — Haiti Anger Index™ is a proprietary research product. +Attribution required when citing results.* diff --git a/haiti_anger_index/__init__.py b/haiti_anger_index/__init__.py new file mode 100644 index 0000000..5c42736 --- /dev/null +++ b/haiti_anger_index/__init__.py @@ -0,0 +1,4 @@ +"""Haiti Anger Index — Python package.""" + +__version__ = "1.0.0" +__author__ = "POLLNEX Insights" diff --git a/haiti_anger_index/__main__.py b/haiti_anger_index/__main__.py new file mode 100644 index 0000000..ae76b2f --- /dev/null +++ b/haiti_anger_index/__main__.py @@ -0,0 +1,5 @@ +"""Allow `python -m haiti_anger_index` to launch the agent.""" + +from haiti_anger_index.agent import main + +main() diff --git a/haiti_anger_index/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0b673af Binary files /dev/null and b/haiti_anger_index/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/__pycache__/__main__.cpython-312.pyc b/haiti_anger_index/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000..6eb33c3 Binary files /dev/null and b/haiti_anger_index/__pycache__/__main__.cpython-312.pyc differ diff --git a/haiti_anger_index/__pycache__/agent.cpython-312.pyc b/haiti_anger_index/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000..27d494f Binary files /dev/null and b/haiti_anger_index/__pycache__/agent.cpython-312.pyc differ diff --git a/haiti_anger_index/__pycache__/config.cpython-312.pyc b/haiti_anger_index/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..0b90f0e Binary files /dev/null and b/haiti_anger_index/__pycache__/config.cpython-312.pyc differ diff --git a/haiti_anger_index/agent.py b/haiti_anger_index/agent.py new file mode 100644 index 0000000..8f9609e --- /dev/null +++ b/haiti_anger_index/agent.py @@ -0,0 +1,287 @@ +""" +Haiti Anger Index — Main Orchestration Agent +============================================ + +Usage +----- +Run once (default): + python -m haiti_anger_index.agent + +Run on a schedule (every 6 hours): + python -m haiti_anger_index.agent --schedule 6h + +Dry run (no DB writes, just print stats): + python -m haiti_anger_index.agent --dry-run + +Options +------- +--db Path to SQLite database (default: haiti_anger_index.db) +--reports Directory for HTML reports (default: reports/) +--schedule Run interval: 1h, 6h, 12h, 24h +--dry-run Print results without saving +--no-model Skip ML model; use keyword-based fallback classifier +--platforms Comma-separated list: twitter,facebook,youtube,reddit,rss + (default: all) +""" + +import argparse +import json +import logging +import os +import sys +import time +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional + +# ── Logging setup ───────────────────────────────────────────────────────────── +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-8s %(name)s — %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler("hai_agent.log", encoding="utf-8"), + ], +) +logger = logging.getLogger("hai.agent") + +from haiti_anger_index import config +from haiti_anger_index.aggregator.data_compiler import DataCompiler +from haiti_anger_index.classifier.language_detector import detect_language +from haiti_anger_index.classifier.sentiment_classifier import SentimentClassifier +from haiti_anger_index.classifier.topic_classifier import classify_topics +from haiti_anger_index.crawlers.facebook_crawler import FacebookCrawler +from haiti_anger_index.crawlers.reddit_crawler import RedditCrawler +from haiti_anger_index.crawlers.rss_crawler import RSSCrawler +from haiti_anger_index.crawlers.twitter_crawler import TwitterCrawler +from haiti_anger_index.crawlers.youtube_crawler import YouTubeCrawler +from haiti_anger_index.index.anger_index import AngerIndexCalculator +from haiti_anger_index.reporting.report_generator import ReportGenerator +from haiti_anger_index.storage.database import Database + +_SCHEDULE_MAP = {"1h": 3600, "6h": 21600, "12h": 43200, "24h": 86400} + +_ALL_PLATFORMS = ["twitter", "facebook", "youtube", "reddit", "rss"] + + +class HaitiAngerIndexAgent: + """ + End-to-end pipeline: + 1. Crawl social media + 2. Classify language + sentiment + topics + 3. Compile statistics + 4. Compute HAI score + 5. Persist to DB + 6. Generate HTML report + """ + + def __init__( + self, + db_path: str = config.DB_PATH, + reports_dir: str = config.REPORTS_DIR, + platforms: Optional[List[str]] = None, + dry_run: bool = False, + ) -> None: + self.dry_run = dry_run + self.platforms = platforms or _ALL_PLATFORMS + + self.db = Database(db_path) if not dry_run else None + self.classifier = SentimentClassifier() + self.compiler = DataCompiler() + self.calculator = AngerIndexCalculator() + self.reporter = ReportGenerator(reports_dir) + + self._crawlers = self._build_crawlers() + + # ── Public API ──────────────────────────────────────────────────────────── + + def run_once(self) -> Dict: + """Execute a full pipeline cycle and return the snapshot dict.""" + run_start = datetime.now(timezone.utc) + logger.info("=" * 60) + logger.info("Haiti Anger Index Agent — run started %s", run_start.isoformat()) + + # Step 1: Crawl + raw_posts = self._crawl_all() + logger.info("Total raw posts collected: %d", len(raw_posts)) + + if not raw_posts: + logger.warning("No posts collected — aborting run.") + return {} + + # Step 2: Persist raw posts + if self.db: + self.db.upsert_posts(raw_posts) + + # Step 3: Classify (language + sentiment + topics) + classified = self._classify(raw_posts) + + # Step 4: Update DB with classifications + if self.db: + for post in classified: + self.db.update_post_classification( + post_id=post["id"], + platform=post["platform"], + language=post.get("language", "unknown"), + sentiment=post["sentiment"], + sentiment_score=post["sentiment_score"], + topics=post["topics"] if isinstance(post["topics"], list) else [], + ) + + # Step 5: Compile statistics + stats = self.compiler.compile(classified) + logger.info( + "Stats → total=%d neg=%d pos=%d neu=%d neg_rate=%.3f", + stats["total"], + stats["negative"], + stats["positive"], + stats["neutral"], + stats["neg_rate"], + ) + + # Step 6: Compute HAI + history = self.db.get_anger_index_history(limit=52) if self.db else [] + period_end = datetime.now(timezone.utc).isoformat() + period_start = (datetime.now(timezone.utc) - timedelta(days=7)).isoformat() + snapshot = self.calculator.compute(stats, period_start, period_end, history) + + logger.info( + "HAI Score: %.1f / 100 [%s] trend=%s", + snapshot["overall_index"], + snapshot["anger_level"], + snapshot.get("trend", {}).get("direction", "stable"), + ) + + # Step 7: Save snapshot + if self.db: + self.db.save_anger_index(snapshot) + + # Step 8: Generate report + report_path = self.reporter.generate(snapshot, history) + summary = ( + f"HAI={snapshot['overall_index']} | {snapshot['anger_level']} | " + f"{stats['total']} posts | neg_rate={stats['neg_rate']:.1%}" + ) + if self.db: + self.db.save_report(report_path, snapshot["overall_index"], summary) + + logger.info("Report: %s", report_path) + logger.info("Run completed in %.1f s", (datetime.now(timezone.utc) - run_start).total_seconds()) + + if self.dry_run: + self._print_summary(snapshot) + + return snapshot + + # ── Crawling ────────────────────────────────────────────────────────────── + + def _crawl_all(self) -> List[Dict]: + posts: List[Dict] = [] + for name, crawler in self._crawlers.items(): + if name not in self.platforms: + continue + result = crawler.run() + logger.info("[%s] → %d posts", name, len(result)) + posts.extend(result) + return posts + + # ── Classification ──────────────────────────────────────────────────────── + + def _classify(self, posts: List[Dict]) -> List[Dict]: + logger.info("Classifying %d posts …", len(posts)) + texts = [p["content"] for p in posts] + + # Language detection + languages = [detect_language(t) for t in texts] + + # Sentiment (batch) + sentiments = self.classifier.classify_batch(texts) + + # Topics + topics_list = [classify_topics(t) for t in texts] + + for i, post in enumerate(posts): + post["language"] = languages[i] + post["sentiment"], post["sentiment_score"] = sentiments[i] + post["topics"] = topics_list[i] + + logger.info( + "Classification done. neg=%d pos=%d neu=%d", + sum(1 for p in posts if p["sentiment"] == "NEGATIVE"), + sum(1 for p in posts if p["sentiment"] == "POSITIVE"), + sum(1 for p in posts if p["sentiment"] == "NEUTRAL"), + ) + return posts + + # ── Builder helpers ─────────────────────────────────────────────────────── + + def _build_crawlers(self) -> Dict: + return { + "twitter": TwitterCrawler(), + "facebook": FacebookCrawler(), + "youtube": YouTubeCrawler(), + "reddit": RedditCrawler(), + "rss": RSSCrawler(), + } + + # ── Printing ────────────────────────────────────────────────────────────── + + @staticmethod + def _print_summary(snapshot: Dict) -> None: + print("\n" + "=" * 60) + print(f" 🇭🇹 HAITI ANGER INDEX (HAI™) — POLLNEX Insights") + print("=" * 60) + print(f" Score : {snapshot['overall_index']} / 100") + print(f" Level : {snapshot['anger_level']}") + print(f" Posts : {snapshot['total_posts']}") + print(f" Negative : {snapshot['negative_count']} " + f"({snapshot['negative_count'] / max(snapshot['total_posts'], 1):.1%})") + print(f" Positive : {snapshot['positive_count']}") + print(f" Neutral : {snapshot['neutral_count']}") + trend = snapshot.get("trend", {}) + direction = trend.get("direction", "stable") + change = trend.get("change", 0) + arrow = "▲" if direction == "rising" else ("▼" if direction == "falling" else "→") + print(f" Trend : {arrow} {direction} ({change:+.1f} pts)") + print("\n Top Drivers:") + for d in snapshot.get("top_drivers", []): + print(f" • {d['label']:30s} anger={d['weighted_anger_score']:.1f}") + print("=" * 60) + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + +def main() -> None: + parser = argparse.ArgumentParser( + description="Haiti Anger Index Agent — POLLNEX Insights" + ) + parser.add_argument("--db", default=config.DB_PATH, help="SQLite database path") + parser.add_argument("--reports", default=config.REPORTS_DIR, help="Reports output directory") + parser.add_argument("--schedule", choices=list(_SCHEDULE_MAP.keys()), default=None, + help="Run interval (e.g. 6h)") + parser.add_argument("--dry-run", action="store_true", help="Print results, don't save") + parser.add_argument("--platforms", default=",".join(_ALL_PLATFORMS), + help="Comma-separated platforms to crawl") + args = parser.parse_args() + + platforms = [p.strip() for p in args.platforms.split(",") if p.strip()] + + agent = HaitiAngerIndexAgent( + db_path=args.db, + reports_dir=args.reports, + platforms=platforms, + dry_run=args.dry_run, + ) + + if args.schedule: + interval = _SCHEDULE_MAP[args.schedule] + logger.info("Running on a %s schedule (every %d seconds)", args.schedule, interval) + while True: + agent.run_once() + logger.info("Next run in %d seconds …", interval) + time.sleep(interval) + else: + agent.run_once() + + +if __name__ == "__main__": + main() diff --git a/haiti_anger_index/aggregator/__init__.py b/haiti_anger_index/aggregator/__init__.py new file mode 100644 index 0000000..53f861e --- /dev/null +++ b/haiti_anger_index/aggregator/__init__.py @@ -0,0 +1 @@ +"""Aggregator sub-package.""" diff --git a/haiti_anger_index/aggregator/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/aggregator/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0721725 Binary files /dev/null and b/haiti_anger_index/aggregator/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/aggregator/__pycache__/data_compiler.cpython-312.pyc b/haiti_anger_index/aggregator/__pycache__/data_compiler.cpython-312.pyc new file mode 100644 index 0000000..65e4cda Binary files /dev/null and b/haiti_anger_index/aggregator/__pycache__/data_compiler.cpython-312.pyc differ diff --git a/haiti_anger_index/aggregator/data_compiler.py b/haiti_anger_index/aggregator/data_compiler.py new file mode 100644 index 0000000..93d903d --- /dev/null +++ b/haiti_anger_index/aggregator/data_compiler.py @@ -0,0 +1,177 @@ +""" +Data compiler: aggregates classified posts into per-platform and +overall statistics used by the Anger Index calculator. +""" + +import json +import logging +import math +from collections import defaultdict +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + +from haiti_anger_index import config + + +class DataCompiler: + """ + Compiles a list of classified post dicts into aggregate statistics. + + Input posts must have: platform, sentiment, sentiment_score, + language, topics (JSON string or list), engagement_score. + """ + + def compile(self, posts: List[Dict]) -> Dict: + """ + Return a statistics dict with keys: + + total, positive, negative, neutral, + platforms, languages, topics, + neg_rate, weighted_neg_rate, + top_negative_keywords + """ + if not posts: + return self._empty() + + total = len(posts) + pos = sum(1 for p in posts if p.get("sentiment") == "POSITIVE") + neg = sum(1 for p in posts if p.get("sentiment") == "NEGATIVE") + neu = sum(1 for p in posts if p.get("sentiment") == "NEUTRAL") + + platforms = self._by_platform(posts) + languages = self._by_language(posts) + topics = self._by_topic(posts) + neg_rate = neg / total if total else 0.0 + weighted_neg_rate = self._weighted_neg_rate(posts) + + return { + "total": total, + "positive": pos, + "negative": neg, + "neutral": neu, + "neg_rate": round(neg_rate, 4), + "weighted_neg_rate": round(weighted_neg_rate, 4), + "platforms": platforms, + "languages": languages, + "topics": topics, + } + + # ── Per-platform breakdown ──────────────────────────────────────────────── + + def _by_platform(self, posts: List[Dict]) -> Dict: + buckets: Dict[str, Dict] = defaultdict(lambda: {"total": 0, "positive": 0, "negative": 0, "neutral": 0}) + for p in posts: + plat = p.get("platform", "unknown") + sent = p.get("sentiment", "NEUTRAL") + buckets[plat]["total"] += 1 + buckets[plat][sent.lower()] = buckets[plat].get(sent.lower(), 0) + 1 + + result = {} + for plat, counts in buckets.items(): + t = counts["total"] + result[plat] = { + "total": t, + "positive": counts.get("positive", 0), + "negative": counts.get("negative", 0), + "neutral": counts.get("neutral", 0), + "neg_rate": round(counts.get("negative", 0) / t, 4) if t else 0.0, + } + return result + + # ── Per-language breakdown ──────────────────────────────────────────────── + + def _by_language(self, posts: List[Dict]) -> Dict: + buckets: Dict[str, Dict] = defaultdict(lambda: {"total": 0, "positive": 0, "negative": 0, "neutral": 0}) + for p in posts: + lang = p.get("language") or "unknown" + sent = p.get("sentiment", "NEUTRAL") + buckets[lang]["total"] += 1 + buckets[lang][sent.lower()] = buckets[lang].get(sent.lower(), 0) + 1 + + result = {} + for lang, counts in buckets.items(): + t = counts["total"] + result[lang] = { + "total": t, + "positive": counts.get("positive", 0), + "negative": counts.get("negative", 0), + "neutral": counts.get("neutral", 0), + "neg_rate": round(counts.get("negative", 0) / t, 4) if t else 0.0, + } + return result + + # ── Per-topic breakdown ─────────────────────────────────────────────────── + + def _by_topic(self, posts: List[Dict]) -> Dict: + buckets: Dict[str, Dict] = defaultdict(lambda: { + "total": 0, "positive": 0, "negative": 0, "neutral": 0, + "neg_rate": 0.0, "weighted_neg_score": 0.0, + }) + + for p in posts: + topics = p.get("topics") or [] + if isinstance(topics, str): + try: + topics = json.loads(topics) + except (json.JSONDecodeError, TypeError): + topics = ["general"] + + sent = p.get("sentiment", "NEUTRAL") + score = float(p.get("sentiment_score") or 0.5) + eng = float(p.get("engagement_score") or 1.0) + + for topic in topics: + buckets[topic]["total"] += 1 + buckets[topic][sent.lower()] = buckets[topic].get(sent.lower(), 0) + 1 + if sent == "NEGATIVE": + weight = config.TOPICS.get(topic, {}).get("weight", 1.0) + buckets[topic]["weighted_neg_score"] += score * eng * weight + + result = {} + for topic, counts in buckets.items(): + t = counts["total"] + result[topic] = { + "total": t, + "positive": counts.get("positive", 0), + "negative": counts.get("negative", 0), + "neutral": counts.get("neutral", 0), + "neg_rate": round(counts.get("negative", 0) / t, 4) if t else 0.0, + "weighted_neg_score": round(counts.get("weighted_neg_score", 0.0), 2), + "label": config.TOPICS.get(topic, {}).get("label", topic.replace("_", " ").title()), + } + return result + + # ── Engagement-weighted negative rate ──────────────────────────────────── + + def _weighted_neg_rate(self, posts: List[Dict]) -> float: + """ + Compute engagement-weighted negative sentiment rate. + + Posts with higher engagement counts more toward the index. + Uses log(1 + engagement) to dampen outliers. + """ + total_weight = 0.0 + neg_weight = 0.0 + for p in posts: + eng = float(p.get("engagement_score") or 1.0) + w = math.log1p(eng) + total_weight += w + if p.get("sentiment") == "NEGATIVE": + score = float(p.get("sentiment_score") or 0.5) + neg_weight += w * score + return (neg_weight / total_weight) if total_weight else 0.0 + + @staticmethod + def _empty() -> Dict: + return { + "total": 0, + "positive": 0, + "negative": 0, + "neutral": 0, + "neg_rate": 0.0, + "weighted_neg_rate": 0.0, + "platforms": {}, + "languages": {}, + "topics": {}, + } diff --git a/haiti_anger_index/classifier/__init__.py b/haiti_anger_index/classifier/__init__.py new file mode 100644 index 0000000..392330c --- /dev/null +++ b/haiti_anger_index/classifier/__init__.py @@ -0,0 +1 @@ +"""Classifier sub-package.""" diff --git a/haiti_anger_index/classifier/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/classifier/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b13be62 Binary files /dev/null and b/haiti_anger_index/classifier/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/classifier/__pycache__/language_detector.cpython-312.pyc b/haiti_anger_index/classifier/__pycache__/language_detector.cpython-312.pyc new file mode 100644 index 0000000..6860c1f Binary files /dev/null and b/haiti_anger_index/classifier/__pycache__/language_detector.cpython-312.pyc differ diff --git a/haiti_anger_index/classifier/__pycache__/sentiment_classifier.cpython-312.pyc b/haiti_anger_index/classifier/__pycache__/sentiment_classifier.cpython-312.pyc new file mode 100644 index 0000000..36ebf6d Binary files /dev/null and b/haiti_anger_index/classifier/__pycache__/sentiment_classifier.cpython-312.pyc differ diff --git a/haiti_anger_index/classifier/__pycache__/topic_classifier.cpython-312.pyc b/haiti_anger_index/classifier/__pycache__/topic_classifier.cpython-312.pyc new file mode 100644 index 0000000..a9f87bb Binary files /dev/null and b/haiti_anger_index/classifier/__pycache__/topic_classifier.cpython-312.pyc differ diff --git a/haiti_anger_index/classifier/language_detector.py b/haiti_anger_index/classifier/language_detector.py new file mode 100644 index 0000000..9645cba --- /dev/null +++ b/haiti_anger_index/classifier/language_detector.py @@ -0,0 +1,84 @@ +""" +Language detector for English (en), French (fr), and Haitian Creole (ht). + +Strategy +-------- +1. Apply a lightweight Haitian Creole keyword heuristic first (since most + off-the-shelf language detectors have weak or no support for 'ht'). +2. Fall back to langdetect for en/fr/other. +""" + +import logging +import re +from typing import Optional + +logger = logging.getLogger(__name__) + +try: + from langdetect import detect, LangDetectException # type: ignore + _LANGDETECT_AVAILABLE = True +except ImportError: + _LANGDETECT_AVAILABLE = False + logger.warning("langdetect not installed — language detection degraded.") + +# ── Haitian Creole heuristics ───────────────────────────────────────────────── +# Common Haitian Creole function words / markers that are absent / rare in French +_HT_MARKERS = re.compile( + r"\b(m|w|li|nou|yo|pa|nan|ak|pou|sou|bay|bèl|mèsi|" + r"avèk|depi|menm|jan|lè|toujou|anko|konsa|gade|wè|" + r"Ayiti|ayisyen|ayisyèn|peyi\s*a|gouvènman|prezidan|" + r"kriz|sekirite|lapolis|gang|vyolans|ekonomi|pòvrete|" + r"grangou|lopital|lekòl|elektrisite|dlo|wout|jwenn|rele|" + r"Bondye|blan|nwa|pitit|fanmi|zanmi|frè|sè)\b", + re.IGNORECASE, +) + +# Minimum fraction of HT-marker words to classify as Haitian Creole +_HT_THRESHOLD = 0.06 + + +def detect_language(text: str) -> str: + """ + Return ISO 639-1 code: 'en', 'fr', 'ht', or 'unknown'. + """ + if not text or not text.strip(): + return "unknown" + + cleaned = text.strip() + + # Step 1: Haitian Creole heuristic + words = re.findall(r"\b\w+\b", cleaned) + if words: + ht_hits = len(_HT_MARKERS.findall(cleaned)) + ratio = ht_hits / len(words) + if ratio >= _HT_THRESHOLD: + return "ht" + + # Step 2: langdetect + if _LANGDETECT_AVAILABLE: + try: + lang = detect(cleaned) + if lang in ("en", "fr"): + return lang + # Creole sometimes misclassified as Portuguese or Spanish + if lang in ("pt", "es", "ca"): + # Re-check with slightly lower threshold + words2 = re.findall(r"\b\w+\b", cleaned) + if words2: + ht_hits2 = len(_HT_MARKERS.findall(cleaned)) + if ht_hits2 / len(words2) >= 0.03: + return "ht" + return lang[:2] if lang else "unknown" + except LangDetectException: + pass + + # Step 3: simple character-frequency heuristic (en vs fr) + accented = len(re.findall(r"[àâäéèêëîïôùûüç]", cleaned.lower())) + if accented / max(len(cleaned), 1) > 0.05: + return "fr" + return "en" + + +def is_relevant_language(lang: str) -> bool: + """Return True if the language is one we analyse (en, fr, ht).""" + return lang in ("en", "fr", "ht") diff --git a/haiti_anger_index/classifier/sentiment_classifier.py b/haiti_anger_index/classifier/sentiment_classifier.py new file mode 100644 index 0000000..090477c --- /dev/null +++ b/haiti_anger_index/classifier/sentiment_classifier.py @@ -0,0 +1,143 @@ +""" +Multilingual sentiment classifier. + +Uses cardiffnlp/twitter-xlm-roberta-base-sentiment which supports 100+ +languages including French. Haitian Creole is close enough to French that +the model performs well on it. + +Falls back to a lightweight keyword-based classifier when transformers is +unavailable (demo / low-resource environments). +""" + +import logging +import re +from typing import Dict, List, Tuple + +logger = logging.getLogger(__name__) + +try: + from transformers import pipeline # type: ignore + _TRANSFORMERS_AVAILABLE = True +except ImportError: + _TRANSFORMERS_AVAILABLE = False + logger.warning("transformers not installed — using keyword-based fallback classifier.") + +from haiti_anger_index import config + +# Label mapping from model output → canonical labels +_LABEL_MAP = { + "positive": "POSITIVE", + "negative": "NEGATIVE", + "neutral": "NEUTRAL", + "label_0": "NEGATIVE", + "label_1": "NEUTRAL", + "label_2": "POSITIVE", +} + + +class SentimentClassifier: + """ + Wraps the XLM-RoBERTa multilingual sentiment model. + + Usage:: + + clf = SentimentClassifier() + results = clf.classify_batch(["I love Haiti", "Violence is terrible"]) + # [('POSITIVE', 0.97), ('NEGATIVE', 0.99)] + """ + + def __init__(self) -> None: + self._pipe = None + if _TRANSFORMERS_AVAILABLE: + try: + self._pipe = pipeline( + "text-classification", + model=config.SENTIMENT_MODEL, + tokenizer=config.SENTIMENT_MODEL, + max_length=config.SENTIMENT_MAX_LENGTH, + truncation=True, + top_k=1, + ) + logger.info("Sentiment model loaded: %s", config.SENTIMENT_MODEL) + except Exception as exc: # noqa: BLE001 + logger.warning("Could not load model (%s) — using keyword fallback.", exc) + + # ── Public API ──────────────────────────────────────────────────────────── + + def classify(self, text: str) -> Tuple[str, float]: + """Classify a single text. Returns (label, score).""" + results = self.classify_batch([text]) + return results[0] + + def classify_batch(self, texts: List[str]) -> List[Tuple[str, float]]: + """ + Classify a batch of texts. + + Returns a list of (label, score) tuples where label is one of + POSITIVE, NEGATIVE, NEUTRAL and score is a confidence in [0, 1]. + """ + if not texts: + return [] + + if self._pipe is not None: + return self._classify_with_model(texts) + return [_keyword_classify(t) for t in texts] + + # ── Private ─────────────────────────────────────────────────────────────── + + def _classify_with_model(self, texts: List[str]) -> List[Tuple[str, float]]: + results: List[Tuple[str, float]] = [] + batch_size = config.SENTIMENT_BATCH_SIZE + for i in range(0, len(texts), batch_size): + batch = texts[i : i + batch_size] + try: + outputs = self._pipe(batch) + for output in outputs: + # pipeline returns list-of-list when top_k=1 + item = output[0] if isinstance(output, list) else output + raw_label = item["label"].lower() + label = _LABEL_MAP.get(raw_label, "NEUTRAL") + score = float(item["score"]) + results.append((label, score)) + except Exception as exc: # noqa: BLE001 + logger.error("Model inference error: %s — using keyword fallback.", exc) + results.extend([_keyword_classify(t) for t in batch]) + return results + + +# ── Keyword-based fallback classifier ──────────────────────────────────────── +# Used when transformers is not available or model fails to load. + +_NEGATIVE_PATTERNS = re.compile( + r"\b(terrible|awful|horrible|bad|poor|sad|angry|outrage|furious|disgusting|" + r"crime|gang|kidnap|violence|murder|shooting|corrupt|crisis|fail|dead|death|" + r"fear|terror|protest|revolt|strike|suffering|starvation|poverty|disaster|" + r"terrible|affreux|horrible|mauvais|triste|colère|corruption|crise|" + r"criminalité|enlèvement|violence|meurtre|fusillade|peur|terreur|grève|" + r"souffrance|famine|pauvreté|catastrophe|mouri|kriz|krim|gang|vyolans|" + r"grangou|pòvrete|mouri|kidnaping|katastwòf|fè mal)\b", + re.IGNORECASE, +) + +_POSITIVE_PATTERNS = re.compile( + r"\b(great|good|excellent|wonderful|amazing|happy|joy|love|peace|hope|" + r"progress|success|victory|improve|better|grow|thrive|solidarity|" + r"bien|excellent|merveilleux|magnifique|heureux|joie|paix|espoir|" + r"progrès|succès|victoire|améliorer|solidarité|" + r"bon|ekselan|bèl|kè kontan|lapè|espwa|pwogrè|siksè|viktwa|amelyore)\b", + re.IGNORECASE, +) + + +def _keyword_classify(text: str) -> Tuple[str, float]: + if not text: + return ("NEUTRAL", 0.5) + neg_hits = len(_NEGATIVE_PATTERNS.findall(text)) + pos_hits = len(_POSITIVE_PATTERNS.findall(text)) + if neg_hits > pos_hits: + score = min(0.5 + neg_hits * 0.1, 0.95) + return ("NEGATIVE", round(score, 3)) + if pos_hits > neg_hits: + score = min(0.5 + pos_hits * 0.1, 0.95) + return ("POSITIVE", round(score, 3)) + return ("NEUTRAL", 0.5) diff --git a/haiti_anger_index/classifier/topic_classifier.py b/haiti_anger_index/classifier/topic_classifier.py new file mode 100644 index 0000000..a0690c5 --- /dev/null +++ b/haiti_anger_index/classifier/topic_classifier.py @@ -0,0 +1,53 @@ +""" +Topic classifier for Haitian social-media posts. + +Each post is assigned to one or more of the nine topic categories defined +in config.TOPICS using a multilingual keyword matching strategy. +""" + +import logging +import re +from typing import List + +logger = logging.getLogger(__name__) + +from haiti_anger_index import config + + +def _build_patterns() -> dict: + """Pre-compile regex patterns for every topic.""" + patterns = {} + for topic_key, topic in config.TOPICS.items(): + combined_kws = ( + topic.get("keywords_en", []) + + topic.get("keywords_fr", []) + + topic.get("keywords_ht", []) + ) + # Escape and join into one alternation pattern + escaped = [re.escape(kw) for kw in combined_kws if kw] + if escaped: + patterns[topic_key] = re.compile( + r"\b(" + "|".join(escaped) + r")\b", + re.IGNORECASE, + ) + return patterns + + +_TOPIC_PATTERNS = _build_patterns() + + +def classify_topics(text: str) -> List[str]: + """ + Return a list of topic keys found in *text*. + + Returns at minimum ['general'] if no specific topic is matched. + """ + if not text: + return ["general"] + + matched = [ + key + for key, pattern in _TOPIC_PATTERNS.items() + if pattern.search(text) + ] + return matched if matched else ["general"] diff --git a/haiti_anger_index/config.py b/haiti_anger_index/config.py new file mode 100644 index 0000000..88d78cc --- /dev/null +++ b/haiti_anger_index/config.py @@ -0,0 +1,185 @@ +""" +Configuration for the Haiti Anger Index Agent. +All API keys and secrets should be set via environment variables. +""" + +import os + +# ─── API credentials ────────────────────────────────────────────────────────── + +TWITTER_BEARER_TOKEN = os.getenv("TWITTER_BEARER_TOKEN", "") +TWITTER_API_KEY = os.getenv("TWITTER_API_KEY", "") +TWITTER_API_SECRET = os.getenv("TWITTER_API_SECRET", "") + +REDDIT_CLIENT_ID = os.getenv("REDDIT_CLIENT_ID", "") +REDDIT_CLIENT_SECRET = os.getenv("REDDIT_CLIENT_SECRET", "") +REDDIT_USER_AGENT = os.getenv("REDDIT_USER_AGENT", "HaitiAngerIndex/1.0 (by POLLNEX)") + +YOUTUBE_API_KEY = os.getenv("YOUTUBE_API_KEY", "") + +FACEBOOK_ACCESS_TOKEN = os.getenv("FACEBOOK_ACCESS_TOKEN", "") + +# ─── Storage ────────────────────────────────────────────────────────────────── + +DB_PATH = os.getenv("HAI_DB_PATH", "haiti_anger_index.db") +REPORTS_DIR = os.getenv("HAI_REPORTS_DIR", "reports") + +# ─── Crawler settings ───────────────────────────────────────────────────────── + +# Max posts to fetch per run per platform +MAX_POSTS_PER_PLATFORM = int(os.getenv("MAX_POSTS_PER_PLATFORM", "200")) + +# Crawl delay in seconds between API calls +CRAWL_DELAY = float(os.getenv("CRAWL_DELAY", "1.0")) + +# ─── Keywords for searching Haitian-related content ─────────────────────────── + +SEARCH_KEYWORDS = [ + # English + "Haiti", "Haitian", "Port-au-Prince", "gang Haiti", "crisis Haiti", + "Haiti government", "Haiti earthquake", "Haiti economy", + # French + "Haïti", "haïtien", "haïtienne", "gouvernement haïtien", + "crise en Haïti", "Port-au-Prince", + # Haitian Creole + "Ayiti", "ayisyen", "peyi a", "leta ayiti", "kriz Ayiti", + "gang yo", "pèp ayisyen", "prezidan Ayiti", +] + +TWITTER_SEARCH_QUERIES = [ + "(Haiti OR Haïti OR Ayiti) lang:en", + "(Haiti OR Haïti OR Ayiti) lang:fr", + "(Haiti OR Haïti OR Ayiti OR ayisyen OR peyi) lang:ht", +] + +REDDIT_SUBREDDITS = [ + "haiti", + "caribbean", + "latinamerica", + "worldnews", + "news", +] + +YOUTUBE_SEARCH_TERMS = [ + "Haiti news", + "Haïti actualité", + "Ayiti nouvèl", + "Haiti crisis 2025", + "Haiti gang violence", +] + +# ─── Haitian news RSS feeds ─────────────────────────────────────────────────── + +RSS_FEEDS = { + "Le Nouvelliste": "https://lenouvelliste.com/feed/", + "Haiti Libre": "https://www.haitilibre.com/rss.xml", + "AlterPresse": "https://www.alterpresse.org/feed/", + "Haiti Info Projet": "https://haitiinfoproject.org/feed/", + "Loop Haiti": "https://loophaiti.com/feed/", + "Rezo Nodwes": "https://rezonodwes.com/feed/", + "Gazette Haiti": "https://gazettehaiti.com/feed/", + "HPN Haiti": "https://www.hpnhaiti.com/feed/", + "Radio Kiskeya": "https://radiokiskeya.com/feed/", + "Haiti Chery": "https://www.haitichery.com/feed/", +} + +# ─── Topic categories and severity weights ──────────────────────────────────── + +TOPICS = { + "government_corruption": { + "label": "Government & Corruption", + "label_fr": "Gouvernement & Corruption", + "label_ht": "Gouvènman & Koripsyon", + "weight": 1.5, + "keywords_en": ["corruption", "government", "president", "parliament", "prime minister", "coup", "democracy"], + "keywords_fr": ["corruption", "gouvernement", "président", "parlement", "premier ministre", "coup d'état"], + "keywords_ht": ["koripsyon", "gouvènman", "prezidan", "palman", "premier minis", "demokrasi"], + }, + "public_safety": { + "label": "Public Safety & Crime", + "label_fr": "Sécurité Publique", + "label_ht": "Sekirite Piblik", + "weight": 1.4, + "keywords_en": ["gang", "kidnapping", "violence", "murder", "shooting", "crime", "police", "security"], + "keywords_fr": ["gang", "enlèvement", "violence", "meurtre", "fusillade", "criminalité", "police", "sécurité"], + "keywords_ht": ["gang", "kidnaping", "vyolans", "mèt", "tire", "krim", "lapolis", "sekirite"], + }, + "economy": { + "label": "Economy & Poverty", + "label_fr": "Économie & Pauvreté", + "label_ht": "Ekonomi & Pòvrete", + "weight": 1.3, + "keywords_en": ["economy", "poverty", "unemployment", "inflation", "price", "money", "dollar", "hunger", "food"], + "keywords_fr": ["économie", "pauvreté", "chômage", "inflation", "prix", "argent", "famine", "nourriture"], + "keywords_ht": ["ekonomi", "pòvrete", "chomaj", "enflasyon", "pri", "lajan", "grangou", "manje"], + }, + "natural_disaster": { + "label": "Natural Disasters", + "label_fr": "Catastrophes Naturelles", + "label_ht": "Katastwòf Natirèl", + "weight": 1.2, + "keywords_en": ["earthquake", "hurricane", "flood", "disaster", "storm", "cyclone"], + "keywords_fr": ["tremblement de terre", "ouragan", "inondation", "catastrophe", "tempête", "cyclone"], + "keywords_ht": ["tranblemanntè", "siklòn", "inondasyon", "katastwòf", "tanpèt"], + }, + "health": { + "label": "Health & Healthcare", + "label_fr": "Santé", + "label_ht": "Sante", + "weight": 1.1, + "keywords_en": ["health", "hospital", "cholera", "disease", "medicine", "doctor", "epidemic"], + "keywords_fr": ["santé", "hôpital", "choléra", "maladie", "médicament", "médecin", "épidémie"], + "keywords_ht": ["sante", "lopital", "kolera", "maladi", "medikaman", "doktè", "epidemi"], + }, + "infrastructure": { + "label": "Infrastructure", + "label_fr": "Infrastructure", + "label_ht": "Enfrastrikti", + "weight": 1.0, + "keywords_en": ["electricity", "water", "road", "infrastructure", "fuel", "gas", "internet"], + "keywords_fr": ["électricité", "eau", "route", "infrastructure", "carburant", "internet"], + "keywords_ht": ["elektrisite", "dlo", "wout", "enfrastrikti", "gaz", "entènèt"], + }, + "human_rights": { + "label": "Human Rights & Diaspora", + "label_fr": "Droits Humains & Diaspora", + "label_ht": "Dwa Moun & Dyaspora", + "weight": 1.3, + "keywords_en": ["human rights", "deportation", "diaspora", "refugee", "abuse", "freedom", "protest"], + "keywords_fr": ["droits humains", "déportation", "diaspora", "réfugié", "abus", "liberté", "manifestation"], + "keywords_ht": ["dwa moun", "depòtasyon", "dyaspora", "refijye", "abi", "libète", "manifestasyon"], + }, + "education": { + "label": "Education", + "label_fr": "Éducation", + "label_ht": "Edikasyon", + "weight": 0.9, + "keywords_en": ["education", "school", "teacher", "university", "students"], + "keywords_fr": ["éducation", "école", "enseignant", "université", "étudiants"], + "keywords_ht": ["edikasyon", "lekòl", "pwofesè", "inivèsite", "elèv"], + }, + "international_relations": { + "label": "International Relations", + "label_fr": "Relations Internationales", + "label_ht": "Relasyon Entènasyonal", + "weight": 0.9, + "keywords_en": ["UN", "United Nations", "USA", "CARICOM", "Kenya", "aid", "international", "sanctions"], + "keywords_fr": ["ONU", "Nations Unies", "États-Unis", "CARICOM", "aide internationale", "sanctions"], + "keywords_ht": ["ONU", "Nasyon Zini", "Etazini", "CARICOM", "èd entènasyonal", "sanksyon"], + }, +} + +# ─── Anger Index scale thresholds ──────────────────────────────────────────── + +ANGER_LEVELS = [ + (0, 25, "Calm", "Calme", "Kalm", "#4CAF50"), + (25, 50, "Concerned", "Préoccupé", "Enkyete", "#FFC107"), + (50, 75, "Agitated", "Agité", "Ajite", "#FF9800"), + (75, 100, "Enraged", "Enragé", "Anraje", "#F44336"), +] + +# ─── Sentiment model ────────────────────────────────────────────────────────── + +SENTIMENT_MODEL = "cardiffnlp/twitter-xlm-roberta-base-sentiment" +SENTIMENT_MAX_LENGTH = 512 +SENTIMENT_BATCH_SIZE = 16 diff --git a/haiti_anger_index/crawlers/__init__.py b/haiti_anger_index/crawlers/__init__.py new file mode 100644 index 0000000..95400bb --- /dev/null +++ b/haiti_anger_index/crawlers/__init__.py @@ -0,0 +1 @@ +"""Crawlers sub-package.""" diff --git a/haiti_anger_index/crawlers/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..2c87bca Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/base.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000..cee4250 Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/base.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/facebook_crawler.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/facebook_crawler.cpython-312.pyc new file mode 100644 index 0000000..68c3826 Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/facebook_crawler.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/reddit_crawler.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/reddit_crawler.cpython-312.pyc new file mode 100644 index 0000000..44a617c Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/reddit_crawler.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/rss_crawler.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/rss_crawler.cpython-312.pyc new file mode 100644 index 0000000..1763a61 Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/rss_crawler.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/twitter_crawler.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/twitter_crawler.cpython-312.pyc new file mode 100644 index 0000000..0ee3b1c Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/twitter_crawler.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/__pycache__/youtube_crawler.cpython-312.pyc b/haiti_anger_index/crawlers/__pycache__/youtube_crawler.cpython-312.pyc new file mode 100644 index 0000000..8233a18 Binary files /dev/null and b/haiti_anger_index/crawlers/__pycache__/youtube_crawler.cpython-312.pyc differ diff --git a/haiti_anger_index/crawlers/base.py b/haiti_anger_index/crawlers/base.py new file mode 100644 index 0000000..21fb7f2 --- /dev/null +++ b/haiti_anger_index/crawlers/base.py @@ -0,0 +1,76 @@ +""" +Abstract base class for all social-media crawlers. +""" + +import logging +import time +from abc import ABC, abstractmethod +from typing import Dict, List + +from haiti_anger_index import config + +logger = logging.getLogger(__name__) + + +class BaseCrawler(ABC): + """ + Every platform crawler inherits from this class. + + Subclasses must implement :py:meth:`fetch` which returns a list of + post dicts with at minimum the keys: + + id, platform, content, created_at + + Optional but encouraged: url, author, likes, shares, comments. + """ + + PLATFORM: str = "unknown" + + def __init__(self) -> None: + self.delay = config.CRAWL_DELAY + self.max_posts = config.MAX_POSTS_PER_PLATFORM + + @abstractmethod + def fetch(self) -> List[Dict]: + """Fetch posts and return normalised dicts.""" + + def _sleep(self) -> None: + time.sleep(self.delay) + + def _make_post( + self, + post_id: str, + content: str, + created_at: str = "", + url: str = "", + author: str = "", + likes: int = 0, + shares: int = 0, + comments: int = 0, + ) -> Dict: + """Return a normalised post dict.""" + return { + "id": str(post_id), + "platform": self.PLATFORM, + "content": content, + "created_at": created_at, + "url": url, + "author": author, + "likes": likes, + "shares": shares, + "comments": comments, + "sentiment": None, + "sentiment_score": None, + "topics": None, + "engagement_score": float(likes + shares * 2 + comments * 1.5), + } + + def run(self) -> List[Dict]: + logger.info("[%s] Starting crawl …", self.PLATFORM) + try: + posts = self.fetch() + logger.info("[%s] Fetched %d posts", self.PLATFORM, len(posts)) + return posts + except Exception as exc: # noqa: BLE001 + logger.error("[%s] Crawl failed: %s", self.PLATFORM, exc) + return [] diff --git a/haiti_anger_index/crawlers/facebook_crawler.py b/haiti_anger_index/crawlers/facebook_crawler.py new file mode 100644 index 0000000..185a3ed --- /dev/null +++ b/haiti_anger_index/crawlers/facebook_crawler.py @@ -0,0 +1,93 @@ +""" +Facebook / Instagram crawler via the Graph API. + +Public page posts are accessible with a valid Page Access Token or +App Access Token. Falls back gracefully when none is configured. + +Note: Direct scraping of Facebook is prohibited by Meta's ToS. + This crawler uses only the official Graph API. +""" + +import logging +from typing import Dict, List +from urllib.parse import urlencode + +import requests + +from haiti_anger_index import config +from haiti_anger_index.crawlers.base import BaseCrawler + +logger = logging.getLogger(__name__) + +# Public Facebook pages / accounts relevant to Haiti +HAITI_FB_PAGES = [ + "LeNouvellisteHaiti", + "HaitiLibreOfficial", + "RadioKiskeya", + "alternativepresshaiti", + "loophaiti", +] + +GRAPH_API_BASE = "https://graph.facebook.com/v19.0" + + +class FacebookCrawler(BaseCrawler): + PLATFORM = "facebook" + + def __init__(self) -> None: + super().__init__() + self._token = config.FACEBOOK_ACCESS_TOKEN + + def fetch(self) -> List[Dict]: + if not self._token: + logger.warning("[Facebook] No access token configured — skipping.") + return [] + + posts: List[Dict] = [] + per_page = max(1, self.max_posts // len(HAITI_FB_PAGES)) + + for page_id in HAITI_FB_PAGES: + if len(posts) >= self.max_posts: + break + posts.extend(self._fetch_page_posts(page_id, per_page)) + self._sleep() + + return posts[: self.max_posts] + + # ── Private ─────────────────────────────────────────────────────────────── + + def _fetch_page_posts(self, page_id: str, limit: int) -> List[Dict]: + params = { + "fields": "id,message,created_time,permalink_url,reactions.summary(true),shares,comments.summary(true)", + "limit": limit, + "access_token": self._token, + } + url = f"{GRAPH_API_BASE}/{page_id}/posts?{urlencode(params)}" + try: + resp = requests.get(url, timeout=15) + resp.raise_for_status() + data = resp.json() + result: List[Dict] = [] + for item in data.get("data", []): + message = item.get("message", "") + if not message: + continue + likes = item.get("reactions", {}).get("summary", {}).get("total_count", 0) + shares = item.get("shares", {}).get("count", 0) + comments = item.get("comments", {}).get("summary", {}).get("total_count", 0) + result.append( + self._make_post( + post_id=item["id"], + content=message, + created_at=item.get("created_time", ""), + url=item.get("permalink_url", ""), + author=page_id, + likes=likes, + shares=shares, + comments=comments, + ) + ) + return result + except Exception as exc: # noqa: BLE001 + logger.error("[Facebook] Page '%s' failed: %s", page_id, exc) + return [] diff --git a/haiti_anger_index/crawlers/reddit_crawler.py b/haiti_anger_index/crawlers/reddit_crawler.py new file mode 100644 index 0000000..edf213c --- /dev/null +++ b/haiti_anger_index/crawlers/reddit_crawler.py @@ -0,0 +1,87 @@ +""" +Reddit crawler using PRAW. + +Searches Haiti-related subreddits for posts and their top-level comments. +Requires REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET env variables. +""" + +import logging +from typing import Dict, List + +from haiti_anger_index import config +from haiti_anger_index.crawlers.base import BaseCrawler + +logger = logging.getLogger(__name__) + +try: + import praw # type: ignore + _PRAW_AVAILABLE = True +except ImportError: + _PRAW_AVAILABLE = False + logger.warning("praw not installed — Reddit crawler disabled.") + + +class RedditCrawler(BaseCrawler): + PLATFORM = "reddit" + + def __init__(self) -> None: + super().__init__() + self._reddit = None + if _PRAW_AVAILABLE and config.REDDIT_CLIENT_ID and config.REDDIT_CLIENT_SECRET: + self._reddit = praw.Reddit( + client_id=config.REDDIT_CLIENT_ID, + client_secret=config.REDDIT_CLIENT_SECRET, + user_agent=config.REDDIT_USER_AGENT, + ) + + def fetch(self) -> List[Dict]: + if self._reddit is None: + logger.warning("[Reddit] Credentials not configured — skipping.") + return [] + + posts: List[Dict] = [] + per_sub = max(1, self.max_posts // len(config.REDDIT_SUBREDDITS)) + + for sub_name in config.REDDIT_SUBREDDITS: + if len(posts) >= self.max_posts: + break + try: + subreddit = self._reddit.subreddit(sub_name) + submissions = list(subreddit.search("Haiti OR Haïti OR Ayiti", limit=per_sub, sort="new")) + for submission in submissions: + from datetime import datetime, timezone + ts = datetime.fromtimestamp(submission.created_utc, tz=timezone.utc).isoformat() + posts.append( + self._make_post( + post_id=submission.id, + content=f"{submission.title}\n{submission.selftext}".strip(), + created_at=ts, + url=f"https://reddit.com{submission.permalink}", + author=str(submission.author or ""), + likes=submission.score, + comments=submission.num_comments, + ) + ) + # Fetch top-level comments (up to 5 per post) + try: + submission.comments.replace_more(limit=0) + for comment in list(submission.comments)[:5]: + posts.append( + self._make_post( + post_id=comment.id, + content=comment.body, + created_at=datetime.fromtimestamp( + comment.created_utc, tz=timezone.utc + ).isoformat(), + url=f"https://reddit.com{comment.permalink}", + author=str(comment.author or ""), + likes=comment.score, + ) + ) + except Exception as exc: # noqa: BLE001 + logger.debug("[Reddit] Comment fetch error: %s", exc) + self._sleep() + except Exception as exc: # noqa: BLE001 + logger.error("[Reddit] Subreddit '%s' failed: %s", sub_name, exc) + + return posts[: self.max_posts] diff --git a/haiti_anger_index/crawlers/rss_crawler.py b/haiti_anger_index/crawlers/rss_crawler.py new file mode 100644 index 0000000..0b0b397 --- /dev/null +++ b/haiti_anger_index/crawlers/rss_crawler.py @@ -0,0 +1,67 @@ +""" +RSS / News crawler. + +Parses Haitian news RSS feeds and returns article summaries as posts. +Uses the feedparser library (pure-Python, no API key needed). +""" + +import hashlib +import logging +from typing import Dict, List + +from haiti_anger_index import config +from haiti_anger_index.crawlers.base import BaseCrawler + +logger = logging.getLogger(__name__) + +try: + import feedparser # type: ignore + _FEEDPARSER_AVAILABLE = True +except ImportError: + _FEEDPARSER_AVAILABLE = False + logger.warning("feedparser not installed — RSS crawler disabled.") + + +class RSSCrawler(BaseCrawler): + PLATFORM = "news_rss" + + def fetch(self) -> List[Dict]: + if not _FEEDPARSER_AVAILABLE: + logger.warning("[RSS] feedparser not available — skipping.") + return [] + + posts: List[Dict] = [] + per_feed = max(1, self.max_posts // max(len(config.RSS_FEEDS), 1)) + + for source_name, feed_url in config.RSS_FEEDS.items(): + if len(posts) >= self.max_posts: + break + try: + feed = feedparser.parse(feed_url) + entries = feed.entries[:per_feed] + for entry in entries: + title = entry.get("title", "") + summary = entry.get("summary", entry.get("description", "")) + content = f"{title}\n{summary}".strip() + if not content: + continue + + link = entry.get("link", "") + published = entry.get("published", entry.get("updated", "")) + # Stable ID from URL hash + post_id = hashlib.md5(link.encode()).hexdigest() if link else hashlib.md5(content[:100].encode()).hexdigest() + + posts.append( + self._make_post( + post_id=f"{source_name}_{post_id}", + content=content, + created_at=published, + url=link, + author=source_name, + ) + ) + self._sleep() + except Exception as exc: # noqa: BLE001 + logger.error("[RSS] Feed '%s' failed: %s", source_name, exc) + + return posts[: self.max_posts] diff --git a/haiti_anger_index/crawlers/twitter_crawler.py b/haiti_anger_index/crawlers/twitter_crawler.py new file mode 100644 index 0000000..cc0bc0a --- /dev/null +++ b/haiti_anger_index/crawlers/twitter_crawler.py @@ -0,0 +1,74 @@ +""" +Twitter / X crawler using the v2 API (tweepy). + +Requires the environment variable TWITTER_BEARER_TOKEN. +Falls back gracefully when no token is configured. +""" + +import logging +from typing import Dict, List + +from haiti_anger_index import config +from haiti_anger_index.crawlers.base import BaseCrawler + +logger = logging.getLogger(__name__) + +try: + import tweepy # type: ignore + _TWEEPY_AVAILABLE = True +except ImportError: + _TWEEPY_AVAILABLE = False + logger.warning("tweepy not installed — Twitter crawler disabled.") + + +class TwitterCrawler(BaseCrawler): + PLATFORM = "twitter" + + def __init__(self) -> None: + super().__init__() + self._client = None + if _TWEEPY_AVAILABLE and config.TWITTER_BEARER_TOKEN: + self._client = tweepy.Client( + bearer_token=config.TWITTER_BEARER_TOKEN, + wait_on_rate_limit=True, + ) + + def fetch(self) -> List[Dict]: + if self._client is None: + logger.warning("[Twitter] No bearer token configured — skipping.") + return [] + + posts: List[Dict] = [] + tweet_fields = ["created_at", "public_metrics", "author_id", "lang"] + + for query in config.TWITTER_SEARCH_QUERIES: + if len(posts) >= self.max_posts: + break + try: + response = self._client.search_recent_tweets( + query=f"{query} -is:retweet", + max_results=min(100, self.max_posts - len(posts)), + tweet_fields=tweet_fields, + ) + if not response.data: + continue + for tweet in response.data: + metrics = tweet.public_metrics or {} + posts.append( + self._make_post( + post_id=str(tweet.id), + content=tweet.text, + created_at=str(tweet.created_at or ""), + url=f"https://twitter.com/i/web/status/{tweet.id}", + author=str(tweet.author_id or ""), + likes=metrics.get("like_count", 0), + shares=metrics.get("retweet_count", 0), + comments=metrics.get("reply_count", 0), + ) + ) + if len(posts) >= self.max_posts: + break + self._sleep() + except Exception as exc: # noqa: BLE001 + logger.error("[Twitter] Query '%s' failed: %s", query, exc) + return posts diff --git a/haiti_anger_index/crawlers/youtube_crawler.py b/haiti_anger_index/crawlers/youtube_crawler.py new file mode 100644 index 0000000..94848e1 --- /dev/null +++ b/haiti_anger_index/crawlers/youtube_crawler.py @@ -0,0 +1,104 @@ +""" +YouTube crawler using the YouTube Data API v3. + +Searches for Haiti-related videos and collects their top-level comments. +Requires the environment variable YOUTUBE_API_KEY. +""" + +import logging +from typing import Dict, List + +from haiti_anger_index import config +from haiti_anger_index.crawlers.base import BaseCrawler + +logger = logging.getLogger(__name__) + +try: + from googleapiclient.discovery import build # type: ignore + _GOOGLE_API_AVAILABLE = True +except ImportError: + _GOOGLE_API_AVAILABLE = False + logger.warning("google-api-python-client not installed — YouTube crawler disabled.") + + +class YouTubeCrawler(BaseCrawler): + PLATFORM = "youtube" + + def __init__(self) -> None: + super().__init__() + self._service = None + if _GOOGLE_API_AVAILABLE and config.YOUTUBE_API_KEY: + self._service = build("youtube", "v3", developerKey=config.YOUTUBE_API_KEY) + + # ── Public API ──────────────────────────────────────────────────────────── + + def fetch(self) -> List[Dict]: + if self._service is None: + logger.warning("[YouTube] No API key configured — skipping.") + return [] + + video_ids = self._search_videos() + posts: List[Dict] = [] + for vid_id in video_ids: + if len(posts) >= self.max_posts: + break + posts.extend(self._fetch_comments(vid_id)) + self._sleep() + return posts[: self.max_posts] + + # ── Private helpers ─────────────────────────────────────────────────────── + + def _search_videos(self) -> List[str]: + video_ids: List[str] = [] + for term in config.YOUTUBE_SEARCH_TERMS: + try: + response = ( + self._service.search() + .list( + q=term, + part="id", + type="video", + maxResults=10, + relevanceLanguage="fr", + order="date", + ) + .execute() + ) + for item in response.get("items", []): + vid_id = item.get("id", {}).get("videoId") + if vid_id and vid_id not in video_ids: + video_ids.append(vid_id) + self._sleep() + except Exception as exc: # noqa: BLE001 + logger.error("[YouTube] Search '%s' failed: %s", term, exc) + return video_ids + + def _fetch_comments(self, video_id: str) -> List[Dict]: + posts: List[Dict] = [] + try: + response = ( + self._service.commentThreads() + .list( + part="snippet", + videoId=video_id, + maxResults=50, + order="relevance", + textFormat="plainText", + ) + .execute() + ) + for item in response.get("items", []): + snippet = item["snippet"]["topLevelComment"]["snippet"] + posts.append( + self._make_post( + post_id=item["id"], + content=snippet.get("textDisplay", ""), + created_at=snippet.get("publishedAt", ""), + url=f"https://www.youtube.com/watch?v={video_id}", + author=snippet.get("authorDisplayName", ""), + likes=snippet.get("likeCount", 0), + ) + ) + except Exception as exc: # noqa: BLE001 + logger.error("[YouTube] Comments for %s failed: %s", video_id, exc) + return posts diff --git a/haiti_anger_index/index/__init__.py b/haiti_anger_index/index/__init__.py new file mode 100644 index 0000000..a16bacf --- /dev/null +++ b/haiti_anger_index/index/__init__.py @@ -0,0 +1 @@ +"""Index sub-package.""" diff --git a/haiti_anger_index/index/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/index/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..e91ee8a Binary files /dev/null and b/haiti_anger_index/index/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/index/__pycache__/anger_index.cpython-312.pyc b/haiti_anger_index/index/__pycache__/anger_index.cpython-312.pyc new file mode 100644 index 0000000..518d296 Binary files /dev/null and b/haiti_anger_index/index/__pycache__/anger_index.cpython-312.pyc differ diff --git a/haiti_anger_index/index/anger_index.py b/haiti_anger_index/index/anger_index.py new file mode 100644 index 0000000..fce45a2 --- /dev/null +++ b/haiti_anger_index/index/anger_index.py @@ -0,0 +1,247 @@ +""" +Haiti Anger Index (HAI) calculator. + +The index mirrors the methodology of Pollara's Rage Index: +- Overall score 0–100. +- Broken down by platform, topic, and language. +- Time-series trend (week-over-week change). +- Identifies the primary drivers of anger. + +Formula +------- +The raw score is a blend of: + 1. Simple negative-sentiment rate (40 %) + 2. Engagement-weighted negative rate (40 %) + 3. Topic-severity adjustment (20 %) + +The blended score is then min-max normalised to [0, 100]. +""" + +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +from haiti_anger_index import config + +logger = logging.getLogger(__name__) + + +class AngerIndexCalculator: + """ + Computes the Haiti Anger Index from compiled statistics. + + Usage:: + + calc = AngerIndexCalculator() + snapshot = calc.compute(stats, period_start, period_end) + """ + + # Weights for the three components + W_NEG_RATE = 0.40 + W_ENG_WEIGHTED = 0.40 + W_TOPIC_SEVERITY = 0.20 + + def compute( + self, + stats: Dict, + period_start: Optional[str] = None, + period_end: Optional[str] = None, + history: Optional[List[Dict]] = None, + ) -> Dict: + """ + Compute the HAI snapshot. + + Parameters + ---------- + stats : dict + Output of DataCompiler.compile(). + period_start / period_end : str (ISO-8601) + Time window covered by this snapshot. + history : list of previous snapshots (used for trend calculation). + + Returns + ------- + A dict ready to be saved via Database.save_anger_index(). + """ + if not stats or stats.get("total", 0) == 0: + return self._zero_snapshot(period_start, period_end) + + raw_score = self._raw_score(stats) + overall_index = round(raw_score * 100, 1) + anger_level, anger_color = self._get_level(overall_index) + + platform_breakdown = self._platform_breakdown(stats["platforms"]) + topic_breakdown = self._topic_breakdown(stats["topics"]) + language_breakdown = self._language_breakdown(stats["languages"]) + + trend = self._compute_trend(overall_index, history) + + return { + "computed_at": datetime.now(timezone.utc).isoformat(), + "period_start": period_start or "", + "period_end": period_end or datetime.now(timezone.utc).isoformat(), + "overall_index": overall_index, + "platform_breakdown": platform_breakdown, + "topic_breakdown": topic_breakdown, + "language_breakdown": language_breakdown, + "total_posts": stats["total"], + "positive_count": stats["positive"], + "negative_count": stats["negative"], + "neutral_count": stats["neutral"], + "anger_level": anger_level, + "anger_level_color": anger_color, + "trend": trend, + "top_drivers": self._top_drivers(stats["topics"]), + "neg_rate": stats["neg_rate"], + "weighted_neg_rate": stats["weighted_neg_rate"], + } + + # ── Score calculation ───────────────────────────────────────────────────── + + def _raw_score(self, stats: Dict) -> float: + neg_rate = stats.get("neg_rate", 0.0) + eng_rate = stats.get("weighted_neg_rate", 0.0) + topic_severity = self._topic_severity_score(stats.get("topics", {})) + + score = ( + self.W_NEG_RATE * neg_rate + + self.W_ENG_WEIGHTED * eng_rate + + self.W_TOPIC_SEVERITY * topic_severity + ) + return min(max(score, 0.0), 1.0) + + def _topic_severity_score(self, topics: Dict) -> float: + """ + Weighted average of per-topic negative rates, where each topic's + weight is taken from config.TOPICS. + """ + total_w = 0.0 + weighted_neg = 0.0 + for topic_key, data in topics.items(): + topic_cfg = config.TOPICS.get(topic_key, {}) + weight = topic_cfg.get("weight", 1.0) + neg_rate = data.get("neg_rate", 0.0) + total_w += weight * data.get("total", 0) + weighted_neg += weight * data.get("total", 0) * neg_rate + return (weighted_neg / total_w) if total_w else 0.0 + + # ── Per-breakdown helpers ───────────────────────────────────────────────── + + def _platform_breakdown(self, platforms: Dict) -> Dict: + result = {} + for plat, data in platforms.items(): + total = data.get("total", 0) + neg = data.get("negative", 0) + pos = data.get("positive", 0) + result[plat] = { + "label": plat.replace("_", " ").title(), + "total": total, + "positive": pos, + "negative": neg, + "neutral": data.get("neutral", 0), + "neg_rate": data.get("neg_rate", 0.0), + "hai_contribution": round(data.get("neg_rate", 0.0) * 100, 1), + } + return result + + def _topic_breakdown(self, topics: Dict) -> Dict: + result = {} + for key, data in topics.items(): + topic_cfg = config.TOPICS.get(key, {}) + neg_rate = data.get("neg_rate", 0.0) + weight = topic_cfg.get("weight", 1.0) + result[key] = { + "label": topic_cfg.get("label", key.replace("_", " ").title()), + "label_fr": topic_cfg.get("label_fr", ""), + "label_ht": topic_cfg.get("label_ht", ""), + "total": data.get("total", 0), + "positive": data.get("positive", 0), + "negative": data.get("negative", 0), + "neutral": data.get("neutral", 0), + "neg_rate": neg_rate, + "severity_weight": weight, + "weighted_anger_score": round(neg_rate * weight * 100, 1), + "weighted_neg_score": data.get("weighted_neg_score", 0.0), + } + return result + + def _language_breakdown(self, languages: Dict) -> Dict: + result = {} + lang_labels = {"en": "English", "fr": "French", "ht": "Haitian Creole", "unknown": "Unknown"} + for lang, data in languages.items(): + result[lang] = { + "label": lang_labels.get(lang, lang.upper()), + "total": data.get("total", 0), + "positive": data.get("positive", 0), + "negative": data.get("negative", 0), + "neutral": data.get("neutral", 0), + "neg_rate": data.get("neg_rate", 0.0), + } + return result + + # ── Trend ───────────────────────────────────────────────────────────────── + + def _compute_trend(self, current: float, history: Optional[List[Dict]]) -> Dict: + if not history: + return {"direction": "stable", "change": 0.0, "previous": None} + prev = history[0].get("overall_index") + if prev is None: + return {"direction": "stable", "change": 0.0, "previous": None} + change = round(current - prev, 1) + if change > 1: + direction = "rising" + elif change < -1: + direction = "falling" + else: + direction = "stable" + return {"direction": direction, "change": change, "previous": round(prev, 1)} + + # ── Top drivers ─────────────────────────────────────────────────────────── + + def _top_drivers(self, topics: Dict, top_n: int = 5) -> List[Dict]: + """Return the top N topics driving anger, sorted by weighted_anger_score.""" + scored = [] + for key, data in topics.items(): + topic_cfg = config.TOPICS.get(key, {}) + neg_rate = data.get("neg_rate", 0.0) + weight = topic_cfg.get("weight", 1.0) + scored.append({ + "key": key, + "label": topic_cfg.get("label", key.replace("_", " ").title()), + "neg_rate": neg_rate, + "weighted_anger_score": round(neg_rate * weight * 100, 1), + "total_posts": data.get("total", 0), + }) + scored.sort(key=lambda x: x["weighted_anger_score"], reverse=True) + return scored[:top_n] + + # ── Level lookup ────────────────────────────────────────────────────────── + + @staticmethod + def _get_level(score: float) -> Tuple[str, str]: + for lo, hi, label, _label_fr, _label_ht, color in config.ANGER_LEVELS: + if lo <= score <= hi: + return label, color + return "Enraged", "#F44336" + + @staticmethod + def _zero_snapshot(period_start: Optional[str], period_end: Optional[str]) -> Dict: + return { + "computed_at": datetime.now(timezone.utc).isoformat(), + "period_start": period_start or "", + "period_end": period_end or datetime.now(timezone.utc).isoformat(), + "overall_index": 0.0, + "platform_breakdown": {}, + "topic_breakdown": {}, + "language_breakdown": {}, + "total_posts": 0, + "positive_count": 0, + "negative_count": 0, + "neutral_count": 0, + "anger_level": "Calm", + "anger_level_color": "#4CAF50", + "trend": {"direction": "stable", "change": 0.0, "previous": None}, + "top_drivers": [], + "neg_rate": 0.0, + "weighted_neg_rate": 0.0, + } diff --git a/haiti_anger_index/reporting/__init__.py b/haiti_anger_index/reporting/__init__.py new file mode 100644 index 0000000..21ff779 --- /dev/null +++ b/haiti_anger_index/reporting/__init__.py @@ -0,0 +1 @@ +"""Reporting sub-package.""" diff --git a/haiti_anger_index/reporting/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/reporting/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..53cff80 Binary files /dev/null and b/haiti_anger_index/reporting/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/reporting/__pycache__/report_generator.cpython-312.pyc b/haiti_anger_index/reporting/__pycache__/report_generator.cpython-312.pyc new file mode 100644 index 0000000..0edd5f0 Binary files /dev/null and b/haiti_anger_index/reporting/__pycache__/report_generator.cpython-312.pyc differ diff --git a/haiti_anger_index/reporting/report_generator.py b/haiti_anger_index/reporting/report_generator.py new file mode 100644 index 0000000..a10fbc0 --- /dev/null +++ b/haiti_anger_index/reporting/report_generator.py @@ -0,0 +1,166 @@ +""" +HTML report generator for the Haiti Anger Index. + +Renders the Jinja2 template with snapshot data and writes an HTML file. +Falls back to simple string formatting when Jinja2 is unavailable. +""" + +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + +try: + from jinja2 import Environment, FileSystemLoader, select_autoescape # type: ignore + _JINJA2_AVAILABLE = True +except ImportError: + _JINJA2_AVAILABLE = False + logger.warning("jinja2 not installed — HTML report generation disabled.") + +_TEMPLATES_DIR = Path(__file__).parent / "templates" + + +class ReportGenerator: + """ + Generates an HTML report from a HAI snapshot dict. + + Usage:: + + gen = ReportGenerator(output_dir="reports") + path = gen.generate(snapshot, history) + """ + + def __init__(self, output_dir: str = "reports") -> None: + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + + self._env = None + if _JINJA2_AVAILABLE: + self._env = Environment( + loader=FileSystemLoader(str(_TEMPLATES_DIR)), + autoescape=select_autoescape(["html"]), + ) + self._env.filters["tojson"] = json.dumps + + # ── Public API ──────────────────────────────────────────────────────────── + + def generate( + self, + snapshot: Dict, + history: Optional[List[Dict]] = None, + ) -> str: + """ + Render the report and return the output file path. + """ + context = self._build_context(snapshot, history or []) + ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + filename = f"hai_report_{ts}.html" + output_path = self.output_dir / filename + + if self._env is not None: + template = self._env.get_template("anger_index_report.html") + html = template.render(**context) + else: + html = self._fallback_html(context) + + output_path.write_text(html, encoding="utf-8") + logger.info("Report written → %s", output_path) + return str(output_path) + + # ── Context builder ─────────────────────────────────────────────────────── + + def _build_context(self, snapshot: Dict, history: List[Dict]) -> Dict: + now = datetime.now(timezone.utc) + total = snapshot.get("total_posts", 0) + pos = snapshot.get("positive_count", 0) + neg = snapshot.get("negative_count", 0) + neu = snapshot.get("neutral_count", 0) + + def pct(n: int) -> float: + return round(n / total * 100, 1) if total else 0.0 + + trend = snapshot.get("trend", {}) + + # Platform chart data + plat_bd = snapshot.get("platform_breakdown", {}) + platform_labels = list(plat_bd.keys()) + platform_neg = [plat_bd[p].get("negative", 0) for p in platform_labels] + platform_pos = [plat_bd[p].get("positive", 0) for p in platform_labels] + platform_neu = [plat_bd[p].get("neutral", 0) for p in platform_labels] + + # Language chart data + lang_bd = snapshot.get("language_breakdown", {}) + lang_label_map = {"en": "English", "fr": "French", "ht": "Creole", "unknown": "Unknown"} + lang_labels = [lang_label_map.get(k, k) for k in lang_bd.keys()] + lang_neg = [lang_bd[k].get("negative", 0) for k in lang_bd.keys()] + + # Historical trend + trend_labels: List[str] = [] + trend_scores: List[float] = [] + for h in reversed(history): + dt_str = h.get("computed_at", "") + try: + dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00")) + trend_labels.append(dt.strftime("%b %d")) + except (ValueError, AttributeError): + trend_labels.append(dt_str[:10]) + trend_scores.append(h.get("overall_index", 0.0)) + # Add current snapshot + trend_labels.append(now.strftime("%b %d")) + trend_scores.append(snapshot.get("overall_index", 0.0)) + + return { + "report_date": now.strftime("%B %d, %Y"), + "year": now.year, + "period_start": snapshot.get("period_start", "")[:10], + "period_end": snapshot.get("period_end", "")[:10] or now.strftime("%Y-%m-%d"), + "overall_index": snapshot.get("overall_index", 0.0), + "anger_color": snapshot.get("anger_level_color", "#4CAF50"), + "anger_level": snapshot.get("anger_level", "Calm"), + "total_posts": total, + "positive_count": pos, + "negative_count": neg, + "neutral_count": neu, + "pos_pct": pct(pos), + "neg_pct": pct(neg), + "neu_pct": pct(neu), + "trend_direction": trend.get("direction", "stable"), + "trend_change": abs(trend.get("change", 0.0)), + "top_drivers": snapshot.get("top_drivers", []), + "platform_breakdown": plat_bd, + "topic_breakdown": snapshot.get("topic_breakdown", {}), + "language_breakdown": lang_bd, + "platform_labels": platform_labels, + "platform_neg": platform_neg, + "platform_pos": platform_pos, + "platform_neu": platform_neu, + "lang_labels": lang_labels, + "lang_neg": lang_neg, + "history": history, + "trend_labels": trend_labels, + "trend_scores": trend_scores, + } + + # ── Minimal fallback ────────────────────────────────────────────────────── + + @staticmethod + def _fallback_html(ctx: Dict) -> str: + return f""" +Haiti Anger Index + +

Haiti Anger Index — {ctx['report_date']}

+

Overall Score: {ctx['overall_index']} / 100 — {ctx['anger_level']}

+

Total posts: {ctx['total_posts']} | + Negative: {ctx['neg_pct']}% | + Positive: {ctx['pos_pct']}% | + Neutral: {ctx['neu_pct']}%

+

Top Drivers

+ +

Generated by POLLNEX Insights

+""" diff --git a/haiti_anger_index/reporting/templates/anger_index_report.html b/haiti_anger_index/reporting/templates/anger_index_report.html new file mode 100644 index 0000000..062b127 --- /dev/null +++ b/haiti_anger_index/reporting/templates/anger_index_report.html @@ -0,0 +1,355 @@ + + + + + +Haiti Anger Index — POLLNEX Insights + + + + + +
+ +
+ Haiti Anger Index™
+ {{ report_date }}
+ Period: {{ period_start }} – {{ period_end }} +
+
+ +
+ + +
+

🇭🇹 Haiti Anger Index (HAI™)

+

+ A real-time composite measure of public sentiment across Haitian social media platforms + in English, French, and Haitian Creole. Powered by POLLNEX Insights. +

+
+ + +
+ +
{{ overall_index }}
+
{{ anger_level }}
+
+ {% if trend_direction == "rising" %} + ▲ +{{ trend_change }} pts vs previous period + {% elif trend_direction == "falling" %} + ▼ {{ trend_change }} pts vs previous period + {% else %} + → Stable vs previous period + {% endif %} +
+

+ Based on {{ total_posts }} posts analysed | + {{ neg_pct }}% negative | + {{ pos_pct }}% positive | + {{ neu_pct }}% neutral +

+
+ + +
+
{{ total_posts }}
Total Posts
+
{{ negative_count }}
Negative
+
{{ positive_count }}
Positive
+
{{ neutral_count }}
Neutral
+
{{ overall_index }}/100
HAI Score
+
+ + +
+ + +
+

🔥 Top Anger Drivers

+ + + + {% for d in top_drivers %} + + + + + + {% endfor %} + +
TopicNeg. RateAnger Score
{{ d.label }}{{ (d.neg_rate * 100) | round(1) }}% +
+
+
+ {{ d.weighted_anger_score }} +
+
+ + +
+

📱 Platform Breakdown

+ +
+ +
+ + +
+ + +
+

🗣️ Language Breakdown

+ +
+ + +
+

📊 Topic Detail

+ + + + {% for key, td in topic_breakdown.items() %} + + + + + + + {% endfor %} + +
TopicPostsNeg%Weighted
{{ td.label }}{{ td.total }}{{ (td.neg_rate * 100) | round(1) }}%{{ td.weighted_anger_score }}
+
+ +
+ + + {% if history %} +
+

📈 Historical HAI Trend

+ +
+ {% endif %} + + +
+

ℹ️ Methodology

+

+ The Haiti Anger Index (HAI™) is computed by POLLNEX Insights as a composite of three weighted components: + (1) raw negative-sentiment rate (40%), (2) engagement-weighted negative rate (40%), and (3) a topic-severity + adjustment that amplifies issues such as kidnapping, corruption, and economic collapse (20%). + Sentiment is classified using a multilingual XLM-RoBERTa model trained on social-media data and validated + for English, French, and Haitian Creole. The index is scaled 0–100 where 0–25 = Calm, 26–50 = Concerned, + 51–75 = Agitated, 76–100 = Enraged. Data is sourced from Twitter/X, Facebook, YouTube, Reddit, and + major Haitian news RSS feeds. +

+
+ +
+ + + + + + + diff --git a/haiti_anger_index/storage/__init__.py b/haiti_anger_index/storage/__init__.py new file mode 100644 index 0000000..703e2a9 --- /dev/null +++ b/haiti_anger_index/storage/__init__.py @@ -0,0 +1 @@ +"""Storage sub-package.""" diff --git a/haiti_anger_index/storage/__pycache__/__init__.cpython-312.pyc b/haiti_anger_index/storage/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..ecf5698 Binary files /dev/null and b/haiti_anger_index/storage/__pycache__/__init__.cpython-312.pyc differ diff --git a/haiti_anger_index/storage/__pycache__/database.cpython-312.pyc b/haiti_anger_index/storage/__pycache__/database.cpython-312.pyc new file mode 100644 index 0000000..f85682c Binary files /dev/null and b/haiti_anger_index/storage/__pycache__/database.cpython-312.pyc differ diff --git a/haiti_anger_index/storage/database.py b/haiti_anger_index/storage/database.py new file mode 100644 index 0000000..58f6dfd --- /dev/null +++ b/haiti_anger_index/storage/database.py @@ -0,0 +1,217 @@ +""" +SQLite storage layer for the Haiti Anger Index. + +Schema +------ +posts — every collected social-media post (deduplicated by id + platform) +anger_index — one snapshot per agent run with the computed HAI score +""" + +import json +import logging +import sqlite3 +from contextlib import contextmanager +from datetime import datetime +from typing import Dict, Generator, List, Optional + +logger = logging.getLogger(__name__) + +DDL = """ +CREATE TABLE IF NOT EXISTS posts ( + id TEXT NOT NULL, + platform TEXT NOT NULL, + url TEXT, + author TEXT, + content TEXT NOT NULL, + language TEXT, + created_at TEXT, + collected_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + likes INTEGER DEFAULT 0, + shares INTEGER DEFAULT 0, + comments INTEGER DEFAULT 0, + sentiment TEXT, + sentiment_score REAL, + topics TEXT, + engagement_score REAL DEFAULT 0, + PRIMARY KEY (id, platform) +); + +CREATE TABLE IF NOT EXISTS anger_index ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + computed_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + period_start TEXT, + period_end TEXT, + overall_index REAL, + platform_breakdown TEXT, + topic_breakdown TEXT, + language_breakdown TEXT, + total_posts INTEGER, + positive_count INTEGER, + negative_count INTEGER, + neutral_count INTEGER, + anger_level TEXT, + anger_level_color TEXT +); + +CREATE TABLE IF NOT EXISTS reports ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + report_path TEXT, + index_value REAL, + summary TEXT +); +""" + + +class Database: + """Thin wrapper around sqlite3 for the Haiti Anger Index.""" + + def __init__(self, db_path: str = "haiti_anger_index.db"): + self.db_path = db_path + self._init_db() + + def _init_db(self) -> None: + with self._conn() as conn: + conn.executescript(DDL) + logger.info("Database initialised at %s", self.db_path) + + @contextmanager + def _conn(self) -> Generator[sqlite3.Connection, None, None]: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + # ── Posts ───────────────────────────────────────────────────────────────── + + def upsert_post(self, post: Dict) -> None: + sql = """ + INSERT OR REPLACE INTO posts + (id, platform, url, author, content, language, created_at, + likes, shares, comments, sentiment, sentiment_score, topics, engagement_score) + VALUES + (:id, :platform, :url, :author, :content, :language, :created_at, + :likes, :shares, :comments, :sentiment, :sentiment_score, :topics, :engagement_score) + """ + row = dict(post) + row.setdefault("url", None) + row.setdefault("author", None) + row.setdefault("language", None) + row.setdefault("created_at", datetime.utcnow().isoformat()) + row.setdefault("likes", 0) + row.setdefault("shares", 0) + row.setdefault("comments", 0) + row.setdefault("sentiment", None) + row.setdefault("sentiment_score", None) + row.setdefault("topics", None) + row.setdefault("engagement_score", 0.0) + if isinstance(row.get("topics"), list): + row["topics"] = json.dumps(row["topics"]) + with self._conn() as conn: + conn.execute(sql, row) + + def upsert_posts(self, posts: List[Dict]) -> None: + for post in posts: + self.upsert_post(post) + logger.info("Upserted %d posts", len(posts)) + + def get_unclassified_posts(self, limit: int = 1000) -> List[Dict]: + sql = "SELECT * FROM posts WHERE sentiment IS NULL LIMIT ?" + with self._conn() as conn: + rows = conn.execute(sql, (limit,)).fetchall() + return [dict(r) for r in rows] + + def update_post_classification( + self, + post_id: str, + platform: str, + language: str, + sentiment: str, + sentiment_score: float, + topics: List[str], + ) -> None: + sql = """ + UPDATE posts + SET language=?, sentiment=?, sentiment_score=?, topics=? + WHERE id=? AND platform=? + """ + with self._conn() as conn: + conn.execute( + sql, + (language, sentiment, sentiment_score, json.dumps(topics), post_id, platform), + ) + + def get_posts_for_period( + self, + period_start: Optional[str] = None, + period_end: Optional[str] = None, + ) -> List[Dict]: + conditions = ["sentiment IS NOT NULL"] + params: List = [] + if period_start: + conditions.append("collected_at >= ?") + params.append(period_start) + if period_end: + conditions.append("collected_at <= ?") + params.append(period_end) + sql = "SELECT * FROM posts WHERE " + " AND ".join(conditions) + with self._conn() as conn: + rows = conn.execute(sql, params).fetchall() + return [dict(r) for r in rows] + + # ── Anger Index ─────────────────────────────────────────────────────────── + + def save_anger_index(self, snapshot: Dict) -> int: + sql = """ + INSERT INTO anger_index + (computed_at, period_start, period_end, overall_index, + platform_breakdown, topic_breakdown, language_breakdown, + total_posts, positive_count, negative_count, neutral_count, + anger_level, anger_level_color) + VALUES + (:computed_at, :period_start, :period_end, :overall_index, + :platform_breakdown, :topic_breakdown, :language_breakdown, + :total_posts, :positive_count, :negative_count, :neutral_count, + :anger_level, :anger_level_color) + """ + row = dict(snapshot) + for key in ("platform_breakdown", "topic_breakdown", "language_breakdown"): + if isinstance(row.get(key), dict): + row[key] = json.dumps(row[key]) + with self._conn() as conn: + cursor = conn.execute(sql, row) + return cursor.lastrowid + + def get_anger_index_history(self, limit: int = 52) -> List[Dict]: + sql = "SELECT * FROM anger_index ORDER BY computed_at DESC LIMIT ?" + with self._conn() as conn: + rows = conn.execute(sql, (limit,)).fetchall() + results = [] + for r in rows: + d = dict(r) + for key in ("platform_breakdown", "topic_breakdown", "language_breakdown"): + if d.get(key): + try: + d[key] = json.loads(d[key]) + except (json.JSONDecodeError, TypeError): + pass + results.append(d) + return results + + def get_latest_anger_index(self) -> Optional[Dict]: + history = self.get_anger_index_history(limit=1) + return history[0] if history else None + + # ── Reports ─────────────────────────────────────────────────────────────── + + def save_report(self, report_path: str, index_value: float, summary: str) -> None: + sql = "INSERT INTO reports (report_path, index_value, summary) VALUES (?, ?, ?)" + with self._conn() as conn: + conn.execute(sql, (report_path, index_value, summary)) + logger.info("Report saved → %s", report_path) diff --git a/reports/hai_report_20260412_223958.html b/reports/hai_report_20260412_223958.html new file mode 100644 index 0000000..8ffba76 --- /dev/null +++ b/reports/hai_report_20260412_223958.html @@ -0,0 +1,428 @@ + + + + + +Haiti Anger Index — POLLNEX Insights + + + + + +
+ +
+ Haiti Anger Index™
+ April 12, 2026
+ Period: 2026-04-05 – 2026-04-12 +
+
+ +
+ + +
+

🇭🇹 Haiti Anger Index (HAI™)

+

+ A real-time composite measure of public sentiment across Haitian social media platforms + in English, French, and Haitian Creole. Powered by POLLNEX Insights. +

+
+ + +
+ +
25.9
+
Concerned
+
+ + → Stable vs previous period + +
+

+ Based on 80 posts analysed | + 30.0% negative | + 2.5% positive | + 67.5% neutral +

+
+ + +
+
80
Total Posts
+
24
Negative
+
2
Positive
+
54
Neutral
+
25.9/100
HAI Score
+
+ + +
+ + +
+

🔥 Top Anger Drivers

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TopicNeg. RateAnger Score
Public Safety & Crime56.5% +
+
+
+ 79.1 +
Education60.0% +
+
+
+ 54.0 +
Health & Healthcare42.9% +
+
+
+ 47.1 +
Government & Corruption26.7% +
+
+
+ 40.0 +
Natural Disasters33.3% +
+
+
+ 40.0 +
+
+ + +
+

📱 Platform Breakdown

+ +
+ +
+ + +
+ + +
+

🗣️ Language Breakdown

+ +
+ + +
+

📊 Topic Detail

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TopicPostsNeg%Weighted
Government & Corruption1526.7%40.0
International Relations1225.0%22.5
Human Rights & Diaspora1118.2%23.6
Public Safety & Crime2356.5%79.1
Economy & Poverty1225.0%32.5
Infrastructure90.0%0.0
Health & Healthcare742.9%47.1
General119.1%9.1
Natural Disasters333.3%40.0
Education560.0%54.0
+
+ +
+ + + + + +
+

ℹ️ Methodology

+

+ The Haiti Anger Index (HAI™) is computed by POLLNEX Insights as a composite of three weighted components: + (1) raw negative-sentiment rate (40%), (2) engagement-weighted negative rate (40%), and (3) a topic-severity + adjustment that amplifies issues such as kidnapping, corruption, and economic collapse (20%). + Sentiment is classified using a multilingual XLM-RoBERTa model trained on social-media data and validated + for English, French, and Haitian Creole. The index is scaled 0–100 where 0–25 = Calm, 26–50 = Concerned, + 51–75 = Agitated, 76–100 = Enraged. Data is sourced from Twitter/X, Facebook, YouTube, Reddit, and + major Haitian news RSS feeds. +

+
+ +
+ + + + + + + \ No newline at end of file diff --git a/reports/hai_report_20260616_085319.html b/reports/hai_report_20260616_085319.html new file mode 100644 index 0000000..68ba2cc --- /dev/null +++ b/reports/hai_report_20260616_085319.html @@ -0,0 +1,458 @@ + + + + + +Haiti Anger Index — POLLNEX Insights + + + + + +
+ +
+ Haiti Anger Index™
+ June 16, 2026
+ Period: 2026-06-09 – 2026-06-16 +
+
+ +
+ + +
+

🇭🇹 Haiti Anger Index (HAI™)

+

+ A real-time composite measure of public sentiment across Haitian social media platforms + in English, French, and Haitian Creole. Powered by POLLNEX Insights. +

+
+ + +
+ +
25.9
+
Concerned
+
+ + → Stable vs previous period + +
+

+ Based on 80 posts analysed | + 30.0% negative | + 2.5% positive | + 67.5% neutral +

+
+ + +
+
80
Total Posts
+
24
Negative
+
2
Positive
+
54
Neutral
+
25.9/100
HAI Score
+
+ + +
+ + +
+

🔥 Top Anger Drivers

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TopicNeg. RateAnger Score
Public Safety & Crime56.5% +
+
+
+ 79.1 +
Education60.0% +
+
+
+ 54.0 +
Health & Healthcare42.9% +
+
+
+ 47.1 +
Government & Corruption26.7% +
+
+
+ 40.0 +
Natural Disasters33.3% +
+
+
+ 40.0 +
+
+ + +
+

📱 Platform Breakdown

+ +
+ +
+ + +
+ + +
+

🗣️ Language Breakdown

+ +
+ + +
+

📊 Topic Detail

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TopicPostsNeg%Weighted
Government & Corruption1526.7%40.0
International Relations1225.0%22.5
Human Rights & Diaspora1118.2%23.6
Public Safety & Crime2356.5%79.1
Economy & Poverty1225.0%32.5
Infrastructure90.0%0.0
Health & Healthcare742.9%47.1
General119.1%9.1
Natural Disasters333.3%40.0
Education560.0%54.0
+
+ +
+ + + +
+

📈 Historical HAI Trend

+ +
+ + + +
+

ℹ️ Methodology

+

+ The Haiti Anger Index (HAI™) is computed by POLLNEX Insights as a composite of three weighted components: + (1) raw negative-sentiment rate (40%), (2) engagement-weighted negative rate (40%), and (3) a topic-severity + adjustment that amplifies issues such as kidnapping, corruption, and economic collapse (20%). + Sentiment is classified using a multilingual XLM-RoBERTa model trained on social-media data and validated + for English, French, and Haitian Creole. The index is scaled 0–100 where 0–25 = Calm, 26–50 = Concerned, + 51–75 = Agitated, 76–100 = Enraged. Data is sourced from Twitter/X, Facebook, YouTube, Reddit, and + major Haitian news RSS feeds. +

+
+ +
+ + + + + + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ba3f1c7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# Core runtime +requests>=2.31.0 +feedparser>=6.0.11 +langdetect>=1.0.9 +jinja2>=3.1.4 + +# Sentiment model (optional — falls back to keyword classifier if absent) +torch>=2.2.0 +transformers>=4.40.0 + +# Social media APIs (all optional — crawler skips gracefully if absent) +tweepy>=4.14.0 # Twitter / X +praw>=7.7.1 # Reddit +google-api-python-client>=2.120.0 # YouTube