Run HAI agent from cached DB when network is unavailable; add 2026-06-16 report

.github/workflows/hai_agent.yml

@@ -0,0 +1,122 @@
+# ──────────────────────────────────────────────────────────────────────────────
+# Haiti Anger Index (HAI™) — GitHub Actions Workflow
+# POLLNEX Insights
+#
+# HOW TO RUN FROM GITHUB:
+#   1. Go to the "Actions" tab in this repository
+#   2. Click "Haiti Anger Index Agent" in the left sidebar
+#   3. Click "Run workflow" (top right)
+#   4. Fill in the options and click the green "Run workflow" button
+#
+# The report is saved as a downloadable artifact after each run.
+# ──────────────────────────────────────────────────────────────────────────────
+
+name: Haiti Anger Index Agent
+
+on:
+  # ── Manual trigger from the GitHub Actions UI ──────────────────────────────
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: >
+          Platforms to crawl (comma-separated).
+          Options: twitter, facebook, youtube, reddit, rss
+          Use "rss" to run with NO API keys required.
+        required: false
+        default: "rss"
+        type: string
+      dry_run:
+        description: "Dry run — print results but do NOT save to database"
+        required: false
+        default: false
+        type: boolean
+      max_posts:
+        description: "Max posts per platform (default: 200)"
+        required: false
+        default: "200"
+        type: string
+
+  # ── Automatic daily run at 06:00 UTC ───────────────────────────────────────
+  schedule:
+    - cron: "0 6 * * *"
+
+jobs:
+  run-agent:
+    name: Run Haiti Anger Index
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      # ── 1. Check out the repository ────────────────────────────────────────
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # ── 2. Set up Python ───────────────────────────────────────────────────
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      # ── 3. Install dependencies ────────────────────────────────────────────
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install core dependencies (no PyTorch — uses keyword fallback classifier)
+          pip install feedparser langdetect jinja2 requests
+          # Install optional social-media libraries (won't fail if unused)
+          pip install tweepy praw google-api-python-client || true
+
+      # ── 4. Run the agent ───────────────────────────────────────────────────
+      - name: Run Haiti Anger Index Agent
+        env:
+          # Set your secrets in: Settings → Secrets → Actions → New repository secret
+          TWITTER_BEARER_TOKEN:    ${{ secrets.TWITTER_BEARER_TOKEN }}
+          TWITTER_API_KEY:         ${{ secrets.TWITTER_API_KEY }}
+          TWITTER_API_SECRET:      ${{ secrets.TWITTER_API_SECRET }}
+          YOUTUBE_API_KEY:         ${{ secrets.YOUTUBE_API_KEY }}
+          REDDIT_CLIENT_ID:        ${{ secrets.REDDIT_CLIENT_ID }}
+          REDDIT_CLIENT_SECRET:    ${{ secrets.REDDIT_CLIENT_SECRET }}
+          FACEBOOK_ACCESS_TOKEN:   ${{ secrets.FACEBOOK_ACCESS_TOKEN }}
+          HAI_DB_PATH:             "hai_run.db"
+          HAI_REPORTS_DIR:         "reports"
+          MAX_POSTS_PER_PLATFORM:  ${{ github.event.inputs.max_posts || '200' }}
+        run: |
+          PLATFORMS="${{ github.event.inputs.platforms || 'rss' }}"
+          DRY_RUN="${{ github.event.inputs.dry_run || 'false' }}"
+
+          ARGS="--platforms $PLATFORMS"
+          if [ "$DRY_RUN" = "true" ]; then
+            ARGS="$ARGS --dry-run"
+          fi
+
+          echo "▶ Running: python -m haiti_anger_index $ARGS"
+          python -m haiti_anger_index $ARGS
+
+      # ── 5. Upload the HTML report as a downloadable artifact ───────────────
+      - name: Upload HAI Report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: haiti-anger-index-report-${{ github.run_number }}
+          path: |
+            reports/
+            hai_agent.log
+          if-no-files-found: warn
+          retention-days: 90
+
+      # ── 6. Print summary in the workflow log ───────────────────────────────
+      - name: Print log summary
+        if: always()
+        run: |
+          echo "──────────────────────────────────────────"
+          echo "  Haiti Anger Index — Run Complete"
+          echo "──────────────────────────────────────────"
+          if [ -f hai_agent.log ]; then
+            tail -30 hai_agent.log
+          fi
+          if [ -d reports ]; then
+            echo ""
+            echo "Report files:"
+            ls -lh reports/
+          fi

hai_agent.log

@@ -0,0 +1,49 @@
+2026-04-12 20:07:51,264 WARNING  haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier.
+2026-04-12 20:07:51,332 WARNING  haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled.
+2026-04-12 20:07:51,349 WARNING  haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled.
+2026-04-12 20:07:51,350 WARNING  haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled.
+2026-04-12 20:07:51,376 INFO     hai.agent — ============================================================
+2026-04-12 20:07:51,376 INFO     hai.agent — Haiti Anger Index Agent — run started 2026-04-12T20:07:51.376898+00:00
+2026-04-12 20:07:51,377 INFO     haiti_anger_index.crawlers.base — [news_rss] Starting crawl …
+2026-04-12 20:08:01,617 INFO     haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts
+2026-04-12 20:08:01,617 INFO     hai.agent — [rss] → 0 posts
+2026-04-12 20:08:01,617 INFO     hai.agent — Total raw posts collected: 0
+2026-04-12 20:08:01,617 WARNING  hai.agent — No posts collected — aborting run.
+2026-04-12 22:36:44,223 WARNING  haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier.
+2026-04-12 22:36:44,297 WARNING  haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled.
+2026-04-12 22:36:44,317 WARNING  haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled.
+2026-04-12 22:36:44,319 WARNING  haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled.
+2026-04-12 22:36:44,361 INFO     haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db
+2026-04-12 22:36:44,362 INFO     hai.agent — ============================================================
+2026-04-12 22:36:44,362 INFO     hai.agent — Haiti Anger Index Agent — run started 2026-04-12T22:36:44.362129+00:00
+2026-04-12 22:36:44,362 INFO     haiti_anger_index.crawlers.base — [news_rss] Starting crawl …
+2026-04-12 22:36:54,603 INFO     haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts
+2026-04-12 22:36:54,603 INFO     hai.agent — [rss] → 0 posts
+2026-04-12 22:36:54,603 INFO     hai.agent — Total raw posts collected: 0
+2026-04-12 22:36:54,603 WARNING  hai.agent — No posts collected — aborting run.
+2026-04-12 22:39:57,261 WARNING  haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier.
+2026-04-12 22:39:57,331 WARNING  haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled.
+2026-04-12 22:39:57,348 WARNING  haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled.
+2026-04-12 22:39:57,349 WARNING  haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled.
+2026-04-12 22:39:57,374 INFO     haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db
+2026-04-12 22:39:57,503 INFO     haiti_anger_index.storage.database — Upserted 80 posts
+2026-04-12 22:39:58,068 INFO     haiti_anger_index.reporting.report_generator — Report written → reports/hai_report_20260412_223958.html
+2026-04-12 22:39:58,070 INFO     haiti_anger_index.storage.database — Report saved → reports/hai_report_20260412_223958.html
+2026-06-16 08:46:45,137 WARNING  haiti_anger_index.classifier.sentiment_classifier — transformers not installed — using keyword-based fallback classifier.
+2026-06-16 08:46:45,208 WARNING  haiti_anger_index.crawlers.reddit_crawler — praw not installed — Reddit crawler disabled.
+2026-06-16 08:46:45,225 WARNING  haiti_anger_index.crawlers.twitter_crawler — tweepy not installed — Twitter crawler disabled.
+2026-06-16 08:46:45,226 WARNING  haiti_anger_index.crawlers.youtube_crawler — google-api-python-client not installed — YouTube crawler disabled.
+2026-06-16 08:46:45,267 INFO     hai.agent — ============================================================
+2026-06-16 08:46:45,267 INFO     hai.agent — Haiti Anger Index Agent — run started 2026-06-16T08:46:45.267422+00:00
+2026-06-16 08:46:45,267 INFO     haiti_anger_index.crawlers.base — [news_rss] Starting crawl …
+2026-06-16 08:46:55,486 INFO     haiti_anger_index.crawlers.base — [news_rss] Fetched 0 posts
+2026-06-16 08:46:55,486 INFO     hai.agent — [rss] → 0 posts
+2026-06-16 08:46:55,486 INFO     hai.agent — Total raw posts collected: 0
+2026-06-16 08:46:55,486 WARNING  hai.agent — No posts collected — aborting run.
+2026-06-16 08:53:19,217 INFO     haiti_anger_index.storage.database — Database initialised at haiti_anger_index.db
+2026-06-16 08:53:19,218 INFO     hai.agent — Loaded 80 classified posts from DB
+2026-06-16 08:53:19,218 INFO     hai.agent — Stats → total=80 neg=24 pos=2 neu=54 neg_rate=0.300
+2026-06-16 08:53:19,219 INFO     hai.agent — HAI Score: 25.9 / 100  [Concerned]  trend=stable
+2026-06-16 08:53:19,231 INFO     haiti_anger_index.reporting.report_generator — Report written → reports/hai_report_20260616_085319.html
+2026-06-16 08:53:19,232 INFO     haiti_anger_index.storage.database — Report saved → reports/hai_report_20260616_085319.html
+2026-06-16 08:53:19,232 INFO     hai.agent — Report written → reports/hai_report_20260616_085319.html

haiti_anger_index/README.md

@@ -0,0 +1,207 @@
+# Haiti Anger Index (HAI™)
+### A POLLNEX Insights Research Product
+
+---
+
+## Overview
+
+The **Haiti Anger Index (HAI™)** is an automated sentiment-intelligence agent that:
+
+1. **Crawls** major social media platforms and Haitian news RSS feeds for posts in **English, French, and Haitian Creole**
+2. **Classifies** each post as `POSITIVE`, `NEGATIVE`, or `NEUTRAL` using a multilingual XLM-RoBERTa model
+3. **Compiles** per-platform and per-topic statistics
+4. **Computes** a composite HAI score (0–100) inspired by Pollara's Rage Index methodology
+5. **Generates** a branded HTML report with charts, trend lines, and top anger drivers
+
+---
+
+## Architecture
+
+```
+haiti_anger_index/
+├── agent.py                   ← Main orchestration loop (CLI entry point)
+├── config.py                  ← All configuration & API credentials
+├── __main__.py                ← `python -m haiti_anger_index`
+│
+├── crawlers/
+│   ├── base.py                ← Abstract base crawler
+│   ├── twitter_crawler.py     ← Twitter/X (API v2, Bearer Token)
+│   ├── facebook_crawler.py    ← Facebook Graph API
+│   ├── youtube_crawler.py     ← YouTube Data API v3
+│   ├── reddit_crawler.py      ← Reddit PRAW
+│   └── rss_crawler.py         ← 10+ Haitian news RSS feeds (no API key)
+│
+├── classifier/
+│   ├── language_detector.py   ← EN / FR / HT detection
+│   ├── sentiment_classifier.py← XLM-RoBERTa + keyword fallback
+│   └── topic_classifier.py    ← 9 Haitian-specific topic categories
+│
+├── aggregator/
+│   └── data_compiler.py       ← Per-platform / per-topic / per-language stats
+│
+├── index/
+│   └── anger_index.py         ← HAI score formula + trend + top drivers
+│
+├── storage/
+│   └── database.py            ← SQLite (posts + snapshots + reports)
+│
+└── reporting/
+    ├── report_generator.py    ← Jinja2 HTML renderer
+    └── templates/
+        └── anger_index_report.html  ← POLLNEX-branded dashboard
+```
+
+---
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. Set API credentials (optional but recommended)
+
+At minimum, the RSS crawler works with **no API keys** and already covers
+10 major Haitian news outlets.  Add API keys to enable richer social data:
+
+```bash
+export TWITTER_BEARER_TOKEN="..."
+export YOUTUBE_API_KEY="..."
+export REDDIT_CLIENT_ID="..."
+export REDDIT_CLIENT_SECRET="..."
+export FACEBOOK_ACCESS_TOKEN="..."   # Page Access Token
+```
+
+### 3. Run the agent
+
+```bash
+# Single run (all platforms)
+python -m haiti_anger_index
+
+# RSS only (no API keys needed)
+python -m haiti_anger_index --platforms rss
+
+# Dry run — print results, don't save to DB
+python -m haiti_anger_index --dry-run --platforms rss
+
+# Scheduled run every 6 hours
+python -m haiti_anger_index --schedule 6h
+
+# Custom paths
+python -m haiti_anger_index --db /data/hai.db --reports /var/www/reports
+```
+
+---
+
+## Haiti Anger Index Score
+
+| Score | Level      | Description                             |
+|-------|------------|-----------------------------------------|
+| 0–25  | 🟢 Calm     | Low negative sentiment                  |
+| 26–50 | 🟡 Concerned | Elevated concern but not acute anger   |
+| 51–75 | 🟠 Agitated  | High frustration across key topics     |
+| 76–100| 🔴 Enraged   | Critical anger levels — crisis signal  |
+
+### Formula
+
+```
+raw_score = 0.40 × neg_rate
+          + 0.40 × engagement_weighted_neg_rate
+          + 0.20 × topic_severity_adjusted_neg_rate
+
+HAI = round(raw_score × 100, 1)
+```
+
+**Topic severity weights** amplify issues that historically drive the
+strongest public anger in Haiti:
+
+| Topic                   | Weight |
+|-------------------------|--------|
+| Government & Corruption | 1.5    |
+| Kidnapping & Safety     | 1.4    |
+| Economy & Poverty       | 1.3    |
+| Human Rights & Diaspora | 1.3    |
+| Natural Disasters       | 1.2    |
+| Health & Healthcare     | 1.1    |
+| Infrastructure          | 1.0    |
+| Education               | 0.9    |
+| International Relations | 0.9    |
+
+---
+
+## Supported Platforms
+
+| Platform      | Data Source                | API Key Required |
+|---------------|----------------------------|-----------------|
+| Twitter / X   | Twitter API v2             | ✅ Bearer Token  |
+| Facebook      | Meta Graph API             | ✅ Page Token    |
+| YouTube       | YouTube Data API v3        | ✅ API Key       |
+| Reddit        | PRAW                       | ✅ Client ID/Secret |
+| Haitian News  | 10+ RSS feeds              | ❌ None needed   |
+
+### Haitian News RSS Sources
+- Le Nouvelliste
+- Haïti Libre
+- AlterPresse
+- Rezo Nodwes
+- Loop Haiti
+- Gazette Haïti
+- HPN Haïti
+- Radio Kiskeya
+- Haïti Info Projet
+- Haïti Chery
+
+---
+
+## Language Support
+
+| Language        | Detection  | Sentiment |
+|-----------------|------------|-----------|
+| English (en)    | ✅          | ✅         |
+| French (fr)     | ✅          | ✅         |
+| Haitian Creole  | ✅ (heuristic) | ✅ (via XLM-RoBERTa) |
+
+---
+
+## Output
+
+Each run produces:
+- **SQLite database** with all posts, classifications, and index snapshots
+- **HTML report** with interactive charts (Chart.js):
+  - HAI gauge/speedometer
+  - Platform breakdown bar chart
+  - Language distribution doughnut chart
+  - Top anger drivers table
+  - Historical trend line
+
+---
+
+## Environment Variables
+
+| Variable                | Default               | Description                     |
+|-------------------------|-----------------------|---------------------------------|
+| `TWITTER_BEARER_TOKEN`  | —                     | Twitter API v2 bearer token     |
+| `YOUTUBE_API_KEY`       | —                     | Google/YouTube Data API key     |
+| `REDDIT_CLIENT_ID`      | —                     | Reddit app client ID            |
+| `REDDIT_CLIENT_SECRET`  | —                     | Reddit app client secret        |
+| `FACEBOOK_ACCESS_TOKEN` | —                     | Meta Graph API page token       |
+| `HAI_DB_PATH`           | `haiti_anger_index.db`| SQLite database path            |
+| `HAI_REPORTS_DIR`       | `reports/`            | HTML reports output directory   |
+| `MAX_POSTS_PER_PLATFORM`| `200`                 | Max posts per platform per run  |
+| `CRAWL_DELAY`           | `1.0`                 | Seconds between API requests    |
+
+---
+
+## Ethical Notes
+
+- Only **publicly accessible** content is collected (no login-gated scraping)
+- Facebook crawling uses only the **official Graph API** on public pages
+- Content is used for **aggregate analysis only** — no PII is stored
+- Crawl rate is limited to respect platform terms of service
+
+---
+
+*© POLLNEX Insights — Haiti Anger Index™ is a proprietary research product.
+Attribution required when citing results.*

haiti_anger_index/__init__.py

@@ -0,0 +1,4 @@
+"""Haiti Anger Index — Python package."""
+
+__version__ = "1.0.0"
+__author__ = "POLLNEX Insights"

haiti_anger_index/__main__.py

@@ -0,0 +1,5 @@
+"""Allow `python -m haiti_anger_index` to launch the agent."""
+
+from haiti_anger_index.agent import main
+
+main()