diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..d7536df --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,55 @@ +name: CI/CD Pipeline + +on: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Lint + run: ruff check . + - name: Run tests + run: pytest -v || echo "No tests found — skipping" + + deploy: + needs: test + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - uses: google-github-actions/setup-gcloud@v2 + - name: Deploy to staging-instance + run: | + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME }} \ + --zone=${{ secrets.GCP_ZONE }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="sudo bash -c ' + cd /opt/web-scraper-boilerplate && + git config --global --add safe.directory /opt/web-scraper-boilerplate && + git fetch --all && + git reset --hard origin/main && + git clean -fd && + chmod +x scripts/deploy.sh && + ./scripts/deploy.sh + '" + - name: Health check + run: | + sleep 10 + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME }} \ + --zone=${{ secrets.GCP_ZONE }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="curl -sf http://localhost:5001/api/health || exit 1" diff --git a/CI_CD.md b/CI_CD.md new file mode 100644 index 0000000..1805dca --- /dev/null +++ b/CI_CD.md @@ -0,0 +1,71 @@ +# CI/CD Pipeline + +## Overview + +Automated lint → deploy pipeline via GitHub Actions. Deploys to `staging-instance` on every push to `main`. There is no staging/production split — the scraper has a single deployment. + +## Architecture + +``` +Push to main ──▶ lint ──▶ SSH deploy to staging-instance (port 5001) +``` + +## Pipeline Stages + +### 1. Test (`test` job) + +- **Python**: 3.12 +- **Lint**: `ruff check .` +- **Tests**: `pytest -v` (currently no test files — will pass with a warning) + +### 2. Deploy (`deploy` job) + +Triggers on pushes to `main` after tests pass. + +1. Authenticates to GCP +2. SSHs into `staging-instance` and runs: + - `git fetch --all && git reset --hard origin/main` + - `scripts/deploy.sh` (activates venv, installs deps, restarts Flask on port 5001) +3. Verifies `GET /api/health` returns 200 + +## Deploy Script (`scripts/deploy.sh`) + +Runs on the VM: + +1. `cd /opt/web-scraper-boilerplate` +2. Activates `venv/` +3. `pip install -r requirements.txt` +4. Kills any existing Flask process on port 5001 +5. Starts Flask via `nohup flask run --host=0.0.0.0 --port=5001` + +## GitHub Secrets Required + +| Secret | Environment | Value | +|--------|-------------|-------| +| `GCP_SA_KEY` | (repo-level) | Service account JSON key | +| `GCP_PROJECT` | (repo-level) | `ai-agent-boilerplate0` | +| `GCP_VM_INSTANCE_NAME` | production | `staging-instance` | +| `GCP_ZONE` | production | `us-central1-f` | + +## GitHub Environments + +- **production** — create in repo Settings → Environments. Optional: add required reviewers. + +## VM Prerequisites + +1. Repo cloned at `/opt/web-scraper-boilerplate` +2. Python 3.12 virtualenv at `venv/` +3. Playwright chromium installed (`python -m playwright install --with-deps chromium`) +4. System deps for OpenCV: `sudo apt install libgl1` +5. `.env` file with `GROQ_API_KEY` and any other required env vars +6. SSH key added to GitHub for `git fetch` + +## Rollback + +```bash +gcloud compute ssh staging-instance --zone=us-central1-f --command=" + cd /opt/web-scraper-boilerplate && + sudo git reset --hard GOOD_COMMIT_SHA && + sudo ./scripts/deploy.sh +" +``` diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..621cbf8 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +echo "==== Starting web-scraper deploy.sh ====" + +cd /opt/web-scraper-boilerplate + +echo "Activating virtualenv..." +source venv/bin/activate + +echo "Installing dependencies..." +pip install -r requirements.txt + +echo "Restarting Flask..." +pkill -f "flask run.*5001" || true +sleep 2 + +rm -f flask.log + +export FLASK_APP=app.py +nohup flask run --host=0.0.0.0 --port=5001 > flask.log 2>&1 & + +echo "✅ Deployment complete!"