From e8235121091f806aae3134ecbcee6afec9749d52 Mon Sep 17 00:00:00 2001 From: Sai Krishna Vennamaneni Date: Wed, 1 Apr 2026 02:38:04 +0530 Subject: [PATCH] ci: add test job, SSH-based deploy, health checks, and CI/CD docs - Add test job with Postgres 16, ruff lint, and pytest - Replace VM stop/start with SSH-based deploy (no downtime) - Add post-deploy health checks - Split staging/production secrets (zone, VM name) - Add CI_CD.md documenting pipeline, secrets, and rollback --- .github/workflows/deploy.yml | 139 ++++++++++++++++++++++++++--------- CI_CD.md | 97 ++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 33 deletions(-) create mode 100644 CI_CD.md diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0679d49..a131121 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,54 +1,127 @@ -name: Deploy Python App to GCP E2 (Staging) +name: CI/CD Pipeline on: push: branches: - main - - staging # change if you want a different branch for staging + - staging jobs: - deploy-staging: - if: github.ref != 'refs/heads/main' + test: runs-on: ubuntu-latest - environment: deploy-staging # matches the environment you created in GitHub + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: test_chatdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + cd code + pip install -r requirements.txt + - name: Lint + run: | + cd code + ruff check . + - name: Run tests + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/" + GROQ_API_KEY: "test-key" + run: | + cd code + pytest test/ -v + deploy-staging: + needs: test + if: github.ref == 'refs/heads/staging' + runs-on: ubuntu-latest + environment: staging steps: - - uses: 'actions/checkout@v4' # Checkout your repository - - uses: 'google-github-actions/auth@v2' # Authenticate to Google Cloud + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v2 with: - project_id: ${{ secrets.GCP_PROJECT }} credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Update startup script and restart staging VM + - uses: google-github-actions/setup-gcloud@v2 + - name: Set branch metadata run: | - gcloud compute instances add-metadata ${{secrets.GCP_VM_INSTANCE_NAME_STAGING}} \ - --zone=${{secrets.GCP_ZONE}} \ - --project=${{secrets.GCP_PROJECT}} \ - --metadata-from-file=startup-script=./scripts/update_app.sh \ - --metadata=BRANCH_NAME=${GITHUB_REF#refs/heads/} - - gcloud compute instances stop ${{secrets.GCP_VM_INSTANCE_NAME_STAGING }} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}} - gcloud compute instances start ${{secrets.GCP_VM_INSTANCE_NAME_STAGING }} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}} + gcloud compute instances add-metadata ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \ + --zone=${{ secrets.GCP_ZONE_STAGING }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --metadata=BRANCH_NAME=staging + - name: Deploy to staging VM + run: | + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \ + --zone=${{ secrets.GCP_ZONE_STAGING }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="sudo bash -c ' + cd /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate && + git config --global --add safe.directory /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate && + git fetch --all && + git reset --hard origin/staging && + git clean -fd && + chmod +x scripts/deploy.sh && + ./scripts/deploy.sh + '" + - name: Health check + run: | + sleep 10 + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \ + --zone=${{ secrets.GCP_ZONE_STAGING }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="curl -sf http://localhost:5000/health || exit 1" deploy-production: + needs: test if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest - environment: deploy-staging # matches the environment you created in GitHub - + environment: production steps: - - uses: 'actions/checkout@v4' # Checkout your repository - - uses: 'google-github-actions/auth@v2' # Authenticate to Google Cloud + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v2 with: - project_id: ${{ secrets.GCP_PROJECT }} # Replace with your actual GCP project ID credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Update startup script and restart production VM + - uses: google-github-actions/setup-gcloud@v2 + - name: Set branch metadata run: | - gcloud compute instances add-metadata ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} \ - --zone=${{secrets.GCP_ZONE}} \ - --project=${{secrets.GCP_PROJECT}} \ - --metadata-from-file=startup-script=./scripts/update_app.sh \ - --metadata=BRANCH_NAME=${GITHUB_REF#refs/heads/} - - gcloud compute instances stop ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}} - gcloud compute instances start ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}} \ No newline at end of file + gcloud compute instances add-metadata ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \ + --zone=${{ secrets.GCP_ZONE_PROD }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --metadata=BRANCH_NAME=main + - name: Deploy to production VM + run: | + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \ + --zone=${{ secrets.GCP_ZONE_PROD }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="sudo bash -c ' + cd /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate && + git config --global --add safe.directory /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate && + git fetch --all && + git reset --hard origin/main && + git clean -fd && + chmod +x scripts/deploy.sh && + ./scripts/deploy.sh + '" + - name: Health check + run: | + sleep 10 + gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \ + --zone=${{ secrets.GCP_ZONE_PROD }} \ + --project=${{ secrets.GCP_PROJECT }} \ + --ssh-flag="-o StrictHostKeyChecking=no" \ + --command="curl -sf http://localhost:5000/health || exit 1" \ No newline at end of file diff --git a/CI_CD.md b/CI_CD.md new file mode 100644 index 0000000..32d82d7 --- /dev/null +++ b/CI_CD.md @@ -0,0 +1,97 @@ +# CI/CD Pipeline + +## Overview + +Automated test → deploy pipeline via GitHub Actions. Pushes to `staging` deploy to the shared staging VM. Pushes to `main` deploy to the dedicated production VM. + +## Architecture + +``` +Push to staging ──▶ test ──▶ SSH deploy to staging-instance (port 5000) +Push to main ──▶ test ──▶ SSH deploy to ai-agent-prod (port 5000) +``` + +## Pipeline Stages + +### 1. Test (`test` job) + +Runs on every push to `main` or `staging`. + +- **Service container**: PostgreSQL 16 on `localhost:5432` (DB: `test_chatdb`) +- **Python**: 3.12 +- **Lint**: `ruff check .` (inside `code/`) +- **Tests**: `pytest test/ -v` (inside `code/`) + +### 2. Deploy Staging (`deploy-staging` job) + +Triggers only on pushes to `staging`. Requires the `test` job to pass. + +1. Authenticates to GCP using the service account key +2. Sets `BRANCH_NAME=staging` in VM instance metadata (used by `config.py` → `get_db_name()` to select `staging_chat_db`) +3. SSHs into the staging VM and runs: + - `git fetch --all && git reset --hard origin/staging` + - `scripts/deploy.sh` (activates venv, installs deps, generates `.env`, restarts Flask on port 5000) +4. Verifies `GET /health` returns 200 + +### 3. Deploy Production (`deploy-production` job) + +Triggers only on pushes to `main`. Requires the `test` job to pass. + +Same as staging but targets `ai-agent-prod` VM and sets `BRANCH_NAME=main` (selects `prod_chat_db`). + +## GitHub Secrets Required + +Configure these in **repo Settings → Secrets and variables → Actions**: + +| Secret | Environment | Value | +|--------|-------------|-------| +| `GCP_SA_KEY` | (repo-level) | Service account JSON key with `compute.instanceAdmin.v1` role | +| `GCP_PROJECT` | (repo-level) | `ai-agent-boilerplate0` | +| `GCP_VM_INSTANCE_NAME_STAGING` | staging | `staging-instance` | +| `GCP_ZONE_STAGING` | staging | `us-central1-f` | +| `GCP_VM_INSTANCE_NAME_PROD` | production | `ai-agent-prod` | +| `GCP_ZONE_PROD` | production | VM zone for production | + +## GitHub Environments + +Create two environments in **repo Settings → Environments**: + +- **staging** — no protection rules needed +- **production** — recommended: add required reviewers for manual approval gate before prod deploys + +## VM Prerequisites + +Each VM must have: + +1. Repo cloned at `/home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate` +2. Python 3.12 virtualenv at `venv/` +3. SSH key added to GitHub for `git fetch` to work +4. `curl` installed (for health checks) + +## Deploy Script + +The existing `scripts/deploy.sh` handles the on-VM deployment: + +1. Activates virtualenv +2. `pip install -r requirements.txt` +3. Generates `.env` via `python3 get_env.py` (pulls secrets from GCP Secret Manager) +4. Kills any existing Flask process +5. Starts Flask on `0.0.0.0:5000` via `nohup` + +## Rollback + +To rollback, push the previous good commit to the target branch: + +```bash +git revert HEAD && git push origin main +``` + +Or manually SSH into the VM and reset: + +```bash +gcloud compute ssh VM_NAME --zone=ZONE --command=" + cd /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate && + sudo git reset --hard GOOD_COMMIT_SHA && + sudo ./scripts/deploy.sh +" +```