ci: add CI/CD pipeline with tests, SSH deploy, and health checks

.github/workflows/deploy.yml

@@ -1,54 +1,127 @@
-name: Deploy Python App to GCP E2 (Staging)
+name: CI/CD Pipeline
 
 on:
   push:
     branches:
       - main
-      - staging   # change if you want a different branch for staging
+      - staging
 
 jobs:
-  deploy-staging:
-    if: github.ref != 'refs/heads/main'
+  test:
     runs-on: ubuntu-latest
-    environment: deploy-staging   # matches the environment you created in GitHub
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_chatdb
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: |
+          cd code
+          pip install -r requirements.txt
+      - name: Lint
+        run: |
+          cd code
+          ruff check .
+      - name: Run tests
+        env:
+          DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/"
+          GROQ_API_KEY: "test-key"
+        run: |
+          cd code
+          pytest test/ -v
 
+  deploy-staging:
+    needs: test
+    if: github.ref == 'refs/heads/staging'
+    runs-on: ubuntu-latest
+    environment: staging
     steps:
-      - uses: 'actions/checkout@v4' # Checkout your repository
-      - uses: 'google-github-actions/auth@v2' # Authenticate to Google Cloud
+      - uses: actions/checkout@v4
+      - uses: google-github-actions/auth@v2
         with:
-          project_id: ${{ secrets.GCP_PROJECT }} 
           credentials_json: ${{ secrets.GCP_SA_KEY }}
-
-      - name: Update startup script and restart staging VM
+      - uses: google-github-actions/setup-gcloud@v2
+      - name: Set branch metadata
         run: |
-          gcloud compute instances add-metadata ${{secrets.GCP_VM_INSTANCE_NAME_STAGING}} \
-            --zone=${{secrets.GCP_ZONE_STAGING}} \
-            --project=${{secrets.GCP_PROJECT}} \
-            --metadata-from-file=startup-script=./scripts/update_app.sh \
-            --metadata=BRANCH_NAME=${GITHUB_REF#refs/heads/}
-
-          gcloud compute instances stop ${{secrets.GCP_VM_INSTANCE_NAME_STAGING }} --zone=${{secrets.GCP_ZONE_STAGING}} --project=${{secrets.GCP_PROJECT}}
-          gcloud compute instances start ${{secrets.GCP_VM_INSTANCE_NAME_STAGING }} --zone=${{secrets.GCP_ZONE_STAGING}} --project=${{secrets.GCP_PROJECT}}
+          gcloud compute instances add-metadata ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \
+            --zone=${{ secrets.GCP_ZONE_STAGING }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --metadata=BRANCH_NAME=staging
+      - name: Deploy to staging VM
+        run: |
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \
+            --zone=${{ secrets.GCP_ZONE_STAGING }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="sudo bash /opt/ai-agent-boilerplate/scripts/update_app.sh"
+      - name: Show deploy logs
+        if: always()
+        run: |
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \
+            --zone=${{ secrets.GCP_ZONE_STAGING }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="sudo cat /var/log/update_app.log || echo 'No log file found'"
+      - name: Health check
+        run: |
+          sleep 10
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_STAGING }} \
+            --zone=${{ secrets.GCP_ZONE_STAGING }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="curl -sf http://localhost:5000/health || exit 1"
 
   deploy-production:
+    needs: test
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
-    environment: deploy-staging  # matches the environment you created in GitHub
-
+    environment: production
     steps:
-      - uses: 'actions/checkout@v4' # Checkout your repository
-      - uses: 'google-github-actions/auth@v2' # Authenticate to Google Cloud
+      - uses: actions/checkout@v4
+      - uses: google-github-actions/auth@v2
         with:
-          project_id: ${{ secrets.GCP_PROJECT }} # Replace with your actual GCP project ID
           credentials_json: ${{ secrets.GCP_SA_KEY }}
-
-      - name: Update startup script and restart production VM
+      - uses: google-github-actions/setup-gcloud@v2
+      - name: Set branch metadata
         run: |
-          gcloud compute instances add-metadata ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} \
-            --zone=${{secrets.GCP_ZONE}} \
-            --project=${{secrets.GCP_PROJECT}} \
-            --metadata-from-file=startup-script=./scripts/update_app.sh \
-            --metadata=BRANCH_NAME=${GITHUB_REF#refs/heads/}
-
-          gcloud compute instances stop ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}}
-          gcloud compute instances start ${{secrets.GCP_VM_INSTANCE_NAME_PROD}} --zone=${{secrets.GCP_ZONE}} --project=${{secrets.GCP_PROJECT}}
+          gcloud compute instances add-metadata ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \
+            --zone=${{ secrets.GCP_ZONE_PROD }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --metadata=BRANCH_NAME=main
+      - name: Deploy to production VM
+        run: |
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \
+            --zone=${{ secrets.GCP_ZONE_PROD }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="sudo bash /opt/ai-agent-boilerplate/scripts/update_app.sh"
+      - name: Show deploy logs
+        if: always()
+        run: |
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \
+            --zone=${{ secrets.GCP_ZONE_PROD }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="sudo cat /var/log/update_app.log || echo 'No log file found'"
+      - name: Health check
+        run: |
+          sleep 10
+          gcloud compute ssh ${{ secrets.GCP_VM_INSTANCE_NAME_PROD }} \
+            --zone=${{ secrets.GCP_ZONE_PROD }} \
+            --project=${{ secrets.GCP_PROJECT }} \
+            --ssh-flag="-o StrictHostKeyChecking=no" \
+            --command="curl -sf http://localhost:5000/health || exit 1"

CI_CD.md

@@ -0,0 +1,97 @@
+# CI/CD Pipeline
+
+## Overview
+
+Automated test → deploy pipeline via GitHub Actions. Pushes to `staging` deploy to the shared staging VM. Pushes to `main` deploy to the dedicated production VM.
+
+## Architecture
+
+```
+Push to staging ──▶ test ──▶ SSH deploy to staging-instance (port 5000)
+Push to main   ──▶ test ──▶ SSH deploy to ai-agent-prod   (port 5000)
+```
+
+## Pipeline Stages
+
+### 1. Test (`test` job)
+
+Runs on every push to `main` or `staging`.
+
+- **Service container**: PostgreSQL 16 on `localhost:5432` (DB: `test_chatdb`)
+- **Python**: 3.12
+- **Lint**: `ruff check .` (inside `code/`)
+- **Tests**: `pytest test/ -v` (inside `code/`)
+
+### 2. Deploy Staging (`deploy-staging` job)
+
+Triggers only on pushes to `staging`. Requires the `test` job to pass.
+
+1. Authenticates to GCP using the service account key
+2. Sets `BRANCH_NAME=staging` in VM instance metadata (used by `config.py` → `get_db_name()` to select `staging_chat_db`)
+3. SSHs into the staging VM and runs:
+   - `git fetch --all && git reset --hard origin/staging`
+   - `scripts/deploy.sh` (activates venv, installs deps, generates `.env`, restarts Flask on port 5000)
+4. Verifies `GET /health` returns 200
+
+### 3. Deploy Production (`deploy-production` job)
+
+Triggers only on pushes to `main`. Requires the `test` job to pass.
+
+Same as staging but targets `ai-agent-prod` VM and sets `BRANCH_NAME=main` (selects `prod_chat_db`).
+
+## GitHub Secrets Required
+
+Configure these in **repo Settings → Secrets and variables → Actions**:
+
+| Secret | Environment | Value |
+|--------|-------------|-------|
+| `GCP_SA_KEY` | (repo-level) | Service account JSON key with `compute.instanceAdmin.v1` role |
+| `GCP_PROJECT` | (repo-level) | `ai-agent-boilerplate0` |
+| `GCP_VM_INSTANCE_NAME_STAGING` | staging | `staging-instance` |
+| `GCP_ZONE_STAGING` | staging | `us-central1-f` |
+| `GCP_VM_INSTANCE_NAME_PROD` | production | `ai-agent-prod` |
+| `GCP_ZONE_PROD` | production | VM zone for production |
+
+## GitHub Environments
+
+Create two environments in **repo Settings → Environments**:
+
+- **staging** — no protection rules needed
+- **production** — recommended: add required reviewers for manual approval gate before prod deploys
+
+## VM Prerequisites
+
+Each VM must have:
+
+1. Repo cloned at `/home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate`
+2. Python 3.12 virtualenv at `venv/`
+3. SSH key added to GitHub for `git fetch` to work
+4. `curl` installed (for health checks)
+
+## Deploy Script
+
+The existing `scripts/deploy.sh` handles the on-VM deployment:
+
+1. Activates virtualenv
+2. `pip install -r requirements.txt`
+3. Generates `.env` via `python3 get_env.py` (pulls secrets from GCP Secret Manager)
+4. Kills any existing Flask process
+5. Starts Flask on `0.0.0.0:5000` via `nohup`
+
+## Rollback
+
+To rollback, push the previous good commit to the target branch:
+
+```bash
+git revert HEAD && git push origin main
+```
+
+Or manually SSH into the VM and reset:
+
+```bash
+gcloud compute ssh VM_NAME --zone=ZONE --command="
+  cd /home/vivek/Ai-agent-boilerplate/ai-agent-boilerplate &&
+  sudo git reset --hard GOOD_COMMIT_SHA &&
+  sudo ./scripts/deploy.sh
+"
+```

scripts/deploy.sh

@@ -28,11 +28,15 @@ python3 get_env.py
 echo "Restarting service..."
 
 echo "Stopping only ai-agent service..."
-pkill -f "/opt/ai-agent-boilerplate"
+if [ -f zero.pid ]; then
+    kill $(cat zero.pid) || true
+    rm zero.pid
+fi
 
+echo "Starting Flask app..."
 cd /opt/ai-agent-boilerplate/code
 export FLASK_APP=app.py
 nohup flask run --host=0.0.0.0 --port=5000 > flask.log 2>&1 &
-
+echo $! > zero.pid
 
 echo "✅ Deployment complete!"