chore: migrate tooling to Ruff, expand CI, add Makefile and IaC workf…

.claude/CLAUDE.md

@@ -0,0 +1,34 @@
+# Torchstack Project Standards
+
+## Project Context
+This is a Torchstack client project. We build backend services, AI/ML pipelines, and cloud infrastructure.
+
+## Code Standards
+- Python 3.12+, type-annotated, Ruff-formatted (line length 100)
+- All functions must have docstrings
+- Tests required for any new feature (pytest, 80% coverage minimum)
+- Infrastructure changes require Terraform plan review before merge
+
+## Architecture Patterns
+- FastAPI for HTTP services, Pydantic v2 for data validation
+- Async-first where I/O-bound
+- 12-factor app configuration (env vars, no hardcoded secrets)
+- PostgreSQL via psycopg for data persistence
+
+## AI Assistant Guidelines
+- Always include type hints on all function signatures
+- When generating Terraform, follow the module structure in `infra/`
+- When generating tests, use pytest fixtures and parametrize for edge cases
+- Always run `ruff check` and `mypy` before suggesting code is complete
+- Flag any hardcoded credentials or secrets immediately
+
+## Git Workflow
+- Branch from `develop`, PR into `develop`, release merges to `main`
+- Commit messages: conventional commits (feat:, fix:, chore:, docs:)
+- PRs require passing CI + 1 approval
+
+## Directory Layout
+- `app/` — FastAPI application (routes, middleware, main entrypoint)
+- `src/` — Shared library code (business logic, utilities)
+- `tests/` — All test files
+- `infra/` — Terraform infrastructure (when applicable)

.flake8

@@ -1,6 +1,4 @@
-[flake8]
-max-line-length = 88
-extend-ignore = E203
-exclude = .git,__pycache__,build,dist
-per-file-ignores =
-    __init__.py:F401 
+# DEPRECATED: This file is no longer used.
+# Flake8 has been replaced by Ruff, which is configured in pyproject.toml.
+# This file is kept temporarily to avoid breaking any local tooling.
+# Safe to delete: rm .flake8

.github/workflows/ci.yml

@@ -1,38 +1,210 @@
-name: CI
+name: CI Pipeline
 
 on:
   push:
-    branches: [ main ]
+    branches: [main, develop]
   pull_request:
-    branches: [ main ]
+    branches: [main, develop]
+
+permissions:
+  contents: read
+  pull-requests: write
+  security-events: write
+
+env:
+  PYTHON_VERSION: "3.12"
+  TERRAFORM_VERSION: "1.7.0"
 
 jobs:
+  # ──────────────────────────────────────────────
+  # 1. CODE QUALITY GATES
+  # ──────────────────────────────────────────────
+  lint-and-format:
+    name: Lint & Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          pip install ruff mypy types-requests types-pyyaml
+
+      - name: Run Ruff (linter + formatter check)
+        run: |
+          ruff check . --output-format=github
+          ruff format --check .
+
+      - name: Run mypy (type checking)
+        run: mypy src/ app/ --ignore-missing-imports
+
+  security-scan:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Run Bandit (Python security)
+        run: |
+          pip install bandit[toml]
+          bandit -r src/ app/ -c pyproject.toml -f json -o bandit-report.json || true
+          bandit -r src/ app/ -c pyproject.toml
+
+      - name: Run Semgrep
+        uses: semgrep/semgrep-action@v1
+        with:
+          config: >-
+            p/python
+            p/security-audit
+            p/secrets
+        env:
+          SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+
+      - name: Dependency vulnerability check
+        run: |
+          pip install pip-audit
+          pip-audit -r requirements-dev.txt --output json || true
+
+  # ──────────────────────────────────────────────
+  # 2. TESTING WITH COVERAGE GATE
+  # ──────────────────────────────────────────────
   test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run pytest with coverage
+        run: |
+          poetry run pytest tests/ \
+            --cov=src --cov=app \
+            --cov-report=xml:coverage.xml \
+            --cov-report=term-missing \
+            --cov-fail-under=80 \
+            -v
+
+      - name: Upload coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.xml
+
+  # ──────────────────────────────────────────────
+  # 3. INFRASTRUCTURE-AS-CODE VALIDATION
+  # ──────────────────────────────────────────────
+  terraform-validate:
+    name: Terraform Validate & Security
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.8, 3.9, "3.10"]
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'infra') ||
+      github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for infra directory
+        id: check_infra
+        run: |
+          if [ -d "infra" ]; then
+            echo "has_infra=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_infra=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Setup Terraform
+        if: steps.check_infra.outputs.has_infra == 'true'
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ env.TERRAFORM_VERSION }}
 
+      - name: Terraform Format Check
+        if: steps.check_infra.outputs.has_infra == 'true'
+        run: terraform fmt -check -recursive infra/
+
+      - name: Terraform Init & Validate
+        if: steps.check_infra.outputs.has_infra == 'true'
+        run: |
+          cd infra/
+          terraform init -backend=false
+          terraform validate
+
+      - name: Run Checkov (policy-as-code)
+        if: steps.check_infra.outputs.has_infra == 'true'
+        uses: bridgecrewio/checkov-action@v12
+        with:
+          directory: infra/
+          framework: terraform
+          output_format: cli,sarif
+          output_file_path: console,checkov-results.sarif
+          soft_fail: true
+
+      - name: Run tfsec
+        if: steps.check_infra.outputs.has_infra == 'true'
+        uses: aquasecurity/tfsec-action@v1.0.3
+        with:
+          working_directory: infra/
+          soft_fail: true
+
+      - name: Infracost cost estimation
+        if: github.event_name == 'pull_request' && steps.check_infra.outputs.has_infra == 'true'
+        uses: infracost/actions/setup@v3
+        with:
+          api-key: ${{ secrets.INFRACOST_API_KEY }}
+
+      - name: Generate Infracost diff
+        if: github.event_name == 'pull_request' && steps.check_infra.outputs.has_infra == 'true'
+        run: |
+          infracost diff \
+            --path=infra/ \
+            --format=json \
+            --out-file=/tmp/infracost.json
+
+      - name: Post Infracost comment
+        if: github.event_name == 'pull_request' && steps.check_infra.outputs.has_infra == 'true'
+        run: |
+          infracost comment github \
+            --path=/tmp/infracost.json \
+            --repo=$GITHUB_REPOSITORY \
+            --github-token=${{ github.token }} \
+            --pull-request=${{ github.event.pull_request.number }} \
+            --behavior=update
+
+  # ──────────────────────────────────────────────
+  # 4. AI-POWERED PR REVIEW (optional)
+  # ──────────────────────────────────────────────
+  ai-review:
+    name: AI Code Review
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements-dev.txt
-    - name: Lint with flake8
-      run: |
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Format check with black
-      run: |
-        black . --check
-    - name: Import sort check with isort
-      run: |
-        isort . --check-only
-    - name: Test with pytest
-      run: |
-        pytest 
+      - uses: actions/checkout@v4
+
+      - name: CodeRabbit AI Review
+        uses: coderabbitai/ai-pr-reviewer@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        with:
+          debug: false
+          review_simple_changes: false
+          review_comment_lgtm: false

.github/workflows/iac-plan.yml

@@ -0,0 +1,100 @@
+name: Terraform Plan on PR
+
+on:
+  pull_request:
+    paths:
+      - 'infra/**'
+      - '.github/workflows/iac-plan.yml'
+
+permissions:
+  contents: read
+  pull-requests: write
+  id-token: write  # For OIDC auth to cloud providers
+
+env:
+  TERRAFORM_VERSION: "1.7.0"
+  TF_WORKING_DIR: "infra"
+
+jobs:
+  plan:
+    name: Terraform Plan
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for infra directory
+        id: check_infra
+        run: |
+          if [ -d "infra" ]; then
+            echo "has_infra=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_infra=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Setup Terraform
+        if: steps.check_infra.outputs.has_infra == 'true'
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ env.TERRAFORM_VERSION }}
+
+      # ── Authenticate to your cloud provider ──
+      # Uncomment the section for your provider:
+
+      # AWS (OIDC - recommended)
+      # - name: Configure AWS credentials
+      #   uses: aws-actions/configure-aws-credentials@v4
+      #   with:
+      #     role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+      #     aws-region: us-east-1
+
+      # GCP (OIDC - recommended)
+      # - name: Authenticate to GCP
+      #   uses: google-github-actions/auth@v2
+      #   with:
+      #     workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
+      #     service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: Terraform Init
+        if: steps.check_infra.outputs.has_infra == 'true'
+        working-directory: ${{ env.TF_WORKING_DIR }}
+        run: terraform init
+
+      - name: Terraform Plan
+        if: steps.check_infra.outputs.has_infra == 'true'
+        id: plan
+        working-directory: ${{ env.TF_WORKING_DIR }}
+        run: |
+          terraform plan -no-color -out=tfplan 2>&1 | tee plan-output.txt
+        continue-on-error: true
+
+      - name: Post plan to PR
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request' && steps.check_infra.outputs.has_infra == 'true'
+        with:
+          script: |
+            const fs = require('fs');
+            const plan = fs.readFileSync('${{ env.TF_WORKING_DIR }}/plan-output.txt', 'utf8');
+            const truncated = plan.length > 60000
+              ? plan.substring(0, 60000) + '\n\n... (truncated)'
+              : plan;
+
+            const body = `## Terraform Plan Output
+            \`\`\`hcl
+            ${truncated}
+            \`\`\`
+
+            **Plan exit code:** \`${{ steps.plan.outcome }}\`
+
+            > Review the plan carefully before approving.`;
+
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: body
+            });
+
+      - name: Fail on plan error
+        if: steps.check_infra.outputs.has_infra == 'true' && steps.plan.outcome == 'failure'
+        run: exit 1