Watchdog #395
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Watchdog | |
| on: | |
| schedule: | |
| - cron: "*/10 * * * *" | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ${{ vars.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} | |
| WATCHDOG_MAX_AGE_SECONDS: ${{ vars.WATCHDOG_MAX_AGE_SECONDS || '4500' }} | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Validate deployment identity configuration | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for name in GCP_PROJECT_ID GCP_WORKLOAD_IDENTITY_PROVIDER GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT; do | |
| if [ -z "${!name:-}" ]; then | |
| echo "::error::Required repository variable ${name} is not configured." | |
| exit 1 | |
| fi | |
| done | |
| readonly EXPECTED_OIDC_IDENTITY_SHA256="68e87a5dc1bbe2e41af33d526514034246c6487fb7b716596cc75fe1c739a6b9" | |
| actual_oidc_identity_sha256="$( | |
| printf '%s\0%s\0%s' \ | |
| "$GCP_PROJECT_ID" \ | |
| "$GCP_WORKLOAD_IDENTITY_PROVIDER" \ | |
| "$GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT" | | |
| sha256sum | | |
| awk '{print $1}' | |
| )" | |
| if [ "$actual_oidc_identity_sha256" != "$EXPECTED_OIDC_IDENTITY_SHA256" ]; then | |
| echo "::error::Repository OIDC identity variables do not match the reviewed identity contract." | |
| exit 1 | |
| fi | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} | |
| create_credentials_file: true | |
| export_environment_variables: true | |
| cleanup_credentials: true | |
| - name: Install deps | |
| run: | | |
| set -euo pipefail | |
| python -m pip install --upgrade pip uv | |
| uv sync --frozen --no-dev | |
| - name: Run watchdog | |
| id: check | |
| run: | | |
| uv run --no-sync python - <<'PY' | |
| import os | |
| from quant_platform_kit.common.health import HealthMonitor, is_heartbeat_fresh | |
| max_age_seconds = int(os.environ.get("WATCHDOG_MAX_AGE_SECONDS", "4500")) | |
| heartbeat = HealthMonitor().read() | |
| alive = is_heartbeat_fresh(heartbeat, max_age_seconds) | |
| if heartbeat is None: | |
| detail = "missing heartbeat" | |
| else: | |
| detail = ( | |
| f"status={heartbeat.status} " | |
| f"timestamp={heartbeat.timestamp or '<missing>'} " | |
| f"cycle_count={heartbeat.cycle_count} " | |
| f"last_error={heartbeat.last_error or '<none>'} " | |
| f"max_age_seconds={max_age_seconds}" | |
| ) | |
| print(f"{'OK' if alive else 'FAIL'} {detail}") | |
| raise SystemExit(0 if alive else 1) | |
| PY |