Add wide_schema to standard benchmark allowlist

.github/workflows/benchmark-main.yml

@@ -0,0 +1,122 @@
+name: Benchmark DataFusion Main
+
+on:
+  schedule:
+    - cron: '0 */6 * * *'
+  workflow_dispatch:
+    inputs:
+      benchmarks:
+        description: 'Benchmarks to run (space-separated)'
+        required: false
+        default: 'clickbench_1'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  REGION: us-central1
+  REGISTRY: us-central1-docker.pkg.dev/${{ vars.GCP_PROJECT_ID }}/benchmarking
+
+jobs:
+  trigger-benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }}
+
+      - uses: google-github-actions/setup-gcloud@v2
+
+      - name: Install gke-gcloud-auth-plugin
+        run: gcloud components install gke-gcloud-auth-plugin
+
+      - name: Get cluster credentials
+        run: gcloud container clusters get-credentials benchmark-cluster --region ${{ env.REGION }}
+
+      - name: Create benchmark job
+        run: |
+          BENCHMARKS="${{ github.event.inputs.benchmarks || 'clickbench_1' }}"
+          JOB_NAME="bench-main-$(date +%s)"
+
+          cat <<EOF | kubectl apply -f -
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: ${JOB_NAME}
+            namespace: benchmarking
+          spec:
+            backoffLimit: 3
+            activeDeadlineSeconds: 3600
+            ttlSecondsAfterFinished: 3600
+            template:
+              spec:
+                serviceAccountName: benchmark-runner
+                restartPolicy: Never
+                nodeSelector:
+                  cloud.google.com/compute-class: Performance
+                  cloud.google.com/machine-family: c4a
+                  kubernetes.io/arch: arm64
+                tolerations:
+                  - key: cloud.google.com/gke-spot
+                    operator: Exists
+                    effect: NoSchedule
+                containers:
+                  - name: runner
+                    image: ${{ env.REGISTRY }}/runner:latest
+                    securityContext:
+                      seccompProfile:
+                        # Unconfined so the DataFusion runner can use
+                        # io_uring (blocked by the default containerd seccomp
+                        # profile). Justified here because the main-tracking
+                        # workflow runs trusted apache/datafusion main.
+                        type: Unconfined
+                    env:
+                      - name: BENCH_TYPE
+                        value: main_tracking
+                      - name: PR_URL
+                        value: "https://github.com/apache/datafusion"
+                      - name: BENCHMARKS
+                        value: "${BENCHMARKS}"
+                      - name: REPO
+                        value: apache/datafusion
+                      - name: GITHUB_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: github-token
+                            key: token
+                      - name: SCCACHE_GCS_BUCKET
+                        value: ${{ vars.GCP_PROJECT_ID }}-sccache
+                      - name: RUSTC_WRAPPER
+                        value: sccache
+                      - name: SCCACHE_GCS_RW_MODE
+                        value: READ_WRITE
+                      - name: DATA_CACHE_BUCKET
+                        value: ${{ vars.GCP_PROJECT_ID }}-benchdata
+                    resources:
+                      requests:
+                        cpu: "12"
+                        memory: "65Gi"
+                        ephemeral-storage: "128Gi"
+                    volumeMounts:
+                      - name: workspace
+                        mountPath: /workspace
+                volumes:
+                  - name: workspace
+                    ephemeral:
+                      volumeClaimTemplate:
+                        spec:
+                          storageClassName: hyperdisk-balanced
+                          accessModes: ["ReadWriteOnce"]
+                          resources:
+                            requests:
+                              storage: 128Gi
+          EOF
+
+          echo "Created job: ${JOB_NAME}"
+
+      - name: Wait for job to start
+        run: |
+          kubectl wait --for=condition=complete --timeout=10s job -l app=benchmark-runner -n benchmarking 2>/dev/null || true
+          echo "Job submitted. Results will be posted when complete."

.github/workflows/build-and-deploy.yml

@@ -0,0 +1,196 @@
+name: Build and Deploy
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'controller/**'
+      - 'runner/**'
+      - 'services/**'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+  workflow_dispatch:
+    inputs:
+      target:
+        description: 'Which image to build'
+        required: false
+        default: 'both'
+        type: choice
+        options: [both, controller, runner]
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  REGION: us-central1
+  REGISTRY: us-central1-docker.pkg.dev/${{ vars.GCP_PROJECT_ID }}/benchmarking
+
+jobs:
+  build-controller:
+    if: >
+      github.event_name == 'push' ||
+      github.event.inputs.target == 'both' ||
+      github.event.inputs.target == 'controller'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: google-github-actions/auth@v2
+        id: auth
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }}
+          token_format: access_token
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Build and push controller
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: controller/Dockerfile
+          push: true
+          tags: ${{ env.REGISTRY }}/controller:${{ github.sha }}
+          cache-from: type=gha,scope=controller
+          cache-to: type=gha,mode=max,scope=controller
+
+  # Build per-platform runner images natively (no QEMU).
+  build-runner-platform:
+    if: >
+      github.event_name == 'push' ||
+      github.event.inputs.target == 'both' ||
+      github.event.inputs.target == 'runner'
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            platform: linux/amd64
+            scope: runner-amd64
+          - runner: ubuntu-24.04-arm
+            platform: linux/arm64
+            scope: runner-arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: google-github-actions/auth@v2
+        id: auth
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }}
+          token_format: access_token
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - name: Extract TPC-H answer files
+        run: |
+          docker create --name tpch-answers --platform linux/amd64 ghcr.io/scalytics/tpch-docker:main
+          docker cp tpch-answers:/opt/tpch/2.18.0_rc2/dbgen/answers/ runner/tpch-answers/
+          docker rm tpch-answers
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Build and push runner (${{ matrix.platform }})
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: runner/Dockerfile
+          push: true
+          platforms: ${{ matrix.platform }}
+          tags: ${{ env.REGISTRY }}/runner:${{ github.sha }}-${{ matrix.scope }}
+          cache-from: type=gha,scope=${{ matrix.scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.scope }}
+
+  # Merge per-platform images into a single multi-arch manifest.
+  build-runner:
+    needs: build-runner-platform
+    if: >
+      github.event_name == 'push' ||
+      github.event.inputs.target == 'both' ||
+      github.event.inputs.target == 'runner'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/auth@v2
+        id: auth
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }}
+          token_format: access_token
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Create multi-arch manifest
+        run: |
+          docker buildx imagetools create -t \
+            ${{ env.REGISTRY }}/runner:${{ github.sha }} \
+            ${{ env.REGISTRY }}/runner:${{ github.sha }}-runner-amd64 \
+            ${{ env.REGISTRY }}/runner:${{ github.sha }}-runner-arm64
+
+  deploy:
+    needs: [build-controller, build-runner]
+    if: always() && !failure() && !cancelled()
+    concurrency:
+      group: pulumi-deploy-services
+      cancel-in-progress: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }}
+
+      - uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: gke-gcloud-auth-plugin
+
+      - name: Install dependencies
+        run: npm ci
+
+      - uses: pulumi/auth-actions@v1
+        with:
+          organization: ${{ vars.PULUMI_ORG }}
+          requested-token-type: urn:pulumi:token-type:access_token:personal
+          scope: user:${{ vars.PULUMI_USER }}
+
+      - name: Configure Pulumi stack
+        working-directory: services
+        run: |
+          pulumi stack select --create ${{ vars.PULUMI_ORG }}/dev
+          pulumi config set gcp:project ${{ vars.GCP_PROJECT_ID }}
+          pulumi config set gcp:region ${{ vars.GCP_REGION }}
+          pulumi config set --secret githubToken '${{ secrets.GH_CONTROLLER_TOKEN }}'
+          if [ -n "${{ secrets.LOGFIRE_TOKEN }}" ]; then
+            pulumi config set --secret logfireToken '${{ secrets.LOGFIRE_TOKEN }}'
+          fi
+          pulumi config set imageTag ${{ github.sha }}
+          pulumi config set runnerRepoUrl '${{ github.server_url }}/${{ github.repository }}'
+
+      - uses: pulumi/actions@v5
+        with:
+          command: up
+          stack-name: ${{ vars.PULUMI_ORG }}/dev
+          work-dir: services