bazelize benchmarks

.github/workflows/pull-request.yml

@@ -72,13 +72,22 @@ jobs:
       issues: write
       pull-requests: write
 
-    env:
-      CAS_ENDPOINT: localhost:9094
-      BENCHMARK_SERVICES: 1
-
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install Podman
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y podman
+          podman --version
+
+      - name: Use Podman for OCI image loading
+        run: |
+          # rules_oci oci_load prefers 'docker' over 'podman'. Make 'docker'
+          # resolve to podman so images are loaded into the Podman store.
+          sudo ln -sf $(command -v podman) /usr/local/bin/docker
+          docker --version
+
       - name: Install Bazel
         uses: bazelbuild/setup-bazelisk@v3
 
@@ -91,19 +100,19 @@ jobs:
             ${{ runner.os }}-bazel-
 
       - name: Build binaries
-        run: bazel build //:all
+        run: bazel build //:server //:worker //:cache //:bes
 
       - name: Run tests
         run: bazel test //...
 
       - name: Build and load images
         run: |
-          for image in server worker cache; do
+          for image in server worker cache bes; do
             echo "=== Building ${image} ==="
             bazel run "//oci:${image}.amd64.image.load"
           done
           echo "=== Images built ==="
-          docker images | grep ferrisrbe || true
+          podman images | grep ferrisrbe || true
 
       - name: Install Kind
         uses: helm/kind-action@v1
@@ -113,9 +122,10 @@ jobs:
 
       - name: Load images into Kind
         run: |
-          kind load docker-image ferrisrbe/server:latest --name ferrisrbe-test
-          kind load docker-image ferrisrbe/worker:latest --name ferrisrbe-test
-          kind load docker-image ferrisrbe/cache:latest --name ferrisrbe-test
+          podman save ferrisrbe/server:latest | kind load image-archive - --name ferrisrbe-test
+          podman save ferrisrbe/worker:latest | kind load image-archive - --name ferrisrbe-test
+          podman save ferrisrbe/cache:latest | kind load image-archive - --name ferrisrbe-test
+          podman save ferrisrbe/bes:latest | kind load image-archive - --name ferrisrbe-test
           echo "=== Verify images in Kind ==="
           kubectl get nodes -o json | jq -r '.items[0].status.images[] | select(.names[] | contains("ferrisrbe")) | .names[0]' || true
 
@@ -128,7 +138,8 @@ jobs:
             --set worker.image.pullPolicy=Never \
             --set cache.image.repository=ferrisrbe/cache \
             --set cache.image.pullPolicy=Never \
-            --set bes.enabled=false \
+            --set bes.image.repository=ferrisrbe/bes \
+            --set bes.image.pullPolicy=Never \
             --set server.livenessProbe.enabled=false \
             --set server.readinessProbe.enabled=false \
             --set worker.livenessProbe.enabled=false \
@@ -149,6 +160,8 @@ jobs:
           kubectl describe deployment ferrisrbe-server -n rbe || true
           echo "=== Describe worker deployment ==="
           kubectl describe deployment ferrisrbe-worker -n rbe || true
+          echo "=== Describe BES deployment ==="
+          kubectl describe deployment ferrisrbe-bes -n rbe || true
 
       - name: Wait for deployment
         run: |
@@ -171,6 +184,13 @@ jobs:
             kubectl logs -l app.kubernetes.io/component=worker -n rbe --tail=100 || true
             exit 1
           }
+          kubectl rollout status deployment/ferrisrbe-bes -n rbe --timeout=120s || {
+            echo "=== BES deployment failed ==="
+            kubectl get pods -n rbe
+            kubectl describe deployment ferrisrbe-bes -n rbe
+            kubectl logs -l app.kubernetes.io/component=bes -n rbe --tail=100 || true
+            exit 1
+          }
 
       - name: Port forward
         run: |
@@ -191,57 +211,22 @@ jobs:
         run: |
           kind delete cluster --name ferrisrbe-test || true
 
-      - name: Install benchmark dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3-pip bc netcat-openbsd
-          pip3 install grpcio grpcio-tools
-
-      - name: Start benchmark cache service
-        run: |
-          docker run -d --name rbe-cache \
-            -p 9094:9094 \
-            -p 8080:8080 \
-            -e RUST_LOG=info \
-            -e RBE_CACHE_PORT=9094 \
-            -e RBE_CACHE_HTTP_PORT=8080 \
-            -e RBE_CACHE_DIR=/data \
-            -e RBE_CACHE_MAX_SIZE_GB=1 \
-            -e RBE_CACHE_AC_MAX_SIZE_GB=1 \
-            ferrisrbe/cache:latest
-
-      - name: Wait for benchmark cache service
-        run: |
-          for i in $(seq 1 30); do
-            if nc -zv localhost 9094 2>/dev/null; then
-              echo "Cache service is ready"
-              exit 0
-            fi
-            echo "Waiting for cache service... ($i/30)"
-            sleep 1
-          done
-          echo "Cache service did not become ready"
-          docker logs rbe-cache
-          exit 1
-
-      - name: Run full benchmark suite
+      - name: Run benchmark suite
         id: benchmark
         run: |
-          cd benchmark
-          ./scripts/benchmark-ci.sh full
+          bazel run //benchmark:run -- full --cache-image ferrisrbe/cache:latest
         continue-on-error: true
 
       - name: Compare PR vs official release
         id: compare
         run: |
-          cd benchmark
-          ./scripts/compare-branches.sh ferrisrbe/server:latest
+          bazel run //benchmark:run -- compare --image ferrisrbe/server:latest
         continue-on-error: true
 
       - name: Check regression thresholds
         id: regression
         run: |
-          python3 benchmark/scripts/check-regression.py benchmark/results/benchmark_data.json
+          bazel run //benchmark/scripts:check_regression -- benchmark/results/benchmark_data.json
         continue-on-error: true
 
       - name: Upload benchmark results

.gitignore

@@ -6,4 +6,9 @@ target
 MODULE.bazel.lock
 node_modules
 ui/dist
-.bes
+.bes
+_site/
+
+# Benchmark generated results
+benchmark/results/*
+!benchmark/results/.gitkeep

MODULE.bazel

@@ -11,6 +11,7 @@ bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
 bazel_dep(name = "platforms", version = "0.0.11")
 bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1")
+bazel_dep(name = "rules_python", version = "1.0.0")
 
 toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
 use_repo(toolchains, "zig_sdk")
@@ -96,3 +97,18 @@ use_repo(
     "debian_bookworm_slim_linux_amd64",
     "debian_bookworm_slim_linux_arm64_v8",
 )
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    is_default = True,
+    python_version = "3.11",
+)
+use_repo(python, "python_3_11", "python_versions")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "benchmark_pip",
+    python_version = "3.11",
+    requirements_lock = "//benchmark:requirements.txt",
+)
+use_repo(pip, "benchmark_pip")

README.md

@@ -179,37 +179,30 @@ Compare to Java-based alternatives that idle at 500MB+ and spike to 4GB+ during
 
 ### Running Benchmarks
 
-The benchmark suite tests eight critical dimensions of RBE performance:
+The benchmark suite tests eight critical dimensions of RBE performance and is now fully Bazelized:
 
 ```bash
-cd benchmark
+# Run the full container-native benchmark suite
+bazel run //benchmark:run -- full
 
-# 1. Memory footprint (baseline)
-./scripts/benchmark.sh
+# Or run the lightweight PR validation suite
+bazel run //benchmark:run -- light
 
-# 2. Execution API throughput (Zero-GC advantage)
-./scripts/execution-load-test.py --actions 1000 --concurrent 50
+# Compare the local image against the official Docker Hub release
+bazel run //benchmark:run -- compare --image ferrisrbe/server:latest
 
-# 3. Action Cache performance (L1 DashMap + L2 disk-backed)
-./scripts/action-cache-test.py --operations 10000 --concurrent 100
-
-# 4. Multi-level scheduler (no head-of-line blocking)
-./scripts/noisy-neighbor-test.py --slow 10 --fast 50
-
-# 5. O(1) Streaming (constant memory with large files)
-./scripts/o1-streaming-test.py --large-sizes 5 10 --small-count 1000
-
-# 6. Connection churn (resource cleanup)
-./scripts/connection-churn-test.py --connections 1000 --disconnect-rate 0.3
+# Generate a markdown report from existing JSON results
+bazel run //benchmark:run -- report
+```
 
-# 7. Cache stampede (thundering herd protection)
-./scripts/cache-stampede-test.py --requests 10000 --concurrent 100
+Individual benchmark scripts are also available as Bazel targets, for example:
 
-# 8. Cold start time (<100ms vs 5-30s JVM)
-./scripts/cold-start-test.sh
+```bash
+bazel run //benchmark/scripts:execution_load_test -- --actions 1000 --concurrent 50
+bazel run //benchmark/scripts:action_cache_test -- --operations 10000 --concurrent 100
 ```
 
-See [benchmark/README.md](benchmark/README.md) for detailed benchmark documentation and [benchmark/results/BENCHMARK_RESULTS.md](benchmark/results/BENCHMARK_RESULTS.md) for results.
+See [benchmark/README.md](benchmark/README.md) for detailed benchmark documentation.
 
 ## Quick Start
 

benchmark/BUILD.bazel

@@ -0,0 +1,22 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+package(default_visibility = ["//visibility:public"])
+
+py_binary(
+    name = "run",
+    srcs = ["orchestrator.py"],
+    main = "orchestrator.py",
+    deps = [
+        "//benchmark/scripts:benchmark_lib",
+    ],
+)
+
+# Convenience target to build every benchmark artefact without running tests.
+filegroup(
+    name = "all",
+    srcs = [
+        ":run",
+        "//benchmark/proto:reapi_py_proto",
+        "//benchmark/scripts:all",
+    ],
+)