From b1afb4355b562ce9e1222a778c8f058b91e26afa Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:28:18 -0500 Subject: [PATCH] controller: build benchmarks without debuginfo by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pods set CARGO_PROFILE_RELEASE_DEBUG=1 to keep DWARF symbols for gdb dumps of hung jobs. That makes rustc and the linker use far more memory. On a live wide_schema run I watched a single rustc climb to ~16 GB and the pod cgroup reach its 65 GiB limit and OOM-kill — during compilation, before any benchmark query executed. (The wide_schema queries themselves use ~1 GB.) The wide_schema "run" step compiles via `cargo bench --bench sql`, so the heavy compile lands inside the monitored window and is repeated per side. Drop the default so builds are fast and lean. The behavior is still available per-run: the trigger comment's shared `env:` block is inherited by the build processes (spawn_command inherits the pod env), so a run can opt back in with: run benchmark env: CARGO_PROFILE_RELEASE_DEBUG: "1" CARGO_BUILD_JOBS: "1" (Per-side baseline:/changed: env only reaches the benchmark run, not the build, so build flags belong in the shared block.) Co-Authored-By: Claude Opus 4.8 (1M context) --- controller/src/benchmarks.rs | 7 ++++++- controller/src/job_manager.rs | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/controller/src/benchmarks.rs b/controller/src/benchmarks.rs index bd373bc..1cf443f 100644 --- a/controller/src/benchmarks.rs +++ b/controller/src/benchmarks.rs @@ -311,10 +311,15 @@ pub fn supported_benchmarks_message(repo_entry: &RepoEntry, requested: &[String] Per-side configuration (`run benchmark tpch` followed by):\n\ ```yaml\n\ env:\n\ - SHARED_SETTING: enabled\n\ + # shared env is inherited by BOTH the build and the run, so build\n\ + # flags go here. Builds default to no debuginfo for speed; opt back\n\ + # in for hung-job gdb dumps and cap jobs to stay within memory:\n\ + CARGO_PROFILE_RELEASE_DEBUG: \"1\"\n\ + CARGO_BUILD_JOBS: \"1\"\n\ baseline:\n\ ref: v45.0.0\n\ env:\n\ + # per-side env only reaches the benchmark run, not the build\n\ DATAFUSION_RUNTIME_MEMORY_LIMIT: 1G\n\ changed:\n\ ref: v46.0.0\n\ diff --git a/controller/src/job_manager.rs b/controller/src/job_manager.rs index 2007353..60b2f95 100644 --- a/controller/src/job_manager.rs +++ b/controller/src/job_manager.rs @@ -385,10 +385,21 @@ async fn create_k8s_job( "RUNNER_JOB_DEADLINE_SECS", config.active_deadline_secs.to_string(), ), - // Keep DWARF symbols in release-built benchmark binaries so live gdb - // dumps from hung jobs resolve to useful frames without changing - // optimization level. - env_var("CARGO_PROFILE_RELEASE_DEBUG", "1"), + // NOTE: we deliberately do NOT set CARGO_PROFILE_RELEASE_DEBUG here. + // Building with debuginfo (`debug=1`) makes rustc and the linker use + // dramatically more memory: a single benchmark compile was observed + // driving one rustc to ~16 GB and the pod cgroup to its 65 GiB limit + // (OOM-kill) — before any benchmark query ran. Default builds stay + // fast and lean. A specific run can re-enable DWARF symbols for live + // gdb dumps from hung jobs by adding a shared `env:` block to the + // trigger comment (shared env is inherited by the build processes, + // unlike per-side `baseline:`/`changed:` env which only reaches the + // benchmark run). To keep memory in check, pair it with a job cap: + // + // run benchmark + // env: + // CARGO_PROFILE_RELEASE_DEBUG: "1" + // CARGO_BUILD_JOBS: "1" ]; // The controller resolves the PR's source branch and hands it to the