perf(ci): run tests in-process multi-threaded (drop --test-threads=1)

.github/workflows/ci.yml

@@ -39,23 +39,16 @@ jobs:
         # regressions are still caught before release, just not on every PR push.
         os: ${{ github.event_name == 'pull_request' && fromJSON('["ubuntu-latest", "windows-latest"]') || fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]') }}
     runs-on: ${{ matrix.os }}
-    # --test-threads=1 (env-race legacy, v3.3.5) serializes ~70 binaries that
-    # now include heavy property/fuzz/benchmark suites (#291, #493, #551) —
-    # well beyond 90 min on hosted runners. Parallelization is tracked
-    # separately; until then the gate gets the time it actually needs.
-    #
-    # Tried switching to cargo-nextest (rust/.config/nextest.toml already
-    # exists and documents the same env-race for 3 known binaries via a
-    # `serial-state` test-group). Reverted: nextest's default parallelism
-    # (test-threads = -4) surfaced several more tests with hidden wall-clock
-    # timing assumptions (e.g. core::cache::tests::hebbian_eviction_bonus_is_wired
-    # depends on two calls landing in the same real-time 500ms window) that a
-    # `max-threads=1` test-group does NOT fix — that only serializes tests
-    # *within* the group against each other, it doesn't protect them from
-    # being scheduling-delayed by the many *other* tests still running in
-    # parallel elsewhere on the same runner. Revisit once those tests are
-    # made robust to scheduling jitter (or the burst window is made
-    # injectable for tests) rather than papering over it with test-groups.
+    # Tests run with libtest's default thread parallelism. The old
+    # `--test-threads=1` (env-race legacy, v3.3.5) serialized ~70 binaries that
+    # now include heavy property/fuzz/benchmark suites (#291, #493, #551) and
+    # dominated the wall clock. What made the flag necessary is fixed at the
+    # source instead: every env-mutating test serializes on the reentrant
+    # `data_dir::test_env_lock()`, and the one test with a real-time window
+    # (`hebbian_eviction_bonus_is_wired`) takes an injected window rather than
+    # depending on scheduling. cargo-nextest was tried first (#1206) and
+    # reverted — 33-80% slower here, since its process-per-test model re-pays
+    # this crate's heavy static-link cost for every test.
     timeout-minutes: 180
     env:
       # ~50 integration-test binaries each statically link the full lib
@@ -145,26 +138,31 @@ jobs:
           if [[ "${{ runner.os }}" == "Windows" ]]; then
             JEMALLOC_OVERRIDE="${{ steps.jemalloc.outputs.jemalloc-override }}" \
             CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-L ${{ steps.jemalloc.outputs.dummy-dir }}" \
-            "$CARGO" test --target x86_64-pc-windows-gnu --all-features -- --test-threads=1
+            "$CARGO" test --target x86_64-pc-windows-gnu --all-features
           elif [[ "${{ runner.os }}" == "Linux" ]]; then
             # `mold -run` redirects child linker invocations to mold without
             # touching RUSTFLAGS (which is pinned to `-Dwarnings` job-wide and
             # would override any .cargo/config.toml rustflags).
-            mold -run "$CARGO" test --all-features -- --test-threads=1
+            mold -run "$CARGO" test --all-features
           else
-            "$CARGO" test --all-features -- --test-threads=1
+            "$CARGO" test --all-features
           fi
 
-      # #581: lock in the parallel BM25 (incremental) rebuild win against silent
-      # regression. Timing is only meaningful on a quiet, single-threaded runner,
-      # so the gate is env-opt-in (no-op in the matrix above) and runs once here.
-      - name: BM25 build-time regression gate (#581)
+      # Wall-clock gates: #581's parallel BM25 rebuild win, the context-kernel
+      # micro-benchmarks and the performance-stress suite. Timing is only
+      # meaningful on a quiet, single-threaded runner, so every one of them is
+      # env-opt-in (no-op in the matrix above, which runs multi-threaded) and
+      # they run serialized here instead.
+      - name: Wall-clock regression gates (#581)
         if: runner.os == 'Linux'
         working-directory: rust
         shell: bash
         env:
           LEAN_CTX_PERF_GATE: "1"
-        run: cargo test --all-features --lib -- --test-threads=1 parallel_incremental_rebuild_perf_gate --nocapture
+        run: |
+          cargo test --all-features --lib -- --test-threads=1 parallel_incremental_rebuild_perf_gate --nocapture
+          cargo test --all-features --lib -- --test-threads=1 context_kernel::perf_benchmark
+          cargo test --all-features --test main -- --test-threads=1 performance_stress
 
   clippy:
     name: Clippy

rust/LOCK_ORDERING.md

@@ -101,12 +101,12 @@ All `std::sync::Mutex` unless noted otherwise.
 
 | # | Lock | File | Purpose |
 |---|------|------|---------|
-| E1 | `ENV_LOCK` | `dashboard/mod.rs:537` | Serialize env-var access in dashboard tests |
 | E2 | `ENV_LOCK` | `core/dense_backend.rs:412` | Serialize env-var access in dense-backend tests |
 | E3 | `ENV_LOCK` | `core/workspace_config.rs:101` | Serialize env-var access in workspace-config tests |
 | E4 | `LOCK` | `core/data_dir.rs:50` | Serialize data-dir creation |
 | E5 | `LOCK` | `core/tokens.rs:190` | Serialize tokenizer tests |
 | E6 | `LOCK` | `core/tokenizer_translation_driver.rs:248` | Serialize tokenizer-translation tests |
+| E7 | `TEST_ENV_MUTEX` | `core/data_dir.rs:161` | `OnceLock<Mutex<()>>` reentrant test env lock — serialize env-mutating tests across threads (`std::env::set_var` is not thread-safe); same thread may re-acquire without deadlocking |
 
 ---
 

rust/src/cli/dispatch/mod.rs

@@ -1065,27 +1065,31 @@ mod tests {
     #[test]
     #[serial]
     fn worker_threads_default_clamps_low() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::remove_var("LEAN_CTX_WORKER_THREADS");
         assert_eq!(resolve_worker_threads(1), 1);
     }
 
     #[test]
     #[serial]
     fn worker_threads_default_clamps_high() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::remove_var("LEAN_CTX_WORKER_THREADS");
         assert_eq!(resolve_worker_threads(32), 4);
     }
 
     #[test]
     #[serial]
     fn worker_threads_default_passthrough() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::remove_var("LEAN_CTX_WORKER_THREADS");
         assert_eq!(resolve_worker_threads(3), 3);
     }
 
     #[test]
     #[serial]
     fn worker_threads_env_override() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::set_var("LEAN_CTX_WORKER_THREADS", "12");
         assert_eq!(resolve_worker_threads(2), 12);
         crate::test_env::remove_var("LEAN_CTX_WORKER_THREADS");
@@ -1094,6 +1098,7 @@ mod tests {
     #[test]
     #[serial]
     fn worker_threads_env_invalid_falls_back() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::set_var("LEAN_CTX_WORKER_THREADS", "not_a_number");
         assert_eq!(resolve_worker_threads(3), 3);
         crate::test_env::remove_var("LEAN_CTX_WORKER_THREADS");

rust/src/core/addons/bootstrap.rs

@@ -630,6 +630,7 @@ mod tests {
     #[test]
     #[cfg(unix)]
     fn ensure_installed_runs_manager_then_verifies() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         let _guard = ENV_LOCK
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
@@ -665,6 +666,7 @@ mod tests {
     #[test]
     #[cfg(unix)]
     fn ensure_installed_propagates_manager_failure() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         let _guard = ENV_LOCK
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner);
@@ -703,6 +705,7 @@ mod tests {
     #[test]
     #[cfg(unix)]
     fn ensure_installed_preflights_a_missing_manager() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         let _guard = ENV_LOCK
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner);

rust/src/core/addons/env_scrub.rs

@@ -61,6 +61,7 @@ mod tests {
 
     #[test]
     fn declared_capabilities_scrub_host_secret() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::set_var("LEAN_CTX_ADDON_TEST_SECRET", "top-secret");
         let caps = AddonCapabilities::default();
         let mut cmd = Command::new("true");
@@ -75,6 +76,7 @@ mod tests {
 
     #[test]
     fn declared_env_name_passes_through() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         crate::test_env::set_var("LEAN_CTX_ADDON_TEST_ALLOWED", "visible");
         let caps = AddonCapabilities {
             network: NetworkAccess::Full,

rust/src/core/budget_tracker.rs

@@ -376,6 +376,7 @@ mod tests {
 
     #[test]
     fn cost_cap_blocks_when_exceeded() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         let t = BudgetTracker::new();
         t.record_cost_usd(6.0);
         // SAFETY: single-threaded test — no concurrent env access.

rust/src/core/cache/mod.rs

@@ -6,8 +6,6 @@ pub use entry::*;
 pub use session::*;
 pub use validation::*;
 
-#[cfg(test)]
-use crate::core::tokens::count_tokens;
 #[cfg(test)]
 use entry::resolve_cache_max_tokens;
 #[cfg(test)]

rust/src/core/cache/session.rs

@@ -57,6 +57,15 @@ impl SessionCache {
         self.co_access.end_burst();
     }
 
+    /// Widen the co-access burst window. Test-only: lets a test keep several
+    /// `store()` calls in one burst without depending on them landing inside
+    /// the real-time 500ms window (a scheduling-jitter flake under parallel
+    /// test execution).
+    #[cfg(test)]
+    pub fn set_co_access_burst_window(&mut self, window: std::time::Duration) {
+        self.co_access.set_burst_window(window);
+    }
+
     /// Per-entry Hebbian eviction bonus (#3): each cached entry that is
     /// co-accessed with the recently-active working set earns a positive bonus
     /// that is added to its RRF score, so clustered files survive eviction

rust/src/core/cache/tests.rs

@@ -180,13 +180,14 @@ fn hebbian_eviction_bonus_is_wired() {
     // #3: files read together build a Hebbian association via store()'s
     // recording, and that association must feed the eviction bonus.
     //
-    // Warm up tiktoken first: the very first count_tokens() in the process
-    // lazily loads the BPE tables (can exceed the 500ms co-access burst
-    // window). store() calls count_tokens() internally, so without warming
-    // up, the two store() calls below straddle that window and never
-    // associate — a flaky-empty bonus. Warming up keeps them in one burst.
-    let _ = count_tokens("warmup");
+    // Widen the burst window so the two store() calls below always land in
+    // one burst — independent of real time. Previously this test warmed up
+    // tiktoken and hoped the calls stayed inside the real 500ms window; that
+    // still flaked under parallel test execution when scheduling jitter
+    // delayed one store() past the window. An injected window removes the
+    // wall-clock dependency entirely.
     let mut cache = SessionCache::new();
+    cache.set_co_access_burst_window(std::time::Duration::from_secs(3600));
     cache.store("/a.rs", "fn a() {}");
     cache.store("/b.rs", "fn b() {}");
     cache.flush_co_access(); // commit the burst → association (a,b) forms
@@ -271,6 +272,7 @@ fn cache_budget_resolver_precedence() {
 
 #[test]
 fn evict_if_needed_removes_lowest_score() {
+    let _env_lock = crate::core::data_dir::test_env_lock();
     crate::test_env::set_var("LEAN_CTX_CACHE_MAX_TOKENS", "50");
     let mut cache = SessionCache::new();
     let big_content = "a]".repeat(30); // ~30 tokens

rust/src/core/content_cache.rs

@@ -404,6 +404,7 @@ mod tests {
 
     #[test]
     fn disabled_via_zero_budget_is_passthrough() {
+        let _env_lock = crate::core::data_dir::test_env_lock();
         let _g = TEST_LOCK
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner);

rust/src/core/context_kernel/config_bridge.rs

@@ -154,7 +154,7 @@ mod tests {
 
     fn setup() -> (
         std::sync::MutexGuard<'static, ()>,
-        std::sync::MutexGuard<'static, ()>,
+        crate::core::data_dir::TestEnvGuard,
         EnvGuard,
     ) {
         let kernel = kernel_config::KERNEL_TEST_LOCK
@@ -174,7 +174,7 @@ mod tests {
 
     #[test]
     fn env_overrides_default() {
-        let _guards = setup();
+        let _guards = setup(); // holds test_env_lock for the test's lifetime
         crate::test_env::set_var("LEAN_CTX_KERNEL_ENABLED", "false");
         let (features, source) = effective_config();
         assert!(!features.enabled);
@@ -183,15 +183,15 @@ mod tests {
 
     #[test]
     fn apply_updates_global() {
-        let _guards = setup();
+        let _guards = setup(); // holds test_env_lock for the test's lifetime
         crate::test_env::set_var("LEAN_CTX_KERNEL_DEDUP", "false");
         apply_config();
         assert!(!kernel_config::features().content_dedup);
     }
 
     #[test]
     fn report_lists_env_vars() {
-        let _guards = setup();
+        let _guards = setup(); // holds test_env_lock for the test's lifetime
         crate::test_env::set_var("LEAN_CTX_KERNEL_MAX_BUDGET", "42");
         assert_eq!(
             config_report().env_vars_detected,

rust/src/core/context_kernel/perf_benchmark.rs

@@ -9,6 +9,14 @@ mod tests {
     };
     use crate::tools::{search_hook, search_kernel};
 
+    /// Wall-clock budgets below only hold on a quiet, single-threaded runner.
+    /// The suite runs multi-threaded, so they are opt-in: CI's perf-gate step
+    /// sets `LEAN_CTX_PERF_GATE=1` and runs them serialized. The work itself
+    /// (and every correctness assertion) still runs on every invocation.
+    fn timing_enforced() -> bool {
+        std::env::var("LEAN_CTX_PERF_GATE").as_deref() == Ok("1")
+    }
+
     fn isolated() -> MutexGuard<'static, ()> {
         let guard = kernel_config::KERNEL_TEST_LOCK
             .lock()
@@ -41,7 +49,7 @@ mod tests {
             let content = format!("unique content {index}");
             assert!(ctx_read_dedup::try_dedup("bench.rs", &content).is_none());
         }
-        assert!(started.elapsed() < Duration::from_millis(500));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(500));
     }
 
     #[test]
@@ -51,7 +59,7 @@ mod tests {
         for _ in 0..10_000 {
             evidence_hook::record_tool_call("ctx_read", 100, 25);
         }
-        assert!(started.elapsed() < Duration::from_millis(200));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(200));
         assert_eq!(evidence_hook::evidence_report().tool_calls, 10_000);
     }
 
@@ -62,7 +70,7 @@ mod tests {
         for _ in 0..100 {
             std::hint::black_box(health::kernel_health());
         }
-        assert!(started.elapsed() < Duration::from_millis(50));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(50));
     }
 
     #[test]
@@ -76,7 +84,7 @@ mod tests {
             search_kernel::record_search(&format!("query-{index}"), 5, 100);
         }
         let summary = search_kernel::search_summary();
-        assert!(started.elapsed() < Duration::from_millis(200));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(200));
         assert_eq!(summary.total_searches, 600);
         assert_eq!(summary.unique_queries, 500);
         assert_eq!(summary.repeated_queries, 100);
@@ -97,7 +105,7 @@ mod tests {
             };
             token_total += std::hint::black_box(envelope).input_tokens;
         }
-        assert!(started.elapsed() < Duration::from_millis(50));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(50));
         assert_eq!(token_total, 12_497_500);
     }
 
@@ -138,6 +146,6 @@ mod tests {
                 "cursor",
             ));
         }
-        assert!(started.elapsed() < Duration::from_millis(500));
+        assert!(!timing_enforced() || started.elapsed() < Duration::from_millis(500));
     }
 }