fix(test): C host + two-level namespace for the Next App Route dylib gate (#8205)

changelog.d/8209-next-gate-c-host-two-level.md

@@ -0,0 +1,15 @@
+### Testing
+
+- `tests/test_next_app_route_dylib.sh` (the #8161 Next App Route dylib gate) no
+  longer aborts on its first request (#8205). The Rust `provider-host.rs` exported
+  rustc's System-allocator shim, and `provider-linker.sh` linked the stdlib
+  provider image with `-flat_namespace`, so the image's `__rust_dealloc` import
+  bound to the host's shim while the runtime image's shim is mimalloc — the first
+  cross-image `Vec<i64>` drop in `js_node_http_server_process_pending` freed a
+  mimalloc pointer with libsystem `free()`. The host is now C
+  (`tests/fixtures/next-app-route/provider-host.c`; same load order, flags, probe
+  check and event loop, no Rust shim in the executable) and the stdlib image is
+  linked two-level so its runtime imports bind to `libperry_runtime.dylib`. The
+  gate's ABI check, forbidden-diagnostic grep and bypass guard are unchanged; the
+  100-batch loop can still surface #8163's default-GC `TypeError` until that
+  lands.

tests/fixtures/next-app-route/provider-host.c

@@ -0,0 +1,91 @@
+/*
+ * dlopen host for the production Next App Route dylib gate
+ * (tests/test_next_app_route_dylib.sh).
+ *
+ * Deliberately C, not Rust (#8205). A Rust executable carries rustc's
+ * allocator shim (`__rust_alloc` / `__rust_dealloc` / ...), backed by the
+ * System allocator. The stdlib provider image imports those same shim symbols
+ * and expects the runtime image's mimalloc-backed definitions; when the main
+ * executable also defines them, a flat lookup binds the stdlib image to the
+ * host's shim, and the first cross-image `Vec` drop frees a mimalloc pointer
+ * with libsystem `free()` and aborts. A C host defines no Rust shim, so the
+ * only definitions in the process are the runtime image's.
+ *
+ * Load order and flags are the contract the gate asserts: both providers are
+ * process-global and eagerly relocated before the app; the app itself is
+ * loaded RTLD_LOCAL with eager relocation, so an unresolved Perry ABI symbol
+ * fails at load time rather than on the first request that reaches it.
+ */
+#include <dlfcn.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef void (*void_fn)(void);
+typedef int (*tick_fn)(void);
+typedef size_t (*probe_fn)(void);
+
+static void *open_image(const char *path, int mode) {
+  void *handle = dlopen(path, mode);
+  if (handle == NULL) {
+    fprintf(stderr, "dlopen failed: %s: %s\n", path, dlerror());
+    exit(1);
+  }
+  return handle;
+}
+
+static void *symbol(void *handle, const char *name) {
+  dlerror();
+  void *address = dlsym(handle, name);
+  const char *error = dlerror();
+  if (address == NULL || error != NULL) {
+    fprintf(stderr, "dlsym failed: %s: %s\n", name,
+            error != NULL ? error : "null address");
+    exit(1);
+  }
+  return address;
+}
+
+int main(int argc, char **argv) {
+  if (argc != 4) {
+    fprintf(stderr, "usage: provider-host runtime stdlib app\n");
+    return 1;
+  }
+
+  void *runtime = open_image(argv[1], RTLD_NOW | RTLD_GLOBAL);
+  void *stdlib = open_image(argv[2], RTLD_NOW | RTLD_GLOBAL);
+  void *app = open_image(argv[3], RTLD_NOW | RTLD_LOCAL);
+
+  void_fn gc_init = (void_fn)symbol(runtime, "js_gc_init");
+  /* The stdlib provider must bind its stateful runtime calls to the runtime
+     image the host loaded, not to a private runtime copy of its own. */
+  probe_fn provider_probe =
+      (probe_fn)symbol(stdlib, "next_app_route_provider_runtime_probe");
+  if (provider_probe() != (size_t)(uintptr_t)gc_init) {
+    fprintf(stderr, "stdlib provider is bound to a different runtime image\n");
+    return 1;
+  }
+
+  void_fn module_init = (void_fn)symbol(app, "perry_module_init");
+  tick_fn run_microtasks =
+      (tick_fn)symbol(runtime, "js_promise_run_microtasks_event_loop");
+  tick_fn timer_tick = (tick_fn)symbol(runtime, "js_timer_tick");
+  tick_fn callback_timer_tick =
+      (tick_fn)symbol(runtime, "js_callback_timer_tick");
+  tick_fn interval_timer_tick =
+      (tick_fn)symbol(runtime, "js_interval_timer_tick");
+  void_fn run_stdlib_pump = (void_fn)symbol(runtime, "js_run_stdlib_pump");
+  void_fn wait_for_event = (void_fn)symbol(runtime, "js_wait_for_event");
+
+  gc_init();
+  module_init();
+  for (;;) {
+    run_microtasks();
+    timer_tick();
+    callback_timer_tick();
+    interval_timer_tick();
+    run_stdlib_pump();
+    wait_for_event();
+  }
+}

tests/fixtures/next-app-route/provider-linker.sh

@@ -103,9 +103,17 @@ if [[ "$host_os" == Darwin ]]; then
     while IFS= read -r symbol; do
         arguments+=("-Wl,-u,$symbol")
     done <"$selected"
+    # Two-level namespace, on purpose (#8205): every undefined symbol in this
+    # image is bound at link time to the image that defines it, so the stdlib
+    # provider's `__rust_alloc`/`__rust_dealloc` imports resolve to the runtime
+    # dylib's mimalloc-backed shim no matter what else the process defines. A
+    # `-flat_namespace` link would instead bind them at load time to the FIRST
+    # definition in the process, which for a Rust host executable is its own
+    # System-allocator shim — a mimalloc buffer freed by libsystem, `abort()`
+    # on the first cross-image `Vec` drop.
     arguments+=(
         '-Wl,-exported_symbols_list' "-Wl,$exports"
-        '-Wl,-rpath,@loader_path' '-Wl,-flat_namespace' '-Wl,-interposable'
+        '-Wl,-rpath,@loader_path'
     )
 else
     version_script="$scratch/exports.map"

tests/test_next_app_route_dylib.sh

@@ -265,23 +265,37 @@ if [[ -s "$missing_symbols" ]]; then
     exit 1
 fi
 
+# The host is C, not Rust (#8205): a Rust executable would carry rustc's
+# System-allocator shim, and the stdlib provider's `__rust_dealloc` import must
+# reach the runtime image's mimalloc-backed shim instead. See provider-host.c.
 host="$scratch/provider-host"
-rustc --edition 2021 -O "$fixture/provider-host.rs" -o "$host"
+"$real_cc" -O2 -o "$host" "$fixture/provider-host.c" -ldl
 provider_abi=$(shasum -a 256 "$available_symbols" | awk '{print $1}')
 echo "Provider ABI hash: $provider_abi"
 
 for cold_start in $(seq 1 "$cold_starts"); do
     host_log="$scratch/host-$cold_start.log"
     (
-        # Next's generated webpack runtime resolves `./chunks/*.js` from the
-        # production server root when it loads an on-demand route chunk.
-        cd "$fixture/.next/server"
+        # Run from the fixture root, the release-tier layout with production
+        # evidence (tests/release/packages/next-app-route/fixture.sh): Next
+        # resolves `.next/routes-manifest.json` against the working directory
+        # on every request, so serving from `.next/server` 500s each request
+        # with ENOENT. The old reason to sit in `.next/server` — the webpack
+        # runtime resolving `./chunks/*.js` from the server root — is gone:
+        # computed relative chunk requires resolve against the route bundle
+        # since #8146.
+        cd "$fixture"
+        # `exec` so `$host_pid` below is the host process itself, not this
+        # subshell. Killing the subshell leaves the host running (observed:
+        # an orphaned provider-host still serving port $port after the gate
+        # exited), and a survivor holds the port, so cold start 2 can never
+        # bind.
         if [[ "$host_os" == Darwin ]]; then
             PORT="$port" HOSTNAME=127.0.0.1 DYLD_LIBRARY_PATH="$providers" \
-                "$host" "$runtime_library" "$stdlib_library" "$app"
+                exec "$host" "$runtime_library" "$stdlib_library" "$app"
         else
             PORT="$port" HOSTNAME=127.0.0.1 LD_LIBRARY_PATH="$providers" \
-                "$host" "$runtime_library" "$stdlib_library" "$app"
+                exec "$host" "$runtime_library" "$stdlib_library" "$app"
         fi
     ) >"$host_log" 2>&1 &
     host_pid=$!