fix(sandbox): verify skipTLSVerify/caFile behavior of kubeconfig under Bun (#1849)

services/sandbox/docs/kubernetes.md

@@ -83,18 +83,18 @@ stream. Keep it out so a stray exec call fails closed.
 
 ## Environment
 
-| Env                                                               | Required   | Notes                                                                                                                                                                           |
-| ----------------------------------------------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `SANDBOX_BACKEND=kubernetes`                                      | yes        | Selects this backend.                                                                                                                                                           |
-| `SANDBOX_SPAWNER_IMAGE`                                           | yes        | The spawner's **own** image ref, used for the `stage`/`harvest` containers. Must match the deployed spawner version.                                                            |
-| `SANDBOX_RUNTIME_IMAGE`                                           | yes        | The `runner` image (`tale-sandbox-runtime:<tag>`).                                                                                                                              |
-| `SANDBOX_K8S_NAMESPACE`                                           | yes        | Namespace the per-exec Pods/Secrets are created in (default `tale-sandbox`).                                                                                                    |
-| `NODE_EXTRA_CA_CERTS`                                             | in-cluster | Point at the SA `ca.crt` (`/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`). **Bun's fetch ignores the kubeconfig CA**, so without this the apiserver TLS isn't trusted.  |
-| `SANDBOX_RUNTIME=runsc`                                           | optional   | Sets the Pod `runtimeClassName` (gVisor) via `SANDBOX_RUNTIME_CLASS` (default `gvisor`).                                                                                        |
-| `SANDBOX_CACHE=pvc`                                               | optional   | Mounts per-org RWX cache PVCs on the runner; needs the PVC RBAC above + an RWX StorageClass. Default `none` (installs fresh each run via the egress proxy).                     |
-| `SANDBOX_K8S_WORKSPACE_SIZE_LIMIT`                                | optional   | `sizeLimit` on the per-exec `/user` emptyDir (default `4Gi`). Bounds deps + temp + outputs; exceeding it evicts the Pod.                                                        |
-| `SANDBOX_EGRESS_PROXY`                                            | optional   | The runner's `HTTPS_PROXY`/`HTTP_PROXY` for `pip`/`npm` (default `http://sandbox-egress:3128`).                                                                                 |
-| `SANDBOX_K8S_SERVER` / `SANDBOX_K8S_TOKEN` / `SANDBOX_K8S_CAFILE` | dev only   | Explicit bearer-token kubeconfig for local Bun dev (kind's client-cert kubeconfig auths as `system:anonymous` under Bun). In-cluster uses the projected SA token automatically. |
+| Env                                                               | Required   | Notes                                                                                                                                                                                                                                                                                                |
+| ----------------------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `SANDBOX_BACKEND=kubernetes`                                      | yes        | Selects this backend.                                                                                                                                                                                                                                                                                |
+| `SANDBOX_SPAWNER_IMAGE`                                           | yes        | The spawner's **own** image ref, used for the `stage`/`harvest` containers. Must match the deployed spawner version.                                                                                                                                                                                 |
+| `SANDBOX_RUNTIME_IMAGE`                                           | yes        | The `runner` image (`tale-sandbox-runtime:<tag>`).                                                                                                                                                                                                                                                   |
+| `SANDBOX_K8S_NAMESPACE`                                           | yes        | Namespace the per-exec Pods/Secrets are created in (default `tale-sandbox`).                                                                                                                                                                                                                         |
+| `NODE_EXTRA_CA_CERTS`                                             | in-cluster | Point at the SA `ca.crt` (`/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`). Kubernetes API calls are trusted via `caFile` → Agent (see TLS note below), but native `fetch()` calls (e.g. harvest presigned-URL uploads) bypass the Agent and require this variable to trust the apiserver CA. |
+| `SANDBOX_RUNTIME=runsc`                                           | optional   | Sets the Pod `runtimeClassName` (gVisor) via `SANDBOX_RUNTIME_CLASS` (default `gvisor`).                                                                                                                                                                                                             |
+| `SANDBOX_CACHE=pvc`                                               | optional   | Mounts per-org RWX cache PVCs on the runner; needs the PVC RBAC above + an RWX StorageClass. Default `none` (installs fresh each run via the egress proxy).                                                                                                                                          |
+| `SANDBOX_K8S_WORKSPACE_SIZE_LIMIT`                                | optional   | `sizeLimit` on the per-exec `/user` emptyDir (default `4Gi`). Bounds deps + temp + outputs; exceeding it evicts the Pod.                                                                                                                                                                             |
+| `SANDBOX_EGRESS_PROXY`                                            | optional   | The runner's `HTTPS_PROXY`/`HTTP_PROXY` for `pip`/`npm` (default `http://sandbox-egress:3128`).                                                                                                                                                                                                      |
+| `SANDBOX_K8S_SERVER` / `SANDBOX_K8S_TOKEN` / `SANDBOX_K8S_CAFILE` | dev only   | Explicit bearer-token kubeconfig for local Bun dev (kind's client-cert kubeconfig auths as `system:anonymous` under Bun — see TLS note below). In-cluster uses the projected SA token automatically.                                                                                                 |
 
 The Pod sets `automountServiceAccountToken: false` — the runtime never gets an
 SA token.
@@ -121,11 +121,39 @@ process is free to ignore. The proxy itself is open at the hostname layer by
 default (`SANDBOX_EGRESS_ALLOWLIST` opt-in), so this NetworkPolicy is the
 _only_ egress fence on k8s; do not run untrusted workloads without it.
 
+## TLS contract under Bun
+
+`@kubernetes/client-node@1.4.0` routes every API call through **node-fetch v2**
+(`gen/http/isomorphic-fetch.js`), which calls `https.request()` with an
+`https.Agent` — **not** Bun's native `fetch()`. Through `node:https`, Bun
+**does** honour TLS options set on the Agent:
+
+| Kubeconfig knob                      | Effect                                        | Honoured by Bun?                                                                           |
+| ------------------------------------ | --------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `skipTLSVerify: true`                | sets `rejectUnauthorized: false` on the Agent | **Yes** — real security relaxation, not dead code                                          |
+| `caFile` / `caData`                  | reads CA bytes into `agent.options.ca`        | **Yes** — CA is used for server-cert verification                                          |
+| `certFile` / `keyFile` (client cert) | sets `cert` / `key` on the Agent              | **No** — Bun's TLS stack does not support mTLS client certs; use bearer-token auth instead |
+
+**Practical consequences:**
+
+- `SANDBOX_K8S_SERVER` + `SANDBOX_K8S_TOKEN` without `SANDBOX_K8S_CAFILE` falls
+  back to `skipTLSVerify: true`. This **disables TLS certificate verification**
+  for the apiserver — intentional for local dev against self-signed clusters, but
+  must never be used in production.
+- `SANDBOX_K8S_CAFILE` enables proper CA verification through the Agent. Set it
+  to the cluster's CA bundle when the apiserver's cert is issued by a private CA.
+- `NODE_EXTRA_CA_CERTS` should **also** be set in-cluster to the SA `ca.crt`
+  path (`/var/run/secrets/kubernetes.io/serviceaccount/ca.crt`) as a
+  belt-and-suspenders measure: any native `fetch()` calls (e.g. harvest
+  presigned-URL uploads) go through Bun's native TLS stack, which reads this env
+  var but ignores `agent.options.ca`.
+
 ## Verification status
 
 Unit-tested (no cluster): the Pod shape + the security invariant (Secret never
-on the runner), the ExecSpec Secret payload + round-trip, and the
-`__TALE_RESULT__` result-line protocol. The on-cluster reliability bar (50+
-consecutive real executions ~100% pass, 2-replica scale, cancel-across-replicas)
-is **pending a healthy cluster** and must be run before enabling this backend in
-production.
+on the runner), the ExecSpec Secret payload + round-trip, the
+`__TALE_RESULT__` result-line protocol, and the `skipTLSVerify`/`caFile` TLS
+knob propagation through the Agent (see `k8s-client.test.ts`). The on-cluster
+reliability bar (50+ consecutive real executions ~100% pass, 2-replica scale,
+cancel-across-replicas) is **pending a healthy cluster** and must be run before
+enabling this backend in production.

services/sandbox/src/backend/kubernetes/k8s-client.test.ts

@@ -3,8 +3,14 @@
 // forever-hung execute()) and withRetry's retryability gate.
 
 import { describe, expect, test } from 'bun:test';
+import { writeFileSync, rmSync } from 'node:fs';
+import type { AgentOptions } from 'node:https';
 
-import { HttpMethod, RequestContext } from '@kubernetes/client-node';
+import {
+  HttpMethod,
+  KubeConfig,
+  RequestContext,
+} from '@kubernetes/client-node';
 
 import { apiTimeout, httpStatusCode, withRetry } from './k8s-client.ts';
 
@@ -110,3 +116,104 @@ describe('withRetry', () => {
     expect(Date.now() - start).toBeLessThan(1_000);
   });
 });
+
+// Verify that @kubernetes/client-node@1.4.0 TLS knobs are NOT inert under Bun.
+//
+// The library routes every request through node-fetch v2
+// (gen/http/isomorphic-fetch.js: `import fetch from "node-fetch"`), which
+// calls https.request() with the configured https.Agent — NOT Bun's native
+// fetch(). Bun's node:https honours rejectUnauthorized and ca on an
+// https.Agent, so skipTLSVerify and caFile ARE effective and carry real
+// security semantics.
+//
+// The limitation documented in the AUTH NOTE of k8s-client.ts is specific to
+// client-cert auth (cert/key options) — Bun's TLS stack does not support
+// mutual TLS client certificates. skipTLSVerify and caFile are distinct knobs
+// that work as intended.
+describe('kubeconfig TLS knobs under Bun', () => {
+  function makeKc(clusterExtra: Record<string, unknown>) {
+    const kc = new KubeConfig();
+    kc.loadFromOptions({
+      clusters: [
+        { name: 'k', server: 'https://k8s.local:6443', ...clusterExtra },
+      ],
+      users: [{ name: 'sa', token: 'tok' }],
+      contexts: [{ name: 'c', cluster: 'k', user: 'sa' }],
+      currentContext: 'c',
+    });
+    return kc;
+  }
+
+  // applySecurityAuthentication constructs an https.Agent whose constructor
+  // options are stored in agent.options (node:https parity with Node.js).
+  // We extract them via a typed cast to AgentOptions to avoid unsafe any.
+  function getAgentOpts(ctx: RequestContext): AgentOptions | undefined {
+    const agent = ctx.getAgent();
+    if (!agent) return undefined;
+    // https.Agent stores its constructor options in agent.options; the property
+    // is present at runtime but narrower than the union type suggests.
+    // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
+    return (agent as unknown as { options: AgentOptions }).options;
+  }
+
+  test('skipTLSVerify:true → rejectUnauthorized:false on the per-request https.Agent', async () => {
+    const kc = makeKc({ skipTLSVerify: true });
+    const ctx = new RequestContext(
+      'https://k8s.local:6443/api/v1/pods',
+      HttpMethod.GET,
+    );
+    await kc.applySecurityAuthentication(ctx);
+    // applySecurityAuthentication calls createAgent(cluster, { rejectUnauthorized: false, … })
+    // and sets the resulting https.Agent on the context. The agent stores
+    // constructor options in agent.options (Node.js / Bun node:https parity).
+    expect(getAgentOpts(ctx)?.rejectUnauthorized).toBe(false);
+  });
+
+  test('caFile → ca buffer present on the per-request https.Agent', async () => {
+    const caPath = '/tmp/k8s-client-test-ca.pem';
+    // Content is a placeholder; bufferFromFileOrString reads the bytes as-is.
+    writeFileSync(caPath, 'placeholder-pem-content');
+    try {
+      const kc = makeKc({ caFile: caPath });
+      const ctx = new RequestContext(
+        'https://k8s.local:6443/api/v1/pods',
+        HttpMethod.GET,
+      );
+      await kc.applySecurityAuthentication(ctx);
+      // applyHTTPSOptions calls fs.readFileSync(caFile) and copies the result
+      // to agentOptions.ca before constructing the https.Agent.
+      const agentOpts = getAgentOpts(ctx);
+      expect(agentOpts?.ca).toBeDefined();
+      expect(Buffer.isBuffer(agentOpts?.ca)).toBe(true);
+      expect(agentOpts?.ca).toEqual(Buffer.from('placeholder-pem-content'));
+    } finally {
+      rmSync(caPath, { force: true });
+    }
+  });
+
+  test('neither skipTLSVerify nor caFile → default TLS (no overrides on the agent)', async () => {
+    const kc = makeKc({});
+    const ctx = new RequestContext(
+      'https://k8s.local:6443/api/v1/pods',
+      HttpMethod.GET,
+    );
+    await kc.applySecurityAuthentication(ctx);
+    const agentOpts = getAgentOpts(ctx);
+    expect(agentOpts).toBeDefined();
+    expect(agentOpts?.rejectUnauthorized).toBeUndefined();
+    expect(agentOpts?.ca).toBeUndefined();
+  });
+
+  test('caData → ca buffer present on the per-request https.Agent', async () => {
+    const caData = Buffer.from('placeholder-pem-content').toString('base64');
+    const kc = makeKc({ caData });
+    const ctx = new RequestContext(
+      'https://k8s.local:6443/api/v1/pods',
+      HttpMethod.GET,
+    );
+    await kc.applySecurityAuthentication(ctx);
+    const agentOpts = getAgentOpts(ctx);
+    expect(agentOpts?.ca).toBeDefined();
+    expect(Buffer.isBuffer(agentOpts?.ca)).toBe(true);
+  });
+});

services/sandbox/src/backend/kubernetes/k8s-client.ts

@@ -6,13 +6,20 @@
 // createNamespacedPod / readNamespacedPodLog / deleteNamespacedPod + presigned-
 // URL I/O done inside the Pod — every primitive below is plain HTTP.
 //
-// AUTH NOTE (Bun): Bun's fetch does NOT apply a kubeconfig's client cert or
-// custom CA, so a client-cert cluster (e.g. kind's default kubeconfig) auths
-// as system:anonymous. The real in-cluster path uses a ServiceAccount BEARER
-// TOKEN (an Authorization header Bun sends fine) + the cluster CA — for Bun to
-// trust that CA, the container must set NODE_EXTRA_CA_CERTS to the SA ca.crt
-// (/var/run/secrets/kubernetes.io/serviceaccount/ca.crt). Local dev needs a
-// token-based kubeconfig, not kind's client-cert one.
+// AUTH NOTE (Bun): @kubernetes/client-node@1.4.0 routes requests through
+// node-fetch v2, which calls https.request() with an https.Agent — NOT Bun's
+// native fetch(). Through node:https, skipTLSVerify (→ rejectUnauthorized:
+// false on the Agent) and caFile (→ ca Buffer on the Agent) ARE honoured by
+// Bun's TLS stack. What is NOT honoured is client-cert auth (cert/key options
+// on the Agent): Bun's TLS layer does not support mutual TLS client
+// certificates, so a client-cert kubeconfig (e.g. kind's default) auths as
+// system:anonymous. Use a ServiceAccount bearer-token kubeconfig instead.
+//
+// In-cluster CA trust: loadFromCluster() sets caFile to the projected SA
+// ca.crt, which is read into the Agent's ca option and works as above. As a
+// belt-and-suspenders measure the container should also set NODE_EXTRA_CA_CERTS
+// to the same path so that any native fetch() calls (e.g. in harvest) trust
+// the apiserver CA without requiring an explicit Agent.
 
 import {
   type ConfigurationOptions,
@@ -77,6 +84,11 @@ export function makeK8sClient(namespace: string): K8sClient {
     // Explicit bearer-token config (dev / Bun-friendly). Bun can't use a
     // client-cert kubeconfig, so point at the apiserver with an SA token; CA
     // trust via SANDBOX_K8S_CAFILE + NODE_EXTRA_CA_CERTS, or skipTLSVerify.
+    if (!process.env.SANDBOX_K8S_CAFILE) {
+      console.warn(
+        '[sandbox.k8s] SANDBOX_K8S_CAFILE not set — TLS certificate verification DISABLED (skipTLSVerify: true). Do not use in production.',
+      );
+    }
     kc.loadFromOptions({
       clusters: [
         {