diff --git a/Makefile b/Makefile index c87e75c..56b5893 100644 --- a/Makefile +++ b/Makefile @@ -193,6 +193,7 @@ define run_agent_container --workspace-tongs "$$workspace_dir/.swarmforge/tongs" \ --workspace "$$workspace_dir" \ --approvals "$(SWARMFORGE_USER_ASSETS_DIR)/approvals.json" \ + --providers "$(SWARMFORGE_USER_ASSETS_DIR)/secret-providers.yaml" \ --anvil-image "$(4)" \ -- \ docker run -it --rm --name "$(1)" \ diff --git a/scripts/run_anvil.py b/scripts/run_anvil.py index 3e60950..0e45432 100644 --- a/scripts/run_anvil.py +++ b/scripts/run_anvil.py @@ -29,10 +29,32 @@ while the long-lived `shared` tongs keep running. A `port` tong's reachability is injected into the anvil as environment; a `none` tong has no anvil-facing surface. -The launcher starts `shared` and `session` tongs reached over the network -(`port`) or with no anvil-facing surface (`none`), carrying no secret references. -A secret reference, an `mcp` or `volume` interface, or a `shared` tong that mounts -the workspace is refused with a clear message rather than started half-wired. +A tong's secret references are resolved on the host (see "Secret delivery") and +handed to the tong as environment, so the launcher starts `shared` and `session` +tongs reached over the network (`port`) or with no anvil-facing surface (`none`), +with or without secrets. An `mcp` or `volume` interface, or a `shared` tong that +mounts the workspace, is refused with a clear message rather than started +half-wired. + +Secret delivery +--------------- +A tong's `env:` values may carry `${secret::}` references. The +launcher resolves them on the host by shelling out to the provider CLIs declared +in the user-layer table passed as `--providers` (defaulting to +`~/.swarmforge/secret-providers.yaml`), so interactive unlocks happen in the +user's terminal before the anvil starts. A resolved secret is never passed to a +tong as a docker `-e` env var (anything holding the docker socket could read it +back via `docker inspect`), never a command-line argument, and never written to +disk. Instead the launcher creates a host FIFO, bind-mounts it read-only into the +tong, and overrides the tong's entrypoint with a `/bin/sh` wrapper that reads the +FIFO, exports each `NAME=value` into its environment, then execs the image's real +entrypoint+command (looked up via `docker inspect`, or declared as +`entrypoint:`/`command:` on the tong). The launcher writes the resolved values +into the FIFO only once the wrapper has opened the read end, so the secrets reach +the real process as ordinary environment variables -- present before it starts, +since the wrapper blocks on the FIFO until delivery -- while the bytes only ever +live in the kernel pipe buffer. A tong with secret env therefore needs a `/bin/sh` +in its image; one without secrets runs its image entrypoint unchanged. First-run approval ------------------ @@ -59,10 +81,14 @@ """ import collections +import errno import importlib.util +import json import os +import shutil import subprocess import sys +import tempfile import time # Load the pure core (layer discovery + name-based merge) by path, the same way @@ -77,7 +103,8 @@ USAGE = ( "usage: run_anvil.py [--user-tongs DIR] [--org-tongs DIR] " "[--repo-tongs DIR] [--workspace-tongs DIR] [--workspace PATH] " - "[--approvals PATH] [--anvil-image IMAGE] [--no-prompt] -- " + "[--approvals PATH] [--providers PATH] [--anvil-image IMAGE] [--no-prompt] " + "-- " ) # Each flag names the host directory for one definition layer. The merge always @@ -92,12 +119,13 @@ # Parsed launcher options. `workspace` is the workspace root used to key approval # of workspace-sourced tongs and to resolve the `workspace` mount word; `approvals` -# is the store path (default resolved in main); `anvil_image` is the image the -# readiness prober runs to dial a tong's network-internal port; `no_prompt` makes -# the approval gate fail closed for scripted runs. +# is the approvals store path and `providers` the secret-provider table path (both +# default-resolved in main); `anvil_image` is the image the readiness prober runs +# to dial a tong's network-internal port; `no_prompt` makes the approval gate fail +# closed for scripted runs. LauncherOptions = collections.namedtuple( "LauncherOptions", - ["layer_dirs", "workspace", "approvals", "anvil_image", "no_prompt"], + ["layer_dirs", "workspace", "approvals", "providers", "anvil_image", "no_prompt"], ) @@ -116,6 +144,7 @@ def parse_args(argv): paths = {} workspace = None approvals = None + providers = None anvil_image = None no_prompt = False index = 0 @@ -127,7 +156,9 @@ def parse_args(argv): raise UsageError("missing anvil command after '--'") layer_dirs = [(layer, paths[layer]) for layer in tongs.LAYERS if layer in paths] return ( - LauncherOptions(layer_dirs, workspace, approvals, anvil_image, no_prompt), + LauncherOptions( + layer_dirs, workspace, approvals, providers, anvil_image, no_prompt + ), anvil_cmd, ) if token in LAYER_FLAGS: @@ -148,6 +179,12 @@ def parse_args(argv): approvals = argv[index + 1] index += 2 continue + if token == "--providers": + if index + 1 >= len(argv): + raise UsageError("--providers requires a path argument") + providers = argv[index + 1] + index += 2 + continue if token == "--anvil-image": if index + 1 >= len(argv): raise UsageError("--anvil-image requires an image argument") @@ -175,6 +212,20 @@ def default_approvals_path(): return os.path.join(base, "approvals.json") +def default_providers_path(): + """Path to the secret-provider table in the user layer when none is passed. + + Mirrors the Makefile's `SWARMFORGE_USER_ASSETS_DIR` default (~/.swarmforge), + so the launcher finds `secret-providers.yaml` even if Make does not pass + `--providers` explicitly. A missing file means no providers configured, which + only matters if a tong actually references a secret. + """ + base = os.environ.get("SWARMFORGE_USER_ASSETS_DIR") or os.path.join( + os.path.expanduser("~"), ".swarmforge" + ) + return os.path.join(base, "secret-providers.yaml") + + def discover_tongs(layer_dirs): """Merged tong set across the given layers ({} when none are present).""" return tongs.merge_tongs(tongs.discover(layer_dirs)) @@ -332,6 +383,113 @@ def resolve(provider, ref): return resolve +# --- Secret delivery channel -------------------------------------------------- +# Resolved secrets reach a tong over a host FIFO bind-mounted into the container, +# not as `-e`/argv/disk. The bytes only ever live in the kernel pipe buffer: the +# tong's `/bin/sh` wrapper blocks reading the FIFO, and the launcher writes the +# `export NAME=value` script once the wrapper has opened the read end, so the +# values are in the process environment before the real entrypoint starts. The +# channel is created behind a factory so `run_with_tongs` can be tested with a +# fake that records the payload instead of touching the filesystem. + + +class SecretChannel: + """A host FIFO handing a tong its secret env through the kernel pipe buffer.""" + + def __init__(self, directory, host_path): + self._dir = directory + self.host_path = host_path + + def deliver(self, payload, *, timeout=30.0, poll=0.05, + sleep=time.sleep, monotonic=time.monotonic): + """Write `payload` once the tong has opened the FIFO's read end. + + Opens the write end non-blocking and retries while no reader is attached + (`ENXIO`), so a tong that never starts times out rather than hanging the + launcher forever. Once the tong's wrapper has opened the read end the open + succeeds; the whole payload is written -- looping over partial writes and + a full pipe buffer, since the channel is non-blocking and a payload can + exceed the pipe capacity -- and the end closed, signalling EOF so the + wrapper's `cat` returns and it execs the real process. Raises + `OrchestrationError` if no reader appears, the buffer never drains within + `timeout`, or the tong closes the read end before delivery completes (so a + truncated secret never reaches the tong silently). + """ + deadline = monotonic() + timeout + while True: + try: + fd = os.open(self.host_path, os.O_WRONLY | os.O_NONBLOCK) + break + except OSError as exc: + if exc.errno == errno.ENXIO and monotonic() < deadline: + sleep(poll) + continue + if exc.errno == errno.ENXIO: + raise OrchestrationError( + "tong did not open its secret channel within %gs" % timeout + ) + raise OrchestrationError("secret channel error: %s" % exc) + try: + data = memoryview(payload.encode("utf-8")) + while data: + try: + data = data[os.write(fd, data):] + except BlockingIOError: + # Pipe buffer full; the tong's `cat` is still draining. Wait, + # bounded by the same deadline, rather than dropping bytes. + if monotonic() >= deadline: + raise OrchestrationError( + "tong did not drain its secret channel within %gs" % timeout + ) + sleep(poll) + except BrokenPipeError: + raise OrchestrationError( + "tong closed its secret channel before delivery completed" + ) + finally: + os.close(fd) + + def cleanup(self): + """Remove the FIFO and its directory (best-effort).""" + shutil.rmtree(self._dir, ignore_errors=True) + + +def open_secret_channel(uid=None): + """Create a host FIFO in a private temp dir, returning a `SecretChannel`. + + The directory is mode 0700 and the FIFO 0600, so only the launcher's user can + open the read end and intercept the secret. When the tong runs as a different + non-root uid (from the image config) the FIFO is `chown`ed to it best-effort so + that user can read it; a container running as root reads it regardless. + """ + directory = tempfile.mkdtemp(prefix="swarmforge-secret-") + os.chmod(directory, 0o700) + host_path = os.path.join(directory, "secret-env") + os.mkfifo(host_path, 0o600) + if uid is not None: + try: + os.chown(host_path, uid, -1) + except OSError: + pass # not permitted (uid differs and launcher is not root); 0600 stands + return SecretChannel(directory, host_path) + + +def _uid_of(image_user): + """Numeric uid from an image's configured user, or None if not a bare uid. + + `docker inspect`'s `.Config.User` may be empty, a numeric `uid[:gid]`, or a + name. Only a bare numeric uid can be `chown`ed to without a passwd lookup; a + name (or empty/root) leaves the FIFO at its default 0600 launcher ownership. + """ + if not image_user: + return None + token = image_user.split(":", 1)[0] + try: + return int(token) + except ValueError: + return None + + # --- Docker seam -------------------------------------------------------------- # Every docker invocation goes through DockerCLI so the orchestration logic can # be unit-tested against a fake. The methods are thin wrappers; the launch @@ -403,6 +561,41 @@ def health_status(self, container): def exec_ok(self, container, command): return self._quiet(["docker", "exec", container] + list(command)) == 0 + def image_exec_config(self, image): + """`(entrypoint, cmd, user)` from an image's config, pulling it if absent. + + Used to reconstruct the process a secret-injecting tong must `exec` once + its `/bin/sh` wrapper has loaded the secret env: overriding `--entrypoint` + for the wrapper drops the image's own entrypoint/command, so they are read + back here. `entrypoint`/`cmd` are argv lists (possibly empty); `user` is + the image's configured user (``""`` when none). A missing image is pulled + once before retrying; a still-missing or unreadable image is a `DockerError`. + """ + info = self._inspect_image(image) + if info is None: + self._checked(["docker", "pull", image]) + info = self._inspect_image(image) + if info is None: + raise DockerError("cannot read image config for %r" % image) + return info + + def _inspect_image(self, image): + fmt = ("{{json .Config.Entrypoint}}\n{{json .Config.Cmd}}\n" + "{{json .Config.User}}") + completed = self._run( + ["docker", "image", "inspect", "--format", fmt, image], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + if completed.returncode != 0: + return None + lines = _decode(completed.stdout).splitlines() + if len(lines) < 3: + return None + entrypoint = json.loads(lines[0]) or [] + cmd = json.loads(lines[1]) or [] + user = json.loads(lines[2]) or "" + return entrypoint, cmd, user + def tcp_probe(self, network, host, port, image): """True if `host:port` accepts a TCP connection from within `network`. @@ -579,10 +772,9 @@ def unsupported_tong_reasons(merged): """Reasons each discovered tong is outside what the launcher can start. The launcher starts `shared` and `session` tongs reached over the network - (`port`) or with no anvil-facing surface (`none`), holding no secret. Refused - here: + (`port`) or with no anvil-facing surface (`none`), resolving any secret + references and delivering them as env over a FIFO. Refused here: - * a secret reference -- it needs tmpfs delivery; * an `mcp` interface -- it needs generated MCP config; * a `volume` interface -- a shared named volume has no consumer yet, so it is not wired into either container; @@ -599,11 +791,6 @@ def unsupported_tong_reasons(merged): for name in sorted(merged): defn = merged[name]["definition"] kind = (defn.get("interface") or {}).get("kind") - if tongs.find_secret_refs(defn): - reasons.append( - "tong '%s' references a secret, which this launcher does not " - "deliver" % name - ) if kind == "mcp": reasons.append( "tong '%s' has an 'mcp' interface, which this launcher does not " @@ -623,64 +810,92 @@ def unsupported_tong_reasons(merged): return reasons -def _start_shared_tong(docker, name, defn, *, container, network, alias, - workspace, label_hash): - """Start one `shared` tong container detached, replacing any old one. - - The launcher only reaches here for a secret-less tong, so the definition's - `env` is passed straight through as `-e`; any existing container of the same - name is removed first so a stale or stopped one is replaced cleanly. +def _start_one_tong(docker, name, defn, *, container, network, alias, + resolver, workspace, label_hash, make_channel): + """Start one tong container detached, delivering any secret env over a FIFO. + + Resolves the definition's secret references through `resolver` and splits the + env into plain (`-e`) and secret. With no secret env the image's own entrypoint + runs unchanged. With secret env, the tong's entrypoint is overridden with a + `/bin/sh` wrapper (built from the image's real entrypoint+command, read via + `docker inspect` or declared on the tong) that reads a bind-mounted host FIFO, + exports each `NAME=value` into its environment, then execs the real process. The + launcher writes the resolved values into the FIFO only once the wrapper has + opened the read end, so the secrets are present in the environment before the + real process starts, while the bytes live only in the kernel pipe buffer -- + never `-e`, argv, or disk. + + Any existing container of the same name is removed first so a stale or stopped + one is replaced cleanly. If anything fails after the container starts -- a + docker error, a delivery timeout, or a Ctrl-C -- the container is removed + before re-raising, so a half-configured `shared` tong (stamped with its + config-hash label) is not reused on the next session despite missing its + secret. The FIFO is always cleaned up. """ - argv = tongs.tong_run_argv( - name, defn, - container_name=container, network=network, alias=alias, - env=defn.get("env") or {}, label_hash=label_hash, workspace=workspace, - ) - docker.rm_force(container) - docker.run_detached(argv) + plan = tongs.plan_tong_secrets(defn.get("env"), resolver) + plain_env = plan["env"] + secrets = plan["secrets"] + + if not secrets: + argv = tongs.tong_run_argv( + name, defn, + container_name=container, network=network, alias=alias, + env=plain_env, label_hash=label_hash, workspace=workspace, + ) + docker.rm_force(container) + docker.run_detached(argv) + return + + image_entrypoint, image_cmd, image_user = docker.image_exec_config(defn["image"]) + try: + target = tongs.resolve_exec_target(defn, image_entrypoint, image_cmd) + except ValueError as exc: + raise OrchestrationError(str(exc)) + entrypoint, command = tongs.secret_inject_argv(target) + payload = tongs.render_secret_exports(secrets) + + channel = make_channel(_uid_of(image_user)) + try: + argv = tongs.tong_run_argv( + name, defn, + container_name=container, network=network, alias=alias, + env=plain_env, label_hash=label_hash, workspace=workspace, + fifo_host_path=channel.host_path, entrypoint=entrypoint, command=command, + ) + docker.rm_force(container) + docker.run_detached(argv) + channel.deliver(payload) + except BaseException: + docker.rm_force(container) + raise + finally: + channel.cleanup() def _ensure_shared_tong(docker, name, defn, *, container, network, alias, - workspace, label_hash): + resolver, workspace, label_hash, make_channel): """Start a `shared` tong, or reuse the running one, recreating it if stale. A `shared` tong is one long-lived container keyed by `shared_container_name`. Its config-hash label answers "did the definition change since it started?": a missing container, a stopped one, or a hash mismatch triggers a fresh start (removing any old container first); a running container with a matching hash - is reused untouched. The hash is over the merged definition, so the same - long-lived container is reused across sessions while the definition is stable. + is reused untouched. The hash is over the merged (pre-resolution) definition, + so the same long-lived container is reused across sessions while the + definition is stable -- and deciding to reuse one never runs a secret-provider + CLI, so a rotated secret behind an unchanged reference does not churn it. """ state = docker.inspect_state(container) if state and state["running"] and state["label"] == label_hash: return - _start_shared_tong( + _start_one_tong( docker, name, defn, container=container, network=network, alias=alias, - workspace=workspace, label_hash=label_hash, + resolver=resolver, workspace=workspace, label_hash=label_hash, + make_channel=make_channel, ) -def _start_session_tong(docker, name, defn, *, container, network, alias, workspace): - """Start one `session` tong container detached on the per-session network. - - A `session` tong is per-session and torn down with the anvil, so it gets a - session-suffixed container name and lives only on the per-session network - under its canonical alias. The launcher only reaches here for a secret-less - tong, so the definition's `env` is passed straight through as `-e`; any - leftover container of the same name (from a crashed prior session) is removed - first so it is replaced cleanly. - """ - argv = tongs.tong_run_argv( - name, defn, - container_name=container, network=network, alias=alias, - env=defn.get("env") or {}, label_hash=tongs.config_hash(defn), - workspace=workspace, - ) - docker.rm_force(container) - docker.run_detached(argv) - - def _injection_pre_image_args(injection): """`-e`/`-v` options the discovered tongs add to the anvil before the image. @@ -696,22 +911,27 @@ def _injection_pre_image_args(injection): return args -def run_with_tongs(merged, anvil_cmd, opts, *, docker, +def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, + make_channel=open_secret_channel, sleep=time.sleep, monotonic=time.monotonic): """Start the discovered tongs, run the anvil, and tear down session state. Only reached when at least one tong was discovered and every tong is startable (the empty case stays a direct exec; unsupported tongs are refused earlier). - `shared` tongs are ensured on the anvil's base network, reusing a running one - whose config hash still matches. When any `session` tong exists a per-session - network is created: the `session` tongs start on it under their canonical - aliases, each network-facing `shared` tong is connected to it for this session, - and the anvil joins it plus the base network (the `NETWORK=` escape hatch). - With no `session` tong the anvil keeps using the base network exactly as - before. Each tong's readiness is probed on the network the anvil will use, - `port` reachability is injected into the anvil argv, then the anvil runs in the - foreground. + Each tong's secret references are resolved through the provider CLIs in + `providers` (the user-layer table) and delivered as env over a FIFO (created by + `make_channel`) as the tong starts; a tong without secrets gets none of that + machinery. `shared` tongs are ensured + on the anvil's base network, reusing a running one whose config hash still + matches (which never re-resolves its secrets). When any `session` tong exists a + per-session network is created: the `session` tongs start on it under their + canonical aliases, each network-facing `shared` tong is connected to it for + this session, and the anvil joins it plus the base network (the `NETWORK=` + escape hatch). With no `session` tong the anvil keeps using the base network + exactly as before. Each tong's readiness is probed on the network the anvil + will use, `port` reachability is injected into the anvil argv, then the anvil + runs in the foreground. On exit -- including SIGINT -- the `session` tongs and the per-session network are torn down (and the connected `shared` tongs disconnected) while the @@ -719,8 +939,10 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, Returns the anvil's exit code. Raises `OrchestrationError` if a tong never becomes ready (the anvil does not run against a half-up environment) or a - `session` tong is discovered with no anvil `--name` to key the session by. + `session` tong is discovered with no anvil `--name` to key the session by, and + `SecretResolutionError` if a secret reference cannot be resolved. """ + resolver = make_secret_resolver(providers or {}) base_network = tongs.anvil_option_value(anvil_cmd, "--network") session_id = tongs.anvil_option_value(anvil_cmd, "--name") @@ -755,10 +977,11 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, alias = tongs.canonical_alias(name, defn) if defn.get("lifecycle") == "session": container = tongs.session_container_name(session_id, name) - _start_session_tong( + _start_one_tong( docker, name, defn, container=container, network=plan["network"], alias=alias, - workspace=opts.workspace, + resolver=resolver, workspace=opts.workspace, + label_hash=tongs.config_hash(defn), make_channel=make_channel, ) started_sessions.append(container) else: @@ -766,7 +989,8 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, _ensure_shared_tong( docker, name, defn, container=container, network=base_network, alias=alias, - workspace=opts.workspace, label_hash=tongs.config_hash(defn), + resolver=resolver, workspace=opts.workspace, + label_hash=tongs.config_hash(defn), make_channel=make_channel, ) ready_checks.append((name, defn, alias, container)) @@ -876,23 +1100,34 @@ def main(argv): return 1 # Refuse anything this launcher cannot start (see unsupported_tong_reasons: - # a secret reference, an MCP or volume interface, or a shared tong mounting - # the workspace) rather than starting it half-wired. Every remaining tong has - # a `port`/`none` interface, whose canonical alias is its unique filename, so - # no two can claim the same network alias. + # an MCP or volume interface, or a shared tong mounting the workspace) rather + # than starting it half-wired. Every remaining tong has a `port`/`none` + # interface, whose canonical alias is its unique filename, so no two can claim + # the same network alias. unsupported = unsupported_tong_reasons(merged) if unsupported: for reason in unsupported: tongs.warn(reason) return 1 + # Load the secret-provider table the resolver shells out to. A malformed file + # stops the launch with a clear message rather than silently dropping a + # provider; a missing file is fine until a tong actually references a secret. + try: + providers = tongs.load_secret_providers(opts.providers or default_providers_path()) + except ValueError as exc: + tongs.warn(str(exc)) + return 1 + # run_with_tongs runs the anvil in the foreground and returns its exit code, # leaving the (long-lived) shared tongs running. A tong that never becomes - # ready stops the launch rather than running the anvil against a half-up - # environment. + # ready, or a secret reference that cannot be resolved, stops the launch + # rather than running the anvil against a half-up environment. try: - return run_with_tongs(merged, anvil_cmd, opts, docker=DockerCLI()) - except (OrchestrationError, DockerError) as exc: + return run_with_tongs( + merged, anvil_cmd, opts, docker=DockerCLI(), providers=providers + ) + except (OrchestrationError, DockerError, SecretResolutionError) as exc: tongs.warn(str(exc)) return 1 except KeyboardInterrupt: diff --git a/scripts/test_run_anvil.py b/scripts/test_run_anvil.py index 95ffdd8..103089b 100644 --- a/scripts/test_run_anvil.py +++ b/scripts/test_run_anvil.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +import threading import unittest from unittest import mock @@ -78,16 +79,19 @@ def test_approval_options_default_to_inert(self): opts, _ = run_anvil.parse_args(["--", "x"]) self.assertIsNone(opts.workspace) self.assertIsNone(opts.approvals) + self.assertIsNone(opts.providers) self.assertIsNone(opts.anvil_image) self.assertFalse(opts.no_prompt) def test_parses_workspace_approvals_and_no_prompt(self): opts, cmd = run_anvil.parse_args( ["--workspace", "/ws", "--approvals", "/a.json", - "--anvil-image", "anvil:img", "--no-prompt", "--", "x"] + "--providers", "/p.yaml", "--anvil-image", "anvil:img", + "--no-prompt", "--", "x"] ) self.assertEqual(opts.workspace, "/ws") self.assertEqual(opts.approvals, "/a.json") + self.assertEqual(opts.providers, "/p.yaml") self.assertEqual(opts.anvil_image, "anvil:img") self.assertTrue(opts.no_prompt) self.assertEqual(cmd, ["x"]) @@ -104,6 +108,10 @@ def test_approvals_without_value_raises(self): with self.assertRaises(run_anvil.UsageError): run_anvil.parse_args(["--approvals"]) + def test_providers_without_value_raises(self): + with self.assertRaises(run_anvil.UsageError): + run_anvil.parse_args(["--providers"]) + def test_missing_separator_raises(self): with self.assertRaises(run_anvil.UsageError): run_anvil.parse_args(["--repo-tongs", "/r", "docker", "run"]) @@ -193,10 +201,12 @@ def test_missing_workspace_tongs_dir_forwards_verbatim(self): self.assertNotIn("tong", stderr) def test_launcher_flags_do_not_leak_into_anvil_argv(self): - # The Makefile always passes --anvil-image; with no tongs it is consumed - # by the launcher and the anvil argv is forwarded unchanged. + # The Makefile always passes --anvil-image and --providers; with no tongs + # they are consumed by the launcher and the anvil argv is forwarded + # unchanged (the secret-provider table is never even read). forwarded, stderr = _run_launcher([ "--anvil-image", "opencode:local", + "--providers", "/nonexistent/secret-providers.yaml", "--repo-tongs", "/nonexistent/tongs", ]) self.assertEqual(forwarded, ANVIL_ARGV) @@ -235,6 +245,30 @@ def test_falls_back_to_home_swarmforge(self): self.assertEqual(run_anvil.default_approvals_path(), expected) +class DefaultProvidersPathTests(unittest.TestCase): + def setUp(self): + self.saved = os.environ.get("SWARMFORGE_USER_ASSETS_DIR") + + def tearDown(self): + if self.saved is None: + os.environ.pop("SWARMFORGE_USER_ASSETS_DIR", None) + else: + os.environ["SWARMFORGE_USER_ASSETS_DIR"] = self.saved + + def test_honors_user_assets_dir(self): + os.environ["SWARMFORGE_USER_ASSETS_DIR"] = "/opt/sf" + self.assertEqual( + run_anvil.default_providers_path(), "/opt/sf/secret-providers.yaml" + ) + + def test_falls_back_to_home_swarmforge(self): + os.environ.pop("SWARMFORGE_USER_ASSETS_DIR", None) + expected = os.path.join( + os.path.expanduser("~"), ".swarmforge", "secret-providers.yaml" + ) + self.assertEqual(run_anvil.default_providers_path(), expected) + + class RenderPrivilegeSummaryTests(unittest.TestCase): def test_renders_requested_privileges(self): text = run_anvil.render_privilege_summary( @@ -394,12 +428,14 @@ def test_error_message_never_contains_the_secret(self): self.assertIn("boom", str(ctx.exception)) def test_drives_plan_tong_secrets_end_to_end(self): - # The resolver is the impure half of tongs.plan_tong_secrets: a secret - # env var ends up as a tmpfs file, never as a -e value. + # The resolver is the impure half of tongs.plan_tong_secrets: a secret env + # var resolves to a value under `secrets`, never the plain `-e` env. resolve = run_anvil.make_secret_resolver({"echo": self._writes("sys.argv[1]")}) - plan = tongs.plan_tong_secrets({"TOKEN": "${secret:echo:s3cr3t}"}, resolve) - self.assertEqual(plan["files"], {"/run/swarmforge/secrets/TOKEN": "s3cr3t"}) - self.assertEqual(plan["env"], {"TOKEN_FILE": "/run/swarmforge/secrets/TOKEN"}) + plan = tongs.plan_tong_secrets( + {"REGION": "us", "TOKEN": "${secret:echo:s3cr3t}"}, resolve + ) + self.assertEqual(plan["env"], {"REGION": "us"}) + self.assertEqual(plan["secrets"], {"TOKEN": "s3cr3t"}) class MainGateTests(unittest.TestCase): @@ -486,22 +522,27 @@ def test_invalid_tong_returns_one_without_exec(self): self.assertEqual(completed.returncode, 1) self.assertEqual(completed.stdout, "") # anvil never ran - def test_secret_tong_refused_without_exec(self): - # A shared tong that references a secret cannot be delivered here, so it - # is refused before docker. + def test_malformed_providers_file_returns_one_without_exec(self): + # The secret-provider table is loaded before any tong starts, so a + # malformed file stops the launch (a clear error, anvil never runs) rather + # than dropping a provider and failing mid-resolution. with tempfile.TemporaryDirectory() as tmp: tongs_dir = os.path.join(tmp, "tongs") os.makedirs(tongs_dir) - with open(os.path.join(tongs_dir, "creds.yaml"), "w") as handle: + with open(os.path.join(tongs_dir, "shipper.yaml"), "w") as handle: handle.write( - "lifecycle: shared\nimage: x\n" - "env:\n TOKEN: ${secret:op:op://Work/t}\n" - "interface:\n kind: none\nreadiness:\n mode: none\n" + "lifecycle: shared\nimage: x\ninterface:\n kind: none\n" + "readiness:\n mode: none\n" ) - completed = _run_launcher_raw(["--repo-tongs", tongs_dir]) + providers = os.path.join(tmp, "secret-providers.yaml") + with open(providers, "w") as handle: + handle.write("providers:\n op: not-a-list\n") + completed = _run_launcher_raw( + ["--repo-tongs", tongs_dir, "--providers", providers] + ) self.assertEqual(completed.returncode, 1) - self.assertEqual(completed.stdout, "") - self.assertIn("secret", completed.stderr) + self.assertEqual(completed.stdout, "") # anvil never ran + self.assertIn("op", completed.stderr) def test_keyboard_interrupt_during_run_returns_130(self): # Ctrl-C while the anvil runs leaves the (long-lived) shared tongs up and @@ -598,10 +639,7 @@ def test_startable_session_tong_has_no_reasons(self): [], ) - def test_secret_mcp_volume_each_refused(self): - self.assertTrue(self._reasons( - {"lifecycle": "shared", "image": "x", "env": {"T": "${secret:op:r}"}, - "interface": {"kind": "none"}, "readiness": {"mode": "none"}})) + def test_mcp_and_volume_each_refused(self): self.assertTrue(self._reasons( {"lifecycle": "shared", "image": "x", "interface": {"kind": "mcp", "name": "g", "port": 8080}, @@ -611,12 +649,20 @@ def test_secret_mcp_volume_each_refused(self): "interface": {"kind": "volume", "volume": "v", "mountpoint": "/m"}, "readiness": {"mode": "none"}})) - def test_session_tong_with_secret_or_mcp_still_refused(self): - # Lifting the session guard does not lift the secret/mcp guards; a session - # tong that needs either is still refused (delivered in a later phase). - self.assertTrue(self._reasons( + def test_secret_tong_is_now_startable(self): + # Secrets are resolved and delivered as env over a FIFO, so a tong that references + # one (and is otherwise reachable over the network or has no surface) is no + # longer refused -- on either lifecycle. + self.assertEqual(self._reasons( + {"lifecycle": "shared", "image": "x", "env": {"T": "${secret:op:r}"}, + "interface": {"kind": "none"}, "readiness": {"mode": "none"}}), []) + self.assertEqual(self._reasons( {"lifecycle": "session", "image": "x", "env": {"T": "${secret:op:r}"}, - "interface": {"kind": "none"}, "readiness": {"mode": "none"}})) + "interface": {"kind": "port", "port": 5432}, "readiness": {"mode": "none"}}), []) + + def test_session_mcp_tong_still_refused(self): + # Lifting the secret guard does not lift the mcp guard; a session tong with + # an mcp interface is still refused (wired up in a later phase). self.assertTrue(self._reasons( {"lifecycle": "session", "image": "x", "interface": {"kind": "mcp", "name": "g", "port": 8080}, @@ -656,12 +702,15 @@ class FakeDocker: """In-process stand-in for DockerCLI that records calls and returns canned results, so orchestration is tested without a docker daemon.""" - def __init__(self, states=None, ready=True, anvil_rc=0): + def __init__(self, states=None, ready=True, anvil_rc=0, + image_config=(["app"], [], "")): self.calls = [] self._states = states or {} # container -> inspect_state dict self._ready = ready self._anvil_rc = anvil_rc + self._image_config = image_config self.run_argvs = [] # detached `docker run` argvs + self.inspected_images = [] # images whose exec config was read self.anvil_argv = None # set when the anvil runs self.anvil_extra_networks = None # extra networks the anvil joined @@ -670,6 +719,12 @@ def rm_force(self, container): def run_detached(self, argv): self.run_argvs.append(argv) + container = argv[argv.index("--name") + 1] if "--name" in argv else None + self.calls.append(("run_detached", container)) + + def image_exec_config(self, image): + self.inspected_images.append(image) + return self._image_config def ensure_network(self, name): self.calls.append(("ensure_network", name)) @@ -708,10 +763,44 @@ def run_foreground(self, argv): return self._anvil_rc +class FakeChannels: + """A `make_channel` stand-in that records secret deliveries without a FIFO. + + Records the uid each channel is opened for, every payload delivered, and how + many channels were cleaned up, so a test can assert what reached the tong (and + that the secret never went through `-e`/argv) without touching the filesystem. + `deliver_error`, if set, is raised from `deliver` to exercise the failure path. + """ + + def __init__(self, deliver_error=None): + self.uids = [] + self.payloads = [] + self.cleanups = 0 + self._deliver_error = deliver_error + + def __call__(self, uid=None): + self.uids.append(uid) + return FakeChannels._Channel(self) + + class _Channel: + host_path = "/fake/swarmforge-secret/secret-env" + + def __init__(self, owner): + self._owner = owner + + def deliver(self, payload, **kwargs): + self._owner.payloads.append(payload) + if self._owner._deliver_error is not None: + raise self._owner._deliver_error + + def cleanup(self): + self._owner.cleanups += 1 + + # Tiny launcher options for driving run_with_tongs directly. def _opts(workspace=None, anvil_image="anvil:img"): return run_anvil.LauncherOptions( - layer_dirs=[], workspace=workspace, approvals=None, + layer_dirs=[], workspace=workspace, approvals=None, providers=None, anvil_image=anvil_image, no_prompt=False, ) @@ -750,6 +839,23 @@ def __call__(self): "readiness": {"mode": "none"}, } +# A secret provider built on the test interpreter (so the suite needs no op/pass +# installed): it echoes the {ref} it is handed, so ${secret:echo:VALUE} resolves +# to "VALUE". +ECHO_PROVIDERS = { + "echo": [sys.executable, "-c", "import sys; sys.stdout.write(sys.argv[1])", "{ref}"] +} + +# A credential-holding shared tong: its token is a secret reference, delivered to +# the running container as env over a FIFO rather than passed as a docker env var. +SHARED_SECRET = { + "lifecycle": "shared", + "image": "github-tong", + "env": {"GITHUB_TOKEN": "${secret:echo:s3cr3t}"}, + "interface": {"kind": "none"}, + "readiness": {"mode": "none"}, +} + class _RecordingRun: """A subprocess.run stand-in that records argvs and returns canned codes. @@ -833,6 +939,112 @@ def test_run_foreground_multi_creates_connects_then_starts(self): ["docker", "start", "--attach", "--interactive", "anvil"] ) + @staticmethod + def _image_run(entrypoint_json, cmd_json, user_json, inspect_codes=(0,)): + """A run() that answers `docker image inspect` with canned JSON. + + `inspect_codes` is the return code for each successive inspect call (so a + test can fail the first and succeed after a pull); other commands return 0. + """ + state = {"calls": 0} + + def run(argv, **kwargs): + if argv[:3] == ["docker", "image", "inspect"]: + idx = min(state["calls"], len(inspect_codes) - 1) + code = inspect_codes[idx] + state["calls"] += 1 + out = ("%s\n%s\n%s" % (entrypoint_json, cmd_json, user_json)).encode() + return subprocess.CompletedProcess(argv, code, stdout=out) + return subprocess.CompletedProcess(argv, 0) + + return run + + def test_image_exec_config_parses_entrypoint_cmd_user(self): + cli = run_anvil.DockerCLI(run=self._image_run('["node"]', '["server.js"]', '"1000"')) + self.assertEqual(cli.image_exec_config("img"), (["node"], ["server.js"], "1000")) + + def test_image_exec_config_treats_null_as_empty(self): + cli = run_anvil.DockerCLI(run=self._image_run("null", "null", "null")) + self.assertEqual(cli.image_exec_config("img"), ([], [], "")) + + def test_image_exec_config_pulls_when_absent_then_succeeds(self): + rec_run = self._image_run('["app"]', "null", '""', inspect_codes=(1, 0)) + cli = run_anvil.DockerCLI(run=rec_run) + self.assertEqual(cli.image_exec_config("img"), (["app"], [], "")) + + def test_image_exec_config_raises_when_still_missing(self): + cli = run_anvil.DockerCLI(run=self._image_run("null", "null", "null", + inspect_codes=(1, 1))) + with self.assertRaises(run_anvil.DockerError): + cli.image_exec_config("img") + + +class UidOfTests(unittest.TestCase): + def test_bare_uid_parses(self): + self.assertEqual(run_anvil._uid_of("1000"), 1000) + self.assertEqual(run_anvil._uid_of("1000:1000"), 1000) + + def test_name_or_empty_is_none(self): + self.assertIsNone(run_anvil._uid_of("appuser")) + self.assertIsNone(run_anvil._uid_of("")) + self.assertIsNone(run_anvil._uid_of(None)) + + +class SecretChannelTests(unittest.TestCase): + def test_times_out_when_no_reader_opens(self): + # No reader ever opens the FIFO, so the non-blocking write open keeps + # getting ENXIO; once the (fake) clock passes the deadline it fails closed + # rather than hanging the launcher. + channel = run_anvil.open_secret_channel() + try: + clock = iter([0.0, 1.0, 2.0, 99.0]) + with self.assertRaises(run_anvil.OrchestrationError): + channel.deliver( + "export X='y'\n", timeout=5.0, poll=0.0, + sleep=lambda _s: None, monotonic=lambda: next(clock), + ) + finally: + channel.cleanup() + + def test_delivers_payload_to_a_reader(self): + # With a reader attached, the payload is written and the reader sees it + # followed by EOF -- the real FIFO round-trip, no docker involved. + channel = run_anvil.open_secret_channel() + received = [] + + def reader(): + with open(channel.host_path, "r") as handle: + received.append(handle.read()) + + thread = threading.Thread(target=reader) + thread.start() + try: + channel.deliver("export TOKEN='s3cr3t'\n") + finally: + thread.join(timeout=5) + channel.cleanup() + self.assertEqual(received, ["export TOKEN='s3cr3t'\n"]) + + def test_delivers_payload_larger_than_pipe_buffer(self): + # A payload bigger than the pipe capacity (~64 KiB) forces several writes + # and a full buffer; every byte must still arrive (no silent truncation). + channel = run_anvil.open_secret_channel() + payload = "export BIG='" + ("x" * 200000) + "'\n" + received = [] + + def reader(): + with open(channel.host_path, "r") as handle: + received.append(handle.read()) + + thread = threading.Thread(target=reader) + thread.start() + try: + channel.deliver(payload) + finally: + thread.join(timeout=10) + channel.cleanup() + self.assertEqual(received, [payload]) + class RunWithTongsTests(unittest.TestCase): def _run(self, docker, merged, anvil=None, workspace=None): @@ -965,6 +1177,134 @@ def test_no_anvil_image_degrades_tcp_to_running_check(self): self.assertEqual(rc, 0) self.assertNotIn("tcp_probe", [c[0] for c in docker.calls]) + # --- Secret resolution + FIFO env delivery ------------------------------ + + def _run_secret(self, docker, merged, providers, channels=None): + return run_anvil.run_with_tongs( + merged, ANVIL_ARGV, _opts(), docker=docker, providers=providers, + make_channel=channels or FakeChannels(), + sleep=lambda _s: None, monotonic=_Clock(), + ) + + def test_secret_delivered_as_env_via_fifo_never_in_argv(self): + # A resolved secret is handed to the tong over the FIFO (an `export` + # script), never as a docker `-e` value; the run argv carries only the + # entrypoint wrapper and the read-only FIFO bind, not the secret. + docker = FakeDocker(image_config=(["node"], ["server.js"], "")) + channels = FakeChannels() + rc = self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), ECHO_PROVIDERS, + channels=channels, + ) + self.assertEqual(rc, 0) + started = docker.run_argvs[0] + # Entrypoint is wrapped and the FIFO is bind-mounted read-only. + self.assertEqual(started[started.index("--entrypoint") + 1], "/bin/sh") + self.assertIn( + "/fake/swarmforge-secret/secret-env:/run/swarmforge/secret-env:ro", started + ) + # The image's real argv is what the wrapper execs (after the image token). + self.assertEqual(started[started.index("github-tong") + 1:], + ["-c", started[started.index("-c") + 1], + "swarmforge-tong", "node", "server.js"]) + # The secret is nowhere in the argv -- it only went through the channel. + self.assertNotIn("s3cr3t", " ".join(started)) + self.assertNotIn("GITHUB_TOKEN=s3cr3t", started) + self.assertEqual(channels.payloads, ["export GITHUB_TOKEN='s3cr3t'\n"]) + self.assertEqual(channels.cleanups, 1) # FIFO cleaned up after delivery + + def test_secret_tong_reads_exec_target_from_image(self): + docker = FakeDocker(image_config=(["entry"], ["arg"], "")) + self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), ECHO_PROVIDERS + ) + self.assertEqual(docker.inspected_images, ["github-tong"]) + + def test_unresolvable_secret_stops_launch_before_anvil(self): + # No provider for the referenced scheme => resolution fails before the tong + # even starts, and the anvil never runs. + docker = FakeDocker() + with self.assertRaises(run_anvil.SecretResolutionError): + self._run_secret(docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), {}) + self.assertEqual(docker.run_argvs, []) # never reached the start + self.assertIsNone(docker.anvil_argv) + + def test_delivery_failure_removes_half_configured_container(self): + # If delivery over the FIFO fails after the container started, the + # container is removed before raising, so a `shared` tong is not left + # stamped with its config-hash label (and reused) while missing its secret. + docker = FakeDocker() + channels = FakeChannels(deliver_error=run_anvil.DockerError("boom")) + with self.assertRaises(run_anvil.DockerError): + self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), ECHO_PROVIDERS, + channels=channels, + ) + # rm_force fires twice: clearing any leftover before start, then removing + # the half-configured container after the failed delivery. + self.assertEqual(docker.calls.count(("rm_force", "swarmforge-shared-gh")), 2) + self.assertEqual(channels.cleanups, 1) # FIFO still cleaned up + self.assertIsNone(docker.anvil_argv) + + def test_interrupt_during_delivery_removes_half_configured_container(self): + # Ctrl-C while delivering must still remove the container, or a `shared` + # tong (stamped with its config-hash label and not tracked for session + # teardown) would be reused next session with a missing secret. + docker = FakeDocker() + channels = FakeChannels(deliver_error=KeyboardInterrupt()) + with self.assertRaises(KeyboardInterrupt): + self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), ECHO_PROVIDERS, + channels=channels, + ) + self.assertEqual(docker.calls.count(("rm_force", "swarmforge-shared-gh")), 2) + self.assertEqual(channels.cleanups, 1) + self.assertIsNone(docker.anvil_argv) + + def test_reused_shared_tong_never_resolves_or_delivers_secrets(self): + # A running shared tong whose hash matches is reused untouched -- deciding + # to reuse must never invoke a secret-provider CLI (which could prompt for + # an unlock every session) or open a channel. + states = {"swarmforge-shared-gh": + {"running": True, "label": tongs.config_hash(SHARED_SECRET)}} + docker = FakeDocker(states=states) + channels = FakeChannels() + boom = {"echo": [sys.executable, "-c", "import sys; sys.exit(1)"]} + rc = self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), boom, + channels=channels, + ) + self.assertEqual(rc, 0) + self.assertEqual(docker.run_argvs, []) # reused, not restarted + self.assertEqual(channels.payloads, []) # no resolution, no delivery + self.assertEqual(docker.inspected_images, []) # no image inspect either + + def test_session_secret_tong_delivered_over_channel(self): + defn = { + "lifecycle": "session", "image": "creds", + "env": {"TOKEN": "${secret:echo:abc}"}, + "interface": {"kind": "none"}, "readiness": {"mode": "none"}, + } + docker = FakeDocker() + channels = FakeChannels() + rc = self._run_secret( + docker, _merged("creds", defn, source=tongs.REPO), ECHO_PROVIDERS, + channels=channels, + ) + self.assertEqual(rc, 0) + self.assertEqual(channels.payloads, ["export TOKEN='abc'\n"]) + + def test_secret_uid_passed_to_channel_factory(self): + # The image's numeric user is passed to the channel factory so the FIFO can + # be chowned to the uid that will read it. + docker = FakeDocker(image_config=(["app"], [], "1000")) + channels = FakeChannels() + self._run_secret( + docker, _merged("gh", SHARED_SECRET, source=tongs.REPO), ECHO_PROVIDERS, + channels=channels, + ) + self.assertEqual(channels.uids, [1000]) + # --- Session lifecycle + per-session networks --------------------------- def test_shared_only_keeps_base_network_and_plain_run(self): diff --git a/scripts/test_tongs.py b/scripts/test_tongs.py index 18668c4..c4d6f59 100644 --- a/scripts/test_tongs.py +++ b/scripts/test_tongs.py @@ -4,6 +4,7 @@ import importlib.util import json import os +import subprocess import tempfile import unittest @@ -255,16 +256,6 @@ def test_bad_interface_kind(self): errors = tongs.validate_tong("t", {"lifecycle": "session", "image": "x", "interface": {"kind": "socket"}}) self.assertTrue(any("interface.kind" in e for e in errors)) - def test_rejects_secret_file_pointer_collision(self): - errors = tongs.validate_tong("t", { - "lifecycle": "session", - "image": "x", - "interface": {"kind": "none"}, - "readiness": {"mode": "none"}, - "env": {"TOKEN": "${secret:op:t}", "TOKEN_FILE": "/declared/path"}, - }) - self.assertTrue(any("TOKEN_FILE" in e for e in errors)) - def test_rejects_invalid_secret_env_name(self): errors = tongs.validate_tong("t", { "lifecycle": "session", @@ -275,6 +266,23 @@ def test_rejects_invalid_secret_env_name(self): }) self.assertTrue(any("a/b" in e for e in errors)) + def test_rejects_non_list_entrypoint_or_command(self): + for field in ("entrypoint", "command"): + errors = tongs.validate_tong("t", { + "lifecycle": "session", "image": "x", + "interface": {"kind": "none"}, "readiness": {"mode": "none"}, + field: "node server.js", # must be a list of strings + }) + self.assertTrue(any(field in e for e in errors), field) + + def test_accepts_list_entrypoint_and_command(self): + errors = tongs.validate_tong("t", { + "lifecycle": "session", "image": "x", + "interface": {"kind": "none"}, "readiness": {"mode": "none"}, + "entrypoint": ["node"], "command": ["server.js"], + }) + self.assertEqual(errors, []) + class SecretRefTests(unittest.TestCase): def test_parse_single_ref_with_inner_colons(self): @@ -390,69 +398,17 @@ def test_partition_empty_env(self): self.assertEqual(tongs.partition_secret_env(None), ({}, {})) self.assertEqual(tongs.partition_secret_env({}), ({}, {})) - def test_delivery_plan_routes_secrets_to_tmpfs_files(self): - plan = tongs.secret_delivery_plan({"TOKEN": "s3cr3t", "API_KEY": "k3y"}) - self.assertEqual(plan["tmpfs"], "/run/swarmforge/secrets") - self.assertEqual( - plan["files"], - {"/run/swarmforge/secrets/API_KEY": "k3y", "/run/swarmforge/secrets/TOKEN": "s3cr3t"}, - ) - self.assertEqual( - plan["env"], - { - "API_KEY_FILE": "/run/swarmforge/secrets/API_KEY", - "TOKEN_FILE": "/run/swarmforge/secrets/TOKEN", - }, - ) - - def test_delivery_plan_empty_has_no_tmpfs(self): - plan = tongs.secret_delivery_plan({}) - self.assertEqual(plan, {"tmpfs": None, "files": {}, "env": {}}) - - def test_plan_tong_secrets_keeps_secret_values_out_of_env(self): + def test_plan_tong_secrets_keeps_secret_values_out_of_plain_env(self): env = {"REGION": "us", "TOKEN": "${secret:op:op://Work/github/token}"} plan = tongs.plan_tong_secrets(env, lambda p, r: "RESOLVED-%s" % r) - # Plain env passes through; the secret becomes a _FILE pointer, never a value. - self.assertEqual( - plan["env"], - {"REGION": "us", "TOKEN_FILE": "/run/swarmforge/secrets/TOKEN"}, - ) - self.assertEqual(plan["tmpfs"], "/run/swarmforge/secrets") - self.assertEqual( - plan["files"], - {"/run/swarmforge/secrets/TOKEN": "RESOLVED-op://Work/github/token"}, - ) - # The resolved secret value reaches files only -- never the -e env plan. + # Plain env passes through; the resolved secret lands only under `secrets`. + self.assertEqual(plan["env"], {"REGION": "us"}) + self.assertEqual(plan["secrets"], {"TOKEN": "RESOLVED-op://Work/github/token"}) self.assertNotIn("RESOLVED-op://Work/github/token", json.dumps(plan["env"])) - def test_delivery_plan_rejects_traversal_env_name(self): - # An env name that would escape the tmpfs dir as a path component is - # refused rather than baked into a file path. - with self.assertRaises(ValueError): - tongs.secret_delivery_plan({"../../etc/cron.d/x": "v"}) - with self.assertRaises(ValueError): - tongs.secret_delivery_plan({"a/b": "v"}) - - def test_plan_tong_secrets_rejects_pointer_collision(self): - # A plain TOKEN_FILE that disagrees with the synthesized pointer would - # make the tong unable to find TOKEN, so the plan fails closed. - env = {"TOKEN_FILE": "/declared/path", "TOKEN": "${secret:op:t}"} - with self.assertRaises(ValueError): - tongs.plan_tong_secrets(env, lambda p, r: "SECRET") - - def test_plan_tong_secrets_allows_matching_declared_pointer(self): - env = { - "TOKEN_FILE": "/run/swarmforge/secrets/TOKEN", - "TOKEN": "${secret:op:t}", - } - plan = tongs.plan_tong_secrets(env, lambda p, r: "SECRET") - self.assertEqual(plan["env"]["TOKEN_FILE"], "/run/swarmforge/secrets/TOKEN") - self.assertEqual(plan["files"], {"/run/swarmforge/secrets/TOKEN": "SECRET"}) - self.assertNotIn("SECRET", json.dumps(plan["env"])) - def test_plan_tong_secrets_inert_without_secrets(self): plan = tongs.plan_tong_secrets({"REGION": "us"}, lambda p, r: "x") - self.assertEqual(plan, {"env": {"REGION": "us"}, "tmpfs": None, "files": {}}) + self.assertEqual(plan, {"env": {"REGION": "us"}, "secrets": {}}) def test_plan_tong_secrets_resolves_each_provider_with_its_ref(self): env = {"A": "${secret:op:a}", "B": "${secret:pass:b}"} @@ -460,6 +416,78 @@ def test_plan_tong_secrets_resolves_each_provider_with_its_ref(self): tongs.plan_tong_secrets(env, lambda p, r: seen.append((p, r)) or "v") self.assertEqual(sorted(seen), [("op", "a"), ("pass", "b")]) + def test_render_secret_exports_quotes_values_safely(self): + # Each value is single-quoted with embedded quotes escaped, so an arbitrary + # value -- here one with a quote, a space, and a newline -- cannot break out + # of its assignment when the wrapper evals the script. + script = tongs.render_secret_exports({"B": "two\nlines", "A": "it's a $X"}) + # Sorted by name; A first. + self.assertEqual( + script, + "export A='it'\\''s a $X'\n" "export B='two\nlines'\n", + ) + + def test_render_secret_exports_eval_round_trips_the_value(self): + # Sanity-check that evaling the rendered script in a real shell reproduces + # the exact bytes, proving the quoting survives metacharacters. + value = "a'b\"c $d `e` \\f\n g" + script = tongs.render_secret_exports({"V": value}) + out = subprocess.run( + ["/bin/sh", "-c", 'eval "$1"; printf %s "$V"', "sh", script], + stdout=subprocess.PIPE, + ).stdout.decode("utf-8") + self.assertEqual(out, value) + + def test_render_secret_exports_rejects_invalid_name(self): + with self.assertRaises(ValueError): + tongs.render_secret_exports({"a/b": "v"}) + + def test_secret_inject_argv_reads_fifo_then_execs_target(self): + entrypoint, command = tongs.secret_inject_argv(["node", "server.js"]) + self.assertEqual(entrypoint, "/bin/sh") + self.assertEqual(command[0], "-c") + self.assertIn("/run/swarmforge/secret-env", command[1]) + self.assertIn("|| exit 1", command[1]) + self.assertIn('exec "$@"', command[1]) + # The target argv is passed after the `$0` placeholder so `"$@"` is it. + self.assertEqual(command[2:], ["swarmforge-tong", "node", "server.js"]) + + def test_secret_inject_argv_does_not_exec_target_when_fifo_read_fails(self): + old_target = tongs.SECRET_FIFO_TARGET + try: + with tempfile.TemporaryDirectory() as tmp: + tongs.SECRET_FIFO_TARGET = os.path.join(tmp, "missing") + entrypoint, command = tongs.secret_inject_argv( + ["/bin/sh", "-c", "printf target-ran"] + ) + completed = subprocess.run( + [entrypoint] + command, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + finally: + tongs.SECRET_FIFO_TARGET = old_target + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, b"") + + def test_resolve_exec_target_uses_image_defaults(self): + self.assertEqual( + tongs.resolve_exec_target({"image": "x"}, ["node"], ["server.js"]), + ["node", "server.js"], + ) + + def test_resolve_exec_target_definition_overrides_image(self): + defn = {"image": "x", "entrypoint": ["tini", "--"], "command": ["app"]} + self.assertEqual( + tongs.resolve_exec_target(defn, ["node"], ["server.js"]), + ["tini", "--", "app"], + ) + + def test_resolve_exec_target_empty_raises(self): + with self.assertRaises(ValueError): + tongs.resolve_exec_target({"image": "x"}, [], []) + class EnvNamingTests(unittest.TestCase): def test_prefix_sanitizes_name(self): @@ -926,13 +954,29 @@ def test_run_argv_mounts_and_resources(self): self.assertIn("--memory", argv) self.assertEqual(argv[argv.index("--memory") + 1], "256m") - def test_run_argv_tmpfs_for_secret_delivery(self): + def test_run_argv_secret_injection_mounts_fifo_wraps_entrypoint(self): + # A secret-bearing tong gets the FIFO bind (read-only), the /bin/sh + # entrypoint override, and the wrapper command appended after the image. + entrypoint, command = tongs.secret_inject_argv(["node", "server.js"]) + argv = tongs.tong_run_argv( + "g", def_of(NONE_TONG), container_name="c", network="n", alias="g", + fifo_host_path="/tmp/sf/secret-env", entrypoint=entrypoint, command=command, + ) + self.assertEqual( + argv[argv.index("--entrypoint") + 1], "/bin/sh" + ) + self.assertIn("/tmp/sf/secret-env:/run/swarmforge/secret-env:ro", argv) + # The wrapper command trails the image (which is NONE_TONG's image). + image = def_of(NONE_TONG)["image"] + self.assertEqual(argv[argv.index(image) + 1 :], command) + + def test_run_argv_without_secrets_omits_entrypoint_and_fifo(self): argv = tongs.tong_run_argv( "g", def_of(NONE_TONG), container_name="c", network="n", alias="g", - tmpfs_dirs=["/run/swarmforge/secrets"], ) - self.assertIn("--tmpfs", argv) - self.assertEqual(argv[argv.index("--tmpfs") + 1], "/run/swarmforge/secrets") + self.assertNotIn("--entrypoint", argv) + self.assertNotIn("secret-env", " ".join(argv)) + self.assertEqual(argv[-1], def_of(NONE_TONG)["image"]) # nothing after image def test_run_argv_does_not_emit_empty_hash_label(self): argv = tongs.tong_run_argv("g", def_of(NONE_TONG), container_name="c", network="n", alias="g") diff --git a/scripts/tongs.py b/scripts/tongs.py index daee351..4752ca8 100644 --- a/scripts/tongs.py +++ b/scripts/tongs.py @@ -270,18 +270,19 @@ def err(msg): if env is not None and not isinstance(env, dict): err("'env' must be a mapping of name -> value") elif isinstance(env, dict): - plain, secret = partition_secret_env(env) + _, secret = partition_secret_env(env) for secret_name in sorted(secret): if not ENV_NAME_RE.match(secret_name): err("invalid secret env name %r (must be a valid identifier)" % secret_name) - continue - pointer_name = secret_name + SECRET_FILE_ENV_SUFFIX - pointer_value = "%s/%s" % (SECRET_TMPFS_DIR, secret_name) - if pointer_name in plain and plain[pointer_name] != pointer_value: - err( - "env %r collides with the secret file pointer for %r" - % (pointer_name, secret_name) - ) + + # `entrypoint`/`command` override what the secret-injection wrapper execs, so + # they must be argv lists of strings if present. + for argvish in ("entrypoint", "command"): + value = defn.get(argvish) + if value is not None and not ( + isinstance(value, list) and all(isinstance(part, str) for part in value) + ): + err("'%s' must be a list of strings" % argvish) for listish in ("mounts", "networks"): value = defn.get(listish) @@ -448,20 +449,25 @@ def secret_provider_command(providers, provider, ref): # --- Secret delivery ---------------------------------------------------------- -# A resolved secret must never reach a tong as a docker `-e` env var: anything -# holding the docker socket (the broker tong) could read it back via -# `docker inspect`. Instead each secret-bearing env var is delivered as a file on -# an in-memory tmpfs the launcher populates at startup, and the tong is pointed -# at the file with a `_FILE` env var (the conventional docker-secret -# indirection). Plain (non-secret) env keeps flowing through `-e` unchanged. - -SECRET_TMPFS_DIR = "/run/swarmforge/secrets" -SECRET_FILE_ENV_SUFFIX = "_FILE" - -# A secret's file lands at SECRET_TMPFS_DIR/, so the env name becomes a -# path component. Restricting it to the POSIX env-name grammar keeps that path -# inside the tmpfs dir -- a name like "../../etc/foo" from an untrusted workspace -# tong cannot escape it -- and is exactly what docker accepts for an env var. +# A resolved secret must never reach a tong as a docker `-e` env var, a command +# argument, or a file on disk: anything holding the docker socket (the broker +# tong) could read an `-e` value back via `docker inspect`. Instead the launcher +# hands the secret-bearing env to the tong over a host FIFO bind-mounted into the +# container, and wraps the tong's entrypoint with a `/bin/sh` prologue that reads +# the FIFO, exports each value into its own environment, then execs the image's +# real entrypoint+command. The bytes travel through the kernel pipe buffer -- never +# a file, an argv, or the container's `Config.Env` -- and arrive as ordinary +# environment variables, so an unmodified server that reads them from its +# environment at startup works unchanged. Plain (non-secret) env keeps flowing +# through `-e`, which is safe because those values are not secret. + +# Where the FIFO is bind-mounted inside the tong, and the shell the wrapper runs. +SECRET_FIFO_TARGET = "/run/swarmforge/secret-env" +SECRET_INJECT_SHELL = "/bin/sh" + +# A secret env name becomes a shell assignment target (`export NAME=...`), so it +# must be a valid identifier -- which is exactly what docker accepts for an env +# var, and what keeps a hostile name from being anything but a variable name. ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @@ -471,8 +477,8 @@ def partition_secret_env(env): `env` is the tong definition's `env` mapping (values may be unresolved `${secret:...}` references). `plain` holds values with no secret reference (safe to pass straight through as `-e`); `secret` holds the keys whose value - contains at least one reference (routed to tmpfs delivery so the resolved - value never appears in `docker inspect`). Order within each is preserved. + contains at least one reference (delivered over the FIFO so the resolved value + never appears in `docker inspect`). Order within each is preserved. """ plain, secret = {}, {} for key, value in (env or {}).items(): @@ -483,69 +489,86 @@ def partition_secret_env(env): return plain, secret -def secret_delivery_plan(resolved_secrets): - """Plan tmpfs delivery for already-resolved secret env values. +def plan_tong_secrets(env, resolver): + """Resolve a tong's secret env and split it from the plain env. - `resolved_secrets` is `{env_name: secret_value}`. Returns plain data the - launcher applies when it starts the tong: + Partitions `env` into plain and secret-bearing values, resolves only the + secret-bearing ones through the injected `resolver(provider, ref) -> str` + (keeping this function pure), and returns: - * `tmpfs` -- the in-tong tmpfs mountpoint to create (`--tmpfs`), so the - secret files live in memory and never touch disk, or `None` - when there are no secrets. - * `files` -- `{absolute_path: secret_value}` the launcher writes into the - running container's tmpfs (never to the host). - * `env` -- `{_FILE: absolute_path}` pointing the tong at each file; - these are paths, not secrets, so they are safe as `-e`. + * `env` -- the plain env vars, safe to pass straight through as `-e`. + * `secrets` -- `{name: resolved value}` to hand the tong over the FIFO. - With no secrets the tmpfs/files/env are all empty, so a tong without secrets - gets no tmpfs mount and no indirection. Raises `ValueError` for an env name - that is not a valid identifier, so a name that would escape the tmpfs dir - when used as a path component stops the launch rather than reaching disk. + Resolved values appear only under `secrets`; nothing here is passed as `-e`, + so no secret is readable through `docker inspect`. """ - files = {} - env = {} + plain, secret = partition_secret_env(env) + resolved = {key: substitute_secrets(value, resolver) for key, value in secret.items()} + return {"env": dict(plain), "secrets": resolved} + + +def render_secret_exports(resolved_secrets): + """POSIX-sh that exports already-resolved secret env, for writing to the FIFO. + + `resolved_secrets` is `{env_name: value}`. Returns one `export NAME='value'` + line per entry (sorted), with the value single-quoted and embedded single + quotes escaped as `'\\''`, so an arbitrary value -- including newlines or shell + metacharacters -- cannot break out of its assignment. The tong's entrypoint + wrapper `eval`s this text, so the launcher (never the secret content) controls + the quoting. Raises `ValueError` for a name that is not a valid identifier. + """ + lines = [] for name in sorted(resolved_secrets): if not ENV_NAME_RE.match(name): raise ValueError("invalid secret env name %r (must be a valid identifier)" % name) - path = "%s/%s" % (SECRET_TMPFS_DIR, name) - files[path] = resolved_secrets[name] - env[name + SECRET_FILE_ENV_SUFFIX] = path - return {"tmpfs": SECRET_TMPFS_DIR if files else None, "files": files, "env": env} + quoted = "'" + resolved_secrets[name].replace("'", "'\\''") + "'" + lines.append("export %s=%s\n" % (name, quoted)) + return "".join(lines) -def plan_tong_secrets(env, resolver): - """Resolve a tong's secret env and plan how the tong receives it. - - Combines the steps the launcher performs for one tong's environment: - partition `env` into plain and secret, resolve only the secret-bearing values - through the injected `resolver(provider, ref) -> str` (keeping this function - pure), then deliver those via tmpfs. Returns: - - * `env` -- plain env vars plus the `_FILE` pointers (never a secret - value). - * `tmpfs` -- the tmpfs mountpoint to create, or `None` when no secrets. - * `files` -- `{absolute_path: secret_value}` to write into the tmpfs. +def secret_inject_argv(target_argv): + """`(entrypoint, command)` that loads FIFO secrets then execs the real argv. - The resolved secret values appear only under `files`, never under `env`, so - nothing the launcher passes as `-e` is readable through `docker inspect`. + `target_argv` is the tong image's real entrypoint+command (the process the + tong would have run without secret injection). Returns the `--entrypoint` + (`/bin/sh`) and the command tokens for `tong_run_argv`: a `-c` prologue that + reads the bind-mounted FIFO, exports each `NAME=value` into the environment, + then `exec`s `target_argv`. The blocking read of the FIFO is also the + synchronization point -- the wrapper waits there until the launcher delivers -- + so the real process never starts before its secret env is set. """ - plain, secret = partition_secret_env(env) - resolved = {key: substitute_secrets(value, resolver) for key, value in secret.items()} - delivery = secret_delivery_plan(resolved) - merged_env = dict(plain) - for key, value in delivery["env"].items(): - # A file pointer (TOKEN_FILE) can collide with a plain env var of the same - # name. Unless it already points at the generated path, fail rather than - # launching a tong that cannot find its secret. - if key in merged_env: - if merged_env[key] == value: - continue - raise ValueError( - "tong env %r collides with the secret file pointer for %r" - % (key, key[:-len(SECRET_FILE_ENV_SUFFIX)]) - ) - merged_env[key] = value - return {"env": merged_env, "tmpfs": delivery["tmpfs"], "files": delivery["files"]} + script = ( + 'secret_env=$(cat %s) || exit 1; ' + 'eval "$secret_env" || exit 1; ' + 'exec "$@"' + ) % SECRET_FIFO_TARGET + return SECRET_INJECT_SHELL, ["-c", script, "swarmforge-tong"] + list(target_argv) + + +def resolve_exec_target(defn, image_entrypoint, image_cmd): + """The argv the tong should ultimately exec, given the image's defaults. + + Overriding `--entrypoint` to inject the secret wrapper drops the image's own + entrypoint/command, so the launcher must restore them. A tong definition may + set them explicitly via `entrypoint:`/`command:` (lists); otherwise the + image's own values (`image_entrypoint`, `image_cmd`, read from + `docker inspect`) are used. The result is `entrypoint + command`. Raises + `ValueError` if that is empty -- there would be no process to exec after the + wrapper, so the definition must declare a `command`. + """ + entrypoint = defn.get("entrypoint") + if entrypoint is None: + entrypoint = image_entrypoint or [] + command = defn.get("command") + if command is None: + command = image_cmd or [] + target = list(entrypoint) + list(command) + if not target: + raise ValueError( + "cannot inject secrets: image %r declares no entrypoint or command to " + "exec; set 'command' in the tong definition" % defn.get("image") + ) + return target # --- Environment-variable naming ---------------------------------------------- @@ -1011,7 +1034,7 @@ def readiness_settings(defn): # --- Docker argv assembly ----------------------------------------------------- -# The launcher turns a validated definition (plus the env/tmpfs/network plan the +# The launcher turns a validated definition (plus the env/secret/network plan the # functions above produce) into the concrete `docker run` argv for a tong, and # rewrites the anvil's own argv to reach the tongs. These builders are pure -- # they return argv lists, run no docker -- so the exact flags can be unit-tested; @@ -1114,10 +1137,12 @@ def tong_run_argv( network, alias, env=None, - tmpfs_dirs=(), label_hash=None, workspace=None, socket_path=DEFAULT_DOCKER_SOCKET, + fifo_host_path=None, + entrypoint=None, + command=(), ): """Full `docker run -d` argv that starts one tong container. @@ -1126,28 +1151,36 @@ def tong_run_argv( named `container_name`, joined to `network` under `alias` as a `--network-alias` (only for network-facing tongs -- `volume`/`none` tongs need no DNS name), and stamped with the tong-name and config-hash labels so a later - launch can detect a stale `shared` container. `env` (plain values plus the - `_FILE` secret pointers from `plan_tong_secrets`) is passed as `-e` in - sorted order; resolved secret *values* never appear here -- they are written - to the `tmpfs_dirs` mounts after start. `mounts:` magic words and `resources:` - are appended, then the image. + launch can detect a stale `shared` container. `env` (the tong's plain, + non-secret values from `plan_tong_secrets`) is passed as `-e` in sorted order; + resolved secret values never appear here -- they arrive over the FIFO instead. + + When the tong has secret env, the launcher passes `fifo_host_path` (bind-mounted + read-only as the secret channel), `entrypoint` (`/bin/sh`), and `command` (the + wrapper that reads the FIFO and execs the image's real argv) -- see + `secret_inject_argv`. With no secrets all three are omitted and the image's own + entrypoint runs unchanged. `mounts:` magic words and `resources:` are appended, + then the image, then any `command` tokens. """ argv = ["docker", "run", "-d", "--name", container_name] if network: argv += ["--network", network] if alias and _is_network_facing(defn): argv += ["--network-alias", alias] + if entrypoint: + argv += ["--entrypoint", entrypoint] argv += ["--label", "%s=%s" % (LABEL_TONG_NAME, name)] if label_hash: argv += ["--label", "%s=%s" % (LABEL_CONFIG_HASH, label_hash)] - for tmpfs_dir in tmpfs_dirs: - argv += ["--tmpfs", tmpfs_dir] + if fifo_host_path: + argv += ["-v", "%s:%s:ro" % (fifo_host_path, SECRET_FIFO_TARGET)] for key in sorted(env or {}): argv += ["-e", "%s=%s" % (key, env[key])] for spec in tong_mount_specs(defn, workspace, socket_path=socket_path): argv += ["-v", spec] argv += tong_resource_flags(defn) argv.append(defn["image"]) + argv += list(command) return argv