From f45551381f6ae53b3b25b3e25ca3c30907223ce8 Mon Sep 17 00:00:00 2001
From: Henry Stern <henry@stern.ca>
Date: Thu, 28 May 2026 14:29:29 -0300
Subject: [PATCH 1/3] feat(agent): worker cloud-init + systemd + bearer auth
 (FJB-94 Phase B)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Lands the deployment story for fjbagent on workers — cloud-init
installs the binary, drops a systemd unit + env file + auth token,
creates the unprivileged `fjb` system user, enables the service.
Opt-in: empty -fjbagent-token-file disables agent install entirely
so this is a no-op for deployments that haven't migrated yet.

Per-deployment knobs:
  - -fjbagent-token-file <path>   shared-secret bearer-token file;
                                  empty = agent install disabled.
  - -fjbagent-url <template>      URL template; {{.Version}} resolves
                                  to this daemon's build version,
                                  $rarch is left for cloud-init to
                                  substitute at boot. Default points
                                  at github releases for the matching
                                  fj-bellows version.

The agent's version is implicitly the same as the orchestrator's
build (linker-stamped main.version) — there is no per-deployment
agent version flag, because agent + orchestrator share a proto and
must come from the same commit. Pre-existing workers from before an
orchestrator upgrade keep running the agent version they were
provisioned with; the proto's wire-compatibility discipline (no
field renumbering, additive only) is the contract that keeps the
upgraded orchestrator able to dial them.

Deploy artifacts (added to cloud-init.yaml.tmpl, gated on
FJBAgentDownloadURL being non-empty):
  - groups: [fjb], users: [{name: fjb, system, nologin}]
  - /etc/fjbagent/auth.token (mode 0600, owned by fjb:fjb)
  - /etc/default/fjbagent (FJBAGENT_LISTEN=0.0.0.0:9001)
  - /etc/systemd/system/fjbagent.service (Type=notify, hardening
    flags from the design doc)
  - /usr/local/bin/fjbagent fetched from the resolved URL
  - systemctl enable --now fjbagent

Agent gets sd_notify support so Type=notify works — net.DialUnix to
$NOTIFY_SOCKET, write "READY=1\n", inline (no go-systemd dep).

NOTE: this PR ships the wiring but does not by itself publish the
fjbagent binary at the default github releases URL. The release
pipeline (.github/workflows/ci.yml) needs a parallel change to
cross-compile fjbagent for linux-amd64 / linux-arm64 and attach the
artifacts to v* tag releases before any deployment can set
-fjbagent-token-file in production. Tracked in FJB-94 phase
follow-ups.

Tests:
  - 5 new bootstrap render tests: backward-compat (no artifacts when
    URL empty), full-render asserts (user creation, token file mode,
    systemd unit text, runcmd), token-required guard, URL resolution
    across default + custom templates.
  - Existing render tests + full repo `go test -race ./...` green.
  - golangci-lint clean.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cmd/fj-bellows/main.go                  |  64 ++++++++++++--
 internal/agent/sdnotify.go              |  33 +++++++
 internal/agent/server.go                |   6 ++
 internal/bootstrap/bootstrap.go         |  89 ++++++++++++++++++-
 internal/bootstrap/bootstrap_test.go    | 113 +++++++++++++++++++++++-
 internal/bootstrap/cloud-init.yaml.tmpl |  50 +++++++++++
 internal/bootstrap/fjbagent.service     |  35 ++++++++
 internal/orchestrator/orchestrator.go   |  27 ++++--
 proto/fjbellows/agent/v1/agent.proto    |  12 ++-
 9 files changed, 410 insertions(+), 19 deletions(-)
 create mode 100644 internal/agent/sdnotify.go
 create mode 100644 internal/bootstrap/fjbagent.service

diff --git a/cmd/fj-bellows/main.go b/cmd/fj-bellows/main.go
index eb8baf5..9b20c40 100644
--- a/cmd/fj-bellows/main.go
+++ b/cmd/fj-bellows/main.go
@@ -45,6 +45,12 @@ func main() {
 	controlListen := flag.String("control-listen", "127.0.0.1:9876", "control plane listen address (TCP); empty disables")
 	controlTokenFile := flag.String("control-token-file", "", "bearer-token file for the control plane (required for non-loopback binds; mode 0600)")
 	enableControlWrites := flag.Bool("enable-control-writes", false, "expose mutating control RPCs (ForceReap, ForceProvision); off by default")
+	// fjbagent (FJB-94). The agent's version implicitly tracks this
+	// orchestrator's build (main.version), so there is no per-deployment
+	// version choice — only a token file (required) and an optional URL
+	// template override (default points at the matching github release).
+	fjbAgentTokenFile := flag.String("fjbagent-token-file", "", "bearer-token file for fjbagent on workers (mode 0600); empty disables agent install")
+	fjbAgentURLTmpl := flag.String("fjbagent-url", "", "URL template for the fjbagent binary; supports {{.Version}} (resolved from this build) and $rarch (resolved by cloud-init). Empty uses the default github releases URL.")
 	flag.Parse()
 
 	// Wrap the stderr text handler with a logbus tee so the control plane's
@@ -65,6 +71,8 @@ func main() {
 		controlListen:       *controlListen,
 		controlTokenFile:    *controlTokenFile,
 		enableControlWrites: *enableControlWrites,
+		fjbAgentTokenFile:   *fjbAgentTokenFile,
+		fjbAgentURLTmpl:     *fjbAgentURLTmpl,
 	}
 	if err := run(opts, log, logBus); err != nil {
 		log.Error("fatal", "err", err)
@@ -82,8 +90,17 @@ type runOpts struct {
 	controlListen       string
 	controlTokenFile    string
 	enableControlWrites bool
+	fjbAgentTokenFile   string
+	fjbAgentURLTmpl     string
 }
 
+// version is the fj-bellows daemon's build version, stamped at link
+// time via -ldflags "-X main.version=<git describe output>". It also
+// drives the version of fjbagent installed on workers via cloud-init,
+// since agent and orchestrator share a proto and must be built from
+// the same commit.
+var version = "dev"
+
 func run(opts runOpts, log *slog.Logger, logBus *logbus.Bus) error {
 	cfg, err := config.Load(opts.configPath)
 	if err != nil {
@@ -169,15 +186,22 @@ func run(opts runOpts, log *slog.Logger, logBus *logbus.Bus) error {
 		return err
 	}
 
+	fjbAgentToken, fjbAgentURL, err := loadFJBAgentSettings(opts, version)
+	if err != nil {
+		return err
+	}
+
 	orch := orchestrator.New(orchestrator.Config{
-		Tag:           cfg.Tag,
-		MaxScale:      cfg.Scale.Max,
-		Labels:        cfg.Forgejo.Labels,
-		PollInterval:  cfg.Poll.Interval.D(),
-		RunnerVersion: opts.runnerVersion,
-		ReadyFile:     bootstrap.DefaultReadyFile,
-		AuthorizedKey: authKey,
-		TransportMode: cfg.Transport.Mode,
+		Tag:                 cfg.Tag,
+		MaxScale:            cfg.Scale.Max,
+		Labels:              cfg.Forgejo.Labels,
+		PollInterval:        cfg.Poll.Interval.D(),
+		RunnerVersion:       opts.runnerVersion,
+		ReadyFile:           bootstrap.DefaultReadyFile,
+		AuthorizedKey:       authKey,
+		TransportMode:       cfg.Transport.Mode,
+		FJBAgentDownloadURL: fjbAgentURL,
+		FJBAgentToken:       fjbAgentToken,
 		Teardown: orchestrator.TeardownPolicy{
 			Model:       prov.BillingModel(),
 			IdleTimeout: cfg.Poll.IdleTimeout.D(),
@@ -743,3 +767,27 @@ func acquireLock(path string) (func(), error) {
 		_ = f.Close()
 	}, nil
 }
+
+// loadFJBAgentSettings reads the fjbagent token from disk (if configured)
+// and resolves the binary download URL against the daemon's own build
+// version. Returns the empty-string pair when -fjbagent-token-file is
+// unset (agent install disabled).
+func loadFJBAgentSettings(opts runOpts, buildVersion string) (token, url string, err error) {
+	if opts.fjbAgentTokenFile == "" {
+		// Empty token file = agent install disabled; nothing to load.
+		return "", "", nil
+	}
+	tok, err := os.ReadFile(opts.fjbAgentTokenFile)
+	if err != nil {
+		return "", "", fmt.Errorf("read fjbagent token: %w", err)
+	}
+	t := strings.TrimSpace(string(tok))
+	if t == "" {
+		return "", "", fmt.Errorf("fjbagent token file %s is empty", opts.fjbAgentTokenFile)
+	}
+	resolvedURL, err := bootstrap.ResolveAgentDownloadURL(opts.fjbAgentURLTmpl, buildVersion)
+	if err != nil {
+		return "", "", fmt.Errorf("resolve fjbagent URL: %w", err)
+	}
+	return t, resolvedURL, nil
+}
diff --git a/internal/agent/sdnotify.go b/internal/agent/sdnotify.go
new file mode 100644
index 0000000..e65988d
--- /dev/null
+++ b/internal/agent/sdnotify.go
@@ -0,0 +1,33 @@
+package agent
+
+import (
+	"net"
+	"os"
+)
+
+// sdNotifyReady signals systemd that the service is ready, when the unit
+// uses Type=notify. Reads NOTIFY_SOCKET from the env (set by systemd) and
+// writes "READY=1\n" to it. No-op when the env var is unset (running
+// outside systemd) so unit tests don't need to fake it.
+//
+// We inline the protocol rather than depend on github.com/coreos/go-systemd:
+// it's a Unix datagram socket and four bytes of payload. Adding the
+// transitive dependency surface isn't worth it.
+func sdNotifyReady() {
+	sock := os.Getenv("NOTIFY_SOCKET")
+	if sock == "" {
+		return
+	}
+	addr := &net.UnixAddr{Name: sock, Net: "unixgram"}
+	conn, err := net.DialUnix("unixgram", nil, addr)
+	if err != nil {
+		// systemd is responsible for cleaning up if we never report
+		// ready; on this failure we just lose the up-signal. Quiet
+		// failure is intentional — the agent should still serve
+		// requests in test/dev environments where NOTIFY_SOCKET
+		// happens to be set but unreachable.
+		return
+	}
+	defer func() { _ = conn.Close() }()
+	_, _ = conn.Write([]byte("READY=1\n"))
+}
diff --git a/internal/agent/server.go b/internal/agent/server.go
index ea89098..3fc7abd 100644
--- a/internal/agent/server.go
+++ b/internal/agent/server.go
@@ -96,6 +96,12 @@ func (s *Server) Run(ctx context.Context) error {
 	}
 	s.log.Info("agent listening", "addr", ln.Addr().String())
 
+	// Tell systemd we're ready as soon as the listener is bound. Under
+	// Type=notify this is what unblocks `systemctl start fjbagent` and
+	// what the orchestrator's readiness gate watches via the unit
+	// active state. No-op outside systemd.
+	sdNotifyReady()
+
 	serveErr := make(chan error, 1)
 	go func() {
 		err := s.srv.Serve(ln)
diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go
index d81317f..de98695 100644
--- a/internal/bootstrap/bootstrap.go
+++ b/internal/bootstrap/bootstrap.go
@@ -8,16 +8,41 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"strings"
 	"text/template"
 )
 
 //go:embed cloud-init.yaml.tmpl
 var cloudInitTemplate string
 
+//go:embed fjbagent.service
+var fjbagentServiceUnit string
+
 // DefaultReadyFile is touched by cloud-init once the worker is provisioned.
 // The orchestrator polls for it over SSH to decide a node is ready.
 const DefaultReadyFile = "/run/fj-bellows-ready"
 
+// DefaultAgentListenPort is the TCP port fjbagent binds on the worker's
+// VPC IP (cache binds the same port on its WG inner address). Exposed as
+// a constant so the orchestrator dialer and the cloud-init renderer agree
+// without an extra config knob.
+const DefaultAgentListenPort = 9001
+
+// DefaultAgentDownloadURLTemplate is the URL the orchestrator resolves
+// against its own build version (via ResolveAgentDownloadURL) before
+// passing the result into Params.FJBAgentDownloadURL. The agent is part
+// of the same fj-bellows codebase as the orchestrator — there is no
+// per-deployment choice of agent version, only a per-build one. {{.Version}}
+// resolves to the orchestrator's main.version (linker-stamped); $rarch
+// stays a shell literal so cloud-init substitutes it at boot from the
+// worker's actual architecture.
+//
+// Pre-existing workers from before an orchestrator upgrade keep running
+// the agent version they were provisioned with — proto-wire compatibility
+// between agent versions is the contract that lets the orchestrator dial
+// older workers without a forced reap.
+const DefaultAgentDownloadURLTemplate = "https://github.com/hstern/fj-bellows/releases/download/v{{.Version}}/fjbagent-linux-$rarch"
+
 // Params fills the cloud-init template.
 type Params struct {
 	// RunnerVersion is the forgejo-runner release to install, without a
@@ -31,6 +56,20 @@ type Params struct {
 	// dial is verified, eliminating the trust-on-first-use window. When empty the
 	// worker keeps the host key cloud-init generates on its own.
 	HostPrivateKey string
+
+	// FJBAgentDownloadURL is the fully-resolved URL the worker fetches the
+	// fjbagent binary from. Use ResolveAgentDownloadURL to substitute the
+	// orchestrator's version into a template first; $rarch stays a shell
+	// literal cloud-init substitutes at boot. Empty disables agent install
+	// entirely (transitional deployments).
+	FJBAgentDownloadURL string
+
+	// FJBAgentToken is the per-deployment shared-secret bearer token the
+	// agent and orchestrator agree on. Written to /etc/fjbagent/auth.token
+	// (mode 0600, owned by the fjb user) and presented by the orchestrator
+	// in the Authorization header on every Connect RPC. Required when
+	// FJBAgentDownloadURL is non-empty.
+	FJBAgentToken string
 }
 
 // Render produces the cloud-init user-data for a worker.
@@ -41,18 +80,66 @@ func Render(p Params) (string, error) {
 	if p.ReadyFile == "" {
 		p.ReadyFile = DefaultReadyFile
 	}
+	if p.FJBAgentDownloadURL != "" && p.FJBAgentToken == "" {
+		return "", errors.New("bootstrap: FJBAgentToken is required when FJBAgentDownloadURL is set")
+	}
+
+	tmplData := struct {
+		Params
+		FJBAgentServiceUnit string
+		FJBAgentListenPort  int
+	}{
+		Params:              p,
+		FJBAgentServiceUnit: fjbagentServiceUnit,
+		FJBAgentListenPort:  DefaultAgentListenPort,
+	}
+
 	funcs := template.FuncMap{
 		"b64enc": func(s string) string {
 			return base64.StdEncoding.EncodeToString([]byte(s))
 		},
+		"indent": func(spaces int, s string) string {
+			prefix := strings.Repeat(" ", spaces)
+			lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
+			for i, line := range lines {
+				lines[i] = prefix + line
+			}
+			return strings.Join(lines, "\n")
+		},
 	}
 	tmpl, err := template.New("cloud-init").Funcs(funcs).Parse(cloudInitTemplate)
 	if err != nil {
 		return "", fmt.Errorf("parse cloud-init template: %w", err)
 	}
 	var buf bytes.Buffer
-	if err := tmpl.Execute(&buf, p); err != nil {
+	if err := tmpl.Execute(&buf, tmplData); err != nil {
 		return "", fmt.Errorf("render cloud-init: %w", err)
 	}
 	return buf.String(), nil
 }
+
+// ResolveAgentDownloadURL substitutes the {{.Version}} placeholder in
+// urlTemplate with the orchestrator's own build version. $rarch stays a
+// shell literal cloud-init substitutes at boot. The agent version is
+// tied to the orchestrator's build by construction — there is no
+// per-deployment version choice.
+func ResolveAgentDownloadURL(urlTemplate, version string) (string, error) {
+	if urlTemplate == "" {
+		urlTemplate = DefaultAgentDownloadURLTemplate
+	}
+	tmpl, err := template.New("agent-url").Parse(urlTemplate)
+	if err != nil {
+		return "", fmt.Errorf("parse agent download URL: %w", err)
+	}
+	var buf bytes.Buffer
+	// Arch stays as a shell variable ($rarch) the cloud-init replaces at
+	// boot time; we only resolve the version here.
+	data := struct {
+		Version string
+		Arch    string
+	}{Version: version, Arch: "$rarch"}
+	if err := tmpl.Execute(&buf, data); err != nil {
+		return "", fmt.Errorf("execute agent download URL: %w", err)
+	}
+	return buf.String(), nil
+}
diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go
index 29a6527..27ffae1 100644
--- a/internal/bootstrap/bootstrap_test.go
+++ b/internal/bootstrap/bootstrap_test.go
@@ -5,6 +5,8 @@ import (
 	"testing"
 )
 
+const testRunnerVersion = "1.0.0"
+
 func TestRender(t *testing.T) {
 	out, err := Render(Params{RunnerVersion: "12.10.1"})
 	if err != nil {
@@ -26,7 +28,7 @@ func TestRender(t *testing.T) {
 }
 
 func TestRenderCustomReadyFile(t *testing.T) {
-	out, err := Render(Params{RunnerVersion: "1.0.0", ReadyFile: "/tmp/custom-ready"})
+	out, err := Render(Params{RunnerVersion: testRunnerVersion, ReadyFile: "/tmp/custom-ready"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -46,7 +48,7 @@ dGVzdC1ub3QtYS1yZWFsLWtleQ==
 -----END OPENSSH PRIVATE KEY-----`
 
 func TestRenderWithHostKey(t *testing.T) {
-	out, err := Render(Params{RunnerVersion: "1.0.0", HostPrivateKey: testHostKeyPEM})
+	out, err := Render(Params{RunnerVersion: testRunnerVersion, HostPrivateKey: testHostKeyPEM})
 	if err != nil {
 		t.Fatalf("Render: %v", err)
 	}
@@ -79,7 +81,7 @@ func TestRenderWithHostKey(t *testing.T) {
 }
 
 func TestRenderWithoutHostKeyUnchanged(t *testing.T) {
-	out, err := Render(Params{RunnerVersion: "1.0.0"})
+	out, err := Render(Params{RunnerVersion: testRunnerVersion})
 	if err != nil {
 		t.Fatalf("Render: %v", err)
 	}
@@ -90,3 +92,108 @@ func TestRenderWithoutHostKeyUnchanged(t *testing.T) {
 		t.Error("arch detection missing")
 	}
 }
+
+func TestRenderWithoutFJBAgent_NoAgentArtifacts(t *testing.T) {
+	out, err := Render(Params{RunnerVersion: testRunnerVersion})
+	if err != nil {
+		t.Fatalf("Render: %v", err)
+	}
+	// When FJBAgentDownloadURL is empty, none of the agent-install
+	// artifacts should be present — keeps the base cloud-init backward
+	// compatible for deployments that haven't enabled the agent yet.
+	for _, needle := range []string{"fjbagent.service", "/etc/fjbagent/auth.token", "useradd", "fjb:fjb"} {
+		if strings.Contains(out, needle) {
+			t.Errorf("agent artifact %q leaked into agent-disabled render:\n%s", needle, out)
+		}
+	}
+}
+
+func TestRenderWithFJBAgent_ProducesAllArtifacts(t *testing.T) {
+	url := "https://example.com/fjbagent-linux-$rarch"
+	out, err := Render(Params{
+		RunnerVersion:       testRunnerVersion,
+		FJBAgentDownloadURL: url,
+		FJBAgentToken:       "deadbeefcafe",
+	})
+	if err != nil {
+		t.Fatalf("Render: %v", err)
+	}
+	wants := []string{
+		// User + group creation via cloud-init's users module.
+		"name: fjb",
+		"primary_group: fjb",
+		"system: true",
+		// Token file with correct mode/owner.
+		"/etc/fjbagent/auth.token",
+		"'0600'",
+		"fjb:fjb",
+		"deadbeefcafe",
+		// Default env file for the systemd unit.
+		"/etc/default/fjbagent",
+		"FJBAGENT_LISTEN=0.0.0.0:9001",
+		// systemd unit text embedded from fjbagent.service.
+		"/etc/systemd/system/fjbagent.service",
+		"Type=notify",
+		"User=fjb",
+		"NoNewPrivileges=true",
+		// runcmd fetches + enables the service.
+		"curl -fsSL -o /usr/local/bin/fjbagent",
+		url,
+		"systemctl enable --now fjbagent",
+	}
+	for _, needle := range wants {
+		if !strings.Contains(out, needle) {
+			t.Errorf("expected %q in cloud-init output:\n%s", needle, out)
+		}
+	}
+}
+
+func TestRenderFJBAgent_RequiresToken(t *testing.T) {
+	_, err := Render(Params{
+		RunnerVersion:       testRunnerVersion,
+		FJBAgentDownloadURL: "https://example.com/fjbagent-linux-$rarch",
+		// FJBAgentToken intentionally unset
+	})
+	if err == nil {
+		t.Fatal("expected error when FJBAgentDownloadURL set without token")
+	}
+	if !strings.Contains(err.Error(), "FJBAgentToken") {
+		t.Errorf("error did not mention token: %v", err)
+	}
+}
+
+func TestResolveAgentDownloadURL(t *testing.T) {
+	tests := []struct {
+		name, urlTmpl, version, want string
+	}{
+		{
+			name:    "default-template",
+			urlTmpl: "",
+			version: "0.6.0",
+			want:    "https://github.com/hstern/fj-bellows/releases/download/v0.6.0/fjbagent-linux-$rarch",
+		},
+		{
+			name:    "custom-template",
+			urlTmpl: "https://internal.example/fjbagent-{{.Version}}-{{.Arch}}",
+			version: "0.6.0",
+			want:    "https://internal.example/fjbagent-0.6.0-$rarch",
+		},
+		{
+			name:    "no-placeholders",
+			urlTmpl: "https://fixed-url.example/fjbagent",
+			version: "0.6.0",
+			want:    "https://fixed-url.example/fjbagent",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ResolveAgentDownloadURL(tt.urlTmpl, tt.version)
+			if err != nil {
+				t.Fatalf("ResolveAgentDownloadURL: %v", err)
+			}
+			if got != tt.want {
+				t.Errorf("got %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/internal/bootstrap/cloud-init.yaml.tmpl b/internal/bootstrap/cloud-init.yaml.tmpl
index 7392423..3a2cd59 100644
--- a/internal/bootstrap/cloud-init.yaml.tmpl
+++ b/internal/bootstrap/cloud-init.yaml.tmpl
@@ -8,6 +8,41 @@ packages:
   - docker.io
   - curl
   - ca-certificates
+{{- if .FJBAgentDownloadURL}}
+# fjbagent (FJB-94) — runs as the unprivileged `fjb` system user. The token
+# below is the per-deployment shared secret; the orchestrator presents it on
+# the Authorization header for every Connect RPC. WG already encrypts and
+# authenticates the orchestrator↔cache leg, and the cache↔worker leg is on
+# the private VPC subnet, so plain HTTP/2 cleartext is acceptable.
+groups:
+  - fjb
+users:
+  - name: fjb
+    system: true
+    no_create_home: true
+    shell: /usr/sbin/nologin
+    primary_group: fjb
+write_files:
+  - path: /etc/fjbagent/auth.token
+    permissions: '0600'
+    owner: 'fjb:fjb'
+    content: |
+      {{.FJBAgentToken}}
+  - path: /etc/default/fjbagent
+    permissions: '0644'
+    content: |
+      # Resolved by cloud-init at boot from the VPC IP cloud-init exposes
+      # via instance-data; the orchestrator dials this address. Bound to
+      # 0.0.0.0 so workers don't need to know their own VPC IP upfront —
+      # the cache-gateway routing in FJB-54 makes the VPC subnet the only
+      # reachable surface, and the WG ACL further constrains what can
+      # land on :9001.
+      FJBAGENT_LISTEN=0.0.0.0:{{.FJBAgentListenPort}}
+  - path: /etc/systemd/system/fjbagent.service
+    permissions: '0644'
+    content: |
+{{.FJBAgentServiceUnit | indent 6}}
+{{- end}}
 runcmd:
   - systemctl enable --now docker
 {{- if .HostPrivateKey}}
@@ -37,4 +72,19 @@ runcmd:
     url="https://code.forgejo.org/forgejo/runner/releases/download/v{{.RunnerVersion}}/forgejo-runner-{{.RunnerVersion}}-linux-${rarch}"
     curl -fsSL -o /usr/local/bin/forgejo-runner "$url"
     chmod +x /usr/local/bin/forgejo-runner
+{{- if .FJBAgentDownloadURL}}
+  - |
+    set -eu
+    arch="$(uname -m)"
+    case "$arch" in
+      x86_64) rarch=amd64 ;;
+      aarch64|arm64) rarch=arm64 ;;
+      *) echo "unsupported arch: $arch" >&2; exit 1 ;;
+    esac
+    url="{{.FJBAgentDownloadURL}}"
+    curl -fsSL -o /usr/local/bin/fjbagent "$url"
+    chmod +x /usr/local/bin/fjbagent
+  - systemctl daemon-reload
+  - systemctl enable --now fjbagent
+{{- end}}
   - touch {{.ReadyFile}}
diff --git a/internal/bootstrap/fjbagent.service b/internal/bootstrap/fjbagent.service
new file mode 100644
index 0000000..a27e17a
--- /dev/null
+++ b/internal/bootstrap/fjbagent.service
@@ -0,0 +1,35 @@
+[Unit]
+Description=fj-bellows agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=notify
+User=fjb
+Group=fjb
+ExecStart=/usr/local/bin/fjbagent -listen ${FJBAGENT_LISTEN} -token-file /etc/fjbagent/auth.token
+Restart=on-failure
+RestartSec=2s
+KillMode=mixed
+TimeoutStopSec=5s
+
+# Hardening — the agent doesn't need any of these.
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictSUIDSGID=true
+RestrictRealtime=true
+LockPersonality=true
+
+# The token file must remain readable; /etc/fjbagent is otherwise read-only.
+ReadOnlyPaths=/etc/fjbagent
+
+# FJBAGENT_LISTEN is templated into /etc/default/fjbagent by cloud-init.
+EnvironmentFile=/etc/default/fjbagent
+
+[Install]
+WantedBy=multi-user.target
diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go
index ed8ad8b..404f01c 100644
--- a/internal/orchestrator/orchestrator.go
+++ b/internal/orchestrator/orchestrator.go
@@ -49,6 +49,17 @@ type Config struct {
 	Teardown      TeardownPolicy
 	AuthorizedKey string
 
+	// FJBAgentDownloadURL is the fully-resolved URL workers fetch fjbagent
+	// from in cloud-init (FJB-94). The agent version implicitly tracks
+	// the orchestrator's build (this is the only design that makes sense
+	// — agent and orchestrator share a proto). Empty disables agent
+	// install entirely.
+	FJBAgentDownloadURL string
+	// FJBAgentToken is the per-deployment shared-secret bearer token.
+	// Required when FJBAgentDownloadURL is set. The orchestrator presents
+	// the same token in the Authorization header when dialing the agent.
+	FJBAgentToken string
+
 	// TransportMode mirrors config.Transport.Mode (FJB-72). Drives the
 	// orchestrator's choice of dial address: empty / "ssh" uses
 	// Node.IP (the legacy public IPv4 path); "cache-gateway" (FJB-54)
@@ -475,9 +486,11 @@ func (o *Orchestrator) doForceProvision(ctx context.Context) forceResult {
 		}
 	}
 	userData, err := bootstrap.Render(bootstrap.Params{
-		RunnerVersion:  o.cfg.RunnerVersion,
-		ReadyFile:      o.cfg.ReadyFile,
-		HostPrivateKey: hostPriv,
+		RunnerVersion:       o.cfg.RunnerVersion,
+		ReadyFile:           o.cfg.ReadyFile,
+		HostPrivateKey:      hostPriv,
+		FJBAgentDownloadURL: o.cfg.FJBAgentDownloadURL,
+		FJBAgentToken:       o.cfg.FJBAgentToken,
 	})
 	if err != nil {
 		o.log.Error("force-provision render cloud-init", "err", err)
@@ -705,9 +718,11 @@ func (o *Orchestrator) provisionOne(ctx context.Context) {
 			}
 		}
 		userData, err := bootstrap.Render(bootstrap.Params{
-			RunnerVersion:  o.cfg.RunnerVersion,
-			ReadyFile:      o.cfg.ReadyFile,
-			HostPrivateKey: hostPriv,
+			RunnerVersion:       o.cfg.RunnerVersion,
+			ReadyFile:           o.cfg.ReadyFile,
+			HostPrivateKey:      hostPriv,
+			FJBAgentDownloadURL: o.cfg.FJBAgentDownloadURL,
+			FJBAgentToken:       o.cfg.FJBAgentToken,
 		})
 		if err != nil {
 			o.log.Error("render cloud-init", "err", err)
diff --git a/proto/fjbellows/agent/v1/agent.proto b/proto/fjbellows/agent/v1/agent.proto
index 3ae137d..393abec 100644
--- a/proto/fjbellows/agent/v1/agent.proto
+++ b/proto/fjbellows/agent/v1/agent.proto
@@ -17,7 +17,17 @@ import "google/protobuf/timestamp.proto";
 // and authenticates the orchestrator↔cache leg, the cache↔worker leg is on
 // the private VPC subnet, and adding TLS would mean another cert lifecycle.
 //
-// FJB-94 ships only Health; FJB-93 will add Exec, FJB-97 will reuse Health
+// **Wire compatibility.** Workers are provisioned with the agent version
+// stamped into the fj-bellows daemon at provision time; an orchestrator
+// upgrade does not retroactively replace the agent on already-running
+// workers. The orchestrator therefore MUST keep speaking older versions of
+// this proto. Discipline:
+//   - Never renumber, rename, or repurpose an existing field.
+//   - New fields use new numbers; readers must tolerate their absence.
+//   - Behaviour changes go through new RPCs or new oneof variants, never
+//     by changing the meaning of an existing one.
+//
+// FJB-94 ships only Health; FJB-93 added Exec, FJB-97 will reuse Health
 // for the gRPC-layer reachability probe.
 service AgentService {
   // Health returns a readiness snapshot of the agent process. Used by the

From 0c6f1dcd2b6dbfcf379b8b20f1d5a83f9d01a9f9 Mon Sep 17 00:00:00 2001
From: Henry Stern <henry@stern.ca>
Date: Thu, 28 May 2026 21:16:33 -0300
Subject: [PATCH 2/3] fix(agent): keep run() under gocyclo budget (FJB-94 Phase
 B)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Phase B token-load added one branch to run() that pushed it from
13 to 14, which is over .golangci.yml's gocyclo budget. Extract the
orchestrator.Config construction (and the FJB-94 token-load that
sits in front of it) into buildOrchestratorConfig so run()'s
complexity is back under.

Locally golangci-lint v2.12.1 didn't flag this in my workspace —
CI's run on Linux did. Cause: probably gocyclo cache differences.
The fix is purely structural; no behaviour change.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cmd/fj-bellows/main.go | 57 +++++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/cmd/fj-bellows/main.go b/cmd/fj-bellows/main.go
index 9f21f0c..4573631 100644
--- a/cmd/fj-bellows/main.go
+++ b/cmd/fj-bellows/main.go
@@ -196,32 +196,11 @@ func run(opts runOpts, log *slog.Logger, logBus *logbus.Bus) error {
 		return err
 	}
 
-	fjbAgentToken, fjbAgentURL, err := loadFJBAgentSettings(opts, version)
+	orchCfg, err := buildOrchestratorConfig(cfg, opts, version, authKey, prov)
 	if err != nil {
 		return err
 	}
-
-	orch := orchestrator.New(orchestrator.Config{
-		Tag:                 cfg.Tag,
-		MaxScale:            cfg.Scale.Max,
-		Labels:              cfg.Forgejo.Labels,
-		PollInterval:        cfg.Poll.Interval.D(),
-		RunnerVersion:       opts.runnerVersion,
-		ReadyFile:           bootstrap.DefaultReadyFile,
-		AuthorizedKey:       authKey,
-		TransportMode:       cfg.Transport.Mode,
-		FJBAgentDownloadURL: fjbAgentURL,
-		FJBAgentToken:       fjbAgentToken,
-		Teardown: orchestrator.TeardownPolicy{
-			Model:       prov.BillingModel(),
-			IdleTimeout: cfg.Poll.IdleTimeout.D(),
-			HourMargin:  cfg.Poll.HourMargin.D(),
-			BillingHour: cfg.Poll.BillingHour.D(),
-		},
-		DrainOnShutdown: opts.drain,
-		DrainTimeout:    opts.drainTimeout,
-		DestroyOnExit:   opts.destroyOnExit,
-	}, prov, fj, dispatcher, log)
+	orch := orchestrator.New(orchCfg, prov, fj, dispatcher, log)
 
 	if err := startControlPlane(ctx, controlOpts{
 		listen:       opts.controlListen,
@@ -824,3 +803,35 @@ func loadFJBAgentSettings(opts runOpts, buildVersion string) (token, url string,
 	}
 	return t, resolvedURL, nil
 }
+
+// buildOrchestratorConfig assembles the orchestrator.Config from the
+// loaded config + flags. Extracted from run() so the latter stays under
+// the gocyclo budget after Phase B added the FJB-94 token-load branch.
+func buildOrchestratorConfig(cfg *config.Config, opts runOpts, buildVersion, authKey string, prov provider.Provider) (orchestrator.Config, error) {
+	fjbAgentToken, fjbAgentURL, err := loadFJBAgentSettings(opts, buildVersion)
+	if err != nil {
+		return orchestrator.Config{}, err
+	}
+	_ = prov // reserved for Phase C: provider-aware SetFJBAgent wiring lives in its own helper there.
+	return orchestrator.Config{
+		Tag:                 cfg.Tag,
+		MaxScale:            cfg.Scale.Max,
+		Labels:              cfg.Forgejo.Labels,
+		PollInterval:        cfg.Poll.Interval.D(),
+		RunnerVersion:       opts.runnerVersion,
+		ReadyFile:           bootstrap.DefaultReadyFile,
+		AuthorizedKey:       authKey,
+		TransportMode:       cfg.Transport.Mode,
+		FJBAgentDownloadURL: fjbAgentURL,
+		FJBAgentToken:       fjbAgentToken,
+		Teardown: orchestrator.TeardownPolicy{
+			Model:       prov.BillingModel(),
+			IdleTimeout: cfg.Poll.IdleTimeout.D(),
+			HourMargin:  cfg.Poll.HourMargin.D(),
+			BillingHour: cfg.Poll.BillingHour.D(),
+		},
+		DrainOnShutdown: opts.drain,
+		DrainTimeout:    opts.drainTimeout,
+		DestroyOnExit:   opts.destroyOnExit,
+	}, nil
+}

From 7610a1e8633d6618b4b7155c4f5b12af49fb40c0 Mon Sep 17 00:00:00 2001
From: Henry Stern <henry@stern.ca>
Date: Thu, 28 May 2026 21:23:04 -0300
Subject: [PATCH 3/3] fix(agent): extract loadSSHKeyForProvider + Linode
 authKey wiring (FJB-94 Phase B)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CI's gocyclo flagged run() at complexity 14 even after the earlier
buildOrchestratorConfig extraction — main has moved forward since
Phase B branched, and the new plumbOrchestratorWGPubkey branch
pushed it back over. Extract two more small helpers to bring it
down: loadSSHKeyForProvider folds the docker-vs-ssh check, and
applyAuthorizedKeyToLinodeProvider folds the Linode duck-type
wiring. Behaviour unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 cmd/fj-bellows/main.go | 54 +++++++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/cmd/fj-bellows/main.go b/cmd/fj-bellows/main.go
index 4573631..6178749 100644
--- a/cmd/fj-bellows/main.go
+++ b/cmd/fj-bellows/main.go
@@ -119,31 +119,16 @@ func run(opts runOpts, log *slog.Logger, logBus *logbus.Bus) error {
 
 	// SSH key is required only for providers that dispatch over SSH. A docker
 	// deployment passes nothing into cloud-init and execs into containers.
-	var (
-		signer  ssh.Signer
-		authKey string
-	)
-	if cfg.Provider != config.ProviderDocker {
-		signer, authKey, err = loadSSHKey(cfg.SSH.PrivateKeyFile)
-		if err != nil {
-			return err
-		}
+	signer, authKey, err := loadSSHKeyForProvider(cfg)
+	if err != nil {
+		return err
 	}
 
 	prov, err := provider.New(cfg.Provider)
 	if err != nil {
 		return err
 	}
-	// Hand the Linode provider the orchestrator's SSH public key so
-	// the managed cache VM (if `cache:` is set) accepts ssh from the
-	// operator for debugging. No tunnel; this is inbound-only debug
-	// access. No-op for non-Linode providers.
-	if l, ok := prov.(*linodeprov.Linode); ok && authKey != "" {
-		// ssh.MarshalAuthorizedKey appends a trailing newline; Linode's
-		// authorized_keys API rejects multi-line values with a 400.
-		// The worker Provision path already does this trim on spec.AuthorizedKey.
-		l.SetSSHAuthorizedKey(strings.TrimSpace(authKey))
-	}
+	applyAuthorizedKeyToLinodeProvider(prov, authKey)
 	// Propagate transport mode into the Linode provider so its managed
 	// firewall synthesizes the right ACCEPT rules (tcp/22 for legacy SSH,
 	// IPsec ports for cache-gateway). Duck-typed so providers that don't
@@ -835,3 +820,34 @@ func buildOrchestratorConfig(cfg *config.Config, opts runOpts, buildVersion, aut
 		DestroyOnExit:   opts.destroyOnExit,
 	}, nil
 }
+
+// applyAuthorizedKeyToLinodeProvider hands the operator's SSH authorized-key
+// line to the Linode provider so the managed cache VM accepts ssh debug
+// access. No-op for non-Linode providers, no-op when authKey is empty
+// (docker-only deployments). Extracted to keep run()'s cyclomatic
+// complexity under the gocyclo budget after the FJB-94 Phase B token-load
+// branch landed.
+func applyAuthorizedKeyToLinodeProvider(prov provider.Provider, authKey string) {
+	if authKey == "" {
+		return
+	}
+	l, ok := prov.(*linodeprov.Linode)
+	if !ok {
+		return
+	}
+	// ssh.MarshalAuthorizedKey appends a trailing newline; Linode's
+	// authorized_keys API rejects multi-line values with a 400. The
+	// worker Provision path already does this trim on spec.AuthorizedKey.
+	l.SetSSHAuthorizedKey(strings.TrimSpace(authKey))
+}
+
+// loadSSHKeyForProvider loads the SSH key only for providers that dispatch
+// over SSH; docker dispatches via container exec and needs nothing. Empty
+// returns are safe for both signer and authKey on the docker path. Extracted
+// to keep run() under the gocyclo budget after FJB-94 Phase B.
+func loadSSHKeyForProvider(cfg *config.Config) (ssh.Signer, string, error) {
+	if cfg.Provider == config.ProviderDocker {
+		return nil, "", nil
+	}
+	return loadSSHKey(cfg.SSH.PrivateKeyFile)
+}