feat(linode): orchestrator-side cache wg-pubkey discovery — Phase B (FJB-99)

cmd/fj-bellows/main.go

@@ -513,12 +513,23 @@ func bootWGStack(ctx context.Context, cfg *config.Config, prov provider.Provider
 		return nil, func() {}, nil
 	}
 	cache := cacheRenderInputs(ctx, cfg, prov)
+	// Only run the FJB-99 bootstrap poll when the operator hasn't
+	// statically configured a peer. Skipping the 3-minute poll under
+	// static config keeps daemon startup fast and matches the
+	// back-compat path the FJB-91 e2e harness exercises against the
+	// persistent test cache.
+	var cachePubkey, cacheEndpoint string
+	if cfg.Transport.WG != nil && (cfg.Transport.WG.Peer.PublicKey == "" || cfg.Transport.WG.Peer.Endpoint == "") {
+		cachePubkey, cacheEndpoint = discoverCachePeer(ctx, prov, log)
+	}
 	stack, err := wgboot.Boot(ctx, wgboot.Config{
-		Transport:  cfg.Transport,
-		ForgejoURL: cfg.Forgejo.URL,
-		ACLSink:    aclSinkFor(prov),
-		Cache:      cache,
-		Logger:     log,
+		Transport:     cfg.Transport,
+		ForgejoURL:    cfg.Forgejo.URL,
+		ACLSink:       aclSinkFor(prov),
+		Cache:         cache,
+		Logger:        log,
+		CachePubkey:   cachePubkey,
+		CacheEndpoint: cacheEndpoint,
 	})
 	if err != nil {
 		return nil, func() {}, fmt.Errorf("transport bootstrap: %w", err)
@@ -530,6 +541,44 @@ func bootWGStack(ctx context.Context, cfg *config.Config, prov provider.Provider
 	}, nil
 }
 
+// discoverCachePeer runs the orchestrator side of the FJB-99 bootstrap
+// loop: poll the managed cache's Object Storage bucket for the cache's
+// WG pubkey (published by cache cloud-init's fjb-wg-bootstrap.sh) and
+// read the cache's public IPv4 from the Linode API to build the WG
+// peer endpoint. Returns empty strings (no error) when the provider
+// can't supply these — wgboot then falls back to the static
+// transport.wg.peer.{public_key,endpoint} config knobs and errors at
+// boot if those are also empty.
+//
+// Bounded at 3 minutes — the cache cloud-init steady-state (package
+// install + wireguard + pubkey upload) typically completes well under
+// 2 minutes, so the bound is generous but not unbounded.
+func discoverCachePeer(ctx context.Context, prov provider.Provider, log *slog.Logger) (pubkey, endpoint string) {
+	type pubkeyWaiter interface {
+		WaitForCacheWGPubkey(ctx context.Context, timeout time.Duration) (string, error)
+	}
+	type endpointReader interface {
+		CachePublicEndpoint(ctx context.Context) (string, error)
+	}
+	if pw, ok := prov.(pubkeyWaiter); ok {
+		start := time.Now()
+		if pk, err := pw.WaitForCacheWGPubkey(ctx, 3*time.Minute); err == nil {
+			log.Info("wgboot: cache wg-pubkey discovered", "elapsed", time.Since(start))
+			pubkey = pk
+		} else {
+			log.Warn("wgboot: cache wg-pubkey discovery failed; falling back to static config", "err", err)
+		}
+	}
+	if er, ok := prov.(endpointReader); ok {
+		if ep, err := er.CachePublicEndpoint(ctx); err == nil {
+			endpoint = ep
+		} else {
+			log.Warn("wgboot: cache public endpoint discovery failed; falling back to static config", "err", err)
+		}
+	}
+	return pubkey, endpoint
+}
+
 // cacheRenderInputs pulls the cache-side facts the renderer needs out
 // of whatever live provider state is available. Today only the Linode
 // provider carries a managed cache; the function returns whatever it

internal/config/transport.go

@@ -254,14 +254,16 @@ func (w *WG) validate() error {
 }
 
 func (p *WGPeer) validate() error {
-	if p.PublicKey == "" {
-		return errors.New("public_key is required")
-	}
-	if p.Endpoint == "" {
-		return errors.New("endpoint is required")
-	}
-	if _, _, err := net.SplitHostPort(p.Endpoint); err != nil {
-		return fmt.Errorf("endpoint %q is not host:port: %w", p.Endpoint, err)
+	// public_key + endpoint are runtime-discoverable from FJB-99's
+	// cache wg-bootstrap loop (the cache writes its pubkey to S3; the
+	// orchestrator reads the cache's public IPv4 from the Linode API).
+	// Either may be empty at config-validate time; wgboot's planBoot
+	// is the load-bearing check that both end up populated by the
+	// time the tunnel is brought up.
+	if p.Endpoint != "" {
+		if _, _, err := net.SplitHostPort(p.Endpoint); err != nil {
+			return fmt.Errorf("endpoint %q is not host:port: %w", p.Endpoint, err)
+		}
 	}
 	if len(p.AllowedIPs) == 0 {
 		return errors.New("allowed_ips must list at least one CIDR")

internal/config/transport_wg_test.go

@@ -156,7 +156,6 @@ func TestWG_DefaultKeepaliveOneSecond(t *testing.T) {
 	}
 }
 
-//nolint:funlen // table-driven: 11 small validation cases inline for readability.
 func TestWG_Validation(t *testing.T) {
 	cases := []struct {
 		name    string
@@ -200,30 +199,6 @@ func TestWG_Validation(t *testing.T) {
 `,
 			wantSub: `local_addr = "not-a-cidr"`,
 		},
-		{
-			name: "missing peer public key",
-			wgBlock: `
-  wg:
-    private_key_file: /tmp/k
-    local_addr: 10.99.0.1/32
-    peer:
-      endpoint: 172.234.203.50:51820
-      allowed_ips: [10.99.0.2/32]
-`,
-			wantSub: "public_key is required",
-		},
-		{
-			name: "missing peer endpoint",
-			wgBlock: `
-  wg:
-    private_key_file: /tmp/k
-    local_addr: 10.99.0.1/32
-    peer:
-      public_key: AbcDefGhiJklMnoPqrStuVwxYzAbcDefGhiJklMnoPqs=
-      allowed_ips: [10.99.0.2/32]
-`,
-			wantSub: "endpoint is required",
-		},
 		{
 			name: "bad peer endpoint shape",
 			wgBlock: `

internal/provider/linode/cache-cloud-init.yaml.tmpl

@@ -105,14 +105,17 @@ write_files:
       EOC
       chmod 0600 /etc/wireguard/wg0.conf
       systemctl enable --now wg-quick@wg0
-      # Publish own pubkey so the orchestrator can read it from S3 and
-      # complete the wg peer config on its side (FJB-99 Phase B).
+      # Publish own pubkey so the orchestrator can read it via plain
+      # HTTPS GET and complete the wg peer config on its side (FJB-99
+      # Phase B). WG public keys are designed to be world-readable —
+      # exposing one over public-read costs nothing.
       AWS_ACCESS_KEY_ID='{{.AccessKey}}' \
       AWS_SECRET_ACCESS_KEY='{{.SecretKey}}' \
       AWS_DEFAULT_REGION='{{.Region}}' \
         aws --endpoint-url '{{.Endpoint}}' \
             s3 cp /etc/wireguard/publickey \
-                  s3://{{.Bucket}}/wg-pubkey.txt
+                  s3://{{.Bucket}}/wg-pubkey.txt \
+            --acl public-read
 {{- end}}
   - path: /etc/systemd/system/zot.service
     permissions: '0644'

internal/provider/linode/cache.go

@@ -7,13 +7,16 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
+	"net/http"
 	"net/netip"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/linode/linodego"
 
@@ -454,6 +457,123 @@ func (m *managedCache) renderIPTablesForCacheGateway() (string, error) {
 	})
 }
 
+// WaitForWGPubkey blocks until the cache's first-boot cloud-init has
+// published its WG public key to the Object Storage bucket, or until
+// ctx is canceled / timeout elapses (whichever comes first). Returns
+// the trimmed public-key string ready to feed into wgboot.Config.
+// FJB-99 Phase B — completes the bootstrap loop that Phase A started
+// from the cache side.
+//
+// The cache writes the object as `wg-pubkey.txt` with `--acl
+// public-read`, so the orchestrator reaches it via plain HTTPS GET —
+// no S3 SDK required, no signed requests. WG public keys are designed
+// to be world-readable; the threat model isn't worse than what's
+// already on the wire during a key exchange.
+//
+// Polls at 5s intervals with exponential backoff up to 30s. First
+// boot typically takes 60-120s (cloud-init + package install +
+// wireguard install).
+func (m *managedCache) WaitForWGPubkey(ctx context.Context, timeout time.Duration) (string, error) {
+	if m.bucket == nil {
+		return "", errors.New("cache: bucket not initialised (no managed cache)")
+	}
+	if m.bucket.endpoint == "" {
+		return "", errors.New("cache: bucket endpoint unset (call Configure first)")
+	}
+	url := fmt.Sprintf("%s/%s/wg-pubkey.txt", strings.TrimRight(m.bucket.endpoint, "/"), m.bucket.label)
+	deadline := time.Now().Add(timeout)
+	delay := 5 * time.Second
+	const maxDelay = 30 * time.Second
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+	for {
+		pub, ready, err := pollWGPubkey(ctx, httpClient, url, m.log)
+		if err != nil {
+			return "", err
+		}
+		if ready {
+			return pub, nil
+		}
+		if time.Now().Add(delay).After(deadline) {
+			return "", fmt.Errorf("cache wg-pubkey poll: timed out after %s waiting for %s", timeout, url)
+		}
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("cache wg-pubkey poll: %w", ctx.Err())
+		case <-time.After(delay):
+		}
+		if delay < maxDelay {
+			delay *= 2
+			if delay > maxDelay {
+				delay = maxDelay
+			}
+		}
+	}
+}
+
+// pollWGPubkey does one HTTPS GET against the bucket URL. Returns
+// (key, true, nil) on 200-with-non-empty-body; (_, false, nil) when the
+// object isn't there yet (404 / connection error — debug-logged for
+// the operator); (_, _, err) when the response is structurally bad
+// (200 with empty body, read failure, request build failure).
+func pollWGPubkey(ctx context.Context, httpClient *http.Client, url string, log *slog.Logger) (string, bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", false, fmt.Errorf("cache wg-pubkey poll: build request: %w", err)
+	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		log.Debug("cache wg-pubkey poll: request error", "url", url, "err", err)
+		return "", false, nil
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		log.Debug("cache wg-pubkey poll: not ready", "url", url, "status", resp.StatusCode)
+		return "", false, nil
+	}
+	body, rerr := io.ReadAll(resp.Body)
+	if rerr != nil {
+		return "", false, fmt.Errorf("cache wg-pubkey poll: read body: %w", rerr)
+	}
+	pub := strings.TrimSpace(string(body))
+	if pub == "" {
+		return "", false, errors.New("cache wg-pubkey poll: 200 OK but body was empty")
+	}
+	return pub, true, nil
+}
+
+// PublicEndpoint returns the cache's WG peer endpoint as host:port
+// (e.g. "172.234.203.50:51820"). The IP comes from the Linode API
+// (the public NIC's assigned IPv4); the port is supplied by the
+// caller — typically transport.wg.listen_port from config or the
+// default 51820 baked into the cache cloud-init. FJB-99 Phase B.
+func (m *managedCache) PublicEndpoint(ctx context.Context, port int) (string, error) {
+	if m.linodeID == 0 {
+		return "", errors.New("cache: linode not provisioned yet")
+	}
+	if port <= 0 {
+		port = defaultCacheWGListenPort
+	}
+	inst, err := m.client.GetInstance(ctx, m.linodeID)
+	if err != nil {
+		return "", fmt.Errorf("cache public endpoint: get instance %d: %w", m.linodeID, err)
+	}
+	for _, ip := range inst.IPv4 {
+		if ip == nil {
+			continue
+		}
+		v4 := ip.To4()
+		if v4 == nil {
+			continue
+		}
+		// Skip the VPC interface IP (RFC1918) — pick the public IPv4.
+		if v4.IsPrivate() {
+			continue
+		}
+		return fmt.Sprintf("%s:%d", v4.String(), port), nil
+	}
+	return "", fmt.Errorf("cache public endpoint: linode %d has no public IPv4 yet", m.linodeID)
+}
+
 // buildCreateOpts assembles the InstanceCreateOptions payload for the
 // cache VM. Public stays primary so outbound (upstream sync, package
 // mirrors, GitHub-zot download) takes the default route; the VPC NIC