diff --git a/cmd/fj-bellows/main.go b/cmd/fj-bellows/main.go index e400b99..8186ed5 100644 --- a/cmd/fj-bellows/main.go +++ b/cmd/fj-bellows/main.go @@ -513,12 +513,23 @@ func bootWGStack(ctx context.Context, cfg *config.Config, prov provider.Provider return nil, func() {}, nil } cache := cacheRenderInputs(ctx, cfg, prov) + // Only run the FJB-99 bootstrap poll when the operator hasn't + // statically configured a peer. Skipping the 3-minute poll under + // static config keeps daemon startup fast and matches the + // back-compat path the FJB-91 e2e harness exercises against the + // persistent test cache. + var cachePubkey, cacheEndpoint string + if cfg.Transport.WG != nil && (cfg.Transport.WG.Peer.PublicKey == "" || cfg.Transport.WG.Peer.Endpoint == "") { + cachePubkey, cacheEndpoint = discoverCachePeer(ctx, prov, log) + } stack, err := wgboot.Boot(ctx, wgboot.Config{ - Transport: cfg.Transport, - ForgejoURL: cfg.Forgejo.URL, - ACLSink: aclSinkFor(prov), - Cache: cache, - Logger: log, + Transport: cfg.Transport, + ForgejoURL: cfg.Forgejo.URL, + ACLSink: aclSinkFor(prov), + Cache: cache, + Logger: log, + CachePubkey: cachePubkey, + CacheEndpoint: cacheEndpoint, }) if err != nil { return nil, func() {}, fmt.Errorf("transport bootstrap: %w", err) @@ -530,6 +541,46 @@ func bootWGStack(ctx context.Context, cfg *config.Config, prov provider.Provider }, nil } +// discoverCachePeer runs the orchestrator side of the FJB-99 bootstrap +// loop: poll the managed cache's Object Storage bucket for the cache's +// WG pubkey (published by cache cloud-init's fjb-wg-bootstrap.sh) and +// read the cache's public IPv4 from the Linode API to build the WG +// peer endpoint. Returns empty strings (no error) when the provider +// can't supply these — wgboot then falls back to the static +// transport.wg.peer.{public_key,endpoint} config knobs and errors at +// boot if those are also empty. +// +// Bounded at 8 minutes — cache cloud-init's apt update + wireguard + +// awscli install dominates the time-to-pubkey, and on a cold Linode +// (no package cache, slow mirror) it can stretch past 5 min; the +// bound is generous so we don't fail a healthy bootstrap on a slow +// mirror day. +func discoverCachePeer(ctx context.Context, prov provider.Provider, log *slog.Logger) (pubkey, endpoint string) { + type pubkeyWaiter interface { + WaitForCacheWGPubkey(ctx context.Context, timeout time.Duration) (string, error) + } + type endpointReader interface { + CachePublicEndpoint(ctx context.Context) (string, error) + } + if pw, ok := prov.(pubkeyWaiter); ok { + start := time.Now() + if pk, err := pw.WaitForCacheWGPubkey(ctx, 8*time.Minute); err == nil { + log.Info("wgboot: cache wg-pubkey discovered", "elapsed", time.Since(start)) + pubkey = pk + } else { + log.Warn("wgboot: cache wg-pubkey discovery failed; falling back to static config", "err", err) + } + } + if er, ok := prov.(endpointReader); ok { + if ep, err := er.CachePublicEndpoint(ctx); err == nil { + endpoint = ep + } else { + log.Warn("wgboot: cache public endpoint discovery failed; falling back to static config", "err", err) + } + } + return pubkey, endpoint +} + // cacheRenderInputs pulls the cache-side facts the renderer needs out // of whatever live provider state is available. Today only the Linode // provider carries a managed cache; the function returns whatever it diff --git a/go.mod b/go.mod index a2a0fe0..5facdd6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26 require ( connectrpc.com/connect v1.20.0 github.com/linode/linodego v1.69.1 + github.com/minio/minio-go/v7 v7.2.0 github.com/prometheus/client_golang v1.23.2 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.52.0 @@ -19,15 +20,27 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-resty/resty/v2 v2.17.2 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-querystring v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.12.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect diff --git a/go.sum b/go.sum index 6a2d855..cd25da0 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -17,10 +19,17 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2A= github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -29,8 +38,16 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/linode/linodego v1.69.1 h1:f45N2MHR/oece2/ktTTCYmrlfse4//k3NgwcF5zbGZ0= github.com/linode/linodego v1.69.1/go.mod h1:Fha0NYsQSx5VZK1HQNJY/z/dIxxkFp+vb5veawbmAUw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs= +github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -41,8 +58,10 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -52,10 +71,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= diff --git a/internal/config/transport.go b/internal/config/transport.go index 808cf9c..48e57c3 100644 --- a/internal/config/transport.go +++ b/internal/config/transport.go @@ -254,14 +254,16 @@ func (w *WG) validate() error { } func (p *WGPeer) validate() error { - if p.PublicKey == "" { - return errors.New("public_key is required") - } - if p.Endpoint == "" { - return errors.New("endpoint is required") - } - if _, _, err := net.SplitHostPort(p.Endpoint); err != nil { - return fmt.Errorf("endpoint %q is not host:port: %w", p.Endpoint, err) + // public_key + endpoint are runtime-discoverable from FJB-99's + // cache wg-bootstrap loop (the cache writes its pubkey to S3; the + // orchestrator reads the cache's public IPv4 from the Linode API). + // Either may be empty at config-validate time; wgboot's planBoot + // is the load-bearing check that both end up populated by the + // time the tunnel is brought up. + if p.Endpoint != "" { + if _, _, err := net.SplitHostPort(p.Endpoint); err != nil { + return fmt.Errorf("endpoint %q is not host:port: %w", p.Endpoint, err) + } } if len(p.AllowedIPs) == 0 { return errors.New("allowed_ips must list at least one CIDR") diff --git a/internal/config/transport_wg_test.go b/internal/config/transport_wg_test.go index 8300fc8..20919b3 100644 --- a/internal/config/transport_wg_test.go +++ b/internal/config/transport_wg_test.go @@ -156,7 +156,6 @@ func TestWG_DefaultKeepaliveOneSecond(t *testing.T) { } } -//nolint:funlen // table-driven: 11 small validation cases inline for readability. func TestWG_Validation(t *testing.T) { cases := []struct { name string @@ -200,30 +199,6 @@ func TestWG_Validation(t *testing.T) { `, wantSub: `local_addr = "not-a-cidr"`, }, - { - name: "missing peer public key", - wgBlock: ` - wg: - private_key_file: /tmp/k - local_addr: 10.99.0.1/32 - peer: - endpoint: 172.234.203.50:51820 - allowed_ips: [10.99.0.2/32] -`, - wantSub: "public_key is required", - }, - { - name: "missing peer endpoint", - wgBlock: ` - wg: - private_key_file: /tmp/k - local_addr: 10.99.0.1/32 - peer: - public_key: AbcDefGhiJklMnoPqrStuVwxYzAbcDefGhiJklMnoPqs= - allowed_ips: [10.99.0.2/32] -`, - wantSub: "endpoint is required", - }, { name: "bad peer endpoint shape", wgBlock: ` diff --git a/internal/provider/linode/bucket.go b/internal/provider/linode/bucket.go index a1d2d07..97fced6 100644 --- a/internal/provider/linode/bucket.go +++ b/internal/provider/linode/bucket.go @@ -53,6 +53,13 @@ type managedBucket struct { // endpoint is the S3 endpoint URL for the bucket's region, looked up // once at ensureAtConfigure. endpoint string + + // accessKey + secretKey are the scoped Object Storage credentials + // minted at ensureAtConfigure. Stored so the orchestrator can sign + // S3 GETs against the bucket for the FJB-99 wg-pubkey discovery + // loop. Cleared on maybeCleanup. + accessKey string + secretKey string } func newManagedBucket(tag, region, label string, client bucketClient, log *slog.Logger) *managedBucket { @@ -114,6 +121,8 @@ func (m *managedBucket) ensureAtConfigure(ctx context.Context) (bucketCreds, err return bucketCreds{}, fmt.Errorf("create scoped object storage key: %w", err) } m.keyID = created.ID + m.accessKey = created.AccessKey + m.secretKey = created.SecretKey return bucketCreds{ Bucket: m.label, diff --git a/internal/provider/linode/cache-cloud-init.yaml.tmpl b/internal/provider/linode/cache-cloud-init.yaml.tmpl index 5cead87..a740a3b 100644 --- a/internal/provider/linode/cache-cloud-init.yaml.tmpl +++ b/internal/provider/linode/cache-cloud-init.yaml.tmpl @@ -23,7 +23,6 @@ packages: {{- end}} {{- if .OrchestratorWGPubkey}} - wireguard - - awscli {{- end}} write_files: - path: /etc/zot/config.json @@ -83,7 +82,12 @@ write_files: # boots reuse the persisted private key. wg0.conf is always # rewritten so a config bump (peer pubkey rotation, port change) # takes effect on reboot without operator intervention. - set -euo pipefail + # + # Logs go to /var/log/fjb-wg-bootstrap.log — read via Lish or + # cat to debug a failed bootstrap post-mortem. + exec >>/var/log/fjb-wg-bootstrap.log 2>&1 + echo "=== fjb-wg-bootstrap.sh starting at $(date -u +%Y-%m-%dT%H:%M:%SZ)" + set -euxo pipefail umask 0077 install -d -m 0700 /etc/wireguard if [ ! -s /etc/wireguard/privatekey ]; then @@ -105,14 +109,21 @@ write_files: EOC chmod 0600 /etc/wireguard/wg0.conf systemctl enable --now wg-quick@wg0 - # Publish own pubkey so the orchestrator can read it from S3 and - # complete the wg peer config on its side (FJB-99 Phase B). - AWS_ACCESS_KEY_ID='{{.AccessKey}}' \ - AWS_SECRET_ACCESS_KEY='{{.SecretKey}}' \ - AWS_DEFAULT_REGION='{{.Region}}' \ - aws --endpoint-url '{{.Endpoint}}' \ - s3 cp /etc/wireguard/publickey \ - s3://{{.Bucket}}/wg-pubkey.txt + # Publish own pubkey via a pre-signed S3 PUT URL the orchestrator + # baked into cloud-init at create time. No awscli on the cache: + # awscli on Debian 13 has a NoneType-iteration bug hitting Linode + # Object Storage that breaks both `s3 cp` and `s3api put-object`, + # so we sidestep it entirely by pre-signing on the orchestrator + # (minio-go) and having the cache just curl the URL. The + # orchestrator's GET path (also minio-go) reads it back. + # --fail-with-body so a 4xx/5xx response prints the XML error + # body to the bootstrap log instead of silently failing. -sS + # silences the progress bar but keeps the error path printable. + curl --fail-with-body -sS \ + --upload-file /etc/wireguard/publickey \ + '{{.WGPubkeyPutURL}}' \ + || { echo "FATAL: curl PUT failed (exit=$?)"; exit 1; } + echo "wg-pubkey upload OK" {{- end}} - path: /etc/systemd/system/zot.service permissions: '0644' diff --git a/internal/provider/linode/cache.go b/internal/provider/linode/cache.go index ccbbe26..72a85d6 100644 --- a/internal/provider/linode/cache.go +++ b/internal/provider/linode/cache.go @@ -7,15 +7,21 @@ import ( "encoding/base64" "errors" "fmt" + "io" "log/slog" + "net/http" "net/netip" + "net/url" "os" "path/filepath" "slices" "strings" "text/template" + "time" "github.com/linode/linodego" + minio "github.com/minio/minio-go/v7" + miniocreds "github.com/minio/minio-go/v7/pkg/credentials" "github.com/hstern/fj-bellows/internal/transport/cachegateway" ) @@ -407,6 +413,20 @@ func (m *managedCache) createFreshCacheLinode(ctx context.Context, pair cacheCer params.OrchestratorWGAddr = defaultOrchestratorWGAddr params.CacheWGAddr = defaultCacheWGAddr params.WGListenPort = defaultCacheWGListenPort + // FJB-99 Phase C — generate a pre-signed PUT URL that the + // cache cloud-init curls to publish its WG pubkey. awscli on + // Debian 13 has a NoneType-iteration bug when hitting Linode + // Object Storage that breaks both `s3 cp` and `s3api put- + // object`; sidestepping awscli entirely by pre-signing here + // and having the cache just curl it is the cleanest dodge. + // 24h expiry covers any plausible cloud-init delay; the URL + // is one-shot in practice (the cache only uploads once on + // first boot). + signedURL, perr := m.presignedWGPubkeyPutURL(24 * time.Hour) + if perr != nil { + return fmt.Errorf("presign wg-pubkey PUT: %w", perr) + } + params.WGPubkeyPutURL = signedURL } userData, err := renderCacheCloudInit(params) if err != nil { @@ -454,6 +474,174 @@ func (m *managedCache) renderIPTablesForCacheGateway() (string, error) { }) } +// WaitForWGPubkey blocks until the cache's first-boot cloud-init has +// published its WG public key to the Object Storage bucket, or until +// ctx is canceled / timeout elapses (whichever comes first). Returns +// the trimmed public-key string ready to feed into wgboot.Config. +// FJB-99 Phase B + Phase C fix. +// +// Uses minio-go for a signed S3 GET because Linode Object Storage +// rejects per-object public-read ACLs (the initial plain-HTTPS path +// errored with `x-amz-acl NotImplemented` and the cache upload +// never landed). The orchestrator signs with the same scoped bucket +// creds the cache already has from cloud-init. +// +// Polls at 5s intervals with exponential backoff up to 30s. First +// boot typically takes 60-120s on a warm Debian mirror; 4-6 min on +// a cold one (apt update + wireguard + awscli install). +func (m *managedCache) WaitForWGPubkey(ctx context.Context, timeout time.Duration) (string, error) { + mc, err := m.buildS3Client() + if err != nil { + return "", err + } + deadline := time.Now().Add(timeout) + delay := 5 * time.Second + const maxDelay = 30 * time.Second + for { + pub, ready, perr := pollWGPubkey(ctx, mc, m.bucket.label, m.log) + if perr != nil { + return "", perr + } + if ready { + return pub, nil + } + if time.Now().Add(delay).After(deadline) { + return "", fmt.Errorf("cache wg-pubkey poll: timed out after %s waiting for s3://%s/wg-pubkey.txt", timeout, m.bucket.label) + } + select { + case <-ctx.Done(): + return "", fmt.Errorf("cache wg-pubkey poll: %w", ctx.Err()) + case <-time.After(delay): + } + if delay < maxDelay { + delay *= 2 + if delay > maxDelay { + delay = maxDelay + } + } + } +} + +// presignedWGPubkeyPutURL returns a pre-signed S3 PUT URL the cache +// cloud-init curls to publish its WG pubkey, dodging awscli's +// "argument of type 'NoneType' is not iterable" bug against Linode +// Object Storage (see comment in createFreshCacheLinode). +func (m *managedCache) presignedWGPubkeyPutURL(expires time.Duration) (string, error) { + mc, err := m.buildS3Client() + if err != nil { + return "", err + } + u, err := mc.PresignedPutObject(context.Background(), m.bucket.label, "wg-pubkey.txt", expires) + if err != nil { + return "", fmt.Errorf("presign PUT: %w", err) + } + return u.String(), nil +} + +// buildS3Client constructs the minio client used by WaitForWGPubkey +// after validating that managedBucket has been Configured. Extracted +// to keep WaitForWGPubkey under the gocyclo budget — the validations +// + Endpoint parse + minio.New chain pushed it over. +func (m *managedCache) buildS3Client() (*minio.Client, error) { + if m.bucket == nil { + return nil, errors.New("cache: bucket not initialised (no managed cache)") + } + if m.bucket.endpoint == "" { + return nil, errors.New("cache: bucket endpoint unset (call Configure first)") + } + if m.bucket.accessKey == "" || m.bucket.secretKey == "" { + return nil, errors.New("cache: bucket credentials unset (call Configure first)") + } + endpointHost, useSSL, err := parseS3Endpoint(m.bucket.endpoint) + if err != nil { + return nil, fmt.Errorf("cache wg-pubkey poll: parse endpoint: %w", err) + } + mc, err := minio.New(endpointHost, &minio.Options{ + Creds: miniocreds.NewStaticV4(m.bucket.accessKey, m.bucket.secretKey, ""), + Secure: useSSL, + Region: m.bucket.region, + }) + if err != nil { + return nil, fmt.Errorf("cache wg-pubkey poll: minio client: %w", err) + } + return mc, nil +} + +// parseS3Endpoint splits "https://us-ord-10.linodeobjects.com" into +// (host, useSSL). minio-go wants the host bare and the scheme as a +// separate flag. +func parseS3Endpoint(raw string) (string, bool, error) { + u, err := url.Parse(raw) + if err != nil { + return "", false, err + } + if u.Host == "" { + return "", false, fmt.Errorf("endpoint %q has no host", raw) + } + return u.Host, u.Scheme == "https", nil +} + +// pollWGPubkey does one signed S3 GET for wg-pubkey.txt. Returns +// (key, true, nil) on hit; (_, false, nil) when the object isn't +// there yet (NoSuchKey / 404 — debug-logged for the operator); (_, +// _, err) when the response is structurally bad (read failure, +// empty body). +func pollWGPubkey(ctx context.Context, mc *minio.Client, bucket string, log *slog.Logger) (string, bool, error) { + obj, err := mc.GetObject(ctx, bucket, "wg-pubkey.txt", minio.GetObjectOptions{}) + if err != nil { + log.Debug("cache wg-pubkey poll: GetObject error", "err", err) + return "", false, nil + } + defer func() { _ = obj.Close() }() + body, rerr := io.ReadAll(obj) + if rerr != nil { + errResp := minio.ToErrorResponse(rerr) + if errResp.Code == "NoSuchKey" || errResp.StatusCode == http.StatusNotFound { + log.Debug("cache wg-pubkey poll: not ready yet", "bucket", bucket) + return "", false, nil + } + return "", false, fmt.Errorf("cache wg-pubkey poll: read object: %w", rerr) + } + pub := strings.TrimSpace(string(body)) + if pub == "" { + return "", false, errors.New("cache wg-pubkey poll: object present but body was empty") + } + return pub, true, nil +} + +// PublicEndpoint returns the cache's WG peer endpoint as host:port +// (e.g. "172.234.203.50:51820"). The IP comes from the Linode API +// (the public NIC's assigned IPv4); the port is supplied by the +// caller — typically transport.wg.listen_port from config or the +// default 51820 baked into the cache cloud-init. FJB-99 Phase B. +func (m *managedCache) PublicEndpoint(ctx context.Context, port int) (string, error) { + if m.linodeID == 0 { + return "", errors.New("cache: linode not provisioned yet") + } + if port <= 0 { + port = defaultCacheWGListenPort + } + inst, err := m.client.GetInstance(ctx, m.linodeID) + if err != nil { + return "", fmt.Errorf("cache public endpoint: get instance %d: %w", m.linodeID, err) + } + for _, ip := range inst.IPv4 { + if ip == nil { + continue + } + v4 := ip.To4() + if v4 == nil { + continue + } + // Skip the VPC interface IP (RFC1918) — pick the public IPv4. + if v4.IsPrivate() { + continue + } + return fmt.Sprintf("%s:%d", v4.String(), port), nil + } + return "", fmt.Errorf("cache public endpoint: linode %d has no public IPv4 yet", m.linodeID) +} + // buildCreateOpts assembles the InstanceCreateOptions payload for the // cache VM. Public stays primary so outbound (upstream sync, package // mirrors, GitHub-zot download) takes the default route; the VPC NIC @@ -657,6 +845,13 @@ type cacheCloudInitParams struct { OrchestratorWGAddr string // e.g. "100.64.0.1" CacheWGAddr string // e.g. "100.64.0.2" WGListenPort int // UDP, e.g. 51820 + + // WGPubkeyPutURL is the pre-signed S3 PUT URL the cache's bootstrap + // script curls to upload its wg-pubkey.txt to the deployment + // bucket (FJB-99 Phase C). Pre-signing the URL on the orchestrator + // (instead of having the cache call awscli) avoids awscli's + // known NoneType-iteration bug against Linode Object Storage. + WGPubkeyPutURL string } // renderCacheCloudInit fills the embedded template. Defaults to the diff --git a/internal/provider/linode/cache_test.go b/internal/provider/linode/cache_test.go index 68edfdb..0880875 100644 --- a/internal/provider/linode/cache_test.go +++ b/internal/provider/linode/cache_test.go @@ -5,10 +5,14 @@ import ( "errors" "fmt" "log/slog" + "net" + "net/http" + "net/http/httptest" "slices" "strings" "sync" "testing" + "time" "github.com/linode/linodego" ) @@ -22,7 +26,10 @@ const ( // Shared cloud-init renderer fixtures used across cache_test.go // and cache_tunnel_test.go — extracted so goconst doesn't flag // repeated literals. - testStubEndpoint = "https://x" + testStubEndpoint = "https://x" + // testFJBCacheLabel is the placeholder bucket label used across the + // renderer + Phase-B WG-bootstrap polling tests. + testFJBCacheLabel = "fjb-cache-test" testStubZotVersion = "1.0.0" ) @@ -393,7 +400,7 @@ func TestRenderCacheCloudInitProducesValidCloudInit(t *testing.T) { const stubCertPEM = "-----BEGIN CERTIFICATE-----\nMOCK\n-----END CERTIFICATE-----\n" const stubKeyPEM = "-----BEGIN EC PRIVATE KEY-----\nMOCK\n-----END EC PRIVATE KEY-----\n" //nolint:gosec // G101: test fixture, not a real key out, err := renderCacheCloudInit(cacheCloudInitParams{ - Bucket: "fjb-cache-test", + Bucket: testFJBCacheLabel, Region: testBucketRegion, Endpoint: testBucketEndpoint, AccessKey: "AK", @@ -407,7 +414,7 @@ func TestRenderCacheCloudInitProducesValidCloudInit(t *testing.T) { } for _, want := range []string{ "#cloud-config", - "fjb-cache-test", + testFJBCacheLabel, testBucketEndpoint, "\"accesskey\": \"AK\"", "\"secretkey\": \"SK\"", @@ -510,14 +517,17 @@ func TestRenderCacheCloudInitIPTablesBakeIn(t *testing.T) { } // FJB-99 Phase A: when OrchestratorWGPubkey is set, the rendered -// cloud-init installs wireguard + awscli, embeds the wg-bootstrap -// script with the orchestrator pubkey baked into [Peer].PublicKey, -// and runs the script from runcmd. When empty (ssh-mode), none of -// those entries leak in. +// cloud-init installs wireguard, embeds the wg-bootstrap script +// with the orchestrator pubkey baked into [Peer].PublicKey, and +// runs the script from runcmd. The bootstrap script curls a +// pre-signed S3 PUT URL (FJB-99 Phase C — sidesteps awscli's +// NoneType bug against Linode Object Storage). When empty +// (ssh-mode), none of those entries leak in. func TestRenderCacheCloudInitWGBootstrap(t *testing.T) { const orchPubkey = "ZmFrZS1vcmNoZXN0cmF0b3ItcHViLWtleS0xMjM0NTY3OD0=" + const fakePresigned = "https://us-ord-10.linodeobjects.com/fjb-cache-test/wg-pubkey.txt?X-Amz-Signature=deadbeef" withWG, err := renderCacheCloudInit(cacheCloudInitParams{ - Bucket: "fjb-cache-test", + Bucket: testFJBCacheLabel, Region: testBucketRegion, Endpoint: "https://us-ord-1.linodeobjects.com", AccessKey: "AK", @@ -529,13 +539,13 @@ func TestRenderCacheCloudInitWGBootstrap(t *testing.T) { OrchestratorWGAddr: "100.64.0.1", CacheWGAddr: "100.64.0.2", WGListenPort: 51820, + WGPubkeyPutURL: fakePresigned, }) if err != nil { t.Fatalf("render with WG: %v", err) } for _, want := range []string{ "- wireguard", - "- awscli", "/usr/local/sbin/fjb-wg-bootstrap.sh", "wg genkey", "Address = 100.64.0.2/32", @@ -544,13 +554,25 @@ func TestRenderCacheCloudInitWGBootstrap(t *testing.T) { "AllowedIPs = 100.64.0.1/32", "PersistentKeepalive = 25", "systemctl enable --now wg-quick@wg0", - "s3://fjb-cache-test/wg-pubkey.txt", - "--endpoint-url 'https://us-ord-1.linodeobjects.com'", + "--upload-file /etc/wireguard/publickey", + fakePresigned, } { if !strings.Contains(withWG, want) { t.Errorf("WG render missing %q\n---\n%s", want, withWG) } } + // Anti-regression: no `aws` invocation should remain in the script + // (a `# awscli` comment explaining why we avoid it is fine — we + // just want to be sure no actual `aws ...` shell command leaked). + for _, bad := range []string{ + "\n aws ", + "\n aws ", + "`aws ", + } { + if strings.Contains(withWG, bad) { + t.Errorf("WG render still invokes awscli (substring %q) — Phase C should be presigned-curl only\n---\n%s", bad, withWG) + } + } withoutWG, err := renderCacheCloudInit(cacheCloudInitParams{ Bucket: "b", @@ -614,6 +636,114 @@ func TestRenderIPTablesForCacheGateway(t *testing.T) { } } +// fakeS3 returns an httptest server that emulates S3's GetObject +// shape for the bucket+key the test cares about. Doesn't verify the +// AWS sigv4 signature — minio-go signs but server-side validation +// would just add ceremony without testing the production path. +type fakeS3State struct { + mu sync.Mutex + hits int + objects map[string][]byte +} + +func newFakeS3(t *testing.T) (*httptest.Server, *fakeS3State) { + t.Helper() + state := &fakeS3State{objects: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + state.mu.Lock() + state.hits++ + body, ok := state.objects[r.URL.Path] + state.mu.Unlock() + if !ok { + // S3 NoSuchKey response shape so minio.ToErrorResponse can + // classify it as "object missing". + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`NoSuchKey`)) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv, state +} + +func bucketWithEndpoint(srvURL, label string) *managedBucket { + return &managedBucket{ + endpoint: srvURL, + label: label, + accessKey: "test-access-key", + secretKey: "test-secret-key", + region: testBucketRegion, + } +} + +// FJB-99 Phase B + Phase C: WaitForWGPubkey signs an S3 GET via +// minio-go, polls until the object appears, and returns its +// trimmed contents. +func TestWaitForWGPubkey_PollsBucketUntilFound(t *testing.T) { + const wantPub = "ZmFrZS1jYWNoZS13Zy1wdWJrZXk=" + srv, state := newFakeS3(t) + go func() { + // Land the object after the first miss so the poller has to + // retry at least once. + time.Sleep(150 * time.Millisecond) + state.mu.Lock() + state.objects["/"+testFJBCacheLabel+"/wg-pubkey.txt"] = []byte(wantPub + "\n") + state.mu.Unlock() + }() + + m := &managedCache{bucket: bucketWithEndpoint(srv.URL, testFJBCacheLabel), log: slog.Default()} + got, err := m.WaitForWGPubkey(t.Context(), 30*time.Second) + if err != nil { + t.Fatalf("WaitForWGPubkey: %v", err) + } + if got != wantPub { + t.Errorf("pubkey = %q, want %q", got, wantPub) + } + state.mu.Lock() + hits := state.hits + state.mu.Unlock() + if hits < 2 { + t.Errorf("expected at least 2 GetObject calls, got %d", hits) + } +} + +// FJB-99 Phase B: WaitForWGPubkey honors the timeout argument. +func TestWaitForWGPubkey_TimeoutSurfacesAsError(t *testing.T) { + srv, _ := newFakeS3(t) + m := &managedCache{bucket: bucketWithEndpoint(srv.URL, testFJBCacheLabel), log: slog.Default()} + _, err := m.WaitForWGPubkey(t.Context(), 50*time.Millisecond) + if err == nil { + t.Fatal("WaitForWGPubkey: want timeout error; got nil") + } + if !strings.Contains(err.Error(), "timed out") { + t.Errorf("err = %v, want substring 'timed out'", err) + } +} + +// FJB-99 Phase B: PublicEndpoint picks the public IPv4 (not the VPC +// /private one) and appends the supplied port. +func TestPublicEndpoint_PicksPublicIPv4(t *testing.T) { + fc := newFakeCacheClient() + publicIP := net.ParseIP("172.234.203.50").To4() + privateIP := net.ParseIP("10.0.0.2").To4() + fc.insts[55] = &linodego.Instance{ + ID: 55, + IPv4: []*net.IP{&publicIP, &privateIP}, + } + m := &managedCache{client: fc, log: slog.Default(), linodeID: 55} + got, err := m.PublicEndpoint(t.Context(), 51820) + if err != nil { + t.Fatalf("PublicEndpoint: %v", err) + } + if got != "172.234.203.50:51820" { + t.Errorf("endpoint = %q, want %q", got, "172.234.203.50:51820") + } +} + // findCacheLinode must not adopt instances whose tag doesn't match — // otherwise a cache from a different deployment could be hijacked. func TestFindCacheLinodeIgnoresOtherDeployments(t *testing.T) { diff --git a/internal/provider/linode/firewall.go b/internal/provider/linode/firewall.go index 2191cad..1d1841e 100644 --- a/internal/provider/linode/firewall.go +++ b/internal/provider/linode/firewall.go @@ -536,10 +536,18 @@ const defaultWGListenPort = 51820 // synthSpecsForTransport returns the list of (proto, ports) tuples each // allow_inbound CIDR should ACCEPT, given the active transport mode and // the configured WG listen port. Empty / "ssh" (legacy default) keeps -// tcp/22; "cache-gateway" (FJB-89) returns a single udp/ -// rule so the cache nanode's WireGuard listener can receive the tunnel -// from the orchestrator's NAT egress. wgListenPort == 0 falls back to -// defaultWGListenPort (51820). +// tcp/22 only; "cache-gateway" emits BOTH udp/ (the +// WireGuard listener that's load-bearing for the cache-gateway data +// path) and tcp/22 (the operator break-glass for first-boot debugging +// when the WG tunnel hasn't come up yet — once wgboot is healthy +// operators can lock down tcp/22 via allow_inbound). wgListenPort == 0 +// falls back to defaultWGListenPort (51820). +// +// Note: an earlier FJB-89 revision dropped tcp/22 entirely under +// cache-gateway. That made the FJB-99 bootstrap-loop failures +// undebuggable — the orchestrator couldn't reach the cache via WG +// because of the very bug it was trying to surface, and tcp/22 was +// closed too. The tcp/22 entry is restored as a debug breakglass. func synthSpecsForTransport(mode string, wgListenPort int) []synthInboundSpec { switch mode { case transportCacheGateway: @@ -555,6 +563,7 @@ func synthSpecsForTransport(mode string, wgListenPort int) []synthInboundSpec { labelTag: "wg", descNote: "udp/" + portStr + " (WireGuard) from allow_inbound", }, + {proto: linodego.TCP, ports: "22", labelTag: "ssh", descNote: "tcp/22 (debug breakglass) from allow_inbound"}, } default: // transportSSH, transportSSHExplicit, or any unrecognised mode return []synthInboundSpec{ diff --git a/internal/provider/linode/firewall_transport_test.go b/internal/provider/linode/firewall_transport_test.go index 0b9fcf8..276e116 100644 --- a/internal/provider/linode/firewall_transport_test.go +++ b/internal/provider/linode/firewall_transport_test.go @@ -56,19 +56,22 @@ func TestSynthSpecsForTransport(t *testing.T) { wantPorts: []string{"22"}, }, { + // FJB-99 Phase C — cache-gateway emits udp/ + tcp/22. + // tcp/22 stays open as the operator debug break-glass; the + // WG tunnel remains the load-bearing access path. name: "cache-gateway default WG port", mode: transportCacheGateway, - wantCount: 1, - wantProtos: []linodego.NetworkProtocol{linodego.UDP}, - wantPorts: []string{"51820"}, + wantCount: 2, + wantProtos: []linodego.NetworkProtocol{linodego.UDP, linodego.TCP}, + wantPorts: []string{"51820", "22"}, }, { name: "cache-gateway custom WG port", mode: transportCacheGateway, wgListenPort: 51821, - wantCount: 1, - wantProtos: []linodego.NetworkProtocol{linodego.UDP}, - wantPorts: []string{"51821"}, + wantCount: 2, + wantProtos: []linodego.NetworkProtocol{linodego.UDP, linodego.TCP}, + wantPorts: []string{"51821", "22"}, }, { // Unrecognised mode falls back to SSH behaviour for safety. @@ -96,8 +99,8 @@ func TestSynthSpecsForTransport(t *testing.T) { } // TestBuildRuleSetCacheGatewayWG verifies the cache-gateway transport -// synthesizes a single ACCEPT rule per address family for udp/ (FJB-89), replacing the IPsec port set that lived here before. +// synthesizes ACCEPT rules per address family for udp/ +// (FJB-89) AND tcp/22 (FJB-99 Phase C debug break-glass). // //nolint:gocyclo // exhaustive single-site assertion of proto/ports/family + leftover-IPsec absence checks. func TestBuildRuleSetCacheGatewayWG(t *testing.T) { @@ -106,33 +109,29 @@ func TestBuildRuleSetCacheGatewayWG(t *testing.T) { if err != nil { t.Fatalf("buildRuleSet: %v", err) } - // 1 spec × (1 v4 chunk + 1 v6 chunk) = 2 inbound rules. - if len(rs.Inbound) != 2 { - t.Fatalf("len(Inbound) = %d, want 2 (1 WG spec × 2 families)", len(rs.Inbound)) + // 2 specs (WG + ssh-debug) × (1 v4 chunk + 1 v6 chunk) = 4 inbound rules. + if len(rs.Inbound) != 4 { + t.Fatalf("len(Inbound) = %d, want 4 (2 specs × 2 families)", len(rs.Inbound)) } - var v4Total, v6Total int + var wgSeen, sshSeen int for _, r := range rs.Inbound { if r.Action != fwActionAccept { t.Errorf("rule %q: action=%q, want ACCEPT", r.Label, r.Action) } - if r.Protocol != linodego.UDP { - t.Errorf("rule %q: proto=%s, want UDP", r.Label, r.Protocol) - } - if r.Ports != "51820" { - t.Errorf("rule %q: ports=%q, want %q", r.Label, r.Ports, "51820") + switch { + case r.Protocol == linodego.UDP && r.Ports == "51820": + wgSeen++ + case r.Protocol == linodego.TCP && r.Ports == "22": + sshSeen++ + default: + t.Errorf("unexpected rule %q: proto=%s, ports=%q", r.Label, r.Protocol, r.Ports) } - v4Total += len(*r.Addresses.IPv4) - v6Total += len(*r.Addresses.IPv6) } - if v4Total != 1 || v6Total != 1 { - t.Errorf("address bucketing: v4=%d v6=%d, want 1 + 1", v4Total, v6Total) + if wgSeen != 2 || sshSeen != 2 { + t.Errorf("rule counts: wg=%d ssh=%d, want 2 each (1 v4 + 1 v6)", wgSeen, sshSeen) } - // No tcp/22 rule should exist in cache-gateway mode. - // Likewise no leftover IPsec (udp/500, udp/4500, ESP) rules. + // No leftover IPsec (udp/500, udp/4500, ESP) rules. for _, r := range rs.Inbound { - if r.Protocol == linodego.TCP && r.Ports == "22" { - t.Errorf("found legacy tcp/22 rule under cache-gateway: %+v", r) - } if r.Protocol == linodego.UDP && (r.Ports == "500" || r.Ports == "4500") { t.Errorf("found stale IPsec rule under cache-gateway: %+v", r) } @@ -148,6 +147,8 @@ func TestBuildRuleSetCacheGatewayWG(t *testing.T) { // TestBuildRuleSetCacheGatewayCustomWGPort verifies the WG listen port // is plumbed through synthSpecsForTransport into the rendered ruleset. +// Now also asserts the tcp/22 debug break-glass (FJB-99 Phase C) is +// present alongside the custom WG port. func TestBuildRuleSetCacheGatewayCustomWGPort(t *testing.T) { const customPort = 51821 fw := testFirewallWithTransportAndPort(firewallConfig{}, transportCacheGateway, customPort) @@ -155,12 +156,20 @@ func TestBuildRuleSetCacheGatewayCustomWGPort(t *testing.T) { if err != nil { t.Fatalf("buildRuleSet: %v", err) } - if len(rs.Inbound) != 1 { - t.Fatalf("len(Inbound) = %d, want 1", len(rs.Inbound)) + if len(rs.Inbound) != 2 { + t.Fatalf("len(Inbound) = %d, want 2 (WG + ssh-debug)", len(rs.Inbound)) } - r := rs.Inbound[0] - if r.Protocol != linodego.UDP || r.Ports != "51821" { - t.Errorf("rule = (%s, %q), want (UDP, %q)", r.Protocol, r.Ports, "51821") + var wgSeen, sshSeen bool + for _, r := range rs.Inbound { + switch { + case r.Protocol == linodego.UDP && r.Ports == "51821": + wgSeen = true + case r.Protocol == linodego.TCP && r.Ports == "22": + sshSeen = true + } + } + if !wgSeen || !sshSeen { + t.Errorf("rules wgSeen=%v sshSeen=%v, want both true", wgSeen, sshSeen) } } @@ -209,9 +218,9 @@ func TestBuildRuleSetCacheGatewayChunking(t *testing.T) { if err != nil { t.Fatalf("buildRuleSet: %v", err) } - // One WG spec × ceil(300/255) = 2 v4 rules, 0 v6. - if len(rs.Inbound) != 2 { - t.Fatalf("len(Inbound) = %d, want 2 (1 WG spec × 2 v4 chunks)", len(rs.Inbound)) + // 2 specs (WG + ssh-debug) × ceil(300/255) = 4 v4 rules, 0 v6. + if len(rs.Inbound) != 4 { + t.Fatalf("len(Inbound) = %d, want 4 (2 specs × 2 v4 chunks)", len(rs.Inbound)) } } diff --git a/internal/provider/linode/linode.go b/internal/provider/linode/linode.go index 22684ea..de12299 100644 --- a/internal/provider/linode/linode.go +++ b/internal/provider/linode/linode.go @@ -252,6 +252,31 @@ func (l *Linode) SetOrchestratorWGPubkey(pubkey string) { l.orchestratorWGPubkey = pubkey } +// WaitForCacheWGPubkey blocks until the cache's first-boot cloud-init +// publishes its WG pubkey to the Object Storage bucket, or until ctx +// is canceled / timeout elapses. Returns the trimmed pubkey for use +// as the wgboot WG peer config. FJB-99 Phase B — orchestrator side +// of the bootstrap loop. Returns ("", error) when no managed cache +// exists or it hasn't been ensured yet. +func (l *Linode) WaitForCacheWGPubkey(ctx context.Context, timeout time.Duration) (string, error) { + if l.cache == nil { + return "", errors.New("linode: no managed cache configured") + } + return l.cache.WaitForWGPubkey(ctx, timeout) +} + +// CachePublicEndpoint returns the cache's WG peer endpoint as +// host:port (e.g. "172.234.203.50:51820") for the wgboot peer config. +// The port comes from l.wgListenPort (operator-configured listen port, +// captured by SetWGListenPort before Configure) or defaults to 51820. +// FJB-99 Phase B. Returns ("", error) when no managed cache exists. +func (l *Linode) CachePublicEndpoint(ctx context.Context) (string, error) { + if l.cache == nil { + return "", errors.New("linode: no managed cache configured") + } + return l.cache.PublicEndpoint(ctx, l.wgListenPort) +} + // CacheStatus returns the managed-cache snapshot consumed by the control // plane's GetCache RPC. Returns nil when no `cache:` block is configured; // the control handler then reports Present=false to the wire. The Linode diff --git a/internal/transport/wgboot/wgboot.go b/internal/transport/wgboot/wgboot.go index ad574b6..2222683 100644 --- a/internal/transport/wgboot/wgboot.go +++ b/internal/transport/wgboot/wgboot.go @@ -111,6 +111,19 @@ type Config struct { // Logger receives boot + runtime events. Falls back to // slog.Default when nil. Logger *slog.Logger + + // CachePubkey + CacheEndpoint are the runtime-discovered peer + // values from the FJB-99 bootstrap loop. When non-empty they + // override Transport.WG.Peer.PublicKey + Transport.WG.Peer.Endpoint + // — the cache nanode generates its keypair at first boot and + // publishes its pubkey to S3; cmd/fj-bellows polls for it + // (managedCache.WaitForWGPubkey) and the public IP comes from + // the Linode API (managedCache.PublicEndpoint). Leaving the static + // transport.wg.peer.{public_key,endpoint} config knobs populated + // remains supported as a back-compat path; the override takes + // precedence when both are present. + CachePubkey string + CacheEndpoint string } // Stack is the running cache-gateway transport. Returned by Boot, @@ -245,11 +258,25 @@ func planBoot(cfg Config, log *slog.Logger) (*bootPlan, error) { if err != nil { return nil, fmt.Errorf("wgboot: parse local_addr %q: %w", cfg.Transport.WG.LocalAddr, err) } - endpoint, err := net.ResolveUDPAddr("udp", cfg.Transport.WG.Peer.Endpoint) + endpointStr := cfg.Transport.WG.Peer.Endpoint + if cfg.CacheEndpoint != "" { + endpointStr = cfg.CacheEndpoint + } + if endpointStr == "" { + return nil, errors.New("wgboot: cache endpoint unset — set transport.wg.peer.endpoint, or provide CacheEndpoint via the FJB-99 bootstrap loop") + } + endpoint, err := net.ResolveUDPAddr("udp", endpointStr) if err != nil { - return nil, fmt.Errorf("wgboot: resolve peer endpoint %q: %w", cfg.Transport.WG.Peer.Endpoint, err) + return nil, fmt.Errorf("wgboot: resolve peer endpoint %q: %w", endpointStr, err) + } + peerKeyStr := cfg.Transport.WG.Peer.PublicKey + if cfg.CachePubkey != "" { + peerKeyStr = cfg.CachePubkey + } + if peerKeyStr == "" { + return nil, errors.New("wgboot: cache pubkey unset — set transport.wg.peer.public_key, or provide CachePubkey via the FJB-99 bootstrap loop") } - peerKey, err := wg.DecodeKey(cfg.Transport.WG.Peer.PublicKey) + peerKey, err := wg.DecodeKey(peerKeyStr) if err != nil { return nil, fmt.Errorf("wgboot: parse peer public_key: %w", err) } diff --git a/internal/transport/wgboot/wgboot_test.go b/internal/transport/wgboot/wgboot_test.go index 7772c05..593e5ec 100644 --- a/internal/transport/wgboot/wgboot_test.go +++ b/internal/transport/wgboot/wgboot_test.go @@ -54,6 +54,41 @@ func TestBoot_RejectsMissingForgejoURL(t *testing.T) { } } +// FJB-99 Phase B: when transport.wg.peer.{public_key,endpoint} are +// empty in config (the new bootstrap-loop default) and the caller +// hasn't supplied CachePubkey/CacheEndpoint overrides either, +// Boot must surface a clear error pointing at the bootstrap loop. +func TestBoot_RejectsMissingPeerWithoutOverrides(t *testing.T) { + cfg := Config{ + Transport: config.Transport{ + Mode: config.TransportCacheGateway, + WG: &config.WG{ + PrivateKeyFile: "/tmp/k-fjb99-missing-peer", + LocalAddr: "100.64.0.1/32", + OverlayPrefix: "100.64.0.0/30", + Peer: config.WGPeer{ + // PublicKey + Endpoint deliberately empty. + AllowedIPs: []string{"100.64.0.2/32"}, + }, + }, + }, + ForgejoURL: "https://git.example.com", + Cache: CacheRenderInputs{ + CacheVPCIP: "10.0.0.2", + WorkerVPCSubnet: "10.0.0.0/24", + }, + } + _, err := Boot(t.Context(), cfg) + if err == nil { + t.Fatal("Boot(no peer + no override) should error; got nil") + } + for _, want := range []string{"FJB-99 bootstrap"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want substring %q", err, want) + } + } +} + func TestBoot_RejectsMissingCacheVPCIP(t *testing.T) { cfg := Config{ Transport: config.Transport{ diff --git a/test/e2e-linode/run-local.sh b/test/e2e-linode/run-local.sh index 2f8764c..44d86c6 100755 --- a/test/e2e-linode/run-local.sh +++ b/test/e2e-linode/run-local.sh @@ -38,8 +38,9 @@ Usage: $0 [--transport=ssh|cache-gateway] --transport=ssh (default) Legacy SSH-on-public-IP dispatch. --transport=cache-gateway WireGuard cache-gateway transport (FJB-54 - verification — requires the persistent cache - nanode at 172.234.203.50 to be reachable). + verification). fj-bellows provisions its + own ephemeral cache per run; no external + persistent infrastructure required. HELP exit 0 ;; @@ -61,13 +62,11 @@ LOG="$WORKDIR/fj-bellows.log" PIDF="$WORKDIR/fj-bellows.pid" FORGEJO_NAME="fjb-e2e-forgejo-$$" -# Cache-gateway persistent fixtures (FJB-91). The cache nanode is a -# long-lived peer outside any e2e tag, hosted at Linode 98209737 with -# the iptables FJB-FORWARD chain pre-installed by FJB-86. The harness -# only verifies it's reachable; it never provisions or destroys it. -CACHE_PUBLIC_IP=172.234.203.50 -CACHE_VPC_IP=10.0.0.2 -CACHE_WG_PUBKEY="BDKfn0TW6W15EqpyoYvVgUxktHk/c9t6kcmDoGTIjig=" +# Cache-gateway runtime fixtures (FJB-99). Each run gets its own +# ephemeral cache provisioned by fj-bellows; the bootstrap loop +# (Phase A + B) handles peer-pubkey discovery + endpoint resolution. +# The values below are just the orchestrator-side knobs that don't +# depend on the cache existing yet. CACHE_WG_PORT=51820 ORCHESTRATOR_WG_PRIVATE_KEY_FILE="${HOME}/.config/fj-bellows/wg-private-key" ORCHESTRATOR_WG_ADDR=100.64.0.1 @@ -169,6 +168,24 @@ cleanup() { log "cleanup (rc=$rc)" [ -s "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null || true docker rm -f "$FORGEJO_NAME" >/dev/null 2>&1 || true + # On failure under cache-gateway, before destroying anything, try to + # SSH into the cache and dump its bootstrap log. The cache is tagged + # `-cache`; its public IPv4 comes from the Linode API. + if [ "$rc" -ne 0 ] && [ "$TRANSPORT" = "cache-gateway" ]; then + cache_tag="$TAG-cache" + cache_ip=$(linode_api GET '/linode/instances?page_size=200' \ + | jq -r --arg t "$cache_tag" '.data[]? | select(.tags|index($t)) | .ipv4[0]' \ + | head -n1) + if [ -n "$cache_ip" ] && [ "$cache_ip" != "null" ]; then + log "dumping cache bootstrap log from $cache_ip" + ssh -i "$KEY" -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ + -o ConnectTimeout=10 \ + root@"$cache_ip" \ + 'echo "--- fjb-wg-bootstrap.log ---"; cat /var/log/fjb-wg-bootstrap.log 2>/dev/null || echo "(not present)"; echo "--- /etc/wireguard ---"; ls -la /etc/wireguard 2>/dev/null || true; cat /etc/wireguard/wg0.conf 2>/dev/null || true; echo "--- wg show ---"; wg show 2>/dev/null || true; echo "--- iptables FJB-FORWARD ---"; iptables -L FJB-FORWARD -n -v --line-numbers 2>/dev/null || true; echo "--- ip route ---"; ip -4 route 2>/dev/null || true; echo "--- ip -4 addr ---"; ip -4 addr 2>/dev/null || true; echo "--- ping worker (10.0.0.3) ---"; ping -c 2 -W 2 10.0.0.3 2>&1 || true; echo "--- nc to worker:22 ---"; timeout 5 bash -c "echo > /dev/tcp/10.0.0.3/22 && echo OPEN || echo CLOSED-or-TIMEOUT" 2>&1 || true; echo "--- cloud-init.log tail ---"; tail -100 /var/log/cloud-init.log 2>/dev/null || true; echo "--- cloud-init-output.log tail ---"; tail -100 /var/log/cloud-init-output.log 2>/dev/null || true' \ + 2>&1 | sed 's/^/[cache] /' >&2 || true + fi + fi destroy_tagged "$TAG" if [ "$rc" -ne 0 ]; then err "FAILED. Workdir kept: $WORKDIR" @@ -213,31 +230,28 @@ export FORGEJO_LABEL=linode-e2e export FORGEJO_WORKFLOW_CONTAINER_OPTS='--network host' FORGEJO_TOKEN=$(bash "$REPO_ROOT/test/e2e-docker/seed.sh") -# Cache-gateway preflight: persistent cache must be SSH-reachable -# (read-only probe) before we burn time provisioning a worker. The -# orchestrator's WG private key file must exist with 0600 perms so the -# embedded wireguard-go bind doesn't refuse to start mid-run. +# Cache-gateway preflight: ensure the orchestrator's WG private key +# file is creatable / has the right perms. The cache itself is now +# ephemeral per run (FJB-99) — fj-bellows provisions it, the cache +# generates its own WG keypair at first boot, publishes the pubkey to +# the deployment's Object Storage bucket, and the orchestrator polls +# for it via plain HTTPS GET (FJB-99 Phase B). No persistent cache +# pre-condition. if [ "$TRANSPORT" = "cache-gateway" ]; then - log "preflight: checking persistent cache at $CACHE_PUBLIC_IP (FJB-91)" - if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new \ - -o UserKnownHostsFile="$KNOWN" \ - root@"$CACHE_PUBLIC_IP" \ - 'wg show wg0 listen-port' >/dev/null 2>&1; then - err "persistent cache $CACHE_PUBLIC_IP is unreachable or wg0 is down" - err "(harness assumes the FJB-86 cache nanode is live; bring it up before retrying)" - exit 1 - fi + log "preflight: checking orchestrator WG private key" if [ ! -r "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE" ]; then - err "missing orchestrator WG private key at $ORCHESTRATOR_WG_PRIVATE_KEY_FILE" - err "(create with: wg genkey > $ORCHESTRATOR_WG_PRIVATE_KEY_FILE && chmod 600 $_)" - exit 1 - fi - perms=$(stat -f '%Lp' "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE" 2>/dev/null || stat -c '%a' "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE" 2>/dev/null || echo unknown) - if [ "$perms" != "600" ]; then - err "orchestrator WG private key has mode $perms (want 0600)" - exit 1 + # First-run convenience: let fj-bellows create the key on its own + # via LoadOrGenerateKey. We just need the parent directory writable. + install -d "$(dirname "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE")" + log " (key file absent — fj-bellows will generate one)" + else + perms=$(stat -f '%Lp' "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE" 2>/dev/null || stat -c '%a' "$ORCHESTRATOR_WG_PRIVATE_KEY_FILE" 2>/dev/null || echo unknown) + if [ "$perms" != "600" ]; then + err "orchestrator WG private key has mode $perms (want 0600)" + exit 1 + fi fi - log "preflight OK (cache wg0 up, orchestrator key mode 0600)" + log "preflight OK" fi # Cache-gateway-only YAML fragments. Empty under ssh mode so the @@ -261,9 +275,11 @@ transport: local_addr: $ORCHESTRATOR_WG_ADDR/32 overlay_prefix: $ORCHESTRATOR_WG_OVERLAY keepalive_interval: 1s + # FJB-99 Phase B: peer.public_key + peer.endpoint left empty — the + # bootstrap loop fills them at runtime (orchestrator polls the + # cache's Object Storage bucket for its pubkey, and reads the + # cache's public IPv4 from the Linode API). peer: - public_key: $CACHE_WG_PUBKEY - endpoint: $CACHE_PUBLIC_IP:$CACHE_WG_PORT allowed_ips: [100.64.0.2/32, 10.0.0.0/24] # ACL gates what workers may reach across the tunnel. The # orchestrator auto-injects an implicit entry for Forgejo + @@ -321,13 +337,11 @@ YAML # VM tag is \$TAG-cache so the worker prefix sweep above also # reaps it; bucket and key sweeps live in destroy_tagged. # -# Under cache-gateway transport (FJB-91), the same `cache:` block -# stays — fj-bellows still provisions the per-run cache for the -# registry+CA. The persistent WG endpoint at 172.234.203.50 is a -# distinct nanode that terminates the orchestrator's WireGuard -# tunnel; the e2e-provisioned cache holds zot + the CA. The two -# nanodes coexist in the VPC for this slice; FJB-87 consolidates -# them onto one host in a follow-up. +# Under cache-gateway transport (FJB-99), the same `cache:` block +# stays — fj-bellows provisions a single per-run cache that holds +# zot + the CA AND terminates the orchestrator's WireGuard tunnel. +# The cache's WG pubkey is discovered at runtime via the bootstrap +# loop landed in FJB-99 Phase A + B. cat >> "$CONFIG" <"$LOG" 2>&1 & echo $! > "$PIDF" -# Wait for the control plane to come up before we depend on it for state. +# Wait for the control plane to come up. Under cache-gateway mode the +# orchestrator first provisions the cache (~30s), runs the WG-pubkey +# poll against the bucket (cache cloud-init takes ~2 min to install +# wireguard + awscli + upload pubkey), then brings up wgboot. Only +# then does the control plane listener bind. 5 min is the safe +# headroom; ssh mode binds the listener within seconds. +if [ "$TRANSPORT" = "cache-gateway" ]; then + # 10 min covers cache provision (~30s) + cloud-init apt + wireguard + # install (~3-5 min) + WaitForWGPubkey poll + wgboot.Boot. The bound + # is generous so a slow Debian mirror day doesn't make the harness + # flake. + CTL_WAIT_SECS=600 +else + CTL_WAIT_SECS=30 +fi ctl_ready=0 -for i in $(seq 1 30); do +for i in $(seq 1 "$CTL_WAIT_SECS"); do if curl -sS --max-time 2 "http://127.0.0.1:${CTL_PORT}/healthz" >/dev/null 2>&1; then log "control plane up after ${i}*1s" ctl_ready=1 @@ -376,7 +404,7 @@ for i in $(seq 1 30); do fi sleep 1 done -[ "$ctl_ready" -eq 1 ] || { err "control plane never came up on :${CTL_PORT}"; exit 1; } +[ "$ctl_ready" -eq 1 ] || { err "control plane never came up on :${CTL_PORT} within ${CTL_WAIT_SECS}s"; exit 1; } # Cache-gateway-only: confirm the WG tunnel + ACL + DNS responder come # up. Each component logs a distinct line that we grep for; on failure @@ -408,21 +436,17 @@ if [ "$TRANSPORT" = "cache-gateway" ]; then ORCH_WG_PUBKEY=$(grep -o 'public_key=[^ ]*' "$LOG" | head -1 | sed 's/public_key=//') log " orchestrator wg pubkey: ${ORCH_WG_PUBKEY:-}" - # FJB-91 PoC scope ends here. The remaining steps (worker provisioning, - # SSH-poll, ListWorkers idle gate, GetCache, ProviderInfo, worker-side - # cache assertions, job-complete wait) all assume the orchestrator can - # actually reach the worker — which today requires the ephemeral cache - # and the worker to share a VPC, which in turn requires a "cache wg - # bootstrap" loop (create cache → fetch its pubkey → reconfigure wg - # peer) that doesn't exist yet. That loop is FJB-94 (fjbagent + - # agent.Health) and FJB-97 (orchestrator → target reachability probe); - # those tickets will replace the SSH-dispatch path entirely. - # - # For this PoC, validating that the WG stack comes up cleanly against - # a known-good cache peer is the assertion that's load-bearing for - # FJB-54 itself. Worker readiness moves with FJB-94. - log "FJB-91 PoC scope complete: wgboot stack validated against persistent cache." - log " (worker provisioning + readiness validation deferred to FJB-94/97)" + # FJB-99 Phase C scope ends here. The bootstrap loop (cache create → + # cache cloud-init publishes wg-pubkey via presigned PUT → orches- + # trator polls bucket → wg tunnel up) is what's load-bearing for the + # ticket. The downstream worker-dispatch via netstack (FJB-92 path) + # depends on cache↔worker VPC routing the orchestrator firewall + # synth doesn't currently provision; that gap is tracked separately + # in FJB-94 (fjbagent replaces SSH dispatch) and a follow-up + # firewall ticket. Reactivating the worker provisioning + idle gate + # here would conflate two distinct issues. + log "FJB-99 Phase C scope complete: ephemeral cache bootstrap validated end-to-end." + log " (worker readiness + job-complete deferred to FJB-94/97)" exit 0 fi