From b5c50c20cffe6a825f630bfb8c6da3ae008404a5 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 4 Jul 2026 20:16:25 +0200 Subject: [PATCH 1/2] fix(unifi): functional provider via Cloud Connector + writable /tmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provider-upjet-unifi has been unable to manage the controller since the migration merged — every managed resource is SYNCED=False. Two root causes, both fixed here: 1. Read-only /tmp: Kyverno forces the provider pod read-only with no /tmp mount, but the Upjet provider writes Terraform workspaces to /tmp/ ("mkdir /tmp/: read-only file system"), so nothing reconciles. Add a writable /tmp emptyDir + explicit hardened securityContext to the DeploymentRuntimeConfig. (Mirrors the identical hunk in #2455; whichever merges first, the other auto-resolves or drops this one file.) 2. No write path: it was pointed at a cloud URL without Cloud Connector enabled. Switch to UniFi Cloud Connector mode (cloud_connector: "true") so the provider authenticates with a Site Manager / Cloud API key and routes writes through https://api.ui.com, which proxies to the controller — the controller need NOT be reachable from Hetzner (no tunnel, no public endpoint). Drop the unused api_url; omit hardware_id (defaults to the first owner=true console). One-time user gate: overwrite api_key at secret/infrastructure/unifi/controller in OpenBao with a WRITE-scoped UniFi Site Manager API key. Co-Authored-By: Claude Opus 4.8 --- .../infrastructure/vault-config/job.yaml | 16 ++++--- .../hetzner/apps/unifi/external-secret.yaml | 43 ++++++++++--------- ...deployment-runtime-config-upjet-unifi.yaml | 27 ++++++++++++ 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index cb7d818e4..8cbd8d167 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -520,15 +520,19 @@ spec: fi # --- 1c. Seed placeholder UniFi controller credentials (only if absent) --- - # The `unifi` tenant's Terraform CR (tofu-controller) reads the UniFi - # API credentials from secret/infrastructure/unifi/controller via the - # unifi-credentials ExternalSecret. We seed PLACEHOLDERS here - # declaratively (no SOPS) ONLY when the secret is absent, so the + # The `unifi` tenant's Crossplane provider (provider-upjet-unifi) reads + # the UniFi credentials from secret/infrastructure/unifi/controller via + # the unifi-controller-credentials ExternalSecret. We seed PLACEHOLDERS + # here declaratively (no SOPS) ONLY when the secret is absent, so the # maintainer can overwrite them in place via the OpenBao UI/CLI and a # later Job re-run won't clobber the real values. OpenBao is backed up # (Raft snapshots → R2 via Velero), so these manually-set values are - # durable — no GitOps re-seed is needed. api_url omits the /api path; - # api_key is a Limited-Admin Local-Access-Only key (UniFi OS >= 9.0.108). + # durable — no GitOps re-seed is needed. GATE: overwrite `api_key` with a + # UniFi Site Manager / Cloud API key that has WRITE scope (unifi.ui.com → + # API, Site + Application scope) — the provider runs in Cloud Connector + # mode (routes through https://api.ui.com), so a local Limited-Admin key + # is NOT used, and `api_url` is left as an unused placeholder for a + # possible future local/tunnel path. if ! bao kv get -mount=secret infrastructure/unifi/controller >/dev/null 2>&1; then echo "Seeding placeholder UniFi credentials at secret/infrastructure/unifi/controller..." bao kv put -mount=secret infrastructure/unifi/controller \ diff --git a/k8s/providers/hetzner/apps/unifi/external-secret.yaml b/k8s/providers/hetzner/apps/unifi/external-secret.yaml index 041470c8e..f5cb13537 100644 --- a/k8s/providers/hetzner/apps/unifi/external-secret.yaml +++ b/k8s/providers/hetzner/apps/unifi/external-secret.yaml @@ -1,16 +1,29 @@ --- -# UniFi controller API credentials from OpenBao, rendered as the single JSON +# UniFi controller credentials from OpenBao, rendered as the single JSON # `credentials` blob the Crossplane ProviderConfig consumes (provider-config.yaml, # secretRef → key `credentials`). The provider forwards it verbatim to the # underlying ubiquiti-community/unifi SDK config. Read via the tenant's NAMESPACED -# `openbao` SecretStore (secret-store.yaml) — NOT the shared cluster store — -# authorised by the infra-unifi-readonly OpenBao policy on the dedicated `unifi` -# role. -# SEEDING: the vault-config Job seeds PLACEHOLDERS at -# secret/infrastructure/unifi/controller on first run (only if absent). GATE: -# overwrite them in place via the OpenBao UI/CLI (api_url omits the /api path; -# Limited-Admin Local-Access-Only key, controller >= 9.0.108). OpenBao is backed -# up (Raft → R2), so the value is durable. +# `openbao` SecretStore (secret-store.yaml) — NOT the shared cluster store. +# +# CLOUD CONNECTOR mode: `cloud_connector: "true"` makes the provider authenticate +# with `api_key` and route every request through the UniFi Cloud +# (https://api.ui.com), which PROXIES to the controller — so the controller does +# NOT need to be reachable from the cluster (no tunnel, no public endpoint). No +# `api_url` is needed (Cloud Connector fixes the endpoint), and `hardware_id` is +# omitted so the provider targets the first console where `owner=true` (add it +# explicitly only if you run multiple consoles). +# +# SEEDING: the vault-config Job seeds a PLACEHOLDER `api_key` at +# secret/infrastructure/unifi/controller (only if absent). GATE: overwrite it in +# place via the OpenBao UI/CLI with a UniFi **Site Manager / Cloud API key that +# has WRITE scope** (unifi.ui.com → API → create key with Site + Application +# scope) — NOT a local Limited-Admin key. OpenBao is backed up (Raft → R2), so the +# value is durable. +# +# The provider unmarshals EVERY credentials value as a Go `string`, so +# `cloud_connector` MUST be the string "true" (not a JSON bool) — a bare bool +# fails `cannot unmarshal bool into Go value of type string`. dict|toJson +# quotes/escapes every value. apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: @@ -28,18 +41,8 @@ spec: engineVersion: v2 type: Opaque data: - # Build the blob with dict|toJson so values are JSON-escaped. The provider - # unmarshals EVERY credentials value as a Go `string`, so allow_insecure - # MUST be the string "false" (not a JSON bool) — a bare bool makes the - # provider fail with `cannot unmarshal bool into Go value of type string` - # (CannotConnectToProvider on every managed resource). api_url/api_key are - # properly quoted/escaped by toJson. - credentials: '{{ dict "api_url" .api_url "api_key" .api_key "site" "default" "allow_insecure" "false" | toJson }}' + credentials: '{{ dict "api_key" .api_key "cloud_connector" "true" "site" "default" | toJson }}' data: - - secretKey: api_url - remoteRef: - key: infrastructure/unifi/controller - property: api_url - secretKey: api_key remoteRef: key: infrastructure/unifi/controller diff --git a/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml b/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml index a5b7fe7c7..c9e094c3b 100644 --- a/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml +++ b/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml @@ -14,9 +14,36 @@ spec: spec: containers: - name: package-runtime + # Kubescape C-0017 (immutable container filesystem) + the hardening + # set Kyverno injects at admission (Kubescape scans this stored spec, + # not the live pod). This provider (devantler-tech provider-upjet-unifi + # v0.1.0, older upjet) DOES write Terraform workspaces to /tmp/, + # so a read-only root with no writable /tmp breaks EVERY managed + # resource — it is failing right now (cluster-wireguard SYNCED=False: + # "mkdir /tmp/: read-only file system"). Kyverno already forces + # the pod read-only with no /tmp mount, so this DRC adds the writable + # /tmp emptyDir that both fixes reconciliation and makes the read-only + # root safe. runAsUser/runAsGroup left to Crossplane's default (2000). + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + seLinuxOptions: + level: s0 resources: requests: cpu: 10m memory: 128Mi limits: memory: 512Mi + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} From e104b75504e24a9d9069c11686b2cf70830843e9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 4 Jul 2026 21:20:28 +0200 Subject: [PATCH 2/2] fix(unifi): cap the provider /tmp emptyDir at 256Mi Co-Authored-By: Claude Fable 5 --- .../crossplane/deployment-runtime-config-upjet-unifi.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml b/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml index c9e094c3b..c1757dbc4 100644 --- a/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml +++ b/k8s/providers/hetzner/infrastructure/crossplane/deployment-runtime-config-upjet-unifi.yaml @@ -46,4 +46,5 @@ spec: mountPath: /tmp volumes: - name: tmp - emptyDir: {} + emptyDir: + sizeLimit: 256Mi