Local environment for testing IPAM against a real Milo control plane

Taskfile.yaml

@@ -6,6 +6,13 @@ vars:
   IPAM_IMAGE_TAG: "dev"
   TEST_INFRA_CLUSTER_NAME: "test-infra"
   TEST_INFRA_REPO_REF: 'v0.6.0'
+  # Local Milo control-plane checkout, used by the milo-integration:* targets to
+  # build/load/deploy Milo into the SAME test-infra kind cluster. Override with
+  # `MILO_REPO=/path/to/milo task milo-integration:up`.
+  MILO_REPO: '{{.MILO_REPO | default "../../datum-cloud/milo"}}'
+  # Static dev token from Milo's auth-tokens secret (admin / system:masters).
+  # Matches the token baked into config/overlays/milo-integration/milo-kubeconfig-secret.yaml.
+  MILO_TOKEN: 'test-admin-token'
 
 includes:
   test-infra:
@@ -146,6 +153,65 @@ tasks:
     cmds:
       - task: test-infra:install-observability
 
+  # ----- Milo integration environment -----
+  #
+  # Stands up IPAM wired to a REAL in-cluster Milo control plane so the full
+  # IPAM<->Milo path can be exercised locally (delegated authn/authz + quota
+  # enforcement + entitlement->grant->claim) — the path the standalone e2e
+  # (--enable-quota=false, no Milo) cannot cover. See docs/milo-integration.md.
+
+  milo-integration:deploy-milo:
+    desc: Build, load and deploy the Milo control plane into the test-infra cluster
+    silent: true
+    cmds:
+      - |
+        set -e
+        echo ">> Deploying Milo from {{.MILO_REPO}} into the test-infra cluster"
+        # The remote test-infra Taskfile resolves the kubeconfig relative to the
+        # milo repo; mirror ours into it so `task dev:deploy` finds the cluster.
+        mkdir -p "{{.MILO_REPO}}/.test-infra"
+        cp .test-infra/kubeconfig "{{.MILO_REPO}}/.test-infra/kubeconfig"
+        cd "{{.MILO_REPO}}" && TASK_X_REMOTE_TASKFILES=1 task dev:build dev:load dev:deploy
+
+  milo-integration:deploy-ipam:
+    desc: Deploy IPAM via the milo-integration overlay (quota ON, delegating to Milo)
+    silent: true
+    cmds:
+      - |
+        set -e
+        # Fresh-create the deployment: switching FROM another overlay's volume
+        # shape (e.g. test-infra tracing) can't be strategic-merged in place.
+        task test-infra:kubectl -- delete deployment ipam-apiserver -n ipam-system --ignore-not-found
+        task test-infra:kubectl -- apply -k config/overlays/milo-integration
+        task test-infra:kubectl -- wait --for=condition=ready pod -l app=ipam-apiserver -n ipam-system --timeout=240s || echo "apiserver pods not ready yet"
+        task test-infra:kubectl -- wait --for=condition=Available apiservice/v1alpha1.ipam.miloapis.com --timeout=180s || echo "APIService not Available yet"
+
+  milo-integration:provision-quota:
+    desc: Register the IPAM quotable type + claim/grant policies in Milo
+    silent: true
+    cmds:
+      - |
+        set -e
+        echo ">> Provisioning IPAM quota in Milo (.milo/kubeconfig)"
+        KUBECONFIG="{{.MILO_REPO}}/.milo/kubeconfig" kubectl apply -k config/overlays/milo-integration/quota
+        # RBAC in Milo for the impersonated tenant user the e2e/demo driver uses.
+        KUBECONFIG="{{.MILO_REPO}}/.milo/kubeconfig" kubectl apply -f config/overlays/milo-integration/rbac-tenant-user.yaml
+
+  milo-integration:up:
+    desc: Bring up the whole Milo integration env (Milo + IPAM overlay + quota provisioning)
+    silent: true
+    cmds:
+      - task: test-infra:cluster-up
+      - task: milo-integration:deploy-milo
+      - task: dev:build
+      - task: dev:load
+      - task: milo-integration:deploy-ipam
+      - task: milo-integration:provision-quota
+      - |
+        echo ""
+        echo "Milo integration environment is up."
+        echo "Drive a full claim with: docs/milo-integration.md (impersonation kubeconfig)"
+
   # ----- E2E -----
 
   # Generate the impersonation kubeconfig the multi-tenant / tenant-isolation

config/overlays/milo-integration/kustomization.yaml

@@ -0,0 +1,65 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# milo-integration overlay.
+#
+# Stands up the IPAM apiserver wired to a real, in-cluster Milo control plane so
+# the FULL IPAM<->milo path can be exercised locally: delegated authn/authz
+# (TokenReview / SubjectAccessReview against milo's IAM) AND quota enforcement
+# (milo's ResourceQuotaEnforcement admission plugin + the quota grant
+# controllers). This is the path the standalone e2e (--enable-quota=false, no
+# milo) cannot cover.
+#
+# Differences from the test-infra overlay:
+#   * --enable-quota=true (test-infra is false)
+#   * all three delegation kubeconfig flags point at the milo-apiserver Service
+#     via a mounted Secret (test-infra uses the in-cluster fallback)
+#   * NetworkPolicy egress opened to milo-system:6443
+#   * NO tracing component (keeps the CPU/feature surface minimal; tracing is a
+#     test-infra concern, orthogonal to the milo path)
+#
+# Reuses test-infra's TLS Certificate (cert-manager, auto-approved) and the
+# postgres component. Prereq: milo must already be deployed in milo-system
+# (see the milo-integration:up Taskfile target / docs/milo-integration.md).
+
+namespace: ipam-system
+
+resources:
+  - ../../base
+  - secret.yaml
+  - tls-certificate.yaml
+  - milo-kubeconfig-secret.yaml
+
+components:
+  - ../../components/namespace
+  - ../../components/api-registration
+  - ../../components/postgres
+
+images:
+  - name: ghcr.io/milo-os/ipam
+    newName: ipam-apiserver
+    newTag: dev
+
+patches:
+  - path: patches/apiservice-patch.yaml
+  - path: patches/deployment-patch.yaml
+    target:
+      kind: Deployment
+      name: ipam-apiserver
+  - path: patches/volumes-patch.yaml
+    target:
+      kind: Deployment
+      name: ipam-apiserver
+  # Open egress from the apiserver to the milo-apiserver (milo-system:6443) so
+  # delegated authn/authz, the quota loopback/per-project clients, and the
+  # quota + APF informers that gate readyz can reach milo. Base is default-deny.
+  - path: patches/milo-egress-networkpolicy-patch.yaml
+    target:
+      kind: NetworkPolicy
+      name: ipam-apiserver
+
+labels:
+  - includeSelectors: false
+    includeTemplates: true
+    pairs:
+      environment: milo-integration

config/overlays/milo-integration/milo-kubeconfig-secret.yaml

@@ -0,0 +1,55 @@
+---
+# Kubeconfig the IPAM apiserver uses to delegate authn/authz to milo and to
+# reach the milo quota backend. All three IPAM delegation flags
+# (--kubeconfig / --authentication-kubeconfig / --authorization-kubeconfig)
+# point at this single file (see patches/deployment-patch.yaml), so milo's IAM
+# resolves every TokenReview / SubjectAccessReview AND the quota admission
+# plugin's loopback config targets the same milo control plane.
+#
+# server:  the in-cluster milo-apiserver Service (NOT the localhost:30443
+#          gateway, which is only reachable from the host). 6443 is the port
+#          the milo-apiserver Service exposes (see milo config/apiserver).
+# token:   the static admin token from milo's auth-tokens secret
+#          (test-admin-token -> admin / system:masters). This is the same
+#          credential milo-controller-manager uses to talk to its own apiserver
+#          (milo-controller-manager-kubeconfig secret), so IPAM gets the same
+#          full-power identity — appropriate for a local integration env, NOT
+#          for production (production mints a scoped ServiceAccount token).
+# insecure-skip-tls-verify: the milo-apiserver serves a cert-manager cert whose
+#          CA IPAM does not bundle here; skipping verification keeps the local
+#          env self-contained. The IPAM<->milo hop stays inside the cluster.
+#
+# This Secret is intentionally checked in with a well-known dev token because
+# the whole overlay is a local integration harness, never deployed anywhere
+# real. Rotate / replace with a minted SA token before reusing against a shared
+# milo.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: milo-kubeconfig
+  namespace: ipam-system
+  labels:
+    app.kubernetes.io/name: ipam
+    app.kubernetes.io/component: milo-integration
+    app.kubernetes.io/part-of: ipam.miloapis.com
+type: Opaque
+stringData:
+  kubeconfig: |
+    apiVersion: v1
+    kind: Config
+    clusters:
+      - name: milo
+        cluster:
+          server: https://milo-apiserver.milo-system.svc.cluster.local:6443
+          insecure-skip-tls-verify: true
+    contexts:
+      - name: milo
+        context:
+          cluster: milo
+          user: ipam
+    current-context: milo
+    preferences: {}
+    users:
+      - name: ipam
+        user:
+          token: test-admin-token

config/overlays/milo-integration/patches/apiservice-patch.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha1.ipam.miloapis.com
+spec:
+  insecureSkipTLSVerify: true

config/overlays/milo-integration/patches/deployment-patch.yaml

@@ -0,0 +1,48 @@
+---
+# milo-integration deployment patch.
+#
+# Flips the IPAM apiserver from the standalone configuration (quota OFF,
+# in-cluster authn/authz fallback) to the FULL milo-delegated configuration:
+#
+#   * --enable-quota=true              -> milo's ResourceQuotaEnforcement
+#                                         admission plugin runs on IPClaim CREATE
+#   * --kubeconfig / --authentication-kubeconfig / --authorization-kubeconfig
+#                                       -> all point at the mounted milo
+#                                          kubeconfig file, so TokenReview /
+#                                          SubjectAccessReview resolve against
+#                                          milo's IAM and the quota plugin's
+#                                          loopback config targets milo.
+#
+# imagePullPolicy: Never because the image is `kind load`-ed, never pulled
+# (same rationale as the test-infra overlay).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ipam-apiserver
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: migrate
+          imagePullPolicy: Never
+      containers:
+        - name: apiserver
+          imagePullPolicy: Never
+          env:
+            - name: ENABLE_QUOTA
+              value: "true"
+            # All three delegation flags read the same mounted kubeconfig file.
+            - name: KUBECONFIG
+              value: "/var/run/ipam-apiserver/milo/kubeconfig"
+            - name: AUTHENTICATION_KUBECONFIG
+              value: "/var/run/ipam-apiserver/milo/kubeconfig"
+            - name: AUTHORIZATION_KUBECONFIG
+              value: "/var/run/ipam-apiserver/milo/kubeconfig"
+            # milo's TokenReview is authoritative; don't require the local host
+            # apiserver to also know the identity.
+            - name: AUTHENTICATION_TOLERATE_LOOKUP_FAILURE
+              value: "true"
+          volumeMounts:
+            - name: milo-kubeconfig
+              mountPath: /var/run/ipam-apiserver/milo
+              readOnly: true

config/overlays/milo-integration/patches/milo-egress-networkpolicy-patch.yaml

@@ -0,0 +1,22 @@
+# Allow the IPAM apiserver to reach the milo-apiserver (milo-system, :6443) for
+# delegated TokenReview / SubjectAccessReview, the quota admission plugin's
+# loopback + per-project control-plane clients, and the quota/APF informers that
+# gate readyz. The base NetworkPolicy is default-deny on egress, so without this
+# rule every milo call (and thus readyz) hangs.
+#
+# namespaceSelector-only (matching kubernetes.io/metadata.name) rather than a
+# podSelector, for the same reason the base policy documents: the kustomization
+# `labels:` transformer rewrites peer matchLabels, which would otherwise force
+# milo's pods to carry IPAM labels and match nothing.
+#
+# JSON6902 append preserves the base egress rules (DNS, postgres, kube-apiserver).
+- op: add
+  path: /spec/egress/-
+  value:
+    to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: milo-system
+    ports:
+      - port: 6443
+        protocol: TCP

config/overlays/milo-integration/patches/volumes-patch.yaml

@@ -0,0 +1,29 @@
+# JSON6902 volume rewrite for the milo-integration overlay.
+#
+# Two changes, applied as explicit array ops so the base default-deny /
+# strategic-merge ordering can't produce a duplicate "tls-certs" volume (which
+# is what happens if a CSI volume and a secret volume of the same name are
+# merged from two different patches):
+#
+#  1. Replace volumes[0] (the base cert-manager CSI TLS volume) with the
+#     cert-manager Certificate Secret (ipam-tls). The built-in cert-manager
+#     approver auto-approves Certificate-issued CertificateRequests; the CSI
+#     driver's requests are NOT approved by it, which hangs the pod in Init.
+#     Same swap the test-infra overlay performs.
+#  2. Append the milo-kubeconfig Secret volume the apiserver mounts for its
+#     delegation kubeconfig.
+#
+# volumes[0] is the TLS volume in config/base/deployment.yaml; keep this index
+# in sync if the base volume order changes.
+- op: replace
+  path: /spec/template/spec/volumes/0
+  value:
+    name: tls-certs
+    secret:
+      secretName: ipam-tls
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: milo-kubeconfig
+    secret:
+      secretName: milo-kubeconfig

config/overlays/milo-integration/quota/claim-creation-policy.yaml

@@ -0,0 +1,44 @@
+# Auto-creates a ResourceClaim whenever an IPClaim is created, so the IPAM
+# apiserver's quota admission plugin has a claim to create-and-wait-for-grant.
+#
+# The IPAM quota admission plugin (milo's ResourceQuotaEnforcement, run inside
+# the IPAM apiserver) looks up the ClaimCreationPolicy whose trigger GVK matches
+# the resource being created (IPClaim), renders this template, creates the
+# ResourceClaim in the project control-plane, and blocks the IPClaim CREATE
+# until that ResourceClaim is Granted (or denies on insufficient quota). The
+# consumerRef is auto-filled by the plugin from the project request context.
+apiVersion: quota.miloapis.com/v1alpha1
+kind: ClaimCreationPolicy
+metadata:
+  name: project-ipclaim-claim-policy
+  labels:
+    app.kubernetes.io/name: ipam
+    app.kubernetes.io/component: milo-integration
+spec:
+  disabled: false
+  trigger:
+    resource:
+      apiVersion: ipam.miloapis.com/v1alpha1
+      kind: IPClaim
+    # No constraints - trigger for every IPClaim.
+  target:
+    resourceClaimTemplate:
+      metadata:
+        # Use requestInfo.name (the admission attrs name), NOT
+        # trigger.metadata.name: the IPClaim object the IPAM aggregated apiserver
+        # hands the quota plugin is converted from an internal Go type and does
+        # not expose a usable `metadata` key to the CEL template engine, so
+        # `trigger.metadata.name` fails to render ("no such key: metadata").
+        # requestInfo.name resolves to the same IPClaim name and is always set.
+        name: "ipclaim-{{requestInfo.name}}"
+        namespace: "{{requestInfo.namespace}}"
+        labels:
+          app.kubernetes.io/name: ipam
+          app.kubernetes.io/component: milo-integration
+        annotations:
+          kubernetes.io/description: "Automatic quota claim for IPClaim creation"
+      spec:
+        # consumerRef auto-filled by the admission plugin from project context.
+        requests:
+          - resourceType: ipam.miloapis.com/ipclaims
+            amount: 1