Add Cilium ClusterMesh scale-test scenario

jobs/competitive-test.yml

@@ -48,6 +48,9 @@ parameters:
 - name: ssh_key_enabled
   type: boolean
   default: true
+- name: skip_publish
+  type: boolean
+  default: false
 
 jobs:
 - job: ${{ parameters.cloud }}
@@ -89,14 +92,15 @@ jobs:
       engine: ${{ parameters.engine }}
       regions: ${{ parameters.regions }}
       engine_input: ${{ parameters.engine_input }}
-  - template: /steps/publish-results.yml
-    parameters:
-      cloud: ${{ parameters.cloud }}
-      topology: ${{ parameters.topology }}
-      engine: ${{ parameters.engine }}
-      regions: ${{ parameters.regions }}
-      engine_input: ${{ parameters.engine_input }}
-      credential_type: ${{ parameters.credential_type }}
+  - ${{ if not(parameters.skip_publish) }}:
+    - template: /steps/publish-results.yml
+      parameters:
+        cloud: ${{ parameters.cloud }}
+        topology: ${{ parameters.topology }}
+        engine: ${{ parameters.engine }}
+        regions: ${{ parameters.regions }}
+        engine_input: ${{ parameters.engine_input }}
+        credential_type: ${{ parameters.credential_type }}
   - template: /steps/cleanup-resources.yml
     parameters:
       cloud: ${{ parameters.cloud }}

modules/python/clusterloader2/clustermesh-scale/config/config.yaml

@@ -0,0 +1,105 @@
+name: clustermesh-scale-test
+
+# Workload: deploy a small fixed number of pods on this cluster (no churn,
+# no traffic). Measurement modules under modules/measurements/ run the actual
+# scale-test instrumentation (cilium agent/operator CPU+memory, kube-apiserver
+# health, mesh-specific PromQL) so each per-cluster JSONL row carries the data
+# needed for cross-cluster comparison in Kusto. The workload is deliberately
+# trivial — fan-out, attribution, and metric coverage are what we're testing
+# in Phase 1; richer workloads land per scenario in Phase 2+.
+
+{{$namespaces := DefaultParam .CL2_NAMESPACES 1}}
+{{$deploymentsPerNamespace := DefaultParam .CL2_DEPLOYMENTS_PER_NAMESPACE 2}}
+{{$replicasPerDeployment := DefaultParam .CL2_REPLICAS_PER_DEPLOYMENT 2}}
+{{$operationTimeout := DefaultParam .CL2_OPERATION_TIMEOUT "15m"}}
+{{$apiServerCallsPerSecond := DefaultParam .CL2_API_SERVER_CALLS_PER_SECOND 5}}
+
+namespace:
+  number: {{$namespaces}}
+  prefix: clustermesh-scale
+  deleteStaleNamespaces: true
+  deleteAutomanagedNamespaces: true
+  enableExistingNamespaces: false
+  deleteNamespaceTimeout: 20m
+
+tuningSets:
+  - name: Sequence
+    parallelismLimitedLoad:
+      parallelismLimit: 1
+  - name: DeploymentCreateQps
+    qpsLoad:
+      qps: {{$apiServerCallsPerSecond}}
+
+steps:
+  # ----- Start measurements -----
+  # control-plane.yaml owns PodStartupLatency + APIResponsivenessPrometheus +
+  # apiserver CPU/mem queries; cilium.yaml owns cilium-agent + cilium-operator
+  # CPU/mem; clustermesh-metrics.yaml owns mesh-specific PromQL (remote-cluster
+  # connectivity, kvstore event rate, identity count, etc.). All three are
+  # gathered later (see "Gather measurements" below) so the steady-state window
+  # is bounded by the workload create/delete pair.
+  - module:
+      path: /modules/measurements/control-plane.yaml
+      params:
+        action: start
+        group: clustermesh-scale-test
+
+  - module:
+      path: /modules/measurements/cilium.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/measurements/clustermesh-metrics.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/clustermesh.yaml
+      params:
+        actionName: create
+        tuningSet: DeploymentCreateQps
+
+  - module:
+      path: /modules/scale-test.yaml
+      params:
+        actionName: create
+        namespaces: {{$namespaces}}
+        deploymentsPerNamespace: {{$deploymentsPerNamespace}}
+        replicasPerDeployment: {{$replicasPerDeployment}}
+        tuningSet: DeploymentCreateQps
+        operationTimeout: {{$operationTimeout}}
+
+  # ----- Gather measurements -----
+  # Mirror the start block above. Order matches network-scale convention.
+  - module:
+      path: /modules/measurements/control-plane.yaml
+      params:
+        action: gather
+        group: clustermesh-scale-test
+
+  - module:
+      path: /modules/measurements/cilium.yaml
+      params:
+        action: gather
+
+  - module:
+      path: /modules/measurements/clustermesh-metrics.yaml
+      params:
+        action: gather
+
+  - module:
+      path: /modules/scale-test.yaml
+      params:
+        actionName: delete
+        namespaces: {{$namespaces}}
+        deploymentsPerNamespace: {{$deploymentsPerNamespace}}
+        replicasPerDeployment: {{$replicasPerDeployment}}
+        tuningSet: DeploymentCreateQps
+        operationTimeout: {{$operationTimeout}}
+
+  - module:
+      path: /modules/clustermesh.yaml
+      params:
+        actionName: delete
+        tuningSet: DeploymentCreateQps

modules/python/clusterloader2/clustermesh-scale/config/event-throughput.yaml

@@ -0,0 +1,166 @@
+name: clustermesh-event-throughput
+
+# Scale scenario #1: Cross-Cluster Event Throughput.
+#
+# Goal (scale testing.txt line 42-54): determine max sustainable and burst
+# event rates for endpoints, services, and identities propagating across
+# the mesh; measure events/sec processed and time-to-convergence proxy.
+#
+# Sequence (every cluster runs this in parallel; CL2 fan-out lives in
+# steps/engine/.../execute.yml):
+#
+#   1. Start measurements (control-plane, cilium, clustermesh-metrics +
+#      scenario-specific clustermesh-throughput + etcd-metrics).
+#   2. Deploy PodMonitor scraping clustermesh-apiserver.
+#   3. Create N pods + N global Services per cluster at a controlled QPS.
+#   4. Warmup sleep — let initial create-flurry settle into steady state.
+#   5. Burst rolling-restart of every Deployment (closes the "burst"
+#      coverage gap from scale testing.txt line 52).
+#   6. Settle sleep — let kvstore queues drain and propagation latency
+#      histograms accumulate steady-state samples.
+#   7. Gather all measurements.
+#   8. Tear down the workload + PodMonitor.
+
+{{$namespaces := DefaultParam .CL2_NAMESPACES 5}}
+{{$deploymentsPerNamespace := DefaultParam .CL2_DEPLOYMENTS_PER_NAMESPACE 4}}
+{{$replicasPerDeployment := DefaultParam .CL2_REPLICAS_PER_DEPLOYMENT 10}}
+{{$operationTimeout := DefaultParam .CL2_OPERATION_TIMEOUT "20m"}}
+{{$apiServerCallsPerSecond := DefaultParam .CL2_API_SERVER_CALLS_PER_SECOND 20}}
+{{$warmupDuration := DefaultParam .CL2_WARMUP_DURATION "30s"}}
+{{$holdDuration := DefaultParam .CL2_HOLD_DURATION "2m"}}
+{{$restartGeneration := DefaultParam .CL2_RESTART_GENERATION 1}}
+
+namespace:
+  number: {{$namespaces}}
+  prefix: clustermesh-et
+  deleteStaleNamespaces: true
+  deleteAutomanagedNamespaces: true
+  enableExistingNamespaces: false
+  deleteNamespaceTimeout: 20m
+
+tuningSets:
+  - name: Sequence
+    parallelismLimitedLoad:
+      parallelismLimit: 1
+  - name: DeploymentCreateQps
+    qpsLoad:
+      qps: {{$apiServerCallsPerSecond}}
+
+steps:
+  # ----- Start measurements -----
+  - module:
+      path: /modules/measurements/control-plane.yaml
+      params:
+        action: start
+        group: clustermesh-event-throughput
+
+  - module:
+      path: /modules/measurements/cilium.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/measurements/clustermesh-metrics.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/measurements/clustermesh-throughput.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/measurements/etcd-metrics.yaml
+      params:
+        action: start
+
+  - module:
+      path: /modules/clustermesh.yaml
+      params:
+        actionName: create
+        tuningSet: DeploymentCreateQps
+
+  # ----- Workload: create -----
+  - module:
+      path: /modules/event-throughput-workload.yaml
+      params:
+        actionName: create
+        generation: 0
+        namespaces: {{$namespaces}}
+        deploymentsPerNamespace: {{$deploymentsPerNamespace}}
+        replicasPerDeployment: {{$replicasPerDeployment}}
+        tuningSet: DeploymentCreateQps
+        operationTimeout: {{$operationTimeout}}
+
+  # ----- Warmup: let the create-flurry settle into steady state -----
+  - name: Warmup before burst
+    measurements:
+      - Identifier: WarmupSleep
+        Method: Sleep
+        Params:
+          duration: {{$warmupDuration}}
+
+  # ----- Burst: rolling-restart of every Deployment -----
+  - module:
+      path: /modules/event-throughput-workload.yaml
+      params:
+        actionName: restart
+        generation: {{$restartGeneration}}
+        namespaces: {{$namespaces}}
+        deploymentsPerNamespace: {{$deploymentsPerNamespace}}
+        replicasPerDeployment: {{$replicasPerDeployment}}
+        tuningSet: DeploymentCreateQps
+        operationTimeout: {{$operationTimeout}}
+
+  # ----- Settle: let kvstore queues drain post-burst -----
+  - name: Settle after burst
+    measurements:
+      - Identifier: SettleSleep
+        Method: Sleep
+        Params:
+          duration: {{$holdDuration}}
+
+  # ----- Gather measurements -----
+  - module:
+      path: /modules/measurements/control-plane.yaml
+      params:
+        action: gather
+        group: clustermesh-event-throughput
+
+  - module:
+      path: /modules/measurements/cilium.yaml
+      params:
+        action: gather
+
+  - module:
+      path: /modules/measurements/clustermesh-metrics.yaml
+      params:
+        action: gather
+
+  - module:
+      path: /modules/measurements/clustermesh-throughput.yaml
+      params:
+        action: gather
+
+  - module:
+      path: /modules/measurements/etcd-metrics.yaml
+      params:
+        action: gather
+
+  # ----- Workload: delete -----
+  - module:
+      path: /modules/event-throughput-workload.yaml
+      params:
+        actionName: delete
+        generation: {{$restartGeneration}}
+        namespaces: {{$namespaces}}
+        deploymentsPerNamespace: {{$deploymentsPerNamespace}}
+        replicasPerDeployment: {{$replicasPerDeployment}}
+        tuningSet: DeploymentCreateQps
+        operationTimeout: {{$operationTimeout}}
+
+  - module:
+      path: /modules/clustermesh.yaml
+      params:
+        actionName: delete
+        tuningSet: DeploymentCreateQps

modules/python/clusterloader2/clustermesh-scale/config/modules/clustermesh.yaml

@@ -0,0 +1,26 @@
+## ClusterMesh module: deploys a PodMonitor for clustermesh-apiserver so the
+## CL2-spawned Prometheus picks up at least one mesh-side metric per cluster.
+## Phase 1 exit criteria require this — see plan.md Phase 1 line 318.
+
+{{$tuningSet := DefaultParam .tuningSet "DeploymentCreateQps"}}
+{{$interval := DefaultParam .interval "15s"}}
+{{ $replicasPerNamespace := 1 }}
+
+{{if eq .actionName "create"}}
+  {{ $replicasPerNamespace = 1 }}
+{{else}}
+  {{ $replicasPerNamespace = 0 }}
+{{end}}
+
+steps:
+  - name: {{.actionName}} ClusterMesh Pod Monitor
+    phases:
+      - namespaceList:
+        - "monitoring"
+        replicasPerNamespace: {{$replicasPerNamespace}}
+        tuningSet: {{$tuningSet}}
+        objectBundle:
+          - objectTemplatePath: "modules/clustermesh/podmonitor.yaml"
+            basename: clustermesh-apiserver
+            templateFillMap:
+              Interval: {{$interval}}

modules/python/clusterloader2/clustermesh-scale/config/modules/clustermesh/podmonitor.yaml

@@ -0,0 +1,35 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: clustermesh-apiserver
+  namespace: monitoring
+spec:
+  # Cilium clustermesh-apiserver exposes metrics on port 9963 (apiserver) and
+  # 9964 (kvstoremesh sidecar) when Prometheus integration is enabled. AKS
+  # managed Cilium uses the same upstream defaults. If a future preview
+  # changes these, override via __address__ relabel below.
+  selector:
+    matchLabels:
+      k8s-app: clustermesh-apiserver
+  namespaceSelector:
+    matchNames:
+      - kube-system
+  podMetricsEndpoints:
+    - interval: {{.Interval}}
+      honorLabels: true
+      path: /metrics
+      relabelings:
+        - sourceLabels: [__address__]
+          action: replace
+          targetLabel: __address__
+          regex: (.+?)(\:\d+)?
+          replacement: $1:9963
+    - interval: {{.Interval}}
+      honorLabels: true
+      path: /metrics
+      relabelings:
+        - sourceLabels: [__address__]
+          action: replace
+          targetLabel: __address__
+          regex: (.+?)(\:\d+)?
+          replacement: $1:9964