Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ generate-configuration:
# Format: example_path::observed_resources_path (observed_resources_path is optional)
EXAMPLES := \
examples/knativestacks/minimal.yaml:: \
examples/knativestacks/nodepool.yaml:: \
examples/knativestacks/standard.yaml::

# Render all examples (parallel execution, output shown per-job when complete)
Expand Down
28 changes: 28 additions & 0 deletions apis/knativestacks/definition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,34 @@ spec:
enum:
- ProviderConfig
- ClusterProviderConfig
nodePool:
description: Optional dedicated Karpenter NodePool for Knative and NATS workloads. When enabled, non-DaemonSet pods are scheduled with workload-type=knative and tolerate knative=true:NoSchedule.
type: object
properties:
enabled:
description: Whether to create a dedicated NodePool. Defaults to false.
type: boolean
default: false
name:
description: NodePool name on the target cluster. Defaults to "hops-knative".
type: string
nodeClassName:
description: EKS Auto Mode NodeClass to reference. Defaults to "hops-default".
type: string
limits:
description: Karpenter NodePool resource limits. Defaults to nodes=10.
type: object
x-kubernetes-preserve-unknown-fields: true
requirements:
description: Karpenter scheduling requirements. Defaults to amd64/linux spot or on-demand nodes.
type: array
items:
type: object
x-kubernetes-preserve-unknown-fields: true
disruption:
description: Karpenter disruption settings. Defaults to WhenEmptyOrUnderutilized after 60s.
type: object
x-kubernetes-preserve-unknown-fields: true
knativeOperator:
description: Configuration for the Knative Operator component.
type: object
Expand Down
11 changes: 11 additions & 0 deletions examples/knativestacks/nodepool.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: hops.ops.com.ai/v1alpha1
kind: KnativeStack
metadata:
name: knative
namespace: default
spec:
clusterName: default
labels:
team: platform
nodePool:
enabled: true
139 changes: 125 additions & 14 deletions functions/render/000-state-init.yaml.gotmpl
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,38 @@
"kind" ($k8sProviderConfigRef.kind | default "ProviderConfig")
}}

# ==============================================================================
# NodePool configuration
# ==============================================================================
{{- $nodePoolSpec := $spec.nodePool | default dict }}
{{- $nodePoolEnabled := false }}
{{- if hasKey $nodePoolSpec "enabled" }}
{{- $nodePoolEnabled = $nodePoolSpec.enabled }}
{{- end }}
{{- $nodePoolName := $nodePoolSpec.name | default "hops-knative" }}
{{- $nodePoolNodeClassName := $nodePoolSpec.nodeClassName | default "hops-default" }}
{{- $nodePoolLimits := $nodePoolSpec.limits | default (dict "nodes" 10) }}
{{- $nodePoolRequirements := $nodePoolSpec.requirements | default (list
(dict "key" "karpenter.sh/capacity-type" "operator" "In" "values" (list "spot" "on-demand"))
(dict "key" "eks.amazonaws.com/instance-category" "operator" "In" "values" (list "c" "m" "r"))
(dict "key" "eks.amazonaws.com/instance-generation" "operator" "Gt" "values" (list "4"))
(dict "key" "eks.amazonaws.com/instance-memory" "operator" "Gt" "values" (list "7999"))
(dict "key" "eks.amazonaws.com/instance-cpu" "operator" "Gt" "values" (list "1"))
(dict "key" "kubernetes.io/arch" "operator" "In" "values" (list "amd64"))
(dict "key" "kubernetes.io/os" "operator" "In" "values" (list "linux"))
) }}
{{- $nodePoolDisruption := $nodePoolSpec.disruption | default (dict "consolidationPolicy" "WhenEmptyOrUnderutilized" "consolidateAfter" "60s") }}
{{- $nodePoolTaintKey := "knative" }}
{{- $nodePoolTaintValue := "true" }}
{{- $nodePoolNodeSelector := dict }}
{{- $nodePoolTolerations := list }}
{{- $nodePoolAffinity := dict }}
{{- if $nodePoolEnabled }}
{{- $nodePoolNodeSelector = dict "workload-type" "knative" }}
{{- $nodePoolTolerations = list (dict "key" $nodePoolTaintKey "operator" "Equal" "value" $nodePoolTaintValue "effect" "NoSchedule") }}
{{- $nodePoolAffinity = dict "nodeAffinity" (dict "requiredDuringSchedulingIgnoredDuringExecution" (dict "nodeSelectorTerms" (list (dict "matchExpressions" (list (dict "key" "workload-type" "operator" "In" "values" (list "knative"))))))) }}
{{- end }}

# ==============================================================================
# Per-component defaults
# ==============================================================================
Expand Down Expand Up @@ -149,6 +181,35 @@
(dict "URL" "https://github.com/knative-extensions/net-gateway-api/releases/download/knative-v1.21.0/release.yaml")
)
}}
{{- $servingDeployments := list }}
{{- range $deploymentDefaults := list
(dict "name" "activator" "container" "activator" "cpu" "15m" "memory" "100Mi")
(dict "name" "autoscaler" "container" "autoscaler" "cpu" "15m" "memory" "100Mi")
(dict "name" "autoscaler-hpa" "container" "autoscaler-hpa" "cpu" "15m" "memory" "100Mi")
(dict "name" "controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
(dict "name" "webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
(dict "name" "net-gateway-api-controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
(dict "name" "net-gateway-api-webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
(dict "name" "net-istio-controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
(dict "name" "net-istio-webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
}}
{{- $deployment := dict
"name" $deploymentDefaults.name
"resources" (list
(dict
"container" $deploymentDefaults.container
"requests" (dict "cpu" $deploymentDefaults.cpu "memory" $deploymentDefaults.memory)
"limits" (dict "memory" "256Mi")
)
)
}}
{{- if $nodePoolEnabled }}
{{- $_ := set $deployment "nodeSelector" $nodePoolNodeSelector }}
{{- $_ := set $deployment "tolerations" $nodePoolTolerations }}
{{- end }}
{{- $servingDeployments = append $servingDeployments $deployment }}
{{- end }}
{{- $_ := set $natsKnServingDefaults "deployments" $servingDeployments }}
{{- $servingSpec := merge ($knServing.spec | default dict) $natsKnServingDefaults }}

# ==============================================================================
Expand Down Expand Up @@ -193,20 +254,40 @@
"container" (dict
"merge" (dict
"resources" (dict
"requests" (dict "cpu" "100m" "memory" "256Mi")
"requests" (dict "cpu" "11m" "memory" "50Mi")
"limits" (dict "cpu" "500m" "memory" "512Mi")
)
)
)
"reloader" (dict
"merge" (dict
"resources" (dict
"requests" (dict "cpu" "10m" "memory" "32Mi")
"requests" (dict "cpu" "11m" "memory" "50Mi")
"limits" (dict "cpu" "50m" "memory" "64Mi")
)
)
)
"natsBox" (dict
"container" (dict
"merge" (dict
"resources" (dict
"requests" (dict "cpu" "15m" "memory" "100Mi")
"limits" (dict "cpu" "100m" "memory" "256Mi")
)
)
)
)
}}
{{- if $nodePoolEnabled }}
{{- $_ := set $natsDefaultValues "podTemplate" (dict
"merge" (dict
"spec" (dict
"nodeSelector" $nodePoolNodeSelector
"tolerations" $nodePoolTolerations
)
)
) }}
{{- end }}
{{- $natsValues := mergeOverwrite (deepCopy $natsDefaultValues) ($nats.values | default dict) }}

# ==============================================================================
Expand All @@ -216,20 +297,37 @@
# mTLS at the node level — eventing doesn't need to know about the mesh.
{{- /* eventing-webhook ships with a 200Mi limit that OOMKills under modest
ApiServerSource / Trigger reconcile load. Bump to 512Mi so it stays up. */}}
{{- $eventingDefaults := dict
"config" dict
"deployments" (list
(dict
"name" "eventing-webhook"
"resources" (list
(dict
"container" "eventing-webhook"
"requests" (dict "memory" "100Mi")
"limits" (dict "memory" "512Mi")
)
{{- $eventingDeployments := list }}
{{- range $deployment := list
(dict "name" "eventing-controller" "container" "eventing-controller" "cpu" "15m" "memory" "284Mi" "limitMemory" "512Mi")
(dict "name" "eventing-webhook" "container" "eventing-webhook" "cpu" "15m" "memory" "309Mi" "limitMemory" "512Mi")
(dict "name" "imc-controller" "container" "controller" "cpu" "23m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "imc-dispatcher" "container" "dispatcher" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "job-sink" "container" "job-sink" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "mt-broker-controller" "container" "mt-broker-controller" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "mt-broker-filter" "container" "filter" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "mt-broker-ingress" "container" "ingress" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
(dict "name" "request-reply" "container" "request-reply" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
}}
{{- $eventingDeployment := dict
"name" $deployment.name
"resources" (list
(dict
"container" $deployment.container
"requests" (dict "cpu" $deployment.cpu "memory" $deployment.memory)
"limits" (dict "memory" $deployment.limitMemory)
)
)
)
}}
{{- if $nodePoolEnabled }}
{{- $_ := set $eventingDeployment "nodeSelector" $nodePoolNodeSelector }}
{{- $_ := set $eventingDeployment "tolerations" $nodePoolTolerations }}
{{- end }}
{{- $eventingDeployments = append $eventingDeployments $eventingDeployment }}
{{- end }}
{{- $eventingDefaults := dict
"config" dict
"deployments" $eventingDeployments
}}
{{- if $natsEnabled }}
{{- $natsNs := $nats.namespace | default "nats" }}
Expand Down Expand Up @@ -257,6 +355,19 @@
"autoTls" (dict "enabled" $autoTlsEnabled "namespaceSelector" $autoTlsSelector)
"helmProviderConfigRef" $helmProviderConfigRef
"kubernetesProviderConfigRef" $k8sProviderConfigRef
"nodePool" (dict
"enabled" $nodePoolEnabled
"name" $nodePoolName
"nodeClassName" $nodePoolNodeClassName
"limits" $nodePoolLimits
"requirements" $nodePoolRequirements
"disruption" $nodePoolDisruption
"taintKey" $nodePoolTaintKey
"taintValue" $nodePoolTaintValue
"nodeSelector" $nodePoolNodeSelector
"tolerations" $nodePoolTolerations
"affinity" $nodePoolAffinity
)
"knativeOperator" (dict
"name" ($knOp.name | default "knative-operator")
"namespace" ($knOp.namespace | default "knative-operator")
Expand Down
91 changes: 91 additions & 0 deletions functions/render/155-nodepool.yaml.gotmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# code: language=yaml
#
# Optional Karpenter NodePool for Knative and NATS workloads.
#

{{- if $state.nodePool.enabled }}
---
apiVersion: kubernetes.m.crossplane.io/v1alpha1
kind: Object
metadata:
name: {{ $state.name }}-nodepool-knative
annotations:
{{ setResourceNameAnnotation "nodepool-knative" }}
labels: {{ $state.labels | toJson }}
spec:
managementPolicies: {{ $state.managementPolicies | toJson }}
forProvider:
manifest:
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
name: {{ $state.nodePool.name }}
spec:
template:
metadata:
labels:
workload-type: knative
spec:
nodeClassRef:
group: eks.amazonaws.com
kind: NodeClass
name: {{ $state.nodePool.nodeClassName }}
taints:
- key: {{ $state.nodePool.taintKey }}
value: {{ $state.nodePool.taintValue | quote }}
effect: NoSchedule
requirements: {{ $state.nodePool.requirements | toJson }}
limits: {{ $state.nodePool.limits | toJson }}
disruption: {{ $state.nodePool.disruption | toJson }}
providerConfigRef:
name: {{ $state.kubernetesProviderConfigRef.name }}
kind: {{ $state.kubernetesProviderConfigRef.kind }}

---
apiVersion: protection.crossplane.io/v1beta1
kind: Usage
metadata:
name: {{ $state.name }}-delete-operator-before-nodepool
annotations:
{{ setResourceNameAnnotation "usage-operator-before-nodepool" }}
labels: {{ $state.labels | toJson }}
spec:
replayDeletion: true
of:
apiVersion: kubernetes.m.crossplane.io/v1alpha1
kind: Object
resourceRef:
name: {{ $state.name }}-nodepool-knative
by:
apiVersion: helm.m.crossplane.io/v1beta1
kind: Release
resourceRef:
name: {{ $state.name }}-{{ $state.knativeOperator.name }}

{{- $natsNodePoolUsageEnabled := $state.nats.enabled }}
{{- if and $state.nats.enabled $state.nats.storageClass.enabled (not $state.observed.natsStorageClass.ready) }}
{{- $natsNodePoolUsageEnabled = false }}
{{- end }}
{{- if $natsNodePoolUsageEnabled }}
---
apiVersion: protection.crossplane.io/v1beta1
kind: Usage
metadata:
name: {{ $state.name }}-delete-nats-before-nodepool
annotations:
{{ setResourceNameAnnotation "usage-nats-before-nodepool" }}
labels: {{ $state.labels | toJson }}
spec:
replayDeletion: true
of:
apiVersion: kubernetes.m.crossplane.io/v1alpha1
kind: Object
resourceRef:
name: {{ $state.name }}-nodepool-knative
by:
apiVersion: helm.m.crossplane.io/v1beta1
kind: Release
resourceRef:
name: {{ $state.name }}-{{ $state.nats.name }}
{{- end }}
{{- end }}
41 changes: 39 additions & 2 deletions functions/render/210-knative-operator.yaml.gotmpl
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,42 @@
#

{{- $knOp := $state.knativeOperator }}
{{- $operatorComponent := dict
"resources" (dict
"requests" (dict "cpu" "15m" "memory" "100Mi")
"limits" (dict "cpu" "1000m" "memory" "512Mi")
)
}}
{{- $webhookComponent := dict
"resources" (dict
"requests" (dict "cpu" "15m" "memory" "284Mi")
"limits" (dict "cpu" "500m" "memory" "512Mi")
)
}}
{{- if $state.nodePool.enabled }}
{{- $_ := set $operatorComponent "affinity" $state.nodePool.affinity }}
{{- $_ := set $operatorComponent "tolerations" $state.nodePool.tolerations }}
{{- $webhookAffinity := deepCopy $state.nodePool.affinity }}
{{- $_ := set $webhookAffinity "podAntiAffinity" (dict
"preferredDuringSchedulingIgnoredDuringExecution" (list
(dict
"weight" 100
"podAffinityTerm" (dict
"labelSelector" (dict "matchLabels" (dict "app" "operator-webhook"))
"topologyKey" "kubernetes.io/hostname"
)
)
)
) }}
{{- $_ := set $webhookComponent "affinity" $webhookAffinity }}
{{- $_ := set $webhookComponent "tolerations" $state.nodePool.tolerations }}
{{- end }}
{{- $operatorDefaults := dict
"knative_operator" (dict
"knative_operator" $operatorComponent
"operator_webhook" $webhookComponent
)
}}

---
apiVersion: helm.m.crossplane.io/v1beta1
Expand All @@ -26,9 +62,10 @@ spec:
{{- if $knOp.overrideAllValues }}
values:
{{- toYaml $knOp.overrideAllValues | nindent 6 }}
{{- else if $knOp.values }}
{{- else }}
{{- $mergedValues := mergeOverwrite $operatorDefaults ($knOp.values | default dict) }}
values:
{{- toYaml $knOp.values | nindent 6 }}
{{- toYaml $mergedValues | nindent 6 }}
{{- end }}
rollbackLimit: 3
providerConfigRef:
Expand Down
Loading
Loading