OIDC token retry under concurrent deploys + cloudExtras.topologySpreadConstraints

docs/docs/examples/kubernetes-affinity/README.md

@@ -76,6 +76,34 @@ stacks:
                 topologyKey: "topology.kubernetes.io/zone"
 ```
 
+### **Spread Replicas Across Nodes (Topology Spread)**
+
+To keep a multi-replica service available across node failures, spread its replicas
+across nodes. On **GKE Autopilot** prefer `topologySpreadConstraints` over
+`podAntiAffinity`: pod anti-affinity forces a 0.5 vCPU minimum request per pod (and is
+*rejected* outright below it), whereas topology spread is not subject to that minimum.
+
+```yaml
+stacks:
+  my-service:
+    type: cloud-compose
+    config:
+      scale:
+        min: 2
+        max: 4
+      cloudExtras:
+        topologySpreadConstraints:
+          - topologyKey: "kubernetes.io/hostname"
+            maxSkew: 1
+            whenUnsatisfiable: "DoNotSchedule"
+            minDomains: 2
+```
+
+`labelSelector` defaults to the deployment's own pods when omitted. On GKE Autopilot,
+`whenUnsatisfiable: DoNotSchedule` together with `minDomains: 2` is required to actually
+force a second node — `ScheduleAnyway` only expresses a preference and lets the scheduler
+bin-pack both replicas onto one node.
+
 ## **Supported Affinity Properties**
 
 ### **Simple Container Properties**
@@ -88,6 +116,13 @@ stacks:
 - **`podAffinity`**: Pod co-location rules
 - **`podAntiAffinity`**: Pod separation and distribution rules
 
+### **Topology Spread Properties (`cloudExtras.topologySpreadConstraints`)**
+- **`topologyKey`** (required): Node label defining a domain (e.g. `kubernetes.io/hostname`)
+- **`maxSkew`**: Max allowed pod-count difference between domains (default `1`)
+- **`whenUnsatisfiable`**: `DoNotSchedule` (default) or `ScheduleAnyway`
+- **`minDomains`**: Minimum eligible domains (only valid with `DoNotSchedule`)
+- **`labelSelector`**: Pods to count; defaults to this deployment's pods when omitted
+
 ## **Implementation Details**
 
 ### **GKE Integration**

pkg/clouds/gcloud/gke_autopilot.go

@@ -109,6 +109,7 @@ func ToGkeAutopilotConfig(tpl any, composeCfg compose.Config, stackCfg *api.Stac
 		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe         // Extract global liveness probe configuration
 		deployCfg.StartupProbe = k8sCloudExtras.StartupProbe           // Extract global startup probe configuration
 		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.TopologySpreadConstraints = k8sCloudExtras.TopologySpreadConstraints
 
 		// Process affinity rules and merge with existing NodeSelector if needed
 		if k8sCloudExtras.Affinity != nil {

pkg/clouds/gcloud/topology_spread_test.go

@@ -0,0 +1,31 @@
+package gcloud
+
+import (
+	"testing"
+
+	"github.com/compose-spec/compose-go/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/clouds/compose"
+)
+
+func TestTopologySpreadThreadedByToGkeAutopilotConfig(t *testing.T) {
+	cloudExtras := any(map[string]any{
+		"topologySpreadConstraints": []map[string]any{
+			{"topologyKey": "kubernetes.io/hostname", "minDomains": 2},
+		},
+	})
+	stackCfg := &api.StackConfigCompose{
+		Runs:        []string{},
+		CloudExtras: &cloudExtras,
+	}
+
+	res, err := ToGkeAutopilotConfig(&GkeAutopilotTemplate{}, compose.Config{Project: &types.Project{}}, stackCfg)
+	require.NoError(t, err)
+	input, ok := res.(*GkeAutopilotInput)
+	require.True(t, ok)
+	require.Len(t, input.Deployment.TopologySpreadConstraints, 1)
+	assert.Equal(t, "kubernetes.io/hostname", input.Deployment.TopologySpreadConstraints[0].TopologyKey)
+}

pkg/clouds/k8s/kube_run.go

@@ -28,6 +28,17 @@ type CloudExtras struct {
 	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`
 	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
 	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
+
+	TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints" yaml:"topologySpreadConstraints"`
+}
+
+// TopologySpreadConstraint spreads pods across nodes without the GKE Autopilot 0.5 vCPU minimum that pod anti-affinity requires.
+type TopologySpreadConstraint struct {
+	MaxSkew           *int           `json:"maxSkew" yaml:"maxSkew"`
+	TopologyKey       string         `json:"topologyKey" yaml:"topologyKey"`
+	WhenUnsatisfiable string         `json:"whenUnsatisfiable" yaml:"whenUnsatisfiable"`
+	LabelSelector     *LabelSelector `json:"labelSelector" yaml:"labelSelector"`
+	MinDomains        *int           `json:"minDomains" yaml:"minDomains"`
 }
 
 // AffinityRules defines pod affinity and anti-affinity rules for node pool isolation
@@ -176,6 +187,7 @@ func ToKubernetesRunConfig(tpl any, composeCfg compose.Config, stackCfg *api.Sta
 		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe         // Extract global liveness probe configuration
 		deployCfg.EphemeralVolumes = k8sCloudExtras.EphemeralVolumes   // Extract generic ephemeral volumes configuration
 		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName // Extract PriorityClass for pod scheduling and preemption
+		deployCfg.TopologySpreadConstraints = k8sCloudExtras.TopologySpreadConstraints
 
 		// Process affinity rules and merge with existing NodeSelector if needed
 		if k8sCloudExtras.Affinity != nil {

pkg/clouds/k8s/topology_spread_test.go

@@ -0,0 +1,67 @@
+package k8s
+
+import (
+	"testing"
+
+	"github.com/compose-spec/compose-go/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/clouds/compose"
+)
+
+func TestTopologySpreadConstraintsExtraction(t *testing.T) {
+	cloudExtras := map[string]any{
+		"topologySpreadConstraints": []map[string]any{
+			{
+				"maxSkew":           1,
+				"topologyKey":       "kubernetes.io/hostname",
+				"whenUnsatisfiable": "DoNotSchedule",
+				"minDomains":        2,
+				"labelSelector": map[string]any{
+					"matchLabels": map[string]any{"tier": "web"},
+				},
+			},
+		},
+	}
+
+	result, err := api.ConvertDescriptor(cloudExtras, &CloudExtras{})
+
+	require.NoError(t, err)
+	require.Len(t, result.TopologySpreadConstraints, 1)
+	c := result.TopologySpreadConstraints[0]
+	assert.Equal(t, "kubernetes.io/hostname", c.TopologyKey)
+	assert.Equal(t, "DoNotSchedule", c.WhenUnsatisfiable)
+	require.NotNil(t, c.MaxSkew)
+	assert.Equal(t, 1, *c.MaxSkew)
+	require.NotNil(t, c.MinDomains)
+	assert.Equal(t, 2, *c.MinDomains)
+	require.NotNil(t, c.LabelSelector)
+	assert.Equal(t, map[string]string{"tier": "web"}, c.LabelSelector.MatchLabels)
+}
+
+func TestTopologySpreadConstraintsAbsent(t *testing.T) {
+	result, err := api.ConvertDescriptor(map[string]any{}, &CloudExtras{})
+	require.NoError(t, err)
+	assert.Nil(t, result.TopologySpreadConstraints)
+}
+
+func TestTopologySpreadThreadedByToKubernetesRunConfig(t *testing.T) {
+	cloudExtras := any(map[string]any{
+		"topologySpreadConstraints": []map[string]any{
+			{"topologyKey": "kubernetes.io/hostname"},
+		},
+	})
+	stackCfg := &api.StackConfigCompose{
+		Runs:        []string{},
+		CloudExtras: &cloudExtras,
+	}
+
+	res, err := ToKubernetesRunConfig(&CloudrunTemplate{}, compose.Config{Project: &types.Project{}}, stackCfg)
+	require.NoError(t, err)
+	input, ok := res.(*KubeRunInput)
+	require.True(t, ok)
+	require.Len(t, input.Deployment.TopologySpreadConstraints, 1)
+	assert.Equal(t, "kubernetes.io/hostname", input.Deployment.TopologySpreadConstraints[0].TopologyKey)
+}

pkg/clouds/k8s/types.go

@@ -33,6 +33,8 @@ type DeploymentConfig struct {
 	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`           // Global startup probe configuration
 	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
 	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
+
+	TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints" yaml:"topologySpreadConstraints"`
 }
 
 type CaddyConfig struct {

pkg/clouds/pulumi/kubernetes/deployment.go

@@ -227,40 +227,41 @@ func DeploySimpleContainer(ctx *sdk.Context, args Args, opts ...sdk.ResourceOpti
 
 	args.Params.Log.Warn(ctx.Context(), "configure simple container deployment for %q in %q", stackName, stackEnv)
 	sc, err := NewSimpleContainer(ctx, &SimpleContainerArgs{
-		KubeProvider:           args.KubeProvider,
-		ComputeContext:         args.ComputeContext,
-		ServiceType:            args.ServiceType,
-		ExternalTrafficPolicy:  args.ExternalTrafficPolicy,
-		UseSSL:                 args.UseSSL,
-		ProvisionIngress:       args.ProvisionIngress,
-		Namespace:              namespace,
-		Service:                deploymentName,
-		Deployment:             deploymentName,
-		ScEnv:                  stackEnv,
-		IngressContainer:       args.Deployment.IngressContainer,
-		Domain:                 lo.FromPtr(args.Deployment.StackConfig).Domain,
-		Prefix:                 lo.FromPtr(args.Deployment.StackConfig).Prefix,
-		ProxyKeepPrefix:        lo.FromPtr(args.Deployment.StackConfig).ProxyKeepPrefix,
-		ParentStack:            lo.If(args.Params.ParentStack != nil, lo.ToPtr(lo.FromPtr(args.Params.ParentStack).FullReference)).Else(nil),
-		ParentEnv:              lo.If(parentEnv != "", lo.ToPtr(parentEnv)).Else(nil),
-		Replicas:               replicas,
-		Headers:                args.Deployment.Headers,
-		SecretEnvs:             mergedSecretEnvs,
-		LbConfig:               args.Deployment.StackConfig.LBConfig,
-		Volumes:                args.Deployment.TextVolumes,
-		PersistentVolumes:      pvs,
-		EphemeralVolumes:       args.Deployment.EphemeralVolumes, // Pass generic ephemeral volumes configuration
-		PriorityClassName:      args.Deployment.PriorityClassName,
-		Containers:             containers,
-		ServiceAccountName:     args.ServiceAccountName,
-		InitContainers:         args.InitContainers,
-		GenerateCaddyfileEntry: args.GenerateCaddyfileEntry,
-		Annotations:            args.Annotations,
-		NodeSelector:           args.NodeSelector,
-		Affinity:               args.Affinity,
-		Sidecars:               args.Sidecars,
-		VPA:                    args.VPA,              // Pass VPA configuration to SimpleContainer
-		Scale:                  args.Deployment.Scale, // Pass Scale configuration to SimpleContainer
+		KubeProvider:              args.KubeProvider,
+		ComputeContext:            args.ComputeContext,
+		ServiceType:               args.ServiceType,
+		ExternalTrafficPolicy:     args.ExternalTrafficPolicy,
+		UseSSL:                    args.UseSSL,
+		ProvisionIngress:          args.ProvisionIngress,
+		Namespace:                 namespace,
+		Service:                   deploymentName,
+		Deployment:                deploymentName,
+		ScEnv:                     stackEnv,
+		IngressContainer:          args.Deployment.IngressContainer,
+		Domain:                    lo.FromPtr(args.Deployment.StackConfig).Domain,
+		Prefix:                    lo.FromPtr(args.Deployment.StackConfig).Prefix,
+		ProxyKeepPrefix:           lo.FromPtr(args.Deployment.StackConfig).ProxyKeepPrefix,
+		ParentStack:               lo.If(args.Params.ParentStack != nil, lo.ToPtr(lo.FromPtr(args.Params.ParentStack).FullReference)).Else(nil),
+		ParentEnv:                 lo.If(parentEnv != "", lo.ToPtr(parentEnv)).Else(nil),
+		Replicas:                  replicas,
+		Headers:                   args.Deployment.Headers,
+		SecretEnvs:                mergedSecretEnvs,
+		LbConfig:                  args.Deployment.StackConfig.LBConfig,
+		Volumes:                   args.Deployment.TextVolumes,
+		PersistentVolumes:         pvs,
+		EphemeralVolumes:          args.Deployment.EphemeralVolumes, // Pass generic ephemeral volumes configuration
+		PriorityClassName:         args.Deployment.PriorityClassName,
+		Containers:                containers,
+		ServiceAccountName:        args.ServiceAccountName,
+		InitContainers:            args.InitContainers,
+		GenerateCaddyfileEntry:    args.GenerateCaddyfileEntry,
+		Annotations:               args.Annotations,
+		NodeSelector:              args.NodeSelector,
+		Affinity:                  args.Affinity,
+		TopologySpreadConstraints: args.Deployment.TopologySpreadConstraints,
+		Sidecars:                  args.Sidecars,
+		VPA:                       args.VPA,              // Pass VPA configuration to SimpleContainer
+		Scale:                     args.Deployment.Scale, // Pass Scale configuration to SimpleContainer
 		PodDisruption: lo.If(args.Deployment.DisruptionBudget != nil, args.Deployment.DisruptionBudget).Else(&k8s.DisruptionBudget{
 			MinAvailable: lo.ToPtr(1),
 		}),

pkg/clouds/pulumi/kubernetes/simple_container.go

@@ -106,24 +106,25 @@ type SimpleContainerArgs struct {
 	KubeProvider           sdk.ProviderResource
 
 	// optional properties
-	PodDisruption         *k8s.DisruptionBudget        `json:"podDisruption" yaml:"podDisruption"`
-	LbConfig              *api.SimpleContainerLBConfig `json:"lbConfig" yaml:"lbConfig"`
-	SecretEnvs            map[string]string            `json:"secretEnvs" yaml:"secretEnvs"`
-	Annotations           map[string]string            `json:"annotations" yaml:"annotations"`
-	NodeSelector          map[string]string            `json:"nodeSelector" yaml:"nodeSelector"`
-	Affinity              *k8s.AffinityRules           `json:"affinity" yaml:"affinity"`
-	PriorityClassName     *string                      `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
-	IngressContainer      *k8s.CloudRunContainer       `json:"ingressContainer" yaml:"ingressContainer"`
-	ServiceType           *string                      `json:"serviceType" yaml:"serviceType"`
-	ExternalTrafficPolicy *string                      `json:"externalTrafficPolicy" yaml:"externalTrafficPolicy"`
-	ProvisionIngress      bool                         `json:"provisionIngress" yaml:"provisionIngress"`
-	Headers               *k8s.Headers                 `json:"headers" yaml:"headers"`
-	Volumes               []k8s.SimpleTextVolume       `json:"volumes" yaml:"volumes"`
-	SecretVolumes         []k8s.SimpleTextVolume       `json:"secretVolumes" yaml:"secretVolumes"`
-	PersistentVolumes     []k8s.PersistentVolume       `json:"persistentVolumes" yaml:"persistentVolumes"`
-	EphemeralVolumes      []k8s.GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"` // Generic ephemeral volumes for large temp storage
-	VPA                   *k8s.VPAConfig               `json:"vpa" yaml:"vpa"`
-	Scale                 *k8s.Scale                   `json:"scale" yaml:"scale"`
+	PodDisruption             *k8s.DisruptionBudget          `json:"podDisruption" yaml:"podDisruption"`
+	LbConfig                  *api.SimpleContainerLBConfig   `json:"lbConfig" yaml:"lbConfig"`
+	SecretEnvs                map[string]string              `json:"secretEnvs" yaml:"secretEnvs"`
+	Annotations               map[string]string              `json:"annotations" yaml:"annotations"`
+	NodeSelector              map[string]string              `json:"nodeSelector" yaml:"nodeSelector"`
+	Affinity                  *k8s.AffinityRules             `json:"affinity" yaml:"affinity"`
+	TopologySpreadConstraints []k8s.TopologySpreadConstraint `json:"topologySpreadConstraints" yaml:"topologySpreadConstraints"`
+	PriorityClassName         *string                        `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
+	IngressContainer          *k8s.CloudRunContainer         `json:"ingressContainer" yaml:"ingressContainer"`
+	ServiceType               *string                        `json:"serviceType" yaml:"serviceType"`
+	ExternalTrafficPolicy     *string                        `json:"externalTrafficPolicy" yaml:"externalTrafficPolicy"`
+	ProvisionIngress          bool                           `json:"provisionIngress" yaml:"provisionIngress"`
+	Headers                   *k8s.Headers                   `json:"headers" yaml:"headers"`
+	Volumes                   []k8s.SimpleTextVolume         `json:"volumes" yaml:"volumes"`
+	SecretVolumes             []k8s.SimpleTextVolume         `json:"secretVolumes" yaml:"secretVolumes"`
+	PersistentVolumes         []k8s.PersistentVolume         `json:"persistentVolumes" yaml:"persistentVolumes"`
+	EphemeralVolumes          []k8s.GenericEphemeralVolume   `json:"ephemeralVolumes" yaml:"ephemeralVolumes"` // Generic ephemeral volumes for large temp storage
+	VPA                       *k8s.VPAConfig                 `json:"vpa" yaml:"vpa"`
+	Scale                     *k8s.Scale                     `json:"scale" yaml:"scale"`
 
 	Log logger.Logger
 	// ...
@@ -594,6 +595,12 @@ func NewSimpleContainer(ctx *sdk.Context, args *SimpleContainerArgs, opts ...sdk
 	convertedAffinity := convertAffinityRulesToKubernetes(args.Affinity)
 	args.Log.Info(ctx.Context(), "🔍 DEBUG: Converted affinity result: %+v", convertedAffinity)
 
+	normalizedTSC, tscErr := normalizeTopologySpreadConstraints(args.TopologySpreadConstraints, appLabels)
+	if tscErr != nil {
+		return nil, tscErr
+	}
+	topologySpread := convertTopologySpreadConstraints(normalizedTSC)
+
 	podSpecArgs := &corev1.PodSpecArgs{
 		NodeSelector: sdk.ToStringMap(args.NodeSelector),
 		Affinity:     convertedAffinity,
@@ -629,6 +636,9 @@ func NewSimpleContainer(ctx *sdk.Context, args *SimpleContainerArgs, opts ...sdk
 	if args.PriorityClassName != nil {
 		podSpecArgs.PriorityClassName = sdk.String(*args.PriorityClassName)
 	}
+	if topologySpread != nil {
+		podSpecArgs.TopologySpreadConstraints = topologySpread
+	}
 	if imagePullSecret != nil {
 		podSpecArgs.ImagePullSecrets = corev1.LocalObjectReferenceArray{
 			corev1.LocalObjectReferenceArgs{
@@ -1427,3 +1437,64 @@ func convertLabelSelector(labelSelector *k8s.LabelSelector) *metav1.LabelSelecto
 
 	return kubeLabelSelector
 }
+
+func normalizeTopologySpreadConstraints(constraints []k8s.TopologySpreadConstraint, appLabels map[string]string) ([]k8s.TopologySpreadConstraint, error) {
+	if len(constraints) == 0 {
+		return nil, nil
+	}
+
+	out := make([]k8s.TopologySpreadConstraint, 0, len(constraints))
+	for i, c := range constraints {
+		if c.TopologyKey == "" {
+			return nil, errors.Errorf("topologySpreadConstraints[%d]: topologyKey is required", i)
+		}
+
+		n := c
+		if n.WhenUnsatisfiable == "" {
+			n.WhenUnsatisfiable = "DoNotSchedule"
+		}
+		if n.WhenUnsatisfiable != "DoNotSchedule" && n.WhenUnsatisfiable != "ScheduleAnyway" {
+			return nil, errors.Errorf("topologySpreadConstraints[%d]: whenUnsatisfiable must be DoNotSchedule or ScheduleAnyway, got %q", i, n.WhenUnsatisfiable)
+		}
+		if n.MaxSkew == nil {
+			n.MaxSkew = lo.ToPtr(1)
+		} else if *n.MaxSkew < 1 {
+			return nil, errors.Errorf("topologySpreadConstraints[%d]: maxSkew must be >= 1, got %d", i, *n.MaxSkew)
+		}
+		if n.MinDomains != nil {
+			if *n.MinDomains < 1 {
+				return nil, errors.Errorf("topologySpreadConstraints[%d]: minDomains must be >= 1, got %d", i, *n.MinDomains)
+			}
+			if n.WhenUnsatisfiable != "DoNotSchedule" {
+				return nil, errors.Errorf("topologySpreadConstraints[%d]: minDomains requires whenUnsatisfiable=DoNotSchedule", i)
+			}
+		}
+		if n.LabelSelector == nil {
+			n.LabelSelector = &k8s.LabelSelector{MatchLabels: appLabels}
+		}
+
+		out = append(out, n)
+	}
+	return out, nil
+}
+
+func convertTopologySpreadConstraints(constraints []k8s.TopologySpreadConstraint) corev1.TopologySpreadConstraintArray {
+	if len(constraints) == 0 {
+		return nil
+	}
+
+	result := make(corev1.TopologySpreadConstraintArray, 0, len(constraints))
+	for _, c := range constraints {
+		constraint := corev1.TopologySpreadConstraintArgs{
+			MaxSkew:           sdk.Int(lo.FromPtr(c.MaxSkew)),
+			TopologyKey:       sdk.String(c.TopologyKey),
+			WhenUnsatisfiable: sdk.String(c.WhenUnsatisfiable),
+			LabelSelector:     convertLabelSelector(c.LabelSelector),
+		}
+		if c.MinDomains != nil {
+			constraint.MinDomains = sdk.Int(*c.MinDomains)
+		}
+		result = append(result, constraint)
+	}
+	return result
+}