fix(security+k8s): retry cosign sign on Rekor 409 conflict; honor cloudExtras startupProbe + periodSeconds

docs/docs/guides/parent-gcp-gke-autopilot.md

@@ -683,6 +683,22 @@ stacks:
           timeoutSeconds: 10
           periodSeconds: 30
           failureThreshold: 3
+
+        # Global startup probe configuration — the cold-start budget.
+        # Readiness/liveness checks are suspended until this probe succeeds,
+        # so a slow first boot (image pull, migrations, JIT warmup) is not
+        # killed mid-start. Total budget here: 10s + 16 × 15s = 250s.
+        # Without it, the startup window defaults to the readiness probe,
+        # whose short thresholds are tuned for steady-state, not first boot.
+        startupProbe:
+          httpGet:
+            path: "/health"
+            port: 8080
+          initialDelaySeconds: 10
+          timeoutSeconds: 5
+          periodSeconds: 15
+          failureThreshold: 16
+          successThreshold: 1
 ```
 
 ## **CloudExtras Field Reference**
@@ -697,6 +713,7 @@ stacks:
 | `vpa`                | `object`            | Vertical Pod Autoscaler                      | Native GKE support             |
 | `readinessProbe`     | `object`            | Global readiness probe                       | Full support                   |
 | `livenessProbe`      | `object`            | Global liveness probe                        | Full support                   |
+| `startupProbe`       | `object`            | Global startup probe (cold-start budget)     | Full support                   |
 | `priorityClassName`  | `string`            | Kubernetes PriorityClass for pod scheduling  | Full support                   |
 | `ephemeralVolumes`   | `[]object`          | Generic ephemeral volumes (>10GB storage)    | Full support                   |
 

docs/docs/troubleshooting/container-security.md

@@ -113,6 +113,30 @@ signing:
    ```
 3. Re-sign image if signature corrupted
 
+### Error: "createLogEntryConflict" (Rekor HTTP 409)
+
+**Problem:** `sc image sign` fails with:
+
+```
+signing bundle: error signing bundle: [POST /api/v1/log/entries][409] createLogEntryConflict
+{"code":409,"message":"an equivalent entry already exists in the transparency log with UUID ..."}
+```
+
+An identical entry is already in the transparency log — typically cosign
+re-uploading an entry whose first attempt timed out client-side but succeeded
+server-side. Common under parallel CI deploys of stacks that share one image.
+The image *is* effectively signed; only the duplicate upload was rejected.
+
+**Solution:**
+
+- `sc` retries the sign automatically with a fresh signature (a fresh
+  invocation cannot conflict with itself), so transient conflicts self-heal.
+- If the error persists after the automatic retries, verify the existing
+  signature instead of re-signing: `cosign tree IMAGE` and `sc image verify`.
+- Persistent conflicts with key-based signing can indicate a deterministic
+  re-sign of an unchanged digest — skip signing when the digest is already
+  signed by the same identity.
+
 ### Warning: "Rekor entry not found"
 
 **Problem:** Keyless signature not in transparency log.

go.mod

@@ -1,6 +1,6 @@
 module github.com/simple-container-com/api
 
-go 1.26.3
+go 1.26.4
 
 require (
 	cloud.google.com/go/storage v1.62.2

pkg/clouds/gcloud/gke_autopilot.go

@@ -107,6 +107,7 @@ func ToGkeAutopilotConfig(tpl any, composeCfg compose.Config, stackCfg *api.Stac
 		deployCfg.VPA = k8sCloudExtras.VPA                             // Extract VPA configuration from CloudExtras
 		deployCfg.ReadinessProbe = k8sCloudExtras.ReadinessProbe       // Extract global readiness probe configuration
 		deployCfg.LivenessProbe = k8sCloudExtras.LivenessProbe         // Extract global liveness probe configuration
+		deployCfg.StartupProbe = k8sCloudExtras.StartupProbe           // Extract global startup probe configuration
 		deployCfg.PriorityClassName = k8sCloudExtras.PriorityClassName // Extract PriorityClass for pod scheduling and preemption
 
 		// Process affinity rules and merge with existing NodeSelector if needed

pkg/clouds/k8s/kube_run.go

@@ -25,6 +25,7 @@ type CloudExtras struct {
 	VPA               *VPAConfig               `json:"vpa" yaml:"vpa"`
 	ReadinessProbe    *CloudRunProbe           `json:"readinessProbe" yaml:"readinessProbe"`
 	LivenessProbe     *CloudRunProbe           `json:"livenessProbe" yaml:"livenessProbe"`
+	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`
 	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
 	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
 }

pkg/clouds/k8s/kube_run_probe_test.go

@@ -0,0 +1,72 @@
+package k8s
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/simple-container-com/api/pkg/api"
+)
+
+// Regression: cloudExtras.startupProbe was silently dropped (no struct field).
+func TestStartupProbeExtraction(t *testing.T) {
+	RegisterTestingT(t)
+
+	cloudExtras := map[string]any{
+		"startupProbe": map[string]any{
+			"httpGet": map[string]any{
+				"path": "/health/",
+				"port": 8000,
+			},
+			"initialDelaySeconds": 10,
+			"timeoutSeconds":      5,
+			"periodSeconds":       15,
+			"failureThreshold":    16,
+			"successThreshold":    1,
+		},
+	}
+
+	result, err := api.ConvertDescriptor(cloudExtras, &CloudExtras{})
+
+	Expect(err).ToNot(HaveOccurred())
+	Expect(result.StartupProbe).ToNot(BeNil())
+	Expect(result.StartupProbe.HttpGet.Path).To(Equal("/health/"))
+	Expect(result.StartupProbe.HttpGet.Port).To(Equal(8000))
+	Expect(*result.StartupProbe.InitialDelaySeconds).To(Equal(10))
+	Expect(*result.StartupProbe.TimeoutSeconds).To(Equal(5))
+	Expect(*result.StartupProbe.PeriodSeconds).To(Equal(15))
+	Expect(*result.StartupProbe.FailureThreshold).To(Equal(16))
+	Expect(*result.StartupProbe.SuccessThreshold).To(Equal(1))
+}
+
+// Regression: periodSeconds was an unknown key and fell back to kubelet defaults.
+func TestProbePeriodSecondsExtraction(t *testing.T) {
+	RegisterTestingT(t)
+
+	cloudExtras := map[string]any{
+		"readinessProbe": map[string]any{
+			"httpGet": map[string]any{
+				"path": "/ready/",
+				"port": 8080,
+			},
+			"periodSeconds":    20,
+			"failureThreshold": 3,
+		},
+	}
+
+	result, err := api.ConvertDescriptor(cloudExtras, &CloudExtras{})
+
+	Expect(err).ToNot(HaveOccurred())
+	Expect(result.ReadinessProbe).ToNot(BeNil())
+	Expect(*result.ReadinessProbe.PeriodSeconds).To(Equal(20))
+	Expect(*result.ReadinessProbe.FailureThreshold).To(Equal(3))
+}
+
+func TestStartupProbeAbsent(t *testing.T) {
+	RegisterTestingT(t)
+
+	result, err := api.ConvertDescriptor(map[string]any{}, &CloudExtras{})
+
+	Expect(err).ToNot(HaveOccurred())
+	Expect(result.StartupProbe).To(BeNil())
+}

pkg/clouds/k8s/types.go

@@ -30,6 +30,7 @@ type DeploymentConfig struct {
 	VPA               *VPAConfig               `json:"vpa" yaml:"vpa"`                             // Vertical Pod Autoscaler configuration
 	ReadinessProbe    *CloudRunProbe           `json:"readinessProbe" yaml:"readinessProbe"`       // Global readiness probe configuration
 	LivenessProbe     *CloudRunProbe           `json:"livenessProbe" yaml:"livenessProbe"`         // Global liveness probe configuration
+	StartupProbe      *CloudRunProbe           `json:"startupProbe" yaml:"startupProbe"`           // Global startup probe configuration
 	EphemeralVolumes  []GenericEphemeralVolume `json:"ephemeralVolumes" yaml:"ephemeralVolumes"`   // Generic ephemeral volumes for large temp storage
 	PriorityClassName *string                  `json:"priorityClassName" yaml:"priorityClassName"` // Kubernetes PriorityClass for pod scheduling and preemption
 }
@@ -133,7 +134,8 @@ type CloudRunProbe struct {
 	HttpGet             ProbeHttpGet   `json:"httpGet" yaml:"httpGet"`
 	Interval            *time.Duration `json:"interval" yaml:"interval"`
 	InitialDelaySeconds *int           `json:"initialDelaySeconds" yaml:"initialDelaySeconds"`
-	IntervaSeconds      *int           `json:"intervaSeconds" yaml:"intervaSeconds"`
+	IntervaSeconds      *int           `json:"intervaSeconds" yaml:"intervaSeconds"` // Deprecated: misspelled and never consumed; use periodSeconds
+	PeriodSeconds       *int           `json:"periodSeconds" yaml:"periodSeconds"`   // k8s-native spelling; takes precedence over Interval
 	FailureThreshold    *int           `json:"failureThreshold" yaml:"failureThreshold"`
 	SuccessThreshold    *int           `json:"successThreshold" yaml:"successThreshold"`
 	TimeoutSeconds      *int           `json:"timeoutSeconds" yaml:"timeoutSeconds"`

pkg/clouds/pulumi/gcp/gke_autopilot_stack.go

@@ -189,6 +189,7 @@ func GkeAutopilotStack(ctx *sdk.Context, stack api.Stack, input api.ResourceInpu
 		VPA:            gkeAutopilotInput.Deployment.VPA,            // Pass VPA configuration to Kubernetes deployment
 		ReadinessProbe: gkeAutopilotInput.Deployment.ReadinessProbe, // Pass global readiness probe configuration
 		LivenessProbe:  gkeAutopilotInput.Deployment.LivenessProbe,  // Pass global liveness probe configuration
+		StartupProbe:   gkeAutopilotInput.Deployment.StartupProbe,   // Pass global startup probe configuration
 		EphemeralSize:  ephemeralSize,
 	}
 

pkg/clouds/pulumi/kubernetes/deployment.go

@@ -46,6 +46,7 @@ type Args struct {
 	VPA                    *k8s.VPAConfig     // Vertical Pod Autoscaler configuration
 	ReadinessProbe         *k8s.CloudRunProbe // Global readiness probe configuration
 	LivenessProbe          *k8s.CloudRunProbe // Global liveness probe configuration
+	StartupProbe           *k8s.CloudRunProbe // Global startup probe configuration
 	EphemeralSize          string
 	// TerminationGracePeriodSeconds overrides pod-level terminationGracePeriodSeconds.
 	TerminationGracePeriodSeconds *int
@@ -177,10 +178,16 @@ func DeploySimpleContainer(ctx *sdk.Context, args Args, opts ...sdk.ResourceOpti
 		}
 
 		var startupProbe *corev1.ProbeArgs
-		if c.Container.StartupProbe == nil && (len(c.Container.Ports) == 1 || c.Container.MainPort != nil) {
+		cStartupProbe := c.Container.StartupProbe
+		// Global fallback for the ingress container only, mirroring readiness/liveness above
+		if cStartupProbe == nil && args.StartupProbe != nil && isIngressContainer {
+			cStartupProbe = args.StartupProbe
+		}
+		if cStartupProbe == nil && (len(c.Container.Ports) == 1 || c.Container.MainPort != nil) {
 			startupProbe = readinessProbe
-		} else if c.Container.StartupProbe != nil && (len(c.Container.Ports) == 1 || c.Container.MainPort != nil) {
-			startupProbe = toProbeArgs(c, c.Container.StartupProbe)
+		} else if cStartupProbe != nil {
+			// Explicit probes resolve their own port — apply even on multi-port containers
+			startupProbe = toProbeArgs(c, cStartupProbe)
 		}
 
 		var resources corev1.ResourceRequirementsArgs
@@ -325,8 +332,13 @@ func toProbeArgs(c *ContainerImage, probe *k8s.CloudRunProbe) *corev1.ProbeArgs
 		probePort = c.Container.Ports[0]
 	}
 
+	// periodSeconds (k8s-native) wins over the legacy duration-typed interval
+	periodSeconds := probe.PeriodSeconds
+	if periodSeconds == nil && probe.Interval != nil {
+		periodSeconds = lo.ToPtr(int(lo.FromPtr(probe.Interval).Seconds()))
+	}
 	probeArgs := &corev1.ProbeArgs{
-		PeriodSeconds:       sdk.IntPtrFromPtr(lo.If(probe.Interval != nil, lo.ToPtr(int(lo.FromPtr(probe.Interval).Seconds()))).Else(nil)),
+		PeriodSeconds:       sdk.IntPtrFromPtr(periodSeconds),
 		InitialDelaySeconds: sdk.IntPtrFromPtr(probe.InitialDelaySeconds),
 		FailureThreshold:    sdk.IntPtrFromPtr(probe.FailureThreshold),
 		SuccessThreshold:    sdk.IntPtrFromPtr(probe.SuccessThreshold),

pkg/clouds/pulumi/kubernetes/deployment_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	sdk "github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	"github.com/samber/lo"
 
 	"github.com/simple-container-com/api/pkg/clouds/k8s"
@@ -326,3 +327,64 @@ func TestToProbeArgs_HeaderPreservation(t *testing.T) {
 	Expect(result).ToNot(BeNil(), "ProbeArgs should not be nil")
 	Expect(result.HttpGet).ToNot(BeNil(), "Should have HTTP GET configured")
 }
+
+// TestToProbeArgs_PeriodSeconds verifies the precedence between the k8s-native
+// periodSeconds field and the legacy duration-typed interval field.
+func TestToProbeArgs_PeriodSeconds(t *testing.T) {
+	container := &ContainerImage{
+		Container: k8s.CloudRunContainer{
+			Name:     "test-container",
+			Ports:    []int{8080},
+			MainPort: lo.ToPtr(8080),
+		},
+	}
+
+	testCases := []struct {
+		name     string
+		probe    *k8s.CloudRunProbe
+		expected sdk.IntPtrInput
+	}{
+		{
+			name: "periodSeconds used directly",
+			probe: &k8s.CloudRunProbe{
+				PeriodSeconds: lo.ToPtr(15),
+			},
+			expected: sdk.IntPtr(15),
+		},
+		{
+			name: "interval converted to seconds when periodSeconds absent",
+			probe: &k8s.CloudRunProbe{
+				Interval: lo.ToPtr(20 * time.Second),
+			},
+			expected: sdk.IntPtr(20),
+		},
+		{
+			name: "periodSeconds wins over interval",
+			probe: &k8s.CloudRunProbe{
+				PeriodSeconds: lo.ToPtr(15),
+				Interval:      lo.ToPtr(99 * time.Second),
+			},
+			expected: sdk.IntPtr(15),
+		},
+		{
+			name:     "neither set leaves kubelet default",
+			probe:    &k8s.CloudRunProbe{},
+			expected: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			RegisterTestingT(t)
+
+			result := toProbeArgs(container, tc.probe)
+
+			Expect(result).ToNot(BeNil())
+			if tc.expected == nil {
+				Expect(result.PeriodSeconds).To(BeNil())
+			} else {
+				Expect(result.PeriodSeconds).To(Equal(tc.expected))
+			}
+		})
+	}
+}

pkg/clouds/pulumi/kubernetes/kube_run.go

@@ -145,6 +145,7 @@ func KubeRun(ctx *sdk.Context, stack api.Stack, input api.ResourceInput, params
 		VPA:            kubeRunInput.Deployment.VPA,            // Pass VPA configuration from DeploymentConfig
 		ReadinessProbe: kubeRunInput.Deployment.ReadinessProbe, // Pass global readiness probe configuration
 		LivenessProbe:  kubeRunInput.Deployment.LivenessProbe,  // Pass global liveness probe configuration
+		StartupProbe:   kubeRunInput.Deployment.StartupProbe,   // Pass global startup probe configuration
 		EphemeralSize:  lo.FromPtr(kubeRunInput.Deployment.StackConfig).Size.Ephemeral,
 	}
 

pkg/security/signing/keybased.go

@@ -15,6 +15,9 @@ type KeyBasedSigner struct {
 	PrivateKey string // Path to private key file or key content
 	Password   string // Optional password for encrypted keys
 	Timeout    time.Duration
+
+	// exec overrides command execution in tests; nil means tools.ExecCommand.
+	exec execFn
 }
 
 // NewKeyBasedSigner creates a new key-based signer
@@ -73,9 +76,12 @@ func (s *KeyBasedSigner) Sign(ctx context.Context, imageRef string) (*SignResult
 
 	// Execute cosign sign command
 	args := []string{"sign", "--key", keyPath, imageRef}
-	stdout, stderr, err := tools.ExecCommand(ctx, "cosign", args, env, s.Timeout)
-	if err != nil {
-		return nil, fmt.Errorf("cosign sign failed: %w\nStderr: %s\nStdout: %s", err, stderr, stdout)
+	exec := s.exec
+	if exec == nil {
+		exec = tools.ExecCommand
+	}
+	if _, err := runCosignSign(ctx, exec, args, env, s.Timeout); err != nil {
+		return nil, err
 	}
 
 	result := &SignResult{

pkg/security/signing/keybased_test.go

@@ -2,6 +2,7 @@ package signing
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -100,3 +101,42 @@ func TestKeyBasedSignerPasswordHandling(t *testing.T) {
 	withPassword := NewKeyBasedSigner("/tmp/cosign.key", "secret123", 5*time.Minute)
 	Expect(strings.Contains("COSIGN_PASSWORD="+withPassword.Password, "COSIGN_PASSWORD=secret123")).To(BeTrue())
 }
+
+func TestKeyBasedSigner_Sign_GivesUpOnPersistentRekorConflict(t *testing.T) {
+	RegisterTestingT(t)
+
+	// Deterministic keys reproduce the same signature: a persistent conflict
+	// must exhaust the loop and fail, never be treated as success.
+	calls := 0
+	signer := NewKeyBasedSigner("test-key-content", "", time.Second)
+	signer.exec = func(ctx context.Context, name string, args []string, env []string, timeout time.Duration) (string, string, error) {
+		calls++
+		return "", "[POST /api/v1/log/entries][409] createLogEntryConflict", fmt.Errorf("exit status 1")
+	}
+
+	_, err := signer.Sign(context.Background(), "registry.example.com/app:1.0.0")
+
+	Expect(err).To(HaveOccurred())
+	Expect(err.Error()).To(ContainSubstring("createLogEntryConflict"))
+	Expect(calls).To(Equal(maxSignAttempts))
+}
+
+func TestKeyBasedSigner_Sign_RetriesOnceOnTransientConflict(t *testing.T) {
+	RegisterTestingT(t)
+
+	calls := 0
+	signer := NewKeyBasedSigner("test-key-content", "", time.Second)
+	signer.exec = func(ctx context.Context, name string, args []string, env []string, timeout time.Duration) (string, string, error) {
+		calls++
+		if calls == 1 {
+			return "", "createLogEntryConflict", fmt.Errorf("exit status 1")
+		}
+		return "", "", nil
+	}
+
+	result, err := signer.Sign(context.Background(), "registry.example.com/app:1.0.0")
+
+	Expect(err).ToNot(HaveOccurred())
+	Expect(result).ToNot(BeNil())
+	Expect(calls).To(Equal(2))
+}