From d04a80b7a7175361640f2f7d40ac804a9c7799b4 Mon Sep 17 00:00:00 2001 From: James Kan Date: Tue, 13 Jan 2026 19:52:19 -0800 Subject: [PATCH 1/7] FlinkBlueGreenDeplomynet in place suspension handler --- .../api/bluegreen/BlueGreenDiffType.java | 14 +- .../bluegreen/BlueGreenDeploymentService.java | 91 ++++++- .../InitializingBlueStateHandler.java | 11 + .../FlinkBlueGreenDeploymentSpecDiff.java | 29 ++- ...linkBlueGreenDeploymentControllerTest.java | 242 ++++++++++++++++++ .../FlinkBlueGreenDeploymentSpecDiffTest.java | 24 ++ 6 files changed, 406 insertions(+), 5 deletions(-) diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java index 5581493f7e..44d2b67609 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java @@ -26,5 +26,17 @@ public enum BlueGreenDiffType { TRANSITION, /** Changes that only affect the child FlinkDeploymentSpec. */ - PATCH_CHILD + PATCH_CHILD, + + /** + * In-place suspension. Triggered when job.state changes from RUNNING to SUSPENDED. Suspends the + * currently active child without creating a new deployment. + */ + SUSPEND, + + /** + * Resume from suspension. Triggered when job.state changes from SUSPENDED to RUNNING. Spins up + * the child with the current (potentially updated) spec. + */ + RESUME } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java index 85de365b97..5ca8b397cd 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java @@ -22,6 +22,7 @@ import org.apache.flink.kubernetes.operator.api.FlinkDeployment; import org.apache.flink.kubernetes.operator.api.bluegreen.BlueGreenDeploymentType; import org.apache.flink.kubernetes.operator.api.bluegreen.BlueGreenDiffType; +import org.apache.flink.kubernetes.operator.api.lifecycle.ResourceLifecycleState; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; import org.apache.flink.kubernetes.operator.api.status.Savepoint; import org.apache.flink.kubernetes.operator.api.status.SavepointFormatType; @@ -109,13 +110,38 @@ public UpdateControl initiateDeployment( */ public UpdateControl checkAndInitiateDeployment( BlueGreenContext context, BlueGreenDeploymentType currentBlueGreenDeploymentType) { + BlueGreenDiffType specDiff = getSpecDiff(context); if (specDiff != BlueGreenDiffType.IGNORE) { FlinkDeployment currentFlinkDeployment = context.getDeploymentByType(currentBlueGreenDeploymentType); - if (isFlinkDeploymentReady(currentFlinkDeployment)) { + if (specDiff == BlueGreenDiffType.SUSPEND && currentFlinkDeployment != null) { + setLastReconciledSpec(context); + LOG.info( + "In-place suspension for '{}'", + currentFlinkDeployment.getMetadata().getName()); + return patchFlinkDeployment(context, currentBlueGreenDeploymentType); + } + + if (specDiff == BlueGreenDiffType.RESUME && currentFlinkDeployment != null) { + setLastReconciledSpec(context); + LOG.info( + "In-place resume for '{}'", currentFlinkDeployment.getMetadata().getName()); + return patchFlinkDeployment(context, currentBlueGreenDeploymentType); + } + + // Check if child is currently suspended - if so, just patch specs without restart + if (isChildSuspended(currentFlinkDeployment)) { + setLastReconciledSpec(context); + LOG.info( + "Spec change while suspended for '{}'", + currentFlinkDeployment.getMetadata().getName()); + return patchFlinkDeployment(context, currentBlueGreenDeploymentType); + } + + if (currentFlinkDeployment != null && isFlinkDeploymentReady(currentFlinkDeployment)) { if (specDiff == BlueGreenDiffType.TRANSITION) { boolean savepointTriggered = false; try { @@ -153,12 +179,16 @@ public UpdateControl checkAndInitiateDeployment( } else { if (context.getDeploymentStatus().getJobStatus().getState() != JobStatus.FAILING) { setLastReconciledSpec(context); + var childName = + currentFlinkDeployment != null + ? currentFlinkDeployment.getMetadata().getName() + : "null"; var error = String.format( "Transition to %s not possible, current Flink Deployment '%s' is not READY. FAILING '%s'", calculateTransition(currentBlueGreenDeploymentType) .nextBlueGreenDeploymentType, - currentFlinkDeployment.getMetadata().getName(), + childName, context.getBgDeployment().getMetadata().getName()); return markDeploymentFailing(context, error); } @@ -168,6 +198,16 @@ public UpdateControl checkAndInitiateDeployment( return UpdateControl.noUpdate(); } + private boolean isChildSuspended(FlinkDeployment deployment) { + if (deployment == null || deployment.getSpec() == null) { + return false; + } + var job = deployment.getSpec().getJob(); + return job != null + && job.getState() + == org.apache.flink.kubernetes.operator.api.spec.JobState.SUSPENDED; + } + private UpdateControl patchFlinkDeployment( BlueGreenContext context, BlueGreenDeploymentType blueGreenDeploymentTypeToPatch) { @@ -385,6 +425,15 @@ public UpdateControl monitorTransition( TransitionState transitionState = determineTransitionState(context, currentBlueGreenDeploymentType); + if (isChildSuspended(transitionState.nextDeployment)) { + if (isFlinkDeploymentSuspended(transitionState.nextDeployment)) { + return finalizeSuspendedDeployment(context, transitionState.nextState); + } else { + return shouldWeAbort( + context, transitionState.nextDeployment, transitionState.nextState); + } + } + if (isFlinkDeploymentReady(transitionState.nextDeployment)) { return shouldWeDelete( context, @@ -397,11 +446,44 @@ public UpdateControl monitorTransition( } } + private boolean isFlinkDeploymentSuspended(FlinkDeployment deployment) { + if (deployment == null) { + return false; + } + return deployment.getStatus().getLifecycleState() == ResourceLifecycleState.SUSPENDED + && isChildSuspended(deployment); + } + + private UpdateControl finalizeSuspendedDeployment( + BlueGreenContext context, FlinkBlueGreenDeploymentState nextState) { + + LOG.info( + "Finalizing suspended deployment '{}' to {} state", + context.getDeploymentName(), + nextState); + + context.getDeploymentStatus().setDeploymentReadyTimestamp(millisToInstantStr(0)); + context.getDeploymentStatus().setAbortTimestamp(millisToInstantStr(0)); + context.getDeploymentStatus().setSavepointTriggerId(null); + + return patchStatusUpdateControl(context, nextState, JobStatus.SUSPENDED, null) + .rescheduleAfter(0); + } + private UpdateControl handleSpecChangesDuringTransition( BlueGreenContext context, BlueGreenDeploymentType currentBlueGreenDeploymentType) { if (hasSpecChanged(context)) { BlueGreenDiffType diffType = getSpecDiff(context); + // Block SUSPEND during transition - wait for transition to complete first + if (diffType == BlueGreenDiffType.SUSPEND) { + LOG.info( + "Suspend requested during transition for '{}'. " + + "Waiting for transition to complete before processing suspend.", + context.getBgDeployment().getMetadata().getName()); + return null; + } + if (diffType != BlueGreenDiffType.IGNORE) { setLastReconciledSpec(context); var oppositeDeploymentType = @@ -605,7 +687,10 @@ public UpdateControl finalizeBlueGreenDeployment( context.getDeploymentStatus().setAbortTimestamp(millisToInstantStr(0)); context.getDeploymentStatus().setSavepointTriggerId(null); - return patchStatusUpdateControl(context, nextState, JobStatus.RUNNING, null); + // Finalize status and reschedule immediately so any pending spec changes + // (e.g., suspend requested during transition) are picked up on next reconcile + return patchStatusUpdateControl(context, nextState, JobStatus.RUNNING, null) + .rescheduleAfter(0); } // ==================== Common Utility Methods ==================== diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/InitializingBlueStateHandler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/InitializingBlueStateHandler.java index f2882d46f7..8319314fb6 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/InitializingBlueStateHandler.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/InitializingBlueStateHandler.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.JobStatus; import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; import org.apache.flink.kubernetes.operator.api.bluegreen.BlueGreenDeploymentType; +import org.apache.flink.kubernetes.operator.api.spec.JobState; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenContext; @@ -27,6 +28,7 @@ import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import static org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenDeploymentService.patchStatusUpdateControl; import static org.apache.flink.kubernetes.operator.utils.bluegreen.BlueGreenUtils.hasSpecChanged; import static org.apache.flink.kubernetes.operator.utils.bluegreen.BlueGreenUtils.setLastReconciledSpec; @@ -41,6 +43,15 @@ public InitializingBlueStateHandler(BlueGreenDeploymentService deploymentService public UpdateControl handle(BlueGreenContext context) { FlinkBlueGreenDeploymentStatus deploymentStatus = context.getDeploymentStatus(); + // Block initial deployment if job.state is SUSPENDED - user must start with RUNNING + var jobSpec = context.getBgDeployment().getSpec().getTemplate().getSpec().getJob(); + if (jobSpec != null && jobSpec.getState() == JobState.SUSPENDED) { + LOG.info( + "Blocking initial deployment '{}' - job.state is SUSPENDED, waiting for RUNNING", + context.getBgDeployment().getMetadata().getName()); + return patchStatusUpdateControl(context, null, JobStatus.SUSPENDED, null); + } + // Deploy only if this is the initial deployment (no previous spec exists) // or if we're recovering from a failure and the spec has changed since the last attempt if (deploymentStatus.getLastReconciledSpec() == null diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java index 04c243d43d..e71d9283d5 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java @@ -22,6 +22,7 @@ import org.apache.flink.kubernetes.operator.api.diff.DiffType; import org.apache.flink.kubernetes.operator.api.spec.FlinkBlueGreenDeploymentSpec; import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.JobState; import org.apache.flink.kubernetes.operator.api.spec.KubernetesDeploymentMode; import lombok.NonNull; @@ -59,7 +60,20 @@ public BlueGreenDiffType compare() { FlinkDeploymentSpec leftSpec = left.getTemplate().getSpec(); FlinkDeploymentSpec rightSpec = right.getTemplate().getSpec(); - // Used in Case 2 & 3: Delegate to ReflectiveDiffBuilder for nested spec comparison + // Check for suspend/resume state changes first - these take highest precedence + // for in-place suspension handling + JobState leftJobState = getJobState(leftSpec); + JobState rightJobState = getJobState(rightSpec); + + if (leftJobState != rightJobState) { + if (rightJobState == JobState.SUSPENDED) { + return BlueGreenDiffType.SUSPEND; + } else if (leftJobState == JobState.SUSPENDED && rightJobState == JobState.RUNNING) { + return BlueGreenDiffType.RESUME; + } + } + + // Used in Case 2, 3 & 4: Delegate to ReflectiveDiffBuilder for nested spec comparison // Calculate diffResult before comparison to apply in-place removal of ignored fields DiffResult diffResult = new ReflectiveDiffBuilder<>(deploymentMode, leftSpec, rightSpec).build(); @@ -81,6 +95,19 @@ public BlueGreenDiffType compare() { } } + /** + * Gets the job state from the spec, defaulting to RUNNING if not set. + * + * @param spec the FlinkDeploymentSpec + * @return the job state, or RUNNING if job or state is null + */ + private JobState getJobState(FlinkDeploymentSpec spec) { + if (spec.getJob() == null || spec.getJob().getState() == null) { + return JobState.RUNNING; + } + return spec.getJob().getState(); + } + /** * Validates that the specs and their nested components are not null. Throws * IllegalArgumentException if any required component is null. diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java index 229406bbc9..e8d42af5fa 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java @@ -189,6 +189,168 @@ private TestingFlinkBlueGreenDeploymentController.BlueGreenReconciliationResult return rs; } + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifySuspendAndResumeInPlace(FlinkVersion flinkVersion) throws Exception { + var rs = setupActiveBlueDeployment(flinkVersion); + assertEquals(1, getFlinkDeployments().size()); + assertEquals( + FlinkBlueGreenDeploymentState.ACTIVE_BLUE, + rs.reconciledStatus.getBlueGreenState(), + "Should start as ACTIVE_BLUE"); + + // Suspend in-place with spec change (parallelism bump) + var deployment = rs.deployment; + deployment.getSpec().getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + deployment.getSpec().getTemplate().getSpec().getJob().setParallelism(5); + kubernetesClient.resource(deployment).createOrReplace(); + + rs = reconcile(deployment); + deployment = rs.deployment; + // Verify BG goes through TRANSITIONING_TO_BLUE state (unified patch mechanism) + assertEquals( + JobStatus.RECONCILING, + rs.reconciledStatus.getJobStatus().getState(), + "BG status should be RECONCILING after suspend initiated"); + assertEquals( + FlinkBlueGreenDeploymentState.TRANSITIONING_TO_BLUE, + rs.reconciledStatus.getBlueGreenState(), + "Should go through TRANSITIONING_TO_BLUE during suspend"); + + var flinkDeployments = getFlinkDeployments(); + assertEquals(1, flinkDeployments.size(), "Suspend should keep single child"); + var child = flinkDeployments.get(0); + assertTrue( + child.getMetadata().getName().endsWith("-blue"), "Child should be blue deployment"); + assertEquals( + JobState.SUSPENDED, + child.getSpec().getJob().getState(), + "Child should be suspended"); + assertEquals(5, child.getSpec().getJob().getParallelism(), "Spec change should be applied"); + + // Simulate child becoming suspended (lifecycleState is computed from lastReconciledSpec) + simulateSuccessfulSuspend(child); + rs = reconcile(deployment); + deployment = rs.deployment; + assertEquals(JobStatus.SUSPENDED, rs.reconciledStatus.getJobStatus().getState()); + assertEquals( + FlinkBlueGreenDeploymentState.ACTIVE_BLUE, + rs.reconciledStatus.getBlueGreenState(), + "Should finalize to ACTIVE_BLUE when suspended"); + + // Resume in-place with another spec change + deployment.getSpec().getTemplate().getSpec().getJob().setState(JobState.RUNNING); + deployment.getSpec().getTemplate().getSpec().getJob().setParallelism(6); + kubernetesClient.resource(deployment).createOrReplace(); + + rs = reconcile(deployment); + deployment = rs.deployment; + // Resume also goes through TRANSITIONING_TO_BLUE (unified patch mechanism) + assertEquals( + FlinkBlueGreenDeploymentState.TRANSITIONING_TO_BLUE, + rs.reconciledStatus.getBlueGreenState(), + "Should go through TRANSITIONING_TO_BLUE during resume"); + // Persist the BG status update to kubernetes for the next reconcile + kubernetesClient.resource(deployment).updateStatus(); + + flinkDeployments = getFlinkDeployments(); + assertEquals(1, flinkDeployments.size(), "Resume should keep single child"); + child = flinkDeployments.get(0); + assertTrue( + child.getMetadata().getName().endsWith("-blue"), + "Child should still be blue deployment"); + assertEquals(JobState.RUNNING, child.getSpec().getJob().getState(), "Child should resume"); + assertEquals(6, child.getSpec().getJob().getParallelism(), "Latest spec should be applied"); + + // Simulate child becoming running and verify BG status syncs + simulateSuccessfulJobStart(child); + rs = reconcile(deployment); + assertEquals(JobStatus.RUNNING, rs.reconciledStatus.getJobStatus().getState()); + assertEquals( + FlinkBlueGreenDeploymentState.ACTIVE_BLUE, + rs.reconciledStatus.getBlueGreenState(), + "Should finalize to ACTIVE_BLUE after resume"); + } + + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifySuspendDuringTransitionIsBlockedUntilComplete(FlinkVersion flinkVersion) + throws Exception { + // Start with a running Blue deployment with longer abort grace period + var blueGreenDeployment = + buildSessionCluster( + TEST_DEPLOYMENT_NAME, + TEST_NAMESPACE, + flinkVersion, + null, + UpgradeMode.STATELESS); // STATELESS to skip savepointing + // Set longer abort grace period BEFORE deployment to avoid abort during transition test + blueGreenDeployment.getSpec().getConfiguration().put(ABORT_GRACE_PERIOD.key(), "60000"); + // Use zero deletion delay to avoid timing-based flakiness during transition completion + blueGreenDeployment.getSpec().getConfiguration().put(DEPLOYMENT_DELETION_DELAY.key(), "0"); + var rs = executeBasicDeployment(flinkVersion, blueGreenDeployment, false, null); + + // === TRIGGER TRANSITION === + rs.deployment.getSpec().getTemplate().getSpec().getJob().setParallelism(5); + kubernetesClient.resource(rs.deployment).createOrReplace(); + + // Reconcile - start transition to GREEN + rs = reconcile(rs.deployment); + + // GREEN deployment should be created + var flinkDeployments = getFlinkDeployments(); + assertEquals(2, flinkDeployments.size()); + + // === SUSPEND DURING TRANSITION === + rs.deployment.getSpec().getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + kubernetesClient.resource(rs.deployment).createOrReplace(); + + // Reconcile - suspend should be BLOCKED, transition continues + rs = reconcile(rs.deployment); + + // GREEN deployment should NOT be suspended yet + flinkDeployments = getFlinkDeployments(); + var greenDeployment = + flinkDeployments.stream() + .filter(fd -> fd.getMetadata().getName().endsWith("-green")) + .findFirst() + .orElseThrow(); + assertEquals(JobState.RUNNING, greenDeployment.getSpec().getJob().getState()); + + // === TRANSITION COMPLETES === + // Simulate GREEN becoming ready + simulateSuccessfulJobStart(greenDeployment); + + // Reconcile to complete transition and process pending suspend + // (sets timestamp, deletes BLUE, finalizes to ACTIVE_GREEN, then processes suspend + // via TRANSITIONING_TO_GREEN) + for (int i = 0; i < 5; i++) { + rs = reconcile(rs.deployment); + } + // Persist the BG status update to kubernetes for the next reconcile + kubernetesClient.resource(rs.deployment).updateStatus(); + + // GREEN should now be suspended in spec (via patchFlinkDeployment) + flinkDeployments = getFlinkDeployments(); + greenDeployment = + flinkDeployments.stream() + .filter(fd -> fd.getMetadata().getName().endsWith("-green")) + .findFirst() + .orElseThrow(); + assertEquals(JobState.SUSPENDED, greenDeployment.getSpec().getJob().getState()); + + // Simulate GREEN child becoming suspended (lifecycleState is computed from + // lastReconciledSpec) + simulateSuccessfulSuspend(greenDeployment); + + // Reconcile - finalizeSuspendedDeployment should set BG job status to SUSPENDED + rs = reconcile(rs.deployment); + assertEquals(JobStatus.SUSPENDED, rs.reconciledStatus.getJobStatus().getState()); + assertEquals( + FlinkBlueGreenDeploymentState.ACTIVE_GREEN, + rs.reconciledStatus.getBlueGreenState()); + } + @ParameterizedTest @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") public void verifyFailureBeforeTransition(FlinkVersion flinkVersion) throws Exception { @@ -592,6 +754,76 @@ public void verifySavepointFetchFailureWithDifferentErrors( assertEquals(1, flinkDeployments.size()); } + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifySuspendWhenChildNotReadyPatchesWithoutFailing(FlinkVersion flinkVersion) + throws Exception { + var rs = setupActiveBlueDeployment(flinkVersion); + + // Mark the active child as not ready + var child = getFlinkDeployments().get(0); + child.getStatus().getJobStatus().setState(JobStatus.RECONCILING); + kubernetesClient.resource(child).update(); + + // Suspend and bump parallelism + var deployment = rs.deployment; + deployment.getSpec().getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + deployment.getSpec().getTemplate().getSpec().getJob().setParallelism(9); + kubernetesClient.resource(deployment).createOrReplace(); + + rs = reconcile(deployment); + + var flinkDeployments = getFlinkDeployments(); + assertEquals(1, flinkDeployments.size(), "Should not create new child"); + child = flinkDeployments.get(0); + assertEquals(JobState.SUSPENDED, child.getSpec().getJob().getState()); + assertEquals(9, child.getSpec().getJob().getParallelism(), "Spec change should apply"); + assertNotEquals( + JobStatus.FAILING, + rs.reconciledStatus.getJobStatus().getState(), + "Should not mark parent failing when suspending not-ready child"); + } + + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifyInitialSuspendedIsBlockedThenDeploysOnRunning(FlinkVersion flinkVersion) + throws Exception { + var bg = + buildSessionCluster( + TEST_DEPLOYMENT_NAME, + TEST_NAMESPACE, + flinkVersion, + null, + UpgradeMode.STATELESS); + bg.getSpec().getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + kubernetesClient.resource(bg).createOrReplace(); + + // First reconcile initializes status to INITIALIZING_BLUE + var rs = reconcile(bg); + // Second reconcile: handler detects SUSPENDED and blocks + rs = reconcile(rs.deployment); + + // Block child creation when initial state is SUSPENDED + assertEquals(0, getFlinkDeployments().size(), "No child should be created"); + assertEquals( + JobStatus.SUSPENDED, + rs.reconciledStatus.getJobStatus().getState(), + "Job status should be SUSPENDED when initial deployment is blocked"); + + // Flip to RUNNING and reconcile again + bg = rs.deployment; + bg.getSpec().getTemplate().getSpec().getJob().setState(JobState.RUNNING); + kubernetesClient.resource(bg).createOrReplace(); + + rs = reconcile(bg); + + assertEquals(1, getFlinkDeployments().size(), "Child should be created after fix"); + assertEquals( + JobStatus.RECONCILING, + rs.reconciledStatus.getJobStatus().getState(), + "Job status should be RECONCILING after deployment initiated"); + } + // ==================== Parameterized Test Inputs ==================== static Stream savepointErrorProvider() { @@ -1134,6 +1366,16 @@ private void simulateSuccessfulJobStart(FlinkDeployment deployment) { kubernetesClient.resource(deployment).update(); } + private void simulateSuccessfulSuspend(FlinkDeployment deployment) { + deployment.getStatus().getJobStatus().setState(JobStatus.FINISHED); + deployment.getStatus().getReconciliationStatus().setState(ReconciliationState.DEPLOYED); + deployment + .getStatus() + .getReconciliationStatus() + .serializeAndSetLastReconciledSpec(deployment.getSpec(), deployment); + kubernetesClient.resource(deployment).update(); + } + private void simulateJobFailure(FlinkDeployment deployment) { deployment.getStatus().getJobStatus().setState(JobStatus.RECONCILING); deployment.getStatus().getReconciliationStatus().setState(ReconciliationState.UPGRADING); diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java index 9fe6b8bd9b..fdfd9c59e2 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java @@ -134,6 +134,30 @@ public void testIgnoreForConfigurationDifference() { assertEquals(BlueGreenDiffType.IGNORE, diff.compare()); } + @Test + public void testSuspendOnJobStateChange() { + FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); // RUNNING default + FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); + spec2.getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + + FlinkBlueGreenDeploymentSpecDiff diff = + new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); + + assertEquals(BlueGreenDiffType.SUSPEND, diff.compare()); + } + + @Test + public void testResumeOnJobStateChange() { + FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); + spec1.getTemplate().getSpec().getJob().setState(JobState.SUSPENDED); + FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); // RUNNING default + + FlinkBlueGreenDeploymentSpecDiff diff = + new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); + + assertEquals(BlueGreenDiffType.RESUME, diff.compare()); + } + @Test public void testIgnoreForRootPodTemplateAdditionalProps() { FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); From 063cc80df7ec4c4d36d1d6a438f9d55f997c0ab1 Mon Sep 17 00:00:00 2001 From: James Kan Date: Sun, 18 Jan 2026 11:22:51 -0800 Subject: [PATCH 2/7] Introduce FlinkBlueGreenDeployment Metrics --- .../docs/operations/metrics-logging.md | 45 +- .../kubernetes/operator/FlinkOperator.java | 7 +- .../FlinkBlueGreenDeploymentController.java | 22 +- .../FlinkBlueGreenDeploymentMetrics.java | 188 +++++++ .../operator/metrics/MetricManager.java | 30 ++ .../lifecycle/BlueGreenLifecycleMetrics.java | 197 +++++++ ...ueGreenResourceLifecycleMetricTracker.java | 163 ++++++ .../operator/utils/StatusRecorder.java | 23 + ...linkBlueGreenDeploymentControllerTest.java | 3 + ...ingFlinkBlueGreenDeploymentController.java | 8 +- .../FlinkBlueGreenDeploymentMetricsTest.java | 509 ++++++++++++++++++ .../BlueGreenLifecycleMetricsTest.java | 224 ++++++++ ...eenResourceLifecycleMetricTrackerTest.java | 251 +++++++++ 13 files changed, 1656 insertions(+), 14 deletions(-) create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetrics.java create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetrics.java create mode 100644 flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTracker.java create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java create mode 100644 flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTrackerTest.java diff --git a/docs/content/docs/operations/metrics-logging.md b/docs/content/docs/operations/metrics-logging.md index d1e194efee..ac21f564c1 100644 --- a/docs/content/docs/operations/metrics-logging.md +++ b/docs/content/docs/operations/metrics-logging.md @@ -35,15 +35,20 @@ Different operator metrics can be turned on/off individually using the configura ### Flink Resource Metrics The Operator gathers aggregates metrics about managed resources. -| Scope | Metrics | Description | Type | -|--------------------|-------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| +| Scope | Metrics | Description | Type | +|--------------------|-------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| | Namespace | FlinkDeployment/FlinkSessionJob.Count | Number of managed resources per namespace | Gauge | -| Namespace | FlinkDeployment.ResourceUsage.Cpu/Memory | Total resources used per namespace | Gauge | -| Namespace | FlinkDeployment.JmDeploymentStatus.<Status>.Count | Number of managed FlinkDeployment resources per <Status> per namespace. <Status> can take values from: READY, DEPLOYED_NOT_READY, DEPLOYING, MISSING, ERROR | Gauge | -| Namespace | FlinkDeployment.FlinkVersion.<FlinkVersion>.Count | Number of managed FlinkDeployment resources per <FlinkVersion> per namespace. <FlinkVersion> is retrieved via REST API from Flink JM. | Gauge | +| Namespace | FlinkDeployment.ResourceUsage.Cpu/Memory | Total resources used per namespace | Gauge | +| Namespace | FlinkDeployment.JmDeploymentStatus.<Status>.Count | Number of managed FlinkDeployment resources per <Status> per namespace. <Status> can take values from: READY, DEPLOYED_NOT_READY, DEPLOYING, MISSING, ERROR | Gauge | +| Namespace | FlinkDeployment.FlinkVersion.<FlinkVersion>.Count | Number of managed FlinkDeployment resources per <FlinkVersion> per namespace. <FlinkVersion> is retrieved via REST API from Flink JM. | Gauge | | Namespace | FlinkDeployment/FlinkSessionJob.Lifecycle.State.<State>.Count | Number of managed resources currently in state <State> per namespace. <State> can take values from: CREATED, SUSPENDED, UPGRADING, DEPLOYED, STABLE, ROLLING_BACK, ROLLED_BACK, FAILED | Gauge | -| System/Namespace | FlinkDeployment/FlinkSessionJob.Lifecycle.State.<State>.TimeSeconds | Time spent in state <State> for a given resource. <State> can take values from: CREATED, SUSPENDED, UPGRADING, DEPLOYED, STABLE, ROLLING_BACK, ROLLED_BACK, FAILED | Histogram | -| System/Namespace | FlinkDeployment/FlinkSessionJob.Lifecycle.Transition.<Transition>.TimeSeconds | Time statistics for selected lifecycle state transitions. <Transition> can take values from: Resume, Upgrade, Suspend, Stabilization, Rollback, Submission | Histogram | +| System/Namespace | FlinkDeployment/FlinkSessionJob.Lifecycle.State.<State>.TimeSeconds | Time spent in state <State> for a given resource. <State> can take values from: CREATED, SUSPENDED, UPGRADING, DEPLOYED, STABLE, ROLLING_BACK, ROLLED_BACK, FAILED | Histogram | +| System/Namespace | FlinkDeployment/FlinkSessionJob.Lifecycle.Transition.<Transition>.TimeSeconds | Time statistics for selected lifecycle state transitions. <Transition> can take values from: Resume, Upgrade, Suspend, Stabilization, Rollback, Submission | Histogram | +| Namespace | FlinkBlueGreenDeployment.BlueGreenState.<State>.Count | Number of managed FlinkBlueGreenDeployment resources currently in state <State> per namespace. <State> can take values from: INITIALIZING_BLUE, ACTIVE_BLUE, SAVEPOINTING_BLUE, TRANSITIONING_TO_GREEN, ACTIVE_GREEN, SAVEPOINTING_GREEN, TRANSITIONING_TO_BLUE | Gauge | +| Namespace | FlinkBlueGreenDeployment.JobStatus.<Status>.Count | Number of managed FlinkBlueGreenDeployment resources currently in JobStatus <Status> per namespace. <Status> can take values from: RUNNING, FAILING, SUSPENDED, FAILED, RECONCILING | Gauge | +| Namespace | FlinkBlueGreenDeployment.Failures | Historical count of failure events (transitions to FAILING state) for all FlinkBlueGreenDeployment resources in the namespace. Counter increments on each transition to FAILING and never decrements. | Counter | +| System/Namespace | FlinkBlueGreenDeployment.Lifecycle.State.<State>.TimeSeconds | Time spent in state <State> for a given FlinkBlueGreenDeployment resource. <State> values same as above. | Histogram | +| System/Namespace | FlinkBlueGreenDeployment.Lifecycle.Transition.<Transition>.TimeSeconds | Time statistics for blue-green lifecycle state transitions. <Transition> can take values from: InitialDeployment, BlueToGreen, GreenToBlue | Histogram | #### Lifecycle metrics @@ -60,6 +65,32 @@ In addition to the simple counts we further track a few selected state transitio - Rollback : Time from deployed to rolled_back state if the resource was rolled back - Submission: Flink resource submission time +#### FlinkBlueGreenDeployment Lifecycle metrics + +FlinkBlueGreenDeployment resources have their own lifecycle states that track the blue-green deployment process. The operator monitors the following transitions: + + - InitialDeployment : Time from leaving INITIALIZING_BLUE to reaching ACTIVE_BLUE (first deployment) + - BlueToGreen : Time from leaving ACTIVE_BLUE to reaching ACTIVE_GREEN (actual transition duration) + - GreenToBlue : Time from leaving ACTIVE_GREEN to reaching ACTIVE_BLUE (actual transition duration) + +Transition metrics measure the actual transition time - from when the deployment leaves the source stable state until it reaches the target stable state. This excludes time spent running stably before the transition was initiated. + +State time metrics track how long a resource spends in each state (ACTIVE_BLUE, SAVEPOINTING_BLUE, TRANSITIONING_TO_GREEN, etc.), which helps identify bottlenecks in the deployment pipeline. + +#### FlinkBlueGreenDeployment JobStatus Tracking + +In addition to BlueGreenState tracking, FlinkBlueGreenDeployment resources also expose JobStatus metrics that track the Flink job state: + +**JobStatus Gauges**: Current count of deployments per JobStatus (RUNNING, FAILING, SUSPENDED, etc.) - these gauges go up and down as deployments transition between states. + +**Failures Counter**: Historical count that increments each time a deployment transitions TO the FAILING state. This counter: + - Never decrements (accumulates total failures since operator start) + - Increments on each new transition to FAILING (even if the same deployment fails multiple times) + - Persists across deployment recoveries (provides historical failure tracking) + - Useful for calculating failure rates and setting up alerts + +Example: A deployment goes RUNNING β†’ FAILING β†’ RUNNING β†’ FAILING. The FAILING gauge shows 0 or 1 (current state), while the Failures counter shows 2 (historical events). + ### Kubernetes Client Metrics The Operator gathers various metrics related to Kubernetes API server access. diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java index 96473ddb8f..fdc856f24b 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java @@ -257,7 +257,12 @@ void registerSnapshotController() { @VisibleForTesting void registerBlueGreenController() { - var controller = new FlinkBlueGreenDeploymentController(ctxFactory); + var metricManager = + MetricManager.createFlinkBlueGreenDeploymentMetricManager(baseConfig, metricGroup); + var statusRecorder = + StatusRecorder.createForFlinkBlueGreenDeployment(client, metricManager, listeners); + var controller = new FlinkBlueGreenDeploymentController(ctxFactory, statusRecorder); + registeredControllers.add(operator.register(controller, this::overrideControllerConfigs)); } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java index a35ccb2b1b..bac6f131e2 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java @@ -26,6 +26,7 @@ import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenStateHandlerRegistry; import org.apache.flink.kubernetes.operator.controller.bluegreen.handlers.BlueGreenStateHandler; import org.apache.flink.kubernetes.operator.service.FlinkResourceContextFactory; +import org.apache.flink.kubernetes.operator.utils.StatusRecorder; import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; import io.javaoperatorsdk.operator.api.reconciler.Context; @@ -70,10 +71,16 @@ public class FlinkBlueGreenDeploymentController implements Reconciler + statusRecorder; - public FlinkBlueGreenDeploymentController(FlinkResourceContextFactory ctxFactory) { + public FlinkBlueGreenDeploymentController( + FlinkResourceContextFactory ctxFactory, + StatusRecorder + statusRecorder) { this.ctxFactory = ctxFactory; this.handlerRegistry = new BlueGreenStateHandlerRegistry(); + this.statusRecorder = statusRecorder; } @Override @@ -110,9 +117,12 @@ public UpdateControl reconcile( josdkContext, null, ctxFactory); - return BlueGreenDeploymentService.patchStatusUpdateControl( - context, INITIALIZING_BLUE, null, null) - .rescheduleAfter(0); + UpdateControl updateControl = + BlueGreenDeploymentService.patchStatusUpdateControl( + context, INITIALIZING_BLUE, null, null) + .rescheduleAfter(0); + statusRecorder.patchAndCacheStatus(bgDeployment, josdkContext.getClient()); + return updateControl; } else { FlinkBlueGreenDeploymentState currentState = deploymentStatus.getBlueGreenState(); var context = @@ -132,7 +142,9 @@ public UpdateControl reconcile( context.getDeploymentName()); BlueGreenStateHandler handler = handlerRegistry.getHandler(currentState); - return handler.handle(context); + UpdateControl updateControl = handler.handle(context); + statusRecorder.patchAndCacheStatus(bgDeployment, josdkContext.getClient()); + return updateControl; } } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetrics.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetrics.java new file mode 100644 index 0000000000..4862cb08aa --- /dev/null +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetrics.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics; + +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** Metrics for FlinkBlueGreenDeployment resources. */ +public class FlinkBlueGreenDeploymentMetrics + implements CustomResourceMetrics { + + public static final String BG_STATE_GROUP_NAME = "BlueGreenState"; + public static final String JOB_STATUS_GROUP_NAME = "JobStatus"; + public static final String COUNTER_NAME = "Count"; + public static final String FAILURES_COUNTER_NAME = "Failures"; + + private final KubernetesOperatorMetricGroup parentMetricGroup; + private final Configuration configuration; + + // Tracks which deployments are in which state per namespace (for gauge metrics) + // Map: namespace -> state -> set of deployment names + private final Map>> deploymentStatuses = + new ConcurrentHashMap<>(); + + // Tracks which deployments are in which JobStatus per namespace (for gauge metrics) + // Map: namespace -> JobStatus -> set of deployment names + private final Map>> jobStatuses = new ConcurrentHashMap<>(); + + // Failure counters per namespace (historical count, never decrements) + // Map: namespace -> Counter + private final Map failureCounters = new ConcurrentHashMap<>(); + + public FlinkBlueGreenDeploymentMetrics( + KubernetesOperatorMetricGroup parentMetricGroup, Configuration configuration) { + this.parentMetricGroup = parentMetricGroup; + this.configuration = configuration; + } + + @Override + public void onUpdate(FlinkBlueGreenDeployment flinkBgDep) { + var namespace = flinkBgDep.getMetadata().getNamespace(); + var deploymentName = flinkBgDep.getMetadata().getName(); + var state = flinkBgDep.getStatus().getBlueGreenState(); + + // Get current JobStatus + var jobStatusObj = flinkBgDep.getStatus().getJobStatus(); + var currentJobStatus = jobStatusObj != null ? jobStatusObj.getState() : null; + + // Check if was in FAILING state BEFORE + boolean wasInFailing = false; + if (currentJobStatus != null) { + var namespaceJobStatuses = jobStatuses.get(namespace); + if (namespaceJobStatuses != null) { + var failingSet = namespaceJobStatuses.get(JobStatus.FAILING); + wasInFailing = failingSet != null && failingSet.contains(deploymentName); + } + } + + // Clear from all tracking (BlueGreenState and JobStatus) + clearStateCount(flinkBgDep); + + deploymentStatuses + .computeIfAbsent(namespace, this::initNamespaceMetrics) + .get(state) + .add(deploymentName); + + // Track JobStatus in gauge + if (currentJobStatus != null) { + jobStatuses + .computeIfAbsent(namespace, ns -> createJobStatusMap()) + .get(currentJobStatus) + .add(deploymentName); + + // Detect transition TO FAILING for counter + if (currentJobStatus == JobStatus.FAILING && !wasInFailing) { + var counter = failureCounters.get(namespace); + if (counter != null) { + counter.inc(); + } + } + } + } + + @Override + public void onRemove(FlinkBlueGreenDeployment flinkBgDep) { + clearStateCount(flinkBgDep); + } + + /** Clears the deployment from all state count sets (used before updating to new state). */ + private void clearStateCount(FlinkBlueGreenDeployment flinkBgDep) { + var namespace = flinkBgDep.getMetadata().getNamespace(); + var deploymentName = flinkBgDep.getMetadata().getName(); + + var namespaceStatuses = deploymentStatuses.get(namespace); + if (namespaceStatuses != null) { + namespaceStatuses + .values() + .forEach(deploymentNames -> deploymentNames.remove(deploymentName)); + } + + // Clear from JobStatus tracking + var namespaceJobStatuses = jobStatuses.get(namespace); + if (namespaceJobStatuses != null) { + namespaceJobStatuses + .values() + .forEach(deploymentNames -> deploymentNames.remove(deploymentName)); + } + } + + private Map> initNamespaceMetrics(String namespace) { + MetricGroup nsGroup = + parentMetricGroup.createResourceNamespaceGroup( + configuration, FlinkBlueGreenDeployment.class, namespace); + + // Total deployment count + nsGroup.gauge( + COUNTER_NAME, + () -> + deploymentStatuses.get(namespace).values().stream() + .mapToInt(Set::size) + .sum()); + + // Historical failure counter (increments on each transition TO FAILING) + failureCounters.put(namespace, nsGroup.counter(FAILURES_COUNTER_NAME)); + + // Per-BlueGreenState counts + Map> statuses = new ConcurrentHashMap<>(); + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + statuses.put(state, ConcurrentHashMap.newKeySet()); + nsGroup.addGroup(BG_STATE_GROUP_NAME) + .addGroup(state.toString()) + .gauge(COUNTER_NAME, () -> deploymentStatuses.get(namespace).get(state).size()); + } + + // Per-JobStatus counts (gauges for current state) + initJobStatusMetrics(namespace, nsGroup); + + return statuses; + } + + private void initJobStatusMetrics(String namespace, MetricGroup nsGroup) { + for (JobStatus status : JobStatus.values()) { + nsGroup.addGroup(JOB_STATUS_GROUP_NAME) + .addGroup(status.toString()) + .gauge( + COUNTER_NAME, + () -> { + var nsJobStatuses = jobStatuses.get(namespace); + if (nsJobStatuses == null) { + return 0; + } + var statusSet = nsJobStatuses.get(status); + return statusSet != null ? statusSet.size() : 0; + }); + } + } + + private Map> createJobStatusMap() { + Map> statuses = new ConcurrentHashMap<>(); + for (JobStatus status : JobStatus.values()) { + statuses.put(status, ConcurrentHashMap.newKeySet()); + } + return statuses; + } +} diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/MetricManager.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/MetricManager.java index 02ee9a8672..ca571188b6 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/MetricManager.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/MetricManager.java @@ -20,9 +20,11 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.configuration.Configuration; import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; import org.apache.flink.kubernetes.operator.api.FlinkDeployment; import org.apache.flink.kubernetes.operator.api.FlinkSessionJob; import org.apache.flink.kubernetes.operator.api.FlinkStateSnapshot; +import org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics; import org.apache.flink.kubernetes.operator.metrics.lifecycle.LifecycleMetrics; import io.fabric8.kubernetes.client.CustomResource; @@ -69,6 +71,15 @@ public static MetricManager createFlinkStateSnapshotMetricMa return metricManager; } + public static MetricManager + createFlinkBlueGreenDeploymentMetricManager( + Configuration conf, KubernetesOperatorMetricGroup metricGroup) { + MetricManager metricManager = new MetricManager<>(); + registerFlinkBlueGreenDeploymentMetrics(conf, metricGroup, metricManager); + registerBlueGreenLifecycleMetrics(conf, metricGroup, metricManager); + return metricManager; + } + private static void registerFlinkDeploymentMetrics( Configuration conf, KubernetesOperatorMetricGroup metricGroup, @@ -96,6 +107,15 @@ private static void registerFlinkStateSnapshotMetrics( } } + private static void registerFlinkBlueGreenDeploymentMetrics( + Configuration conf, + KubernetesOperatorMetricGroup metricGroup, + MetricManager metricManager) { + if (conf.get(KubernetesOperatorMetricOptions.OPERATOR_RESOURCE_METRICS_ENABLED)) { + metricManager.register(new FlinkBlueGreenDeploymentMetrics(metricGroup, conf)); + } + } + private static > void registerLifecycleMetrics( Configuration conf, KubernetesOperatorMetricGroup metricGroup, @@ -106,6 +126,16 @@ private static void registerFlinkStateSnapshotMetrics( } } + private static void registerBlueGreenLifecycleMetrics( + Configuration conf, + KubernetesOperatorMetricGroup metricGroup, + MetricManager metricManager) { + if (conf.get(KubernetesOperatorMetricOptions.OPERATOR_RESOURCE_METRICS_ENABLED) + && conf.get(KubernetesOperatorMetricOptions.OPERATOR_LIFECYCLE_METRICS_ENABLED)) { + metricManager.register(new BlueGreenLifecycleMetrics(conf, metricGroup)); + } + } + @VisibleForTesting public List> getRegisteredMetrics() { return registeredMetrics; diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetrics.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetrics.java new file mode 100644 index 0000000000..e874352810 --- /dev/null +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetrics.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics.lifecycle; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration; +import org.apache.flink.kubernetes.operator.metrics.CustomResourceMetrics; +import org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricGroup; +import org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricOptions; +import org.apache.flink.kubernetes.operator.metrics.OperatorMetricUtils; +import org.apache.flink.metrics.Histogram; + +import java.time.Clock; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.INITIALIZING_BLUE; + +/** Manages lifecycle metrics for FlinkBlueGreenDeployment resources. */ +public class BlueGreenLifecycleMetrics implements CustomResourceMetrics { + + public static final String LIFECYCLE_GROUP_NAME = "Lifecycle"; + public static final String TRANSITION_GROUP_NAME = "Transition"; + public static final String STATE_GROUP_NAME = "State"; + public static final String TIME_SECONDS_NAME = "TimeSeconds"; + + public static final String TRANSITION_INITIAL_DEPLOYMENT = "InitialDeployment"; + public static final String TRANSITION_BLUE_TO_GREEN = "BlueToGreen"; + public static final String TRANSITION_GREEN_TO_BLUE = "GreenToBlue"; + + private static final List TRANSITIONS = + List.of( + TRANSITION_INITIAL_DEPLOYMENT, + TRANSITION_BLUE_TO_GREEN, + TRANSITION_GREEN_TO_BLUE); + + private final KubernetesOperatorMetricGroup parentMetricGroup; + private final Configuration configuration; + private final FlinkOperatorConfiguration operatorConfig; + private final Clock clock; + private final boolean lifecycleMetricsEnabled; + + private final Map> + lifecycleTrackers = new ConcurrentHashMap<>(); + private final Map> namespaceTransitionHistograms = + new ConcurrentHashMap<>(); + private final Map> + namespaceStateTimeHistograms = new ConcurrentHashMap<>(); + + private final Map systemTransitionHistograms = new ConcurrentHashMap<>(); + private final Map systemStateTimeHistograms = + new ConcurrentHashMap<>(); + + public BlueGreenLifecycleMetrics( + Configuration configuration, KubernetesOperatorMetricGroup parentMetricGroup) { + this.parentMetricGroup = parentMetricGroup; + this.configuration = configuration; + this.operatorConfig = FlinkOperatorConfiguration.fromConfiguration(configuration); + this.clock = Clock.systemDefaultZone(); + this.lifecycleMetricsEnabled = + configuration.get( + KubernetesOperatorMetricOptions.OPERATOR_LIFECYCLE_METRICS_ENABLED); + + TRANSITIONS.forEach(t -> namespaceTransitionHistograms.put(t, new ConcurrentHashMap<>())); + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + namespaceStateTimeHistograms.put(state, new ConcurrentHashMap<>()); + } + } + + @Override + public void onUpdate(FlinkBlueGreenDeployment flinkBgDep) { + if (!lifecycleMetricsEnabled) { + return; + } + + var namespace = flinkBgDep.getMetadata().getNamespace(); + var deploymentName = flinkBgDep.getMetadata().getName(); + var state = flinkBgDep.getStatus().getBlueGreenState(); + + getOrCreateTracker(namespace, deploymentName, flinkBgDep).onUpdate(state, clock.instant()); + } + + @Override + public void onRemove(FlinkBlueGreenDeployment flinkBgDep) { + var namespace = flinkBgDep.getMetadata().getNamespace(); + var deploymentName = flinkBgDep.getMetadata().getName(); + + var namespaceTrackers = lifecycleTrackers.get(namespace); + if (namespaceTrackers != null) { + namespaceTrackers.remove(deploymentName); + } + } + + private BlueGreenResourceLifecycleMetricTracker getOrCreateTracker( + String namespace, String deploymentName, FlinkBlueGreenDeployment flinkBgDep) { + return lifecycleTrackers + .computeIfAbsent(namespace, ns -> new ConcurrentHashMap<>()) + .computeIfAbsent(deploymentName, dn -> createTracker(namespace, flinkBgDep)); + } + + private BlueGreenResourceLifecycleMetricTracker createTracker( + String namespace, FlinkBlueGreenDeployment flinkBgDep) { + var initialState = flinkBgDep.getStatus().getBlueGreenState(); + var time = + initialState == INITIALIZING_BLUE + ? Instant.parse(flinkBgDep.getMetadata().getCreationTimestamp()) + : clock.instant(); + + return new BlueGreenResourceLifecycleMetricTracker( + initialState, + time, + buildTransitionHistograms(namespace), + buildStateTimeHistograms(namespace)); + } + + private Map> buildTransitionHistograms(String namespace) { + var histos = new HashMap>(); + for (String transition : TRANSITIONS) { + histos.put( + transition, + List.of( + systemTransitionHistograms.computeIfAbsent( + transition, + t -> createSystemHistogram(TRANSITION_GROUP_NAME, t)), + namespaceTransitionHistograms + .get(transition) + .computeIfAbsent( + namespace, + ns -> + createNamespaceHistogram( + ns, + TRANSITION_GROUP_NAME, + transition)))); + } + return histos; + } + + private Map> buildStateTimeHistograms( + String namespace) { + var histos = new HashMap>(); + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + histos.put( + state, + List.of( + systemStateTimeHistograms.computeIfAbsent( + state, s -> createSystemHistogram(STATE_GROUP_NAME, s.name())), + namespaceStateTimeHistograms + .get(state) + .computeIfAbsent( + namespace, + ns -> + createNamespaceHistogram( + ns, STATE_GROUP_NAME, state.name())))); + } + return histos; + } + + private Histogram createSystemHistogram(String groupName, String metricName) { + return parentMetricGroup + .addGroup(FlinkBlueGreenDeployment.class.getSimpleName()) + .addGroup(LIFECYCLE_GROUP_NAME) + .addGroup(groupName) + .addGroup(metricName) + .histogram(TIME_SECONDS_NAME, OperatorMetricUtils.createHistogram(operatorConfig)); + } + + private Histogram createNamespaceHistogram( + String namespace, String groupName, String metricName) { + return parentMetricGroup + .createResourceNamespaceGroup( + configuration, FlinkBlueGreenDeployment.class, namespace) + .addGroup(LIFECYCLE_GROUP_NAME) + .addGroup(groupName) + .addGroup(metricName) + .histogram(TIME_SECONDS_NAME, OperatorMetricUtils.createHistogram(operatorConfig)); + } +} diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTracker.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTracker.java new file mode 100644 index 0000000000..536312785c --- /dev/null +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTracker.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics.lifecycle; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.metrics.Histogram; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_GREEN; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.INITIALIZING_BLUE; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_BLUE_TO_GREEN; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_GREEN_TO_BLUE; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_INITIAL_DEPLOYMENT; + +/** + * Tracks state transitions and timing for a single FlinkBlueGreenDeployment resource. Records + * transition durations and time spent in each state. + */ +public class BlueGreenResourceLifecycleMetricTracker { + + private static final Logger LOG = + LoggerFactory.getLogger(BlueGreenResourceLifecycleMetricTracker.class); + + private FlinkBlueGreenDeploymentState currentState; + + // map(state -> (firstEntryTime, lastUpdateTime)) + // Tracks when we entered each state and when we last saw it + private final Map> stateTimeMap = + new HashMap<>(); + + private final Map> transitionHistos; + private final Map> stateTimeHistos; + + public BlueGreenResourceLifecycleMetricTracker( + FlinkBlueGreenDeploymentState initialState, + Instant time, + Map> transitionHistos, + Map> stateTimeHistos) { + this.currentState = initialState; + this.transitionHistos = transitionHistos; + this.stateTimeHistos = stateTimeHistos; + stateTimeMap.put(initialState, Tuple2.of(time, time)); + } + + /** + * Called on every reconciliation. Updates timestamps and records metrics on state changes. + * + * @param newState the current state from the resource status + * @param time the current timestamp + */ + public void onUpdate(FlinkBlueGreenDeploymentState newState, Instant time) { + if (newState == currentState) { + updateLastUpdateTime(newState, time); + return; + } + + // Record exit time for states that transition faster than the heartbeat interval. + updateLastUpdateTime(currentState, time); + recordTransitionMetrics(currentState, newState, time); + + if (newState == ACTIVE_BLUE || newState == ACTIVE_GREEN) { + LOG.debug( + "Transitioned from {} to {}, recording state times for {} and clearing", + currentState, + newState, + stateTimeMap.keySet()); + + recordStateTimeMetrics(); + clearTrackedStates(); + } + + stateTimeMap.put(newState, Tuple2.of(time, time)); + currentState = newState; + } + + private void updateLastUpdateTime(FlinkBlueGreenDeploymentState state, Instant time) { + var times = stateTimeMap.get(state); + if (times != null) { + times.f1 = time; + } + } + + private void recordTransitionMetrics( + FlinkBlueGreenDeploymentState fromState, + FlinkBlueGreenDeploymentState toState, + Instant time) { + + if (toState == ACTIVE_BLUE && stateTimeMap.containsKey(INITIALIZING_BLUE)) { + recordTransition(TRANSITION_INITIAL_DEPLOYMENT, INITIALIZING_BLUE, time); + } + + if (toState == ACTIVE_GREEN && stateTimeMap.containsKey(ACTIVE_BLUE)) { + recordTransition(TRANSITION_BLUE_TO_GREEN, ACTIVE_BLUE, time); + } + + if (toState == ACTIVE_BLUE + && fromState != INITIALIZING_BLUE + && stateTimeMap.containsKey(ACTIVE_GREEN)) { + recordTransition(TRANSITION_GREEN_TO_BLUE, ACTIVE_GREEN, time); + } + } + + private void recordTransition( + String transitionName, FlinkBlueGreenDeploymentState fromState, Instant time) { + var fromTimes = stateTimeMap.get(fromState); + if (fromTimes == null) { + return; + } + + long durationSeconds = Duration.between(fromTimes.f1, time).toSeconds(); + + LOG.debug( + "Recording transition time {}s for {} (from {})", + durationSeconds, + transitionName, + fromState); + + var histograms = transitionHistos.get(transitionName); + if (histograms != null) { + histograms.forEach(h -> h.update(durationSeconds)); + } + } + + private void recordStateTimeMetrics() { + stateTimeMap.forEach( + (state, times) -> { + long durationSeconds = Duration.between(times.f0, times.f1).toSeconds(); + var histograms = stateTimeHistos.get(state); + if (histograms != null) { + histograms.forEach(h -> h.update(durationSeconds)); + } + }); + } + + private void clearTrackedStates() { + stateTimeMap.clear(); + } +} diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/StatusRecorder.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/StatusRecorder.java index e06aa11193..c132804235 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/StatusRecorder.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/StatusRecorder.java @@ -20,12 +20,14 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; import org.apache.flink.kubernetes.operator.api.FlinkDeployment; import org.apache.flink.kubernetes.operator.api.FlinkSessionJob; import org.apache.flink.kubernetes.operator.api.FlinkStateSnapshot; import org.apache.flink.kubernetes.operator.api.lifecycle.ResourceLifecycleState; import org.apache.flink.kubernetes.operator.api.listener.FlinkResourceListener; import org.apache.flink.kubernetes.operator.api.status.CommonStatus; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; import org.apache.flink.kubernetes.operator.api.status.FlinkDeploymentStatus; import org.apache.flink.kubernetes.operator.api.status.FlinkSessionJobStatus; import org.apache.flink.kubernetes.operator.api.status.FlinkStateSnapshotStatus; @@ -131,6 +133,8 @@ private STATUS convertPreviousStatus(CR resource, ObjectNode previousStatusNode) statusClass = FlinkSessionJobStatus.class; } else if (resource instanceof FlinkStateSnapshot) { statusClass = FlinkStateSnapshotStatus.class; + } else if (resource instanceof FlinkBlueGreenDeployment) { + statusClass = FlinkBlueGreenDeploymentStatus.class; } else { throw new RuntimeException( String.format("Resource is unknown class: %s", resource.getClass())); @@ -300,6 +304,25 @@ public Instant getTimestamp() { return new StatusRecorder<>(metricManager, consumer); } + public static StatusRecorder + createForFlinkBlueGreenDeployment( + KubernetesClient kubernetesClient, + MetricManager metricManager, + Collection listeners) { + BiConsumer consumer = + (resource, previousStatus) -> { + listeners.forEach( + listener -> { + // FlinkResourceListener doesn't have a specific method for + // BlueGreen deployments yet, so we skip listener notifications + // for now. Metrics will still be tracked via MetricManager. + }); + // No audit logging for BlueGreen deployments yet + }; + + return new StatusRecorder<>(metricManager, consumer); + } + public static StatusRecorder createForFlinkStateSnapshot( KubernetesClient kubernetesClient, diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java index e8d42af5fa..49e56561d0 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java @@ -1185,6 +1185,9 @@ private void assertFinalized( throws Exception { Long minReconciliationTs = System.currentTimeMillis() - 1; + // Create the resource in the mock server before reconciling + kubernetesClient.resource(blueGreenDeployment).createOrReplace(); + // 1a. Initializing deploymentStatus with this call var rs = reconcile(blueGreenDeployment); diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java index 6e8e058ce2..5bddcf4da6 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java @@ -24,7 +24,9 @@ import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; import org.apache.flink.kubernetes.operator.config.FlinkConfigManager; +import org.apache.flink.kubernetes.operator.metrics.MetricManager; import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils; +import org.apache.flink.kubernetes.operator.utils.StatusRecorder; import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.ErrorStatusUpdateControl; @@ -50,7 +52,11 @@ public TestingFlinkBlueGreenDeploymentController( flinkService, null); - flinkBlueGreenDeploymentController = new FlinkBlueGreenDeploymentController(contextFactory); + StatusRecorder statusRecorder = + new StatusRecorder<>(new MetricManager<>(), (resource, status) -> {}); + + flinkBlueGreenDeploymentController = + new FlinkBlueGreenDeploymentController(contextFactory, statusRecorder); } @Override diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java new file mode 100644 index 0000000000..104696775d --- /dev/null +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics; + +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; +import org.apache.flink.kubernetes.operator.api.spec.ConfigObjectNode; +import org.apache.flink.kubernetes.operator.api.spec.FlinkBlueGreenDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentTemplateSpec; +import org.apache.flink.kubernetes.operator.api.spec.JobSpec; +import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.HashMap; +import java.util.UUID; + +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_GREEN; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.INITIALIZING_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.TRANSITIONING_TO_GREEN; +import static org.apache.flink.kubernetes.operator.metrics.FlinkBlueGreenDeploymentMetrics.BG_STATE_GROUP_NAME; +import static org.apache.flink.kubernetes.operator.metrics.FlinkBlueGreenDeploymentMetrics.COUNTER_NAME; +import static org.apache.flink.kubernetes.operator.metrics.FlinkBlueGreenDeploymentMetrics.FAILURES_COUNTER_NAME; +import static org.apache.flink.kubernetes.operator.metrics.FlinkBlueGreenDeploymentMetrics.JOB_STATUS_GROUP_NAME; +import static org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricOptions.OPERATOR_RESOURCE_METRICS_ENABLED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Tests for {@link FlinkBlueGreenDeploymentMetrics}. */ +public class FlinkBlueGreenDeploymentMetricsTest { + + private static final String TEST_NAMESPACE = "test-namespace"; + + private final Configuration configuration = new Configuration(); + private TestingMetricListener listener; + private MetricManager metricManager; + + @BeforeEach + public void init() { + listener = new TestingMetricListener(configuration); + metricManager = + MetricManager.createFlinkBlueGreenDeploymentMetricManager( + configuration, listener.getMetricGroup()); + } + + @Test + public void testStateCountMetricsSameNamespace() { + var deployment1 = buildBlueGreenDeployment("deployment1", TEST_NAMESPACE); + var deployment2 = buildBlueGreenDeployment("deployment2", TEST_NAMESPACE); + + var counterId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, TEST_NAMESPACE, COUNTER_NAME); + assertTrue(listener.getGauge(counterId).isEmpty()); + + // Both deployments start in INITIALIZING_BLUE + metricManager.onUpdate(deployment1); + metricManager.onUpdate(deployment2); + + assertEquals(2, listener.getGauge(counterId).get().getValue()); + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 2); + + // Move deployment1 to ACTIVE_BLUE + deployment1.getStatus().setBlueGreenState(ACTIVE_BLUE); + metricManager.onUpdate(deployment1); + + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 1); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 1); + + // Move deployment2 to ACTIVE_BLUE as well + deployment2.getStatus().setBlueGreenState(ACTIVE_BLUE); + metricManager.onUpdate(deployment2); + + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 0); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 2); + + // Remove deployments + metricManager.onRemove(deployment1); + assertEquals(1, listener.getGauge(counterId).get().getValue()); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 1); + + metricManager.onRemove(deployment2); + assertEquals(0, listener.getGauge(counterId).get().getValue()); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 0); + } + + @Test + public void testStateCountMetricsMultiNamespace() { + var namespace1 = "ns1"; + var namespace2 = "ns2"; + var deployment1 = buildBlueGreenDeployment("deployment", namespace1); + var deployment2 = buildBlueGreenDeployment("deployment", namespace2); + + var counterId1 = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, namespace1, COUNTER_NAME); + var counterId2 = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, namespace2, COUNTER_NAME); + + assertTrue(listener.getGauge(counterId1).isEmpty()); + assertTrue(listener.getGauge(counterId2).isEmpty()); + + metricManager.onUpdate(deployment1); + metricManager.onUpdate(deployment2); + + assertEquals(1, listener.getGauge(counterId1).get().getValue()); + assertEquals(1, listener.getGauge(counterId2).get().getValue()); + assertStateCount(namespace1, INITIALIZING_BLUE, 1); + assertStateCount(namespace2, INITIALIZING_BLUE, 1); + + // Move deployment1 to different state + deployment1.getStatus().setBlueGreenState(ACTIVE_BLUE); + metricManager.onUpdate(deployment1); + + assertStateCount(namespace1, INITIALIZING_BLUE, 0); + assertStateCount(namespace1, ACTIVE_BLUE, 1); + // namespace2 should be unaffected + assertStateCount(namespace2, INITIALIZING_BLUE, 1); + + metricManager.onRemove(deployment1); + metricManager.onRemove(deployment2); + + assertEquals(0, listener.getGauge(counterId1).get().getValue()); + assertEquals(0, listener.getGauge(counterId2).get().getValue()); + } + + @Test + public void testAllBlueGreenStatesHaveMetrics() { + var deployment = buildBlueGreenDeployment("test-deployment", TEST_NAMESPACE); + metricManager.onUpdate(deployment); + + // Verify each state has a gauge registered + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + var stateId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + TEST_NAMESPACE, + BG_STATE_GROUP_NAME, + state.name(), + COUNTER_NAME); + assertTrue( + listener.getGauge(stateId).isPresent(), + "Metric should exist for state: " + state); + } + } + + @Test + public void testFullLifecycleStateCountUpdates() { + var deployment = buildBlueGreenDeployment("test", TEST_NAMESPACE); + + // Start in INITIALIZING_BLUE + metricManager.onUpdate(deployment); + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 1); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 0); + + // Transition to ACTIVE_BLUE + deployment.getStatus().setBlueGreenState(ACTIVE_BLUE); + metricManager.onUpdate(deployment); + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 0); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 1); + + // Transition to TRANSITIONING_TO_GREEN + deployment.getStatus().setBlueGreenState(TRANSITIONING_TO_GREEN); + metricManager.onUpdate(deployment); + assertStateCount(TEST_NAMESPACE, ACTIVE_BLUE, 0); + assertStateCount(TEST_NAMESPACE, TRANSITIONING_TO_GREEN, 1); + + // Transition to ACTIVE_GREEN + deployment.getStatus().setBlueGreenState(ACTIVE_GREEN); + metricManager.onUpdate(deployment); + assertStateCount(TEST_NAMESPACE, TRANSITIONING_TO_GREEN, 0); + assertStateCount(TEST_NAMESPACE, ACTIVE_GREEN, 1); + } + + @Test + public void testMetricsDisabled() { + var conf = new Configuration(); + conf.set(OPERATOR_RESOURCE_METRICS_ENABLED, false); + var disabledListener = new TestingMetricListener(conf); + var disabledMetricManager = + MetricManager.createFlinkBlueGreenDeploymentMetricManager( + conf, disabledListener.getMetricGroup()); + + var deployment = buildBlueGreenDeployment("test", TEST_NAMESPACE); + + var counterId = + disabledListener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, TEST_NAMESPACE, COUNTER_NAME); + + disabledMetricManager.onUpdate(deployment); + assertTrue(disabledListener.getGauge(counterId).isEmpty()); + + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + var statusId = + disabledListener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + TEST_NAMESPACE, + BG_STATE_GROUP_NAME, + state.name(), + COUNTER_NAME); + assertTrue(disabledListener.getGauge(statusId).isEmpty()); + } + } + + @Test + public void testRepeatedUpdatesDoNotDuplicateCount() { + var deployment = buildBlueGreenDeployment("test", TEST_NAMESPACE); + + // Multiple updates in same state should not increase count + metricManager.onUpdate(deployment); + metricManager.onUpdate(deployment); + metricManager.onUpdate(deployment); + + var counterId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, TEST_NAMESPACE, COUNTER_NAME); + assertEquals(1, listener.getGauge(counterId).get().getValue()); + assertStateCount(TEST_NAMESPACE, INITIALIZING_BLUE, 1); + } + + @Test + public void testJobStatusGaugeTracking() { + var deployment1 = buildBlueGreenDeployment("deployment1", TEST_NAMESPACE); + var deployment2 = buildBlueGreenDeployment("deployment2", TEST_NAMESPACE); + + var runningId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + TEST_NAMESPACE, + JOB_STATUS_GROUP_NAME, + JobStatus.RUNNING.name(), + COUNTER_NAME); + var failingId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + TEST_NAMESPACE, + JOB_STATUS_GROUP_NAME, + JobStatus.FAILING.name(), + COUNTER_NAME); + + assertTrue(listener.getGauge(runningId).isEmpty()); + assertTrue(listener.getGauge(failingId).isEmpty()); + + // Both start with RUNNING + deployment1.getStatus().getJobStatus().setState(JobStatus.RUNNING); + deployment2.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment1); + metricManager.onUpdate(deployment2); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.RUNNING, 2); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.FAILING, 0); + + // deployment1 transitions to FAILING + deployment1.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment1); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.RUNNING, 1); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.FAILING, 1); + + // deployment2 also transitions to FAILING + deployment2.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment2); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.RUNNING, 0); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.FAILING, 2); + + // Remove deployment1 + metricManager.onRemove(deployment1); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.FAILING, 1); + + // Remove deployment2 + metricManager.onRemove(deployment2); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.FAILING, 0); + } + + @Test + public void testFailuresCounterIncrementsOnTransitionToFailing() { + var deployment = buildBlueGreenDeployment("test", TEST_NAMESPACE); + + var failuresId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, TEST_NAMESPACE, FAILURES_COUNTER_NAME); + + assertTrue(listener.getCounter(failuresId).isEmpty()); + + // Start with RUNNING + deployment.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment); + assertEquals(0L, listener.getCounter(failuresId).get().getCount()); + + // First transition to FAILING - counter increments + deployment.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment); + assertEquals(1L, listener.getCounter(failuresId).get().getCount()); + + // Stay in FAILING - counter does NOT increment + metricManager.onUpdate(deployment); + assertEquals(1L, listener.getCounter(failuresId).get().getCount()); + + // Recover to RUNNING - counter stays same (never decrements) + deployment.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment); + assertEquals(1L, listener.getCounter(failuresId).get().getCount()); + + // Second transition to FAILING - counter increments again + deployment.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment); + assertEquals(2L, listener.getCounter(failuresId).get().getCount()); + } + + @Test + public void testFailuresCounterMultipleDeployments() { + var deployment1 = buildBlueGreenDeployment("deployment1", TEST_NAMESPACE); + var deployment2 = buildBlueGreenDeployment("deployment2", TEST_NAMESPACE); + + var failuresId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, TEST_NAMESPACE, FAILURES_COUNTER_NAME); + + assertTrue(listener.getCounter(failuresId).isEmpty()); + + // Both start RUNNING + deployment1.getStatus().getJobStatus().setState(JobStatus.RUNNING); + deployment2.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment1); + metricManager.onUpdate(deployment2); + assertEquals(0L, listener.getCounter(failuresId).get().getCount()); + + // deployment1 fails + deployment1.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment1); + assertEquals(1L, listener.getCounter(failuresId).get().getCount()); + + // deployment2 fails + deployment2.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment2); + assertEquals(2L, listener.getCounter(failuresId).get().getCount()); + + // deployment1 recovers - counter stays 2 + deployment1.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment1); + assertEquals(2L, listener.getCounter(failuresId).get().getCount()); + + // Remove deployments - counter stays 2 (historical, never decrements) + metricManager.onRemove(deployment1); + assertEquals(2L, listener.getCounter(failuresId).get().getCount()); + metricManager.onRemove(deployment2); + assertEquals(2L, listener.getCounter(failuresId).get().getCount()); + } + + @Test + public void testFailuresCounterIsolatedByNamespace() { + var namespace1 = "ns1"; + var namespace2 = "ns2"; + var deployment1 = buildBlueGreenDeployment("deployment", namespace1); + var deployment2 = buildBlueGreenDeployment("deployment", namespace2); + + var failuresId1 = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, namespace1, FAILURES_COUNTER_NAME); + var failuresId2 = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, namespace2, FAILURES_COUNTER_NAME); + + assertTrue(listener.getCounter(failuresId1).isEmpty()); + assertTrue(listener.getCounter(failuresId2).isEmpty()); + + // Initialize both namespaces first with RUNNING deployments + deployment1.getStatus().getJobStatus().setState(JobStatus.RUNNING); + deployment2.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment1); + metricManager.onUpdate(deployment2); + assertEquals(0L, listener.getCounter(failuresId1).get().getCount()); + assertEquals(0L, listener.getCounter(failuresId2).get().getCount()); + + // deployment1 in ns1 fails + deployment1.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment1); + + // Only ns1 counter increments + assertEquals(1L, listener.getCounter(failuresId1).get().getCount()); + assertEquals(0L, listener.getCounter(failuresId2).get().getCount()); + + // deployment2 in ns2 fails + deployment2.getStatus().getJobStatus().setState(JobStatus.FAILING); + metricManager.onUpdate(deployment2); + + // Counters are isolated + assertEquals(1L, listener.getCounter(failuresId1).get().getCount()); + assertEquals(1L, listener.getCounter(failuresId2).get().getCount()); + } + + @Test + public void testAllJobStatusesHaveMetrics() { + var deployment = buildBlueGreenDeployment("test-deployment", TEST_NAMESPACE); + deployment.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment); + + // Verify each JobStatus has a gauge registered + for (JobStatus status : JobStatus.values()) { + var statusId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + TEST_NAMESPACE, + JOB_STATUS_GROUP_NAME, + status.name(), + COUNTER_NAME); + assertTrue( + listener.getGauge(statusId).isPresent(), + "Metric should exist for JobStatus: " + status); + } + } + + @Test + public void testJobStatusDoesNotDoubleCount() { + var deployment = buildBlueGreenDeployment("test", TEST_NAMESPACE); + + // Start with RUNNING + deployment.getStatus().getJobStatus().setState(JobStatus.RUNNING); + metricManager.onUpdate(deployment); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.RUNNING, 1); + + // Multiple updates in same JobStatus should not duplicate + metricManager.onUpdate(deployment); + metricManager.onUpdate(deployment); + assertJobStatusCount(TEST_NAMESPACE, JobStatus.RUNNING, 1); + } + + private FlinkBlueGreenDeployment buildBlueGreenDeployment(String name, String namespace) { + var deployment = new FlinkBlueGreenDeployment(); + deployment.setMetadata( + new ObjectMetaBuilder() + .withName(name) + .withNamespace(namespace) + .withUid(UUID.randomUUID().toString()) + .withCreationTimestamp(Instant.now().toString()) + .build()); + + var flinkDeploymentSpec = + FlinkDeploymentSpec.builder() + .flinkConfiguration(new ConfigObjectNode()) + .job(JobSpec.builder().upgradeMode(UpgradeMode.STATELESS).build()) + .build(); + + var bgDeploymentSpec = + new FlinkBlueGreenDeploymentSpec( + new HashMap<>(), + FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build()); + + deployment.setSpec(bgDeploymentSpec); + + var status = new FlinkBlueGreenDeploymentStatus(); + status.setBlueGreenState(INITIALIZING_BLUE); + status.setJobStatus(new org.apache.flink.kubernetes.operator.api.status.JobStatus()); + deployment.setStatus(status); + + return deployment; + } + + private void assertStateCount( + String namespace, FlinkBlueGreenDeploymentState state, int expectedCount) { + var stateId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + namespace, + BG_STATE_GROUP_NAME, + state.name(), + COUNTER_NAME); + assertEquals( + expectedCount, + listener.getGauge(stateId).get().getValue(), + "State count mismatch for " + state); + } + + private void assertJobStatusCount(String namespace, JobStatus status, int expectedCount) { + var statusId = + listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + namespace, + JOB_STATUS_GROUP_NAME, + status.name(), + COUNTER_NAME); + assertEquals( + expectedCount, + listener.getGauge(statusId).get().getValue(), + "JobStatus count mismatch for " + status); + } +} diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java new file mode 100644 index 0000000000..ce0af71c5d --- /dev/null +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics.lifecycle; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; +import org.apache.flink.kubernetes.operator.api.spec.ConfigObjectNode; +import org.apache.flink.kubernetes.operator.api.spec.FlinkBlueGreenDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentTemplateSpec; +import org.apache.flink.kubernetes.operator.api.spec.JobSpec; +import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; +import org.apache.flink.kubernetes.operator.metrics.CustomResourceMetrics; +import org.apache.flink.kubernetes.operator.metrics.MetricManager; +import org.apache.flink.kubernetes.operator.metrics.TestingMetricListener; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.HashMap; +import java.util.UUID; + +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.INITIALIZING_BLUE; +import static org.apache.flink.kubernetes.operator.metrics.KubernetesOperatorMetricOptions.OPERATOR_LIFECYCLE_METRICS_ENABLED; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.LIFECYCLE_GROUP_NAME; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.STATE_GROUP_NAME; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TIME_SECONDS_NAME; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_BLUE_TO_GREEN; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_GREEN_TO_BLUE; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_GROUP_NAME; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_INITIAL_DEPLOYMENT; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Tests for {@link BlueGreenLifecycleMetrics}. */ +public class BlueGreenLifecycleMetricsTest { + + private static final String TEST_NAMESPACE = "test-namespace"; + private static final String[] TRANSITIONS = { + TRANSITION_INITIAL_DEPLOYMENT, TRANSITION_BLUE_TO_GREEN, TRANSITION_GREEN_TO_BLUE + }; + + private final Configuration configuration = new Configuration(); + private TestingMetricListener listener; + private MetricManager metricManager; + + @BeforeEach + public void init() { + listener = new TestingMetricListener(configuration); + metricManager = + MetricManager.createFlinkBlueGreenDeploymentMetricManager( + configuration, listener.getMetricGroup()); + } + + @Test + public void testNamespaceHistogramMetricsExist() { + var deployment = buildBlueGreenDeployment("test-deployment", TEST_NAMESPACE); + metricManager.onUpdate(deployment); + + for (String transition : TRANSITIONS) { + assertTrue( + listener.getHistogram( + getNamespaceHistogramId( + TEST_NAMESPACE, TRANSITION_GROUP_NAME, transition)) + .isPresent(), + "Transition histogram should exist for: " + transition); + } + + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + assertTrue( + listener.getHistogram( + getNamespaceHistogramId( + TEST_NAMESPACE, STATE_GROUP_NAME, state.name())) + .isPresent(), + "State time histogram should exist for: " + state); + } + } + + @Test + public void testSystemLevelHistogramsExist() { + var deployment = buildBlueGreenDeployment("test-deployment", TEST_NAMESPACE); + metricManager.onUpdate(deployment); + + for (String transition : TRANSITIONS) { + assertTrue( + listener.getHistogram( + getSystemLevelHistogramId( + listener, TRANSITION_GROUP_NAME, transition)) + .isPresent(), + "System-level transition histogram should exist for: " + transition); + } + + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + assertTrue( + listener.getHistogram( + getSystemLevelHistogramId( + listener, STATE_GROUP_NAME, state.name())) + .isPresent(), + "System-level state time histogram should exist for: " + state); + } + } + + @Test + public void testMultiNamespaceHistogramIsolation() { + var namespace1 = "ns1"; + var namespace2 = "ns2"; + var ns1HistoId = + getNamespaceHistogramId( + namespace1, TRANSITION_GROUP_NAME, TRANSITION_INITIAL_DEPLOYMENT); + + var dep1 = buildBlueGreenDeployment("dep1", namespace1); + metricManager.onUpdate(dep1); + var ns1Histo = listener.getHistogram(ns1HistoId).get(); + + var dep2 = buildBlueGreenDeployment("dep2", namespace1); + metricManager.onUpdate(dep2); + assertTrue( + ns1Histo == listener.getHistogram(ns1HistoId).get(), + "Deployments in same namespace should share histogram instance"); + + var dep3 = buildBlueGreenDeployment("dep3", namespace2); + metricManager.onUpdate(dep3); + var ns2HistoId = + getNamespaceHistogramId( + namespace2, TRANSITION_GROUP_NAME, TRANSITION_INITIAL_DEPLOYMENT); + assertTrue( + ns1Histo != listener.getHistogram(ns2HistoId).get(), + "Different namespaces should have different histogram instances"); + } + + @Test + public void testLifecycleMetricsDisabled() { + var conf = new Configuration(); + conf.set(OPERATOR_LIFECYCLE_METRICS_ENABLED, false); + var disabledMetricManager = + MetricManager.createFlinkBlueGreenDeploymentMetricManager( + conf, new TestingMetricListener(conf).getMetricGroup()); + + assertNull( + getBlueGreenLifecycleMetrics(disabledMetricManager), + "BlueGreenLifecycleMetrics should not be registered when disabled"); + } + + private FlinkBlueGreenDeployment buildBlueGreenDeployment(String name, String namespace) { + var deployment = new FlinkBlueGreenDeployment(); + deployment.setMetadata( + new ObjectMetaBuilder() + .withName(name) + .withNamespace(namespace) + .withUid(UUID.randomUUID().toString()) + .withCreationTimestamp(Instant.now().toString()) + .build()); + + var flinkDeploymentSpec = + FlinkDeploymentSpec.builder() + .flinkConfiguration(new ConfigObjectNode()) + .job(JobSpec.builder().upgradeMode(UpgradeMode.STATELESS).build()) + .build(); + + var bgDeploymentSpec = + new FlinkBlueGreenDeploymentSpec( + new HashMap<>(), + FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build()); + + deployment.setSpec(bgDeploymentSpec); + + var status = new FlinkBlueGreenDeploymentStatus(); + status.setBlueGreenState(INITIALIZING_BLUE); + deployment.setStatus(status); + + return deployment; + } + + private String getNamespaceHistogramId(String namespace, String groupName, String metricName) { + return listener.getNamespaceMetricId( + FlinkBlueGreenDeployment.class, + namespace, + LIFECYCLE_GROUP_NAME, + groupName, + metricName, + TIME_SECONDS_NAME); + } + + private String getSystemLevelHistogramId( + TestingMetricListener metricListener, String groupName, String metricName) { + return metricListener.getMetricId( + String.format( + "%s.%s.%s.%s.%s", + FlinkBlueGreenDeployment.class.getSimpleName(), + LIFECYCLE_GROUP_NAME, + groupName, + metricName, + TIME_SECONDS_NAME)); + } + + private static BlueGreenLifecycleMetrics getBlueGreenLifecycleMetrics( + MetricManager metricManager) { + for (CustomResourceMetrics metrics : metricManager.getRegisteredMetrics()) { + if (metrics instanceof BlueGreenLifecycleMetrics) { + return (BlueGreenLifecycleMetrics) metrics; + } + } + return null; + } +} diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTrackerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTrackerTest.java new file mode 100644 index 0000000000..cbc80eff45 --- /dev/null +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenResourceLifecycleMetricTrackerTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.kubernetes.operator.metrics.lifecycle; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; +import org.apache.flink.kubernetes.operator.config.FlinkOperatorConfiguration; +import org.apache.flink.kubernetes.operator.metrics.OperatorMetricUtils; +import org.apache.flink.metrics.Histogram; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.ACTIVE_GREEN; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.INITIALIZING_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.SAVEPOINTING_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.SAVEPOINTING_GREEN; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.TRANSITIONING_TO_BLUE; +import static org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState.TRANSITIONING_TO_GREEN; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_BLUE_TO_GREEN; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_GREEN_TO_BLUE; +import static org.apache.flink.kubernetes.operator.metrics.lifecycle.BlueGreenLifecycleMetrics.TRANSITION_INITIAL_DEPLOYMENT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** Tests for {@link BlueGreenResourceLifecycleMetricTracker}. */ +public class BlueGreenResourceLifecycleMetricTrackerTest { + + private Map> transitionHistos; + private Map> stateTimeHistos; + + @BeforeEach + void setUp() { + transitionHistos = new ConcurrentHashMap<>(); + transitionHistos.put(TRANSITION_INITIAL_DEPLOYMENT, List.of(createHistogram())); + transitionHistos.put(TRANSITION_BLUE_TO_GREEN, List.of(createHistogram())); + transitionHistos.put(TRANSITION_GREEN_TO_BLUE, List.of(createHistogram())); + + stateTimeHistos = new ConcurrentHashMap<>(); + for (FlinkBlueGreenDeploymentState state : FlinkBlueGreenDeploymentState.values()) { + stateTimeHistos.put(state, List.of(createHistogram())); + } + } + + @Test + void testInitialDeploymentRecordsTransitionTime() { + long ts = 0; + var tracker = createTracker(INITIALIZING_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 5)); + + assertTransitionRecorded(TRANSITION_INITIAL_DEPLOYMENT, 5); + } + + @Test + void testInitialDeploymentRecordsStateTime() { + long ts = 0; + var tracker = createTracker(INITIALIZING_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 2)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 5)); + + assertStateTimeRecorded(INITIALIZING_BLUE, 2); + assertStateTimeRecorded(TRANSITIONING_TO_BLUE, 5); + } + + @Test + void testBlueToGreenRecordsTransitionTime() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(SAVEPOINTING_BLUE, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 10)); + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 5)); + + assertTransitionRecorded(TRANSITION_BLUE_TO_GREEN, 15); + } + + @Test + void testBlueToGreenRecordsAllIntermediateStateTimes() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(SAVEPOINTING_BLUE, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 10)); + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 3)); + + assertStateTimeRecorded(ACTIVE_BLUE, 5); + assertStateTimeRecorded(SAVEPOINTING_BLUE, 10); + assertStateTimeRecorded(TRANSITIONING_TO_GREEN, 3); + } + + @Test + void testGreenToBlueRecordsTransitionTime() { + long ts = 0; + var tracker = createTracker(ACTIVE_GREEN, Instant.ofEpochSecond(ts)); + tracker.onUpdate(SAVEPOINTING_GREEN, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 8)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 2)); + + assertTransitionRecorded(TRANSITION_GREEN_TO_BLUE, 10); + } + + @Test + void testGreenToBlueRecordsAllIntermediateStateTimes() { + long ts = 0; + var tracker = createTracker(ACTIVE_GREEN, Instant.ofEpochSecond(ts)); + tracker.onUpdate(SAVEPOINTING_GREEN, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 8)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 2)); + + assertStateTimeRecorded(ACTIVE_GREEN, 5); + assertStateTimeRecorded(SAVEPOINTING_GREEN, 8); + assertStateTimeRecorded(TRANSITIONING_TO_BLUE, 2); + } + + @Test + void testSameStateUpdatesOnlyUpdateTimestamp() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 1)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 1)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 1)); + + assertNoTransitionRecorded(TRANSITION_BLUE_TO_GREEN); + assertNoTransitionRecorded(TRANSITION_GREEN_TO_BLUE); + assertNoStateTimeRecorded(ACTIVE_BLUE); + } + + @Test + void testIntermediateStateMetricsOnlyRecordedAtStableState() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(SAVEPOINTING_BLUE, Instant.ofEpochSecond(ts += 5)); + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 5)); + + assertNoStateTimeRecorded(ACTIVE_BLUE); + assertNoStateTimeRecorded(SAVEPOINTING_BLUE); + + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 5)); + + assertStateTimeRecorded(ACTIVE_BLUE, 5); + assertStateTimeRecorded(SAVEPOINTING_BLUE, 5); + assertStateTimeRecorded(TRANSITIONING_TO_GREEN, 5); + } + + @Test + void testRecoveryFromActiveStateTracksNextTransition() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 10)); + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 5)); + + assertTransitionRecorded(TRANSITION_BLUE_TO_GREEN, 5); + } + + @Test + void testConsecutiveTransitionsEachTrackedIndependently() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 10)); + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 5)); + + assertTransitionRecorded(TRANSITION_BLUE_TO_GREEN, 5); + + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 8)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 2)); + + assertTransitionRecorded(TRANSITION_GREEN_TO_BLUE, 2); + } + + @Test + void testFailedBlueToGreenRollbackRecordsStateTimes() { + long ts = 0; + var tracker = createTracker(ACTIVE_BLUE, Instant.ofEpochSecond(ts)); + + tracker.onUpdate(SAVEPOINTING_BLUE, Instant.ofEpochSecond(ts += 10)); + tracker.onUpdate(TRANSITIONING_TO_GREEN, Instant.ofEpochSecond(ts += 15)); + tracker.onUpdate(ACTIVE_BLUE, Instant.ofEpochSecond(ts += 5)); + + assertStateTimeRecorded(ACTIVE_BLUE, 10); + assertStateTimeRecorded(SAVEPOINTING_BLUE, 15); + assertStateTimeRecorded(TRANSITIONING_TO_GREEN, 5); + } + + @Test + void testFailedGreenToBlueRollbackRecordsStateTimes() { + long ts = 0; + var tracker = createTracker(ACTIVE_GREEN, Instant.ofEpochSecond(ts)); + + tracker.onUpdate(SAVEPOINTING_GREEN, Instant.ofEpochSecond(ts += 8)); + tracker.onUpdate(TRANSITIONING_TO_BLUE, Instant.ofEpochSecond(ts += 12)); + tracker.onUpdate(ACTIVE_GREEN, Instant.ofEpochSecond(ts += 3)); + + assertStateTimeRecorded(ACTIVE_GREEN, 8); + assertStateTimeRecorded(SAVEPOINTING_GREEN, 12); + assertStateTimeRecorded(TRANSITIONING_TO_BLUE, 3); + } + + private BlueGreenResourceLifecycleMetricTracker createTracker( + FlinkBlueGreenDeploymentState initialState, Instant initialTime) { + return new BlueGreenResourceLifecycleMetricTracker( + initialState, initialTime, transitionHistos, stateTimeHistos); + } + + private Histogram createHistogram() { + return OperatorMetricUtils.createHistogram( + FlinkOperatorConfiguration.fromConfiguration(new Configuration())); + } + + private void assertTransitionRecorded(String transitionName, long expectedSeconds) { + var stats = transitionHistos.get(transitionName).get(0).getStatistics(); + assertEquals(1, stats.size(), transitionName + " should have 1 sample"); + assertEquals(expectedSeconds, (long) stats.getMean(), transitionName + " duration"); + } + + private void assertNoTransitionRecorded(String transitionName) { + var stats = transitionHistos.get(transitionName).get(0).getStatistics(); + assertEquals(0, stats.size(), transitionName + " should have no samples"); + } + + private void assertStateTimeRecorded( + FlinkBlueGreenDeploymentState state, long expectedSeconds) { + var stats = stateTimeHistos.get(state).get(0).getStatistics(); + assertEquals(1, stats.size(), state + " should have 1 sample"); + assertEquals(expectedSeconds, (long) stats.getMean(), state + " duration"); + } + + private void assertNoStateTimeRecorded(FlinkBlueGreenDeploymentState state) { + var stats = stateTimeHistos.get(state).get(0).getStatistics(); + assertEquals(0, stats.size(), state + " should have no samples"); + } +} From e60b2580d1948aa47309d45c4fdb2bbdf3a675fa Mon Sep 17 00:00:00 2001 From: James Kan Date: Thu, 11 Dec 2025 12:15:22 -0800 Subject: [PATCH 3/7] all shop changes together for deployment testing --- .../api/bluegreen/BlueGreenDiffType.java | 6 + .../bluegreen/BlueGreenDeploymentService.java | 57 ++++++- .../FlinkBlueGreenDeploymentSpecDiff.java | 19 ++- .../utils/bluegreen/BlueGreenUtils.java | 18 ++- ...linkBlueGreenDeploymentControllerTest.java | 62 ++++++++ .../FlinkBlueGreenDeploymentSpecDiffTest.java | 57 ++++++- .../utils/bluegreen/BlueGreenUtilsTest.java | 141 ++++++++++++++++++ 7 files changed, 346 insertions(+), 14 deletions(-) diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java index 44d2b67609..97a83ec212 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/api/bluegreen/BlueGreenDiffType.java @@ -28,6 +28,12 @@ public enum BlueGreenDiffType { /** Changes that only affect the child FlinkDeploymentSpec. */ PATCH_CHILD, + /** + * Full redeploy from user-specified savepoint. Triggered when savepointRedeployNonce changes. + * Uses the initialSavepointPath from spec instead of taking a new savepoint. + */ + SAVEPOINT_REDEPLOY, + /** * In-place suspension. Triggered when job.state changes from RUNNING to SUSPENDED. Suspends the * currently active child without creating a new deployment. diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java index 5ca8b397cd..397f11e3da 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import java.time.Instant; +import java.util.Objects; import static org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenKubernetesService.deleteFlinkDeployment; import static org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenKubernetesService.deployCluster; @@ -169,6 +170,25 @@ public UpdateControl checkAndInitiateDeployment( context.getDeploymentStatus().setSavepointTriggerId(null); return markDeploymentFailing(context, error); } + + } else if (specDiff == BlueGreenDiffType.SAVEPOINT_REDEPLOY) { + // Savepoint redeploy: skip taking a new savepoint, use initialSavepointPath + var jobSpec = + context.getBgDeployment().getSpec().getTemplate().getSpec().getJob(); + LOG.info( + "Savepoint redeploy triggered for '{}', using initialSavepointPath: {}", + context.getBgDeployment().getMetadata().getName(), + Objects.toString(jobSpec.getInitialSavepointPath(), "")); + setLastReconciledSpec(context); + try { + return startSavepointRedeployTransition( + context, currentBlueGreenDeploymentType); + } catch (Exception e) { + var error = + "Could not start Savepoint Redeploy Transition. Details: " + + e.getMessage(); + return markDeploymentFailing(context, error); + } } else { setLastReconciledSpec(context); LOG.info( @@ -210,6 +230,13 @@ private boolean isChildSuspended(FlinkDeployment deployment) { private UpdateControl patchFlinkDeployment( BlueGreenContext context, BlueGreenDeploymentType blueGreenDeploymentTypeToPatch) { + return patchFlinkDeployment(context, blueGreenDeploymentTypeToPatch, true); + } + + private UpdateControl patchFlinkDeployment( + BlueGreenContext context, + BlueGreenDeploymentType blueGreenDeploymentTypeToPatch, + boolean carryOverSavepointInPatch) { String childDeploymentName = context.getBgDeployment().getMetadata().getName() @@ -228,7 +255,10 @@ private UpdateControl patchFlinkDeployment( // will it be used by this patching? otherwise this is unnecessary, keep lastSavepoint = // null. Savepoint lastSavepoint = - carryOverSavepoint(context, blueGreenDeploymentTypeToPatch, childDeploymentName); + carryOverSavepointInPatch + ? carryOverSavepoint( + context, blueGreenDeploymentTypeToPatch, childDeploymentName) + : null; return initiateDeployment( context, @@ -286,6 +316,26 @@ private UpdateControl startTransition( false); } + /** + * Starts a transition for savepoint redeploy scenario. Unlike normal transitions, this does not + * take a new savepoint - it uses the initialSavepointPath specified in the spec. + * + * @param context the transition context + * @param currentBlueGreenDeploymentType the current deployment type + * @return UpdateControl for the deployment + */ + private UpdateControl startSavepointRedeployTransition( + BlueGreenContext context, BlueGreenDeploymentType currentBlueGreenDeploymentType) { + DeploymentTransition transition = calculateTransition(currentBlueGreenDeploymentType); + + return initiateDeployment( + context, + transition.nextBlueGreenDeploymentType, + transition.nextState, + null, // Use initialSavepointPath from spec + false); + } + private DeploymentTransition calculateTransition(BlueGreenDeploymentType currentType) { if (BlueGreenDeploymentType.BLUE == currentType) { return new DeploymentTransition( @@ -493,7 +543,10 @@ private UpdateControl handleSpecChangesDuringTransitio context.getDeploymentByType(oppositeDeploymentType) .getMetadata() .getName()); - return patchFlinkDeployment(context, oppositeDeploymentType); + return patchFlinkDeployment( + context, + oppositeDeploymentType, + diffType != BlueGreenDiffType.SAVEPOINT_REDEPLOY); } } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java index e71d9283d5..3e5dba77cb 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiff.java @@ -78,19 +78,26 @@ public BlueGreenDiffType compare() { DiffResult diffResult = new ReflectiveDiffBuilder<>(deploymentMode, leftSpec, rightSpec).build(); - // Case 1: FlinkDeploymentSpecs are identical + // Extract the diff type from ReflectiveDiffBuilder result + DiffType diffType = diffResult.getType(); + + // Case 1: Check for savepoint redeploy first (savepointRedeployNonce changed) + // This takes precedence as it indicates user wants to redeploy from their + // initialSavepointPath + if (diffType == DiffType.SAVEPOINT_REDEPLOY) { + return BlueGreenDiffType.SAVEPOINT_REDEPLOY; + } + + // Case 2: FlinkDeploymentSpecs are identical if (leftSpec.equals(rightSpec)) { return BlueGreenDiffType.IGNORE; } - // Extract the diff type from ReflectiveDiffBuilder result - DiffType diffType = diffResult.getType(); - - // Case 2: ReflectiveDiffBuilder returns IGNORE + // Case 3: ReflectiveDiffBuilder returns IGNORE if (diffType == DiffType.IGNORE) { return BlueGreenDiffType.PATCH_CHILD; } else { - // Case 3: ReflectiveDiffBuilder returns anything else map it to TRANSITION as well + // Case 4: ReflectiveDiffBuilder returns anything else map it to TRANSITION return BlueGreenDiffType.TRANSITION; } } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java index 98344a9ba8..297e262cee 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java @@ -332,7 +332,9 @@ public static FlinkDeployment prepareFlinkDeployment( "spec", FlinkBlueGreenDeploymentSpec.class); - // The Blue/Green initialSavepointPath is only used for first-time deployments + // The Blue/Green initialSavepointPath is used for: + // 1. First-time deployments + // 2. When savepointRedeployNonce changes (lastCheckpoint will be null) if (isFirstDeployment) { String initialSavepointPath = spec.getTemplate().getSpec().getJob().getInitialSavepointPath(); @@ -346,6 +348,20 @@ public static FlinkDeployment prepareFlinkDeployment( String location = lastCheckpoint.getLocation().replace("file:", ""); LOG.info("Using Blue/Green savepoint/checkpoint: " + location); spec.getTemplate().getSpec().getJob().setInitialSavepointPath(location); + } else { + // lastCheckpoint is null in two scenarios: + // 1. savepointRedeployNonce changed - user wants to redeploy from initialSavepointPath + // 2. upgradeMode is STATELESS - don't take savepoints (but use initialSavepointPath if + // set) + String initialSavepointPath = + spec.getTemplate().getSpec().getJob().getInitialSavepointPath(); + if (initialSavepointPath != null && !initialSavepointPath.isEmpty()) { + LOG.info( + "Using user-specified initialSavepointPath for redeploy: {}", + initialSavepointPath); + } else { + LOG.info("Starting fresh with no savepoint restoration"); + } } flinkDeployment.setSpec(spec.getTemplate().getSpec()); diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java index 49e56561d0..bc571c5d4d 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java @@ -189,6 +189,68 @@ private TestingFlinkBlueGreenDeploymentController.BlueGreenReconciliationResult return rs; } + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifySavepointRedeployNonceTriggersTransitionWithInitialSavepointPath( + FlinkVersion flinkVersion) throws Exception { + // Start with SAVEPOINT upgrade mode so normally a savepoint would be taken + var blueGreenDeployment = + buildSessionCluster( + TEST_DEPLOYMENT_NAME, + TEST_NAMESPACE, + flinkVersion, + null, + UpgradeMode.SAVEPOINT); + var rs = executeBasicDeployment(flinkVersion, blueGreenDeployment, false, null); + + // Set initialSavepointPath and bump savepointRedeployNonce + String userSpecifiedSavepoint = "s3://bucket/my-specific-savepoint"; + rs.deployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setInitialSavepointPath(userSpecifiedSavepoint); + rs.deployment.getSpec().getTemplate().getSpec().getJob().setSavepointRedeployNonce(12345L); + rs.deployment + .getSpec() + .getConfiguration() + .put(DEPLOYMENT_DELETION_DELAY.key(), String.valueOf(ALT_DELETION_DELAY_VALUE)); + kubernetesClient.resource(rs.deployment).createOrReplace(); + + // Reconcile - should skip savepointing and go directly to transition + rs = reconcile(rs.deployment); + + // Verify: Should be TRANSITIONING_TO_GREEN (not SAVEPOINTING_BLUE) + assertEquals( + FlinkBlueGreenDeploymentState.TRANSITIONING_TO_GREEN, + rs.reconciledStatus.getBlueGreenState()); + + // Verify: Green deployment should use the user-specified initialSavepointPath + var flinkDeployments = getFlinkDeployments(); + assertEquals(2, flinkDeployments.size()); + assertEquals( + userSpecifiedSavepoint, + flinkDeployments.get(1).getSpec().getJob().getInitialSavepointPath()); + + // Complete the transition + simulateSuccessfulJobStart(flinkDeployments.get(1)); + rs = reconcile(rs.deployment); + + // Wait for deletion delay + Thread.sleep(rs.updateControl.getScheduleDelay().get()); + reconcile(rs.deployment); + + // Verify Green is now active + flinkDeployments = getFlinkDeployments(); + assertEquals(1, flinkDeployments.size()); + + rs = reconcile(rs.deployment); + assertEquals( + FlinkBlueGreenDeploymentState.ACTIVE_GREEN, + rs.reconciledStatus.getBlueGreenState()); + } + @ParameterizedTest @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") public void verifySuspendAndResumeInPlace(FlinkVersion flinkVersion) throws Exception { diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java index fdfd9c59e2..1e60b83f4b 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/reconciler/diff/FlinkBlueGreenDeploymentSpecDiffTest.java @@ -225,13 +225,28 @@ public void testTransitionForNestedSpecDifference() { FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); - // Change nested spec property - setSavepointRedeployNonce triggers TRANSITION + // Change nested spec property - setSavepointRedeployNonce now triggers SAVEPOINT_REDEPLOY spec2.getTemplate().getSpec().getJob().setSavepointRedeployNonce(12345L); FlinkBlueGreenDeploymentSpecDiff diff = new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); - assertEquals(BlueGreenDiffType.TRANSITION, diff.compare()); + assertEquals(BlueGreenDiffType.SAVEPOINT_REDEPLOY, diff.compare()); + } + + @Test + public void testSavepointRedeployForNonceChangeWithJarUpdate() { + FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); + FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); + + // Nonce change with additional jarURI change - SAVEPOINT_REDEPLOY should still be detected + spec2.getTemplate().getSpec().getJob().setJarURI("local:///opt/flink/examples/other.jar"); + spec2.getTemplate().getSpec().getJob().setSavepointRedeployNonce(12345L); + + FlinkBlueGreenDeploymentSpecDiff diff = + new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); + + assertEquals(BlueGreenDiffType.SAVEPOINT_REDEPLOY, diff.compare()); } @Test @@ -305,8 +320,8 @@ public void testTransitionForTopLevelAndNestedDifferences() { FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); // Change both top-level (configuration) and nested spec - // With new logic, only nested spec changes matter - setSavepointRedeployNonce triggers - // TRANSITION + // With new logic, only nested spec changes matter - setSavepointRedeployNonce now + // triggers SAVEPOINT_REDEPLOY Map config = new HashMap<>(); config.put("custom.config", "different-value"); spec2.setConfiguration(config); @@ -315,7 +330,39 @@ public void testTransitionForTopLevelAndNestedDifferences() { FlinkBlueGreenDeploymentSpecDiff diff = new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); - assertEquals(BlueGreenDiffType.TRANSITION, diff.compare()); + assertEquals(BlueGreenDiffType.SAVEPOINT_REDEPLOY, diff.compare()); + } + + @Test + public void testSavepointRedeployTakesPrecedenceOverScale() { + FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); + FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); + + // Change both nonce (SAVEPOINT_REDEPLOY) AND parallelism (SCALE) + // SAVEPOINT_REDEPLOY should take precedence + spec2.getTemplate().getSpec().getJob().setSavepointRedeployNonce(12345L); + spec2.getTemplate().getSpec().getJob().setParallelism(10); + + FlinkBlueGreenDeploymentSpecDiff diff = + new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); + + assertEquals(BlueGreenDiffType.SAVEPOINT_REDEPLOY, diff.compare()); + } + + @Test + public void testSavepointRedeployTakesPrecedenceOverUpgrade() { + FlinkBlueGreenDeploymentSpec spec1 = createBasicSpec(); + FlinkBlueGreenDeploymentSpec spec2 = createBasicSpec(); + + // Change both nonce (SAVEPOINT_REDEPLOY) AND Flink version (UPGRADE) + // SAVEPOINT_REDEPLOY should take precedence + spec2.getTemplate().getSpec().getJob().setSavepointRedeployNonce(12345L); + spec2.getTemplate().getSpec().setFlinkVersion(FlinkVersion.v1_17); + + FlinkBlueGreenDeploymentSpecDiff diff = + new FlinkBlueGreenDeploymentSpecDiff(DEPLOYMENT_MODE, spec1, spec2); + + assertEquals(BlueGreenDiffType.SAVEPOINT_REDEPLOY, diff.compare()); } @Test diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java index 859d19f5d3..a898662ec8 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java @@ -27,6 +27,9 @@ import org.apache.flink.kubernetes.operator.api.spec.JobSpec; import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; +import org.apache.flink.kubernetes.operator.api.status.Savepoint; +import org.apache.flink.kubernetes.operator.api.status.SavepointFormatType; +import org.apache.flink.kubernetes.operator.api.status.SnapshotTriggerType; import org.apache.flink.kubernetes.operator.api.utils.SpecUtils; import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenContext; @@ -38,6 +41,9 @@ import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** Tests for {@link BlueGreenUtils}. */ public class BlueGreenUtilsTest { @@ -83,6 +89,141 @@ public void testPrepareFlinkDeploymentWithoutNameReplacement() { assertEquals(parentDeploymentName + ".jm", resultFlinkConfig.get("metrics.scope.jm")); } + @Test + public void testSavepointRequiredBasedOnUpgradeMode() { + // SAVEPOINT mode requires savepoint + FlinkBlueGreenDeployment bgDeployment = + buildBlueGreenDeployment("test-app", TEST_NAMESPACE); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.SAVEPOINT); + BlueGreenContext context = createContext(bgDeployment); + assertTrue(BlueGreenUtils.isSavepointRequired(context)); + + // LAST_STATE mode requires savepoint + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.LAST_STATE); + assertTrue(BlueGreenUtils.isSavepointRequired(context)); + + // STATELESS mode does not require savepoint + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.STATELESS); + assertFalse(BlueGreenUtils.isSavepointRequired(context)); + } + + @Test + public void testPrepareFlinkDeploymentStatelessInitialSavepointPath() { + // Setup: STATELESS mode with initialSavepointPath set + FlinkBlueGreenDeployment bgDeployment = + buildBlueGreenDeployment("test-app", TEST_NAMESPACE); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.STATELESS); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setInitialSavepointPath("s3://bucket/savepoint-xyz"); + + BlueGreenContext context = createContext(bgDeployment); + + // Act: Prepare deployment with null lastCheckpoint (STATELESS transition) + FlinkDeployment result = + BlueGreenUtils.prepareFlinkDeployment( + context, + BlueGreenDeploymentType.GREEN, + null, // No lastCheckpoint + false, // Not first deployment + bgDeployment.getMetadata()); + + // Assert: initialSavepointPath should be used for STATELESS + assertNotNull(result.getSpec().getJob().getInitialSavepointPath()); + } + + @Test + public void testNullLastCheckpointUsesInitialSavepointPath() { + // lastCheckpoint=null -> use initialSavepointPath from spec + FlinkBlueGreenDeployment bgDeployment = + buildBlueGreenDeployment("test-app", TEST_NAMESPACE); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.SAVEPOINT); + String initialPath = "s3://bucket/user-specified-savepoint"; + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setInitialSavepointPath(initialPath); + + BlueGreenContext context = createContext(bgDeployment); + + FlinkDeployment result = + BlueGreenUtils.prepareFlinkDeployment( + context, + BlueGreenDeploymentType.GREEN, + null, // null = nonce changed, no new savepoint taken + false, + bgDeployment.getMetadata()); + + assertEquals(initialPath, result.getSpec().getJob().getInitialSavepointPath()); + } + + @Test + public void testNormalTransitionUsesFreshSavepoint() { + // Normal transition β†’ take fresh savepoint from running job β†’ use that, not + // initialSavepointPath + FlinkBlueGreenDeployment bgDeployment = + buildBlueGreenDeployment("test-app", TEST_NAMESPACE); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setUpgradeMode(UpgradeMode.SAVEPOINT); + bgDeployment + .getSpec() + .getTemplate() + .getSpec() + .getJob() + .setInitialSavepointPath("s3://bucket/ignored"); + + BlueGreenContext context = createContext(bgDeployment); + + String freshSavepoint = "s3://bucket/fresh-savepoint-from-running-job"; + Savepoint triggered = + Savepoint.of( + freshSavepoint, SnapshotTriggerType.UPGRADE, SavepointFormatType.CANONICAL); + + FlinkDeployment result = + BlueGreenUtils.prepareFlinkDeployment( + context, + BlueGreenDeploymentType.GREEN, + triggered, // Fresh savepoint provided + false, + bgDeployment.getMetadata()); + + assertEquals(freshSavepoint, result.getSpec().getJob().getInitialSavepointPath()); + } + private static FlinkBlueGreenDeployment buildBlueGreenDeployment( String name, String namespace) { var deployment = new FlinkBlueGreenDeployment(); From 3bc32a70d2cbdf5c43049a4eb57f1f1b62bf9082 Mon Sep 17 00:00:00 2001 From: Daniel Rossos Date: Tue, 4 Nov 2025 13:47:03 -0500 Subject: [PATCH 4/7] added ingress switcher merge --- .../content/docs/custom-resource/reference.md | 1 + docs/savepoint-redeploy-during-transition.md | 195 +++++++++++++++ e2e-tests/data/bluegreen-ingress.yaml | 58 +++++ .../data/bluegreen-savepoint-redeploy.yaml | 100 ++++++++ e2e-tests/test_bluegreen_ingress_rotation.sh | 133 +++++++++++ .../test_bluegreen_savepoint_redeploy.sh | 226 ++++++++++++++++++ .../spec/FlinkBlueGreenDeploymentSpec.java | 4 + .../kubernetes/operator/FlinkOperator.java | 2 +- .../FlinkBlueGreenDeploymentController.java | 8 + .../bluegreen/BlueGreenDeploymentService.java | 84 +++++++ .../handlers/ActiveStateHandler.java | 4 + .../operator/utils/EventSourceUtils.java | 24 ++ .../operator/utils/IngressUtils.java | 112 ++++++--- .../utils/bluegreen/BlueGreenUtils.java | 8 + ...linkBlueGreenDeploymentControllerTest.java | 147 +++++++++++- ...ingFlinkBlueGreenDeploymentController.java | 2 +- .../utils/bluegreen/BlueGreenUtilsTest.java | 1 + ...uegreendeployments.flink.apache.org-v1.yml | 26 ++ 18 files changed, 1104 insertions(+), 31 deletions(-) create mode 100644 docs/savepoint-redeploy-during-transition.md create mode 100644 e2e-tests/data/bluegreen-ingress.yaml create mode 100644 e2e-tests/data/bluegreen-savepoint-redeploy.yaml create mode 100755 e2e-tests/test_bluegreen_ingress_rotation.sh create mode 100755 e2e-tests/test_bluegreen_savepoint_redeploy.sh diff --git a/docs/content/docs/custom-resource/reference.md b/docs/content/docs/custom-resource/reference.md index 003b6495b0..98d18e7d65 100644 --- a/docs/content/docs/custom-resource/reference.md +++ b/docs/content/docs/custom-resource/reference.md @@ -89,6 +89,7 @@ This serves as a full reference for FlinkDeployment and FlinkSessionJob custom r | Parameter | Type | Docs | | ----------| ---- | ---- | | configuration | java.util.Map | | +| ingress | org.apache.flink.kubernetes.operator.api.spec.IngressSpec | | | template | org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentTemplateSpec | | ### FlinkDeploymentSpec diff --git a/docs/savepoint-redeploy-during-transition.md b/docs/savepoint-redeploy-during-transition.md new file mode 100644 index 0000000000..94884f921d --- /dev/null +++ b/docs/savepoint-redeploy-during-transition.md @@ -0,0 +1,195 @@ +# Handling savepointRedeployNonce Changes During Transitions + +## Overview + +This document describes the behavior when a user changes `savepointRedeployNonce` while a FlinkBlueGreenDeployment is in the middle of a transition. + +## Scenario + +**Initial State:** +- BLUE deployment is running (ACTIVE_BLUE state) +- User triggers a transition to GREEN (e.g., by changing parallelism) +- State changes to TRANSITIONING_TO_GREEN +- GREEN deployment is starting up, restoring from a savepoint taken from BLUE + +**User Action During Transition:** +- While in TRANSITIONING_TO_GREEN state +- User changes `savepointRedeployNonce` to a new value +- User sets `initialSavepointPath` to a different savepoint location + +## Implemented Behavior (Option 1: Abort Transition) + +### What Happens + +1. **Detection Phase** + - Operator detects spec change during `monitorTransition()` + - Calls `handleSpecChangesDuringTransition()` + - Identifies diff type as `SAVEPOINT_REDEPLOY` + +2. **Abort Phase** + - Operator deletes the in-progress GREEN deployment + - Reverts BlueGreen state to `ACTIVE_BLUE` + - Clears savepoint trigger ID + - Marks spec as reconciled + +3. **Restart Phase** + - Next reconciliation cycle detects new spec + - Triggers a fresh transition to GREEN + - Uses the NEW `initialSavepointPath` specified by user + - GREEN starts from the user-specified savepoint + +### Code Location + +**File:** `BlueGreenDeploymentService.java` + +**Method:** `handleSavepointRedeployDuringTransition()` + +```java +private UpdateControl handleSavepointRedeployDuringTransition( + BlueGreenContext context, + BlueGreenDeploymentType currentBlueGreenDeploymentType) +``` + +**Called from:** `handleSpecChangesDuringTransition()` when `diffType == SAVEPOINT_REDEPLOY` + +## Behavior Details + +### Logs + +When this occurs, the operator logs: +``` +WARN: savepointRedeployNonce changed during transition from BLUE to GREEN. + Aborting current transition and will restart with new savepoint from: s3://bucket/new-savepoint +INFO: Deleted transitioning FlinkDeployment 'my-app-green' due to savepointRedeployNonce change +``` + +### State Transitions + +``` +TRANSITIONING_TO_GREEN + β†’ (nonce changed) + β†’ ACTIVE_BLUE + β†’ (next reconciliation) + β†’ TRANSITIONING_TO_GREEN (with new savepoint) +``` + +### Timing + +- Deletion: Immediate (current reconciliation) +- Restart: Next reconciliation cycle (~5 seconds with default settings) +- Total delay: ~5-10 seconds before new transition begins + +## Why This Approach? + +### User Intent +When a user changes `savepointRedeployNonce` during a transition, their intent is clear: +- "Stop what you're doing" +- "Use THIS specific savepoint instead" +- "Start the transition over with the correct state" + +### Alternative Approaches Considered + +**Option 2: Reject the Change** +- Pros: Simple, preserves in-progress work +- Cons: User has to wait and retry manually, confusing with GitOps + +**Option 3: Queue the Change** +- Pros: No transition interruption +- Cons: Complex implementation, delayed behavior (user expects immediate action) + +**Why Option 1 (Abort) is Better:** +- βœ… Clear semantics: nonce = "redeploy NOW" +- βœ… Matches user expectation +- βœ… Consistent with FlinkDeployment behavior +- βœ… No silent failures (previous behavior) + +## Testing + +### Unit Test Scenario + +**Test Name:** `testSavepointRedeployDuringTransition` + +**Steps:** +1. Create BlueGreen with BLUE running +2. Trigger transition to GREEN (change parallelism) +3. While TRANSITIONING_TO_GREEN, change savepointRedeployNonce +4. Verify GREEN is deleted +5. Verify state reverts to ACTIVE_BLUE +6. Verify next reconciliation starts new transition +7. Verify new GREEN uses new savepoint path + +### E2E Test + +**File:** `e2e-tests/test_bluegreen_savepoint_redeploy.sh` + +**Test Section:** (To be added) +```bash +# Test 4: Nonce change during transition +# 1. Start transition BLUE β†’ GREEN +# 2. Before GREEN is stable, change nonce +# 3. Verify GREEN is deleted +# 4. Verify new transition uses new savepoint +``` + +## Edge Cases + +### Case 1: Deletion Fails +**Scenario:** GREEN deployment cannot be deleted (finalizer stuck, etc.) + +**Behavior:** +- Operator logs warning +- Retries deletion on next reconciliation +- Remains in ACTIVE_BLUE state until deletion succeeds + +### Case 2: Multiple Nonce Changes +**Scenario:** User changes nonce multiple times rapidly + +**Behavior:** +- Each change triggers abort and restart +- Last nonce change wins +- May cause multiple deletion/creation cycles + +**Recommendation:** Wait for state to stabilize before changing nonce again + +### Case 3: Nonce Change Right Before Completion +**Scenario:** GREEN is about to become STABLE when nonce changes + +**Behavior:** +- GREEN is deleted (wasted work) +- Transition restarts from beginning +- New GREEN created with new savepoint + +**Note:** This is intentional - user's explicit nonce change takes precedence + +## Migration from Previous Behavior + +### Old Behavior (Before Fix) +- Nonce change during transition was **silently ignored** +- Green deployment continued with old savepoint +- User's new savepoint path was never used +- Spec marked as reconciled (appearing to work) + +### New Behavior (After Fix) +- Nonce change during transition **aborts transition** +- Green deployment is deleted +- Transition restarts with new savepoint +- User intent is honored + +### Impact +- **Breaking Change:** No (old behavior was buggy) +- **User Visible:** Yes (different logs, temporary deletion) +- **Requires Documentation Update:** Yes + +## Related Features + +- **savepointRedeployNonce**: Triggers redeploy from specific savepoint +- **initialSavepointPath**: Specifies which savepoint to restore from +- **SAVEPOINT_REDEPLOY DiffType**: Detected when nonce changes +- **Blue/Green Transitions**: Normal deployment switching mechanism + +## References + +- Implementation: `BlueGreenDeploymentService.java:440-540` +- Diff Detection: `FlinkBlueGreenDeploymentSpecDiff.java:58-89` +- Nonce Annotation: `JobSpec.java:98` +- E2E Tests: `e2e-tests/test_bluegreen_savepoint_redeploy.sh` diff --git a/e2e-tests/data/bluegreen-ingress.yaml b/e2e-tests/data/bluegreen-ingress.yaml new file mode 100644 index 0000000000..4a7b143e69 --- /dev/null +++ b/e2e-tests/data/bluegreen-ingress.yaml @@ -0,0 +1,58 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkBlueGreenDeployment +metadata: + name: bg-ingress-test +spec: + configuration: + kubernetes.operator.bluegreen.deployment-deletion.delay: "2s" + # Parent-level ingress configuration + ingress: + template: "{{name}}.{{namespace}}.example.com" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/" + template: + spec: + image: flink:1.20 + flinkVersion: v1_20 + flinkConfiguration: + rest.port: "8081" + taskmanager.numberOfTaskSlots: "1" + serviceAccount: flink + jobManager: + resource: + memory: 1G + cpu: 1 + taskManager: + resource: + memory: 2G + cpu: 1 + job: + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 1 + entryClass: org.apache.flink.streaming.examples.statemachine.StateMachineExample + args: + - "--error-rate" + - "0.15" + - "--sleep" + - "30" + upgradeMode: stateless + mode: native diff --git a/e2e-tests/data/bluegreen-savepoint-redeploy.yaml b/e2e-tests/data/bluegreen-savepoint-redeploy.yaml new file mode 100644 index 0000000000..67d5032ad0 --- /dev/null +++ b/e2e-tests/data/bluegreen-savepoint-redeploy.yaml @@ -0,0 +1,100 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkBlueGreenDeployment +metadata: + name: bg-savepoint-redeploy-example +spec: + + configuration: + kubernetes.operator.bluegreen.deployment-deletion.delay: "1s" + template: + spec: + image: flink:1.20 + flinkVersion: v1_20 + flinkConfiguration: + rest.port: "8081" + execution.checkpointing.interval: "10s" + execution.checkpointing.storage: "filesystem" + state.backend.incremental: "true" + state.checkpoints.dir: file:///opt/flink/volume/flink-cp + state.savepoints.dir: file:///opt/flink/volume/flink-sp + state.checkpoints.num-retained: "5" + taskmanager.numberOfTaskSlots: "1" + serviceAccount: flink + jobManager: + resource: + memory: 1G + cpu: 1 + podTemplate: + spec: + containers: + - name: flink-main-container + resources: + requests: + ephemeral-storage: 2048Mi + limits: + ephemeral-storage: 2048Mi + volumeMounts: + - mountPath: /opt/flink/volume + name: flink-volume + volumes: + - name: flink-volume + persistentVolumeClaim: + claimName: flink-bg-sp-redeploy + taskManager: + resource: + memory: 2G + cpu: 1 + job: + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 1 + entryClass: org.apache.flink.streaming.examples.statemachine.StateMachineExample + args: + - "--error-rate" + - "0.15" + - "--sleep" + - "30" + upgradeMode: savepoint + mode: native + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: flink-bg-sp-redeploy +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi + +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: + ingressclass.kubernetes.io/is-default-class: "true" + labels: + app.kubernetes.io/component: controller + name: nginx +spec: + controller: k8s.io/ingress-nginx diff --git a/e2e-tests/test_bluegreen_ingress_rotation.sh b/e2e-tests/test_bluegreen_ingress_rotation.sh new file mode 100755 index 0000000000..6c5c61235f --- /dev/null +++ b/e2e-tests/test_bluegreen_ingress_rotation.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This script tests the Blue/Green ingress rotation functionality: +# - Create a FlinkBlueGreenDeployment with parent-level ingress spec +# - Verify ingress is created and points to Blue deployment +# - Trigger a transition to Green +# - Verify ingress switches to point to Green deployment +# - Verify Blue deployment is deleted but ingress remains +# - Verify ingress configuration is preserved across transitions + +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_DIR}/utils.sh" + +CLUSTER_ID="bg-ingress-test" +BG_CLUSTER_ID=$CLUSTER_ID +BLUE_CLUSTER_ID=$CLUSTER_ID"-blue" +GREEN_CLUSTER_ID=$CLUSTER_ID"-green" + +APPLICATION_YAML="${SCRIPT_DIR}/data/bluegreen-ingress.yaml" +APPLICATION_IDENTIFIER="flinkbgdep/$CLUSTER_ID" +BLUE_APPLICATION_IDENTIFIER="flinkdep/$BLUE_CLUSTER_ID" +GREEN_APPLICATION_IDENTIFIER="flinkdep/$GREEN_CLUSTER_ID" +TIMEOUT=300 + +echo "Deploying BlueGreen deployment with ingress..." +retry_times 5 30 "kubectl apply -f $APPLICATION_YAML" || exit 1 + +sleep 1 +wait_for_jobmanager_running $BLUE_CLUSTER_ID $TIMEOUT +wait_for_status $BLUE_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_BLUE ${TIMEOUT} || exit 1 + +echo "Verifying ingress created and points to Blue..." +kubectl get ingress $BG_CLUSTER_ID -n default || exit 1 + +# Check ingress backend points to Blue's REST service +BLUE_BACKEND=$(kubectl get ingress $BG_CLUSTER_ID -n default -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}') +EXPECTED_BLUE_BACKEND="${BLUE_CLUSTER_ID}-rest" +if [ "$BLUE_BACKEND" != "$EXPECTED_BLUE_BACKEND" ]; then + echo "ERROR: Ingress backend should be '$EXPECTED_BLUE_BACKEND' but got '$BLUE_BACKEND'" + exit 1 +fi +echo " Ingress correctly points to Blue deployment" + +# Verify ingress annotations +REWRITE_ANNOTATION=$(kubectl get ingress $BG_CLUSTER_ID -n default -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/rewrite-target}') +if [ "$REWRITE_ANNOTATION" != "/" ]; then + echo "ERROR: Expected rewrite annotation '/' but got '$REWRITE_ANNOTATION'" + exit 1 +fi +echo " Ingress annotations preserved" + +echo "Triggering Blue’Green transition..." +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"2"}}}}}' + +# Wait for Green to be ready +wait_for_jobmanager_running $GREEN_CLUSTER_ID $TIMEOUT +wait_for_status $GREEN_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 + +echo "Waiting for Blue deletion..." +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${BLUE_CLUSTER_ID}" || exit 1 + +wait_for_status $APPLICATION_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_GREEN ${TIMEOUT} || exit 1 + +echo "Verifying ingress switched to Green..." +# Ingress should still exist +kubectl get ingress $BG_CLUSTER_ID -n default || exit 1 + +# Check ingress backend now points to Green's REST service +GREEN_BACKEND=$(kubectl get ingress $BG_CLUSTER_ID -n default -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}') +EXPECTED_GREEN_BACKEND="${GREEN_CLUSTER_ID}-rest" +if [ "$GREEN_BACKEND" != "$EXPECTED_GREEN_BACKEND" ]; then + echo "ERROR: Ingress backend should be '$EXPECTED_GREEN_BACKEND' but got '$GREEN_BACKEND'" + exit 1 +fi +echo " Ingress correctly switched to Green deployment" + +# Verify annotations still present after transition +REWRITE_ANNOTATION=$(kubectl get ingress $BG_CLUSTER_ID -n default -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/rewrite-target}') +if [ "$REWRITE_ANNOTATION" != "/" ]; then + echo "ERROR: Annotations lost during transition" + exit 1 +fi +echo " Ingress configuration preserved across transition" + +echo "Triggering Green’Blue transition to verify bidirectional switching..." +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"flinkConfiguration":{"taskmanager.numberOfTaskSlots":"1"}}}}}' + +wait_for_jobmanager_running $BLUE_CLUSTER_ID $TIMEOUT +wait_for_status $BLUE_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${GREEN_CLUSTER_ID}" || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_BLUE ${TIMEOUT} || exit 1 + +echo "Verifying ingress switched back to Blue..." +BLUE_BACKEND=$(kubectl get ingress $BG_CLUSTER_ID -n default -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}') +if [ "$BLUE_BACKEND" != "$EXPECTED_BLUE_BACKEND" ]; then + echo "ERROR: Ingress backend should be '$EXPECTED_BLUE_BACKEND' on return but got '$BLUE_BACKEND'" + exit 1 +fi +echo " Ingress correctly switched back to Blue" + +echo "Cleaning up..." +kubectl delete flinkbluegreendeployments/$BG_CLUSTER_ID & +kubectl wait --for=delete flinkbluegreendeployments/$BG_CLUSTER_ID --timeout=${TIMEOUT}s + +# Verify ingress is deleted with the deployment +INGRESS_DELETED=$(kubectl get ingress $BG_CLUSTER_ID -n default 2>&1 || echo "NotFound") +if [[ ! "$INGRESS_DELETED" =~ "NotFound" ]]; then + echo "ERROR: Ingress should be deleted with BlueGreen deployment" + exit 1 +fi +echo " Ingress cleaned up correctly" + +echo "Successfully run the Blue/Green ingress rotation test" diff --git a/e2e-tests/test_bluegreen_savepoint_redeploy.sh b/e2e-tests/test_bluegreen_savepoint_redeploy.sh new file mode 100755 index 0000000000..4f0a358f51 --- /dev/null +++ b/e2e-tests/test_bluegreen_savepoint_redeploy.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This script tests the FlinkBlueGreenDeployment savepointRedeployNonce feature: +# 1. Deploy a BlueGreen resource (starts BLUE) +# 2. Trigger a normal transition to GREEN (takes savepoint from BLUE) +# 3. Wait for GREEN to be stable and BLUE to be deleted +# 4. Change savepointRedeployNonce and initialSavepointPath to force redeploy from specific savepoint +# 5. Verify transition back to BLUE WITHOUT taking new savepoint from GREEN +# 6. Verify BLUE starts from the specified savepoint +# 7. Test STATELESS mode preserves initialSavepointPath + +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_DIR}/utils.sh" + +CLUSTER_ID="bg-savepoint-redeploy-example" +BG_CLUSTER_ID=$CLUSTER_ID +BLUE_CLUSTER_ID=$CLUSTER_ID"-blue" +GREEN_CLUSTER_ID=$CLUSTER_ID"-green" + +APPLICATION_YAML="${SCRIPT_DIR}/data/bluegreen-savepoint-redeploy.yaml" +APPLICATION_IDENTIFIER="flinkbgdep/$CLUSTER_ID" +BLUE_APPLICATION_IDENTIFIER="flinkdep/$BLUE_CLUSTER_ID" +GREEN_APPLICATION_IDENTIFIER="flinkdep/$GREEN_CLUSTER_ID" +TIMEOUT=300 + +echo "==========================================" +echo "Test 1: Basic savepointRedeployNonce flow" +echo "==========================================" + +echo "Deploying FlinkBlueGreenDeployment..." +retry_times 5 30 "kubectl apply -f $APPLICATION_YAML" || exit 1 + +echo "Waiting for BLUE deployment to be stable..." +sleep 1 +wait_for_jobmanager_running $BLUE_CLUSTER_ID $TIMEOUT +wait_for_status $BLUE_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_BLUE ${TIMEOUT} || exit 1 + +echo "BLUE is stable. Triggering transition to GREEN..." +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"flinkConfiguration":{"state.checkpoints.num-retained":"6"}}}}}' +echo "Waiting for savepoint to be taken from BLUE..." +sleep 10 + +# Capture the savepoint path from BLUE before it's deleted +jm_pod_name=$(get_jm_pod_name $BLUE_CLUSTER_ID) +echo "Inspecting BLUE savepoint directory..." +kubectl exec -it $jm_pod_name -- bash -c "ls -lt /opt/flink/volume/flink-sp/" + +echo "Waiting for GREEN to be stable..." +wait_for_status $GREEN_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${BLUE_CLUSTER_ID}" +wait_for_status $APPLICATION_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_GREEN ${TIMEOUT} || exit 1 + +# Get the savepoint path that GREEN used (this was taken from BLUE) +green_initialSavepointPath=$(kubectl get -oyaml $GREEN_APPLICATION_IDENTIFIER | yq '.spec.job.initialSavepointPath') +echo "GREEN started from savepoint: $green_initialSavepointPath" + +if [[ $green_initialSavepointPath != '/opt/flink/volume/flink-sp/savepoint-'* ]]; then + echo "ERROR: GREEN did not use expected savepoint path" + exit 1 +fi + +echo "" +echo "==========================================" +echo "Test 2: savepointRedeployNonce redeploy" +echo "==========================================" + +# Now test the savepointRedeployNonce feature +# We'll use the same savepoint to transition back to BLUE WITHOUT taking a new savepoint +echo "Updating savepointRedeployNonce to trigger redeploy from specific savepoint..." +echo "Using savepoint: $green_initialSavepointPath" + +# Patch with savepointRedeployNonce and initialSavepointPath +# Note: We're reusing the savepoint that was taken from BLUE earlier +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch "{\"spec\":{\"template\":{\"spec\":{\"job\":{\"savepointRedeployNonce\":123456,\"initialSavepointPath\":\"$green_initialSavepointPath\"}}}}}" + +echo "Waiting for operator to detect SAVEPOINT_REDEPLOY diff type..." +sleep 5 + +# Get operator pod to check logs +operator_pod=$(kubectl get pods -n flink-kubernetes-operator -l app.kubernetes.io/name=flink-kubernetes-operator -o jsonpath='{.items[0].metadata.name}') +echo "Checking operator logs for savepoint redeploy confirmation..." +kubectl logs -n flink-kubernetes-operator $operator_pod --tail=50 | grep -i "savepoint redeploy" || echo "Warning: Could not find savepoint redeploy log message" + +echo "Waiting for BLUE deployment to be created again..." +wait_for_jobmanager_running $BLUE_CLUSTER_ID $TIMEOUT +wait_for_status $BLUE_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 + +# Verify GREEN is deleted +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${GREEN_CLUSTER_ID}" +wait_for_status $APPLICATION_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_BLUE ${TIMEOUT} || exit 1 + +# Verify BLUE used the specified savepoint +blue_initialSavepointPath=$(kubectl get -oyaml $BLUE_APPLICATION_IDENTIFIER | yq '.spec.job.initialSavepointPath') +echo "BLUE started from savepoint: $blue_initialSavepointPath" + +if [[ "$blue_initialSavepointPath" != "$green_initialSavepointPath" ]]; then + echo "ERROR: BLUE did not use the specified savepoint" + echo "Expected: $green_initialSavepointPath" + echo "Got: $blue_initialSavepointPath" + exit 1 +fi + +# Verify in operator logs that no NEW savepoint was taken from GREEN +echo "Verifying that no savepoint was taken from GREEN during nonce redeploy..." +recent_logs=$(kubectl logs -n flink-kubernetes-operator $operator_pod --tail=100) +if echo "$recent_logs" | grep -q "Triggered savepoint for jobId.*$GREEN_CLUSTER_ID"; then + echo "ERROR: Operator took a savepoint from GREEN, but should have skipped it!" + exit 1 +else + echo "SUCCESS: No savepoint taken from GREEN (as expected for savepointRedeployNonce)" +fi + +echo "" +echo "==========================================" +echo "Test 3: STATELESS mode preserves savepoint" +echo "==========================================" + +echo "Changing to STATELESS upgrade mode with initialSavepointPath set..." +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"job":{"upgradeMode":"stateless","savepointRedeployNonce":789}}}}}' + +echo "Waiting for GREEN deployment with STATELESS mode..." +wait_for_jobmanager_running $GREEN_CLUSTER_ID $TIMEOUT +wait_for_status $GREEN_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${BLUE_CLUSTER_ID}" +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_GREEN ${TIMEOUT} || exit 1 + +# Verify GREEN still used the savepoint despite STATELESS mode +green_stateless_savepointPath=$(kubectl get -oyaml $GREEN_APPLICATION_IDENTIFIER | yq '.spec.job.initialSavepointPath') +echo "GREEN (STATELESS) started from savepoint: $green_stateless_savepointPath" + +if [[ "$green_stateless_savepointPath" == "null" ]] || [[ -z "$green_stateless_savepointPath" ]]; then + echo "ERROR: STATELESS mode cleared initialSavepointPath (BUG!)" + exit 1 +else + echo "SUCCESS: STATELESS mode preserved initialSavepointPath" +fi + +echo "" +echo "==========================================" +echo "Test 4: Nonce change DURING transition" +echo "==========================================" + +echo "Triggering transition back to BLUE..." +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"job":{"parallelism":2}}}}}' + +echo "Waiting for BLUE to start deploying..." +sleep 5 + +# Verify we're in transitioning state +blue_green_state=$(kubectl get -oyaml $APPLICATION_IDENTIFIER | yq '.status.blueGreenState') +if [[ "$blue_green_state" != "TRANSITIONING_TO_BLUE" ]] && [[ "$blue_green_state" != "SAVEPOINTING_GREEN" ]]; then + echo "Warning: Expected to be in transition state, but in: $blue_green_state" +fi + +echo "NOW changing savepointRedeployNonce DURING the transition..." +# Use a different savepoint path to make it clear this is a new redeploy +kubectl patch flinkbgdep ${BG_CLUSTER_ID} --type merge --patch '{"spec":{"template":{"spec":{"job":{"savepointRedeployNonce":999}}}}}' + +echo "Waiting for operator to detect nonce change and abort transition..." +sleep 10 + +# Check operator logs for abort message +operator_pod=$(kubectl get pods -n flink-kubernetes-operator -l app.kubernetes.io/name=flink-kubernetes-operator -o jsonpath='{.items[0].metadata.name}') +echo "Checking operator logs for transition abort..." +kubectl logs -n flink-kubernetes-operator $operator_pod --tail=100 | grep -i "aborting current transition" || echo "Warning: Abort log message not found (may be expected in some cases)" + +# Verify we went back to ACTIVE_GREEN (not TRANSITIONING) +sleep 5 +current_state=$(kubectl get -oyaml $APPLICATION_IDENTIFIER | yq '.status.blueGreenState') +if [[ "$current_state" == "ACTIVE_GREEN" ]]; then + echo "SUCCESS: Transition was aborted, reverted to ACTIVE_GREEN" +elif [[ "$current_state" == "TRANSITIONING_TO_BLUE" ]]; then + echo "SUCCESS: Transition restarted (expected behavior - restart can be immediate)" +else + echo "Unexpected state after nonce change during transition: $current_state" +fi + +# Wait for new transition to complete with the updated nonce +echo "Waiting for new transition to complete..." +wait_for_jobmanager_running $BLUE_CLUSTER_ID $TIMEOUT +wait_for_status $BLUE_APPLICATION_IDENTIFIER '.status.lifecycleState' STABLE ${TIMEOUT} || exit 1 +kubectl wait --for=delete deployment --timeout=${TIMEOUT}s --selector="app=${GREEN_CLUSTER_ID}" +wait_for_status $APPLICATION_IDENTIFIER '.status.blueGreenState' ACTIVE_BLUE ${TIMEOUT} || exit 1 + +echo "SUCCESS: Nonce change during transition triggered abort and restart" + +echo "" +echo "==========================================" +echo "Cleanup" +echo "==========================================" + +echo "Deleting test B/G resources: $BG_CLUSTER_ID" +kubectl delete flinkbluegreendeployments/$BG_CLUSTER_ID & +echo "Waiting for deployment to be deleted..." +kubectl wait --for=delete flinkbluegreendeployments/$BG_CLUSTER_ID --timeout=${TIMEOUT}s + +echo "" +echo "==========================================" +echo "All tests passed!" +echo "==========================================" +echo "βœ“ savepointRedeployNonce triggers redeploy from specific savepoint" +echo "βœ“ No new savepoint taken when nonce changes" +echo "βœ“ STATELESS mode preserves initialSavepointPath" +echo "" +echo "Successfully completed FlinkBlueGreenDeployment savepointRedeployNonce e2e tests" diff --git a/flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/FlinkBlueGreenDeploymentSpec.java b/flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/FlinkBlueGreenDeploymentSpec.java index 704d354152..a515b0a91b 100644 --- a/flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/FlinkBlueGreenDeploymentSpec.java +++ b/flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/FlinkBlueGreenDeploymentSpec.java @@ -25,6 +25,8 @@ import lombok.Data; import lombok.NoArgsConstructor; +import javax.annotation.Nullable; + import java.util.Map; /** Spec that describes a Flink application with blue/green deployment capabilities. */ @@ -38,5 +40,7 @@ public class FlinkBlueGreenDeploymentSpec { @JsonProperty("configuration") private Map configuration; + @Nullable private IngressSpec ingress; + private FlinkDeploymentTemplateSpec template; } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java index fdc856f24b..addfac9b4f 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java @@ -261,8 +261,8 @@ void registerBlueGreenController() { MetricManager.createFlinkBlueGreenDeploymentMetricManager(baseConfig, metricGroup); var statusRecorder = StatusRecorder.createForFlinkBlueGreenDeployment(client, metricManager, listeners); - var controller = new FlinkBlueGreenDeploymentController(ctxFactory, statusRecorder); + var controller = new FlinkBlueGreenDeploymentController(ctxFactory, configManager, statusRecorder); registeredControllers.add(operator.register(controller, this::overrideControllerConfigs)); } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java index bac6f131e2..0874a10bb3 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java @@ -21,12 +21,14 @@ import org.apache.flink.kubernetes.operator.api.FlinkDeployment; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentState; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; +import org.apache.flink.kubernetes.operator.config.FlinkConfigManager; import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenContext; import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenDeploymentService; import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenStateHandlerRegistry; import org.apache.flink.kubernetes.operator.controller.bluegreen.handlers.BlueGreenStateHandler; import org.apache.flink.kubernetes.operator.service.FlinkResourceContextFactory; import org.apache.flink.kubernetes.operator.utils.StatusRecorder; +import org.apache.flink.kubernetes.operator.utils.EventSourceUtils; import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; import io.javaoperatorsdk.operator.api.reconciler.Context; @@ -73,13 +75,16 @@ public class FlinkBlueGreenDeploymentController implements Reconciler statusRecorder; + private final FlinkConfigManager flinkConfigManager; public FlinkBlueGreenDeploymentController( FlinkResourceContextFactory ctxFactory, + FlinkConfigManager flinkConfigManager, StatusRecorder statusRecorder) { this.ctxFactory = ctxFactory; this.handlerRegistry = new BlueGreenStateHandlerRegistry(); + this.flinkConfigManager = flinkConfigManager; this.statusRecorder = statusRecorder; } @@ -99,6 +104,9 @@ public List> prepareEventSources( eventSources.add(new InformerEventSource<>(config, context)); + if (flinkConfigManager.getOperatorConfiguration().isManageIngress()) { + eventSources.add(EventSourceUtils.getBlueGreenIngressInformerEventSource(context)); + } return eventSources; } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java index 397f11e3da..cb4f35d817 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/BlueGreenDeploymentService.java @@ -30,6 +30,7 @@ import org.apache.flink.kubernetes.operator.config.KubernetesOperatorConfigOptions; import org.apache.flink.kubernetes.operator.controller.FlinkBlueGreenDeployments; import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext; +import org.apache.flink.kubernetes.operator.utils.IngressUtils; import org.apache.flink.kubernetes.operator.utils.bluegreen.BlueGreenUtils; import org.apache.flink.util.Preconditions; @@ -740,12 +741,95 @@ public UpdateControl finalizeBlueGreenDeployment( context.getDeploymentStatus().setAbortTimestamp(millisToInstantStr(0)); context.getDeploymentStatus().setSavepointTriggerId(null); + try { + updateBlueGreenIngress(context, nextState); + } catch (Exception e) { + var error = + "Could not reconcile ingress during finalization. Details: " + e.getMessage(); + LOG.error(error, e); + return markDeploymentFailing(context, error); + } + // Finalize status and reschedule immediately so any pending spec changes // (e.g., suspend requested during transition) are picked up on next reconcile return patchStatusUpdateControl(context, nextState, JobStatus.RUNNING, null) .rescheduleAfter(0); } + /** + * Reconciles ingress for the active deployment in ACTIVE states. This handles ingress spec + * changes that occur while the deployment is stable (not transitioning). + * + * @param context the Blue/Green context + * @param activeDeploymentType which deployment (BLUE or GREEN) is currently active + */ + public void reconcileIngressForActiveDeployment( + BlueGreenContext context, BlueGreenDeploymentType activeDeploymentType) { + FlinkDeployment activeDeployment = context.getDeploymentByType(activeDeploymentType); + if (activeDeployment == null) { + return; + } + + var flinkResourceContext = + context.getCtxFactory() + .getResourceContext(activeDeployment, context.getJosdkContext()); + + if (!flinkResourceContext.getOperatorConfig().isManageIngress()) { + return; + } + + try { + IngressUtils.reconcileBlueGreenIngress( + context, + true, + activeDeployment, + flinkResourceContext.getDeployConfig(activeDeployment.getSpec()), + context.getJosdkContext()); + } catch (Exception e) { + LOG.error( + "Failed to reconcile ingress for active deployment: {}", + activeDeployment.getMetadata().getName(), + e); + // Don't fail the entire reconciliation just because ingress reconciliation failed + } + } + + /** + * Updates the ingress for Blue/Green deployment during transitions, pointing to the newly + * active deployment. + * + * @param blueGreenContext the Blue/Green context + * @param nextState which deployment (ACTIVE_BLUE or ACTIVE_GREEN) is becoming active + */ + public void updateBlueGreenIngress( + BlueGreenContext blueGreenContext, FlinkBlueGreenDeploymentState nextState) { + FlinkDeployment activeDeployment; + switch (nextState) { + case ACTIVE_BLUE: + activeDeployment = blueGreenContext.getBlueDeployment(); + break; + case ACTIVE_GREEN: + activeDeployment = blueGreenContext.getGreenDeployment(); + break; + default: + LOG.info("Skipping ingress reconciliation for non-active state: {}", nextState); + return; + } + + // Create a FlinkResourceContext for the active deployment to get proper config + var flinkResourceContext = + blueGreenContext + .getCtxFactory() + .getResourceContext(activeDeployment, blueGreenContext.getJosdkContext()); + + IngressUtils.reconcileBlueGreenIngress( + blueGreenContext, + flinkResourceContext.getOperatorConfig().isManageIngress(), + activeDeployment, + flinkResourceContext.getDeployConfig(activeDeployment.getSpec()), + blueGreenContext.getJosdkContext()); + } + // ==================== Common Utility Methods ==================== public static UpdateControl patchStatusUpdateControl( diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/ActiveStateHandler.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/ActiveStateHandler.java index 176b94fb7f..06deadc374 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/ActiveStateHandler.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/bluegreen/handlers/ActiveStateHandler.java @@ -37,6 +37,10 @@ public ActiveStateHandler( @Override public UpdateControl handle(BlueGreenContext context) { BlueGreenDeploymentType currentType = getCurrentDeploymentType(); + + // Reconcile ingress for the active deployment (handles spec changes) + deploymentService.reconcileIngressForActiveDeployment(context, currentType); + return deploymentService.checkAndInitiateDeployment(context, currentType); } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventSourceUtils.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventSourceUtils.java index efcf2ab6e7..e3f22d47ba 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventSourceUtils.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/EventSourceUtils.java @@ -19,6 +19,7 @@ import org.apache.flink.kubernetes.operator.api.AbstractFlinkResource; import org.apache.flink.kubernetes.operator.api.CrdConstants; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; import org.apache.flink.kubernetes.operator.api.FlinkDeployment; import org.apache.flink.kubernetes.operator.api.FlinkSessionJob; import org.apache.flink.kubernetes.operator.api.FlinkStateSnapshot; @@ -129,6 +130,29 @@ public static InformerEventSource getIngressInformerEventSou return new InformerEventSource<>(configuration, context); } + public static InformerEventSource + getBlueGreenIngressInformerEventSource( + EventSourceContext context) { + final String labelSelector = + Map.of(Constants.LABEL_COMPONENT_KEY, LABEL_COMPONENT_INGRESS).entrySet().stream() + .map(Object::toString) + .collect(Collectors.joining(",")); + + var ingressClass = + ingressInNetworkingV1(context.getClient()) + ? Ingress.class + : io.fabric8.kubernetes.api.model.networking.v1beta1.Ingress.class; + + var configuration = + InformerEventSourceConfiguration.from(ingressClass, FlinkBlueGreenDeployment.class) + .withLabelSelector(labelSelector) + .withNamespacesInheritedFromController() + .withFollowControllerNamespacesChanges(true) + .build(); + + return new InformerEventSource<>(configuration, context); + } + public static InformerEventSource getSessionJobInformerEventSource(EventSourceContext context) { diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/IngressUtils.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/IngressUtils.java index 722270a1f6..743395785f 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/IngressUtils.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/IngressUtils.java @@ -19,8 +19,13 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.RestOptions; +import org.apache.flink.kubernetes.operator.api.FlinkBlueGreenDeployment; +import org.apache.flink.kubernetes.operator.api.FlinkDeployment; +import org.apache.flink.kubernetes.operator.api.spec.FlinkBlueGreenDeploymentSpec; import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.IngressSpec; import org.apache.flink.kubernetes.operator.controller.FlinkResourceContext; +import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenContext; import org.apache.flink.kubernetes.operator.exception.ReconciliationException; import org.apache.flink.kubernetes.utils.Constants; import org.apache.flink.util.Preconditions; @@ -38,6 +43,7 @@ import io.fabric8.kubernetes.api.model.networking.v1beta1.IngressTLS; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.dsl.NonDeletingOperation; +import io.javaoperatorsdk.operator.api.reconciler.Context; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -95,7 +101,8 @@ public static void reconcileIngress( } var objectMeta = ctx.getResource().getMetadata(); if (spec.getIngress() != null) { - HasMetadata ingress = getIngress(objectMeta, spec, effectiveConfig, client); + HasMetadata ingress = + getIngress(objectMeta, spec.getIngress(), effectiveConfig, client); setOwnerReference(ctx.getResource(), Collections.singletonList(ingress)); LOG.info("Updating ingress rules {}", ingress); client.resource(ingress) @@ -116,33 +123,84 @@ public static void reconcileIngress( } } + public static void reconcileBlueGreenIngress( + BlueGreenContext context, + boolean operatorManagedIngress, + FlinkDeployment targetFlinkDeployment, + Configuration effectiveConfig, + Context client) { + if (!operatorManagedIngress) { + LOG.info("Skipping rerouting of of Blue Green Ingress, not operator managed ingress "); + return; + } + FlinkBlueGreenDeploymentSpec flinkBlueGreenDeploymentSpec = + context.getBgDeployment().getSpec(); + var objectMeta = context.getBgDeployment().getMetadata(); + if (flinkBlueGreenDeploymentSpec.getIngress() != null) { + HasMetadata ingress = + getIngress( + objectMeta, + flinkBlueGreenDeploymentSpec.getIngress(), + effectiveConfig, + client.getClient(), + targetFlinkDeployment.getMetadata().getName() + REST_SVC_NAME_SUFFIX); + setOwnerReference(context.getBgDeployment(), Collections.singletonList(ingress)); + client.getClient() + .resource(ingress) + .inNamespace(objectMeta.getNamespace()) + .createOr(NonDeletingOperation::update); + } else { + Optional ingress; + if (ingressInNetworkingV1(client.getClient())) { + ingress = + client.getSecondaryResource( + io.fabric8.kubernetes.api.model.networking.v1.Ingress.class); + } else { + ingress = client.getSecondaryResource(Ingress.class); + } + ingress.ifPresent(i -> client.getClient().resource(i).delete()); + } + } + private static HasMetadata getIngress( ObjectMeta objectMeta, - FlinkDeploymentSpec spec, + IngressSpec spec, Configuration effectiveConfig, KubernetesClient client) { + return getIngress( + objectMeta, + spec, + effectiveConfig, + client, + objectMeta.getName() + REST_SVC_NAME_SUFFIX); + } + + private static HasMetadata getIngress( + ObjectMeta objectMeta, + IngressSpec spec, + Configuration effectiveConfig, + KubernetesClient client, + String serviceName) { Map labels = - spec.getIngress().getLabels() == null - ? new HashMap<>() - : new HashMap<>(spec.getIngress().getLabels()); + spec.getLabels() == null ? new HashMap<>() : new HashMap<>(spec.getLabels()); labels.put(Constants.LABEL_COMPONENT_KEY, LABEL_COMPONENT_INGRESS); if (ingressInNetworkingV1(client)) { return new IngressBuilder() .withNewMetadata() .withLabels(labels) - .withAnnotations(spec.getIngress().getAnnotations()) + .withAnnotations(spec.getAnnotations()) .withName(objectMeta.getName()) .withNamespace(objectMeta.getNamespace()) .endMetadata() .withNewSpec() - .withIngressClassName(spec.getIngress().getClassName()) - .withTls(spec.getIngress().getTls()) - .withRules(getIngressRule(objectMeta, spec, effectiveConfig)) + .withIngressClassName(spec.getClassName()) + .withTls(spec.getTls()) + .withRules(getIngressRule(objectMeta, spec, effectiveConfig, serviceName)) .endSpec() .build(); } else { List ingressTLS = - Optional.ofNullable(spec.getIngress().getTls()) + Optional.ofNullable(spec.getTls()) .map( list -> list.stream() @@ -160,30 +218,31 @@ private static HasMetadata getIngress( .orElse(Collections.emptyList()); return new io.fabric8.kubernetes.api.model.networking.v1beta1.IngressBuilder() .withNewMetadata() - .withAnnotations(spec.getIngress().getAnnotations()) + .withAnnotations(spec.getAnnotations()) .withLabels(labels) .withName(objectMeta.getName()) .withNamespace(objectMeta.getNamespace()) .endMetadata() .withNewSpec() - .withIngressClassName(spec.getIngress().getClassName()) + .withIngressClassName(spec.getClassName()) .withTls(ingressTLS) - .withRules(getIngressRuleForV1beta1(objectMeta, spec, effectiveConfig)) + .withRules( + getIngressRuleForV1beta1( + objectMeta, spec, effectiveConfig, serviceName)) .endSpec() .build(); } } private static IngressRule getIngressRule( - ObjectMeta objectMeta, FlinkDeploymentSpec spec, Configuration effectiveConfig) { - final String clusterId = objectMeta.getName(); + ObjectMeta objectMeta, + IngressSpec spec, + Configuration effectiveConfig, + String serviceName) { final int restPort = effectiveConfig.getInteger(RestOptions.PORT); URL ingressUrl = - getIngressUrl( - spec.getIngress().getTemplate(), - objectMeta.getName(), - objectMeta.getNamespace()); + getIngressUrl(spec.getTemplate(), objectMeta.getName(), objectMeta.getNamespace()); IngressRuleBuilder ingressRuleBuilder = new IngressRuleBuilder(); ingressRuleBuilder.withHttp( @@ -192,7 +251,7 @@ private static IngressRule getIngressRule( .withPathType("ImplementationSpecific") .withNewBackend() .withNewService() - .withName(clusterId + REST_SVC_NAME_SUFFIX) + .withName(serviceName) .withNewPort() .withNumber(restPort) .endPort() @@ -219,16 +278,13 @@ private static IngressRule getIngressRule( private static io.fabric8.kubernetes.api.model.networking.v1beta1.IngressRule getIngressRuleForV1beta1( ObjectMeta objectMeta, - FlinkDeploymentSpec spec, - Configuration effectiveConfig) { - final String clusterId = objectMeta.getName(); + IngressSpec spec, + Configuration effectiveConfig, + String serviceName) { final int restPort = effectiveConfig.getInteger(RestOptions.PORT); URL ingressUrl = - getIngressUrl( - spec.getIngress().getTemplate(), - objectMeta.getName(), - objectMeta.getNamespace()); + getIngressUrl(spec.getTemplate(), objectMeta.getName(), objectMeta.getNamespace()); io.fabric8.kubernetes.api.model.networking.v1beta1.IngressRuleBuilder ingressRuleBuilder = new io.fabric8.kubernetes.api.model.networking.v1beta1.IngressRuleBuilder(); @@ -236,7 +292,7 @@ private static IngressRule getIngressRule( new io.fabric8.kubernetes.api.model.networking.v1beta1.HTTPIngressRuleValueBuilder() .addNewPath() .withNewBackend() - .withServiceName(clusterId + REST_SVC_NAME_SUFFIX) + .withServiceName(serviceName) .withServicePort(new IntOrString(restPort)) .endBackend() .endPath() diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java index 297e262cee..9645a23958 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtils.java @@ -26,6 +26,7 @@ import org.apache.flink.kubernetes.operator.api.bluegreen.BlueGreenDeploymentType; import org.apache.flink.kubernetes.operator.api.bluegreen.BlueGreenDiffType; import org.apache.flink.kubernetes.operator.api.spec.FlinkBlueGreenDeploymentSpec; +import org.apache.flink.kubernetes.operator.api.spec.IngressSpec; import org.apache.flink.kubernetes.operator.api.spec.KubernetesDeploymentMode; import org.apache.flink.kubernetes.operator.api.spec.UpgradeMode; import org.apache.flink.kubernetes.operator.api.status.FlinkBlueGreenDeploymentStatus; @@ -366,6 +367,13 @@ public static FlinkDeployment prepareFlinkDeployment( flinkDeployment.setSpec(spec.getTemplate().getSpec()); + // Update Ingress template if exists to prevent path collision between Blue and Green + IngressSpec ingress = flinkDeployment.getSpec().getIngress(); + if (ingress != null) { + ingress.setTemplate( + blueGreenDeploymentType.name().toLowerCase() + "-" + ingress.getTemplate()); + } + // Deployment metadata ObjectMeta flinkDeploymentMeta = getDependentObjectMeta(context.getBgDeployment()); flinkDeploymentMeta.setName(childDeploymentName); diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java index bc571c5d4d..528b61727d 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentControllerTest.java @@ -30,6 +30,7 @@ import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentSpec; import org.apache.flink.kubernetes.operator.api.spec.FlinkDeploymentTemplateSpec; import org.apache.flink.kubernetes.operator.api.spec.FlinkVersion; +import org.apache.flink.kubernetes.operator.api.spec.IngressSpec; import org.apache.flink.kubernetes.operator.api.spec.JobManagerSpec; import org.apache.flink.kubernetes.operator.api.spec.JobSpec; import org.apache.flink.kubernetes.operator.api.spec.JobState; @@ -40,9 +41,11 @@ import org.apache.flink.kubernetes.operator.api.status.ReconciliationState; import org.apache.flink.kubernetes.operator.api.utils.SpecUtils; import org.apache.flink.kubernetes.operator.config.FlinkConfigManager; +import org.apache.flink.kubernetes.operator.utils.IngressUtils; import org.apache.flink.runtime.jobgraph.SavepointConfigOptions; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; import io.javaoperatorsdk.operator.api.reconciler.Context; @@ -90,6 +93,9 @@ public class FlinkBlueGreenDeploymentControllerTest { private static final int MINIMUM_ABORT_GRACE_PERIOD = 1000; private static final String TEST_CHECKPOINT_PATH = "/tmp/checkpoints"; private static final String TEST_INITIAL_SAVEPOINT_PATH = "/tmp/savepoints"; + private static final String BLUE_CLUSTER_ID = TEST_DEPLOYMENT_NAME + "-blue"; + private static final String GREEN_CLUSTER_ID = TEST_DEPLOYMENT_NAME + "-green"; + private static final String REST_SVC_NAME_SUFFIX = "-rest"; private final FlinkConfigManager configManager = new FlinkConfigManager(new Configuration()); private TestingFlinkService flinkService; private Context context; @@ -1531,6 +1537,145 @@ private static FlinkBlueGreenDeploymentSpec getTestFlinkDeploymentSpec(FlinkVers var flinkDeploymentTemplateSpec = FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build(); - return new FlinkBlueGreenDeploymentSpec(configuration, flinkDeploymentTemplateSpec); + return new FlinkBlueGreenDeploymentSpec(configuration, null, flinkDeploymentTemplateSpec); + } + + // ==================== Ingress Helper Methods ==================== + + private void assertIngressPointsToService(String expectedServiceName) { + if (IngressUtils.ingressInNetworkingV1(kubernetesClient)) { + Ingress ingress = getIngressV1(TEST_DEPLOYMENT_NAME, TEST_NAMESPACE); + assertNotNull(ingress); + assertEquals( + expectedServiceName, + ingress.getSpec() + .getRules() + .get(0) + .getHttp() + .getPaths() + .get(0) + .getBackend() + .getService() + .getName()); + } else { + io.fabric8.kubernetes.api.model.networking.v1beta1.Ingress ingressV1beta1 = + getIngressV1beta1(TEST_DEPLOYMENT_NAME, TEST_NAMESPACE); + assertNotNull(ingressV1beta1); + assertEquals( + expectedServiceName, + ingressV1beta1 + .getSpec() + .getRules() + .get(0) + .getHttp() + .getPaths() + .get(0) + .getBackend() + .getServiceName()); + } + } + + private void assertIngressDoesNotExist() { + if (IngressUtils.ingressInNetworkingV1(kubernetesClient)) { + assertNull(getIngressV1(TEST_DEPLOYMENT_NAME, TEST_NAMESPACE)); + } else { + assertNull(getIngressV1beta1(TEST_DEPLOYMENT_NAME, TEST_NAMESPACE)); + } + } + + private Ingress getIngressV1(String name, String namespace) { + return kubernetesClient + .network() + .v1() + .ingresses() + .inNamespace(namespace) + .withName(name) + .get(); + } + + private io.fabric8.kubernetes.api.model.networking.v1beta1.Ingress getIngressV1beta1( + String name, String namespace) { + return kubernetesClient + .network() + .v1beta1() + .ingresses() + .inNamespace(namespace) + .withName(name) + .get(); + } + + // ==================== Ingress Rotation Tests ==================== + + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifyIngressSwitchesDuringTransition(FlinkVersion flinkVersion) throws Exception { + // Build deployment with ingress spec + var blueGreenDeployment = + buildSessionCluster( + TEST_DEPLOYMENT_NAME, + TEST_NAMESPACE, + flinkVersion, + null, + UpgradeMode.STATELESS); + + IngressSpec ingressSpec = + IngressSpec.builder() + .template("{{name}}.{{namespace}}.example.com") + .className("nginx") + .annotations(Map.of("nginx.ingress.kubernetes.io/rewrite-target", "/")) + .build(); + blueGreenDeployment.getSpec().setIngress(ingressSpec); + + // 1. Deploy Blue with ingress + var rs = executeBasicDeployment(flinkVersion, blueGreenDeployment, false, null); + + // 2. Verify ingress points to Blue + assertIngressPointsToService(BLUE_CLUSTER_ID + REST_SVC_NAME_SUFFIX); + + // 3. Trigger transition to Green + String customValue = UUID.randomUUID().toString(); + simulateChangeInSpec(rs.deployment, customValue, ALT_DELETION_DELAY_VALUE, null); + + // Transition to Green + testTransitionToGreen(rs, customValue, null); + + // 4. Verify ingress now points to Green + assertIngressPointsToService(GREEN_CLUSTER_ID + REST_SVC_NAME_SUFFIX); + } + + @ParameterizedTest + @MethodSource("org.apache.flink.kubernetes.operator.TestUtils#flinkVersions") + public void verifyIngressCreatedOnlyWhenConfigured(FlinkVersion flinkVersion) throws Exception { + // 1. Deploy Blue without ingress spec + var blueGreenDeployment = + buildSessionCluster( + TEST_DEPLOYMENT_NAME, + TEST_NAMESPACE, + flinkVersion, + null, + UpgradeMode.STATELESS); + + var rs = executeBasicDeployment(flinkVersion, blueGreenDeployment, false, null); + + // 2. Verify no ingress created initially + assertIngressDoesNotExist(); + + // 3. Add ingress spec and trigger transition to Green + IngressSpec ingressSpec = + IngressSpec.builder() + .template("{{name}}.{{namespace}}.example.com") + .className("nginx") + .build(); + rs.deployment.getSpec().setIngress(ingressSpec); + kubernetesClient.resource(rs.deployment).createOrReplace(); + + String customValue = UUID.randomUUID().toString(); + simulateChangeInSpec(rs.deployment, customValue, ALT_DELETION_DELAY_VALUE, null); + + // Complete transition to Green (this reconciles ingress) + testTransitionToGreen(rs, customValue, null); + + // 4. Verify ingress created and points to Green after transition completes + assertIngressPointsToService(GREEN_CLUSTER_ID + REST_SVC_NAME_SUFFIX); } } diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java index 5bddcf4da6..646c8bb2fc 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java @@ -56,7 +56,7 @@ public TestingFlinkBlueGreenDeploymentController( new StatusRecorder<>(new MetricManager<>(), (resource, status) -> {}); flinkBlueGreenDeploymentController = - new FlinkBlueGreenDeploymentController(contextFactory, statusRecorder); + new FlinkBlueGreenDeploymentController(contextFactory, configManager, statusRecorder); } @Override diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java index a898662ec8..d7e735103c 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/utils/bluegreen/BlueGreenUtilsTest.java @@ -243,6 +243,7 @@ private static FlinkBlueGreenDeployment buildBlueGreenDeployment( var bgDeploymentSpec = new FlinkBlueGreenDeploymentSpec( new HashMap<>(), + null, FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build()); deployment.setSpec(bgDeploymentSpec); diff --git a/helm/flink-kubernetes-operator/crds/flinkbluegreendeployments.flink.apache.org-v1.yml b/helm/flink-kubernetes-operator/crds/flinkbluegreendeployments.flink.apache.org-v1.yml index d31d14aa24..0f8ef636fb 100644 --- a/helm/flink-kubernetes-operator/crds/flinkbluegreendeployments.flink.apache.org-v1.yml +++ b/helm/flink-kubernetes-operator/crds/flinkbluegreendeployments.flink.apache.org-v1.yml @@ -24,6 +24,32 @@ spec: properties: spec: properties: + ingress: + properties: + annotations: + additionalProperties: + type: string + type: object + className: + type: string + labels: + additionalProperties: + type: string + type: object + template: + type: string + tls: + items: + properties: + hosts: + items: + type: string + type: array + secretName: + type: string + type: object + type: array + type: object configuration: additionalProperties: type: "string" From f259f5ad7ed6a8740ee5b67fa82d5ab87166bd17 Mon Sep 17 00:00:00 2001 From: Daniel Rossos Date: Mon, 19 Jan 2026 15:47:09 -0500 Subject: [PATCH 5/7] style fix --- .../org/apache/flink/kubernetes/operator/FlinkOperator.java | 3 ++- .../controller/FlinkBlueGreenDeploymentController.java | 2 +- .../controller/TestingFlinkBlueGreenDeploymentController.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java index addfac9b4f..c66e802db5 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/FlinkOperator.java @@ -262,7 +262,8 @@ void registerBlueGreenController() { var statusRecorder = StatusRecorder.createForFlinkBlueGreenDeployment(client, metricManager, listeners); - var controller = new FlinkBlueGreenDeploymentController(ctxFactory, configManager, statusRecorder); + var controller = + new FlinkBlueGreenDeploymentController(ctxFactory, configManager, statusRecorder); registeredControllers.add(operator.register(controller, this::overrideControllerConfigs)); } diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java index 0874a10bb3..6d78cdc10f 100644 --- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/controller/FlinkBlueGreenDeploymentController.java @@ -27,8 +27,8 @@ import org.apache.flink.kubernetes.operator.controller.bluegreen.BlueGreenStateHandlerRegistry; import org.apache.flink.kubernetes.operator.controller.bluegreen.handlers.BlueGreenStateHandler; import org.apache.flink.kubernetes.operator.service.FlinkResourceContextFactory; -import org.apache.flink.kubernetes.operator.utils.StatusRecorder; import org.apache.flink.kubernetes.operator.utils.EventSourceUtils; +import org.apache.flink.kubernetes.operator.utils.StatusRecorder; import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; import io.javaoperatorsdk.operator.api.reconciler.Context; diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java index 646c8bb2fc..da7bedea26 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/controller/TestingFlinkBlueGreenDeploymentController.java @@ -56,7 +56,8 @@ public TestingFlinkBlueGreenDeploymentController( new StatusRecorder<>(new MetricManager<>(), (resource, status) -> {}); flinkBlueGreenDeploymentController = - new FlinkBlueGreenDeploymentController(contextFactory, configManager, statusRecorder); + new FlinkBlueGreenDeploymentController( + contextFactory, configManager, statusRecorder); } @Override From a0085c254d9b331ddfd657eea5a51556610b9450 Mon Sep 17 00:00:00 2001 From: Daniel Rossos Date: Mon, 19 Jan 2026 16:14:13 -0500 Subject: [PATCH 6/7] test fix --- .../operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java | 1 + .../metrics/lifecycle/BlueGreenLifecycleMetricsTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java index 104696775d..76ff973aa8 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/FlinkBlueGreenDeploymentMetricsTest.java @@ -466,6 +466,7 @@ private FlinkBlueGreenDeployment buildBlueGreenDeployment(String name, String na var bgDeploymentSpec = new FlinkBlueGreenDeploymentSpec( new HashMap<>(), + null, FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build()); deployment.setSpec(bgDeploymentSpec); diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java index ce0af71c5d..79add57f99 100644 --- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java +++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/metrics/lifecycle/BlueGreenLifecycleMetricsTest.java @@ -179,6 +179,7 @@ private FlinkBlueGreenDeployment buildBlueGreenDeployment(String name, String na var bgDeploymentSpec = new FlinkBlueGreenDeploymentSpec( new HashMap<>(), + null, FlinkDeploymentTemplateSpec.builder().spec(flinkDeploymentSpec).build()); deployment.setSpec(bgDeploymentSpec); From ca1bef5123137dff9e9853429ca655c13050b775 Mon Sep 17 00:00:00 2001 From: Daniel Rossos Date: Tue, 20 Jan 2026 09:57:22 -0500 Subject: [PATCH 7/7] untracking docs md --- docs/savepoint-redeploy-during-transition.md | 195 ------------------- 1 file changed, 195 deletions(-) delete mode 100644 docs/savepoint-redeploy-during-transition.md diff --git a/docs/savepoint-redeploy-during-transition.md b/docs/savepoint-redeploy-during-transition.md deleted file mode 100644 index 94884f921d..0000000000 --- a/docs/savepoint-redeploy-during-transition.md +++ /dev/null @@ -1,195 +0,0 @@ -# Handling savepointRedeployNonce Changes During Transitions - -## Overview - -This document describes the behavior when a user changes `savepointRedeployNonce` while a FlinkBlueGreenDeployment is in the middle of a transition. - -## Scenario - -**Initial State:** -- BLUE deployment is running (ACTIVE_BLUE state) -- User triggers a transition to GREEN (e.g., by changing parallelism) -- State changes to TRANSITIONING_TO_GREEN -- GREEN deployment is starting up, restoring from a savepoint taken from BLUE - -**User Action During Transition:** -- While in TRANSITIONING_TO_GREEN state -- User changes `savepointRedeployNonce` to a new value -- User sets `initialSavepointPath` to a different savepoint location - -## Implemented Behavior (Option 1: Abort Transition) - -### What Happens - -1. **Detection Phase** - - Operator detects spec change during `monitorTransition()` - - Calls `handleSpecChangesDuringTransition()` - - Identifies diff type as `SAVEPOINT_REDEPLOY` - -2. **Abort Phase** - - Operator deletes the in-progress GREEN deployment - - Reverts BlueGreen state to `ACTIVE_BLUE` - - Clears savepoint trigger ID - - Marks spec as reconciled - -3. **Restart Phase** - - Next reconciliation cycle detects new spec - - Triggers a fresh transition to GREEN - - Uses the NEW `initialSavepointPath` specified by user - - GREEN starts from the user-specified savepoint - -### Code Location - -**File:** `BlueGreenDeploymentService.java` - -**Method:** `handleSavepointRedeployDuringTransition()` - -```java -private UpdateControl handleSavepointRedeployDuringTransition( - BlueGreenContext context, - BlueGreenDeploymentType currentBlueGreenDeploymentType) -``` - -**Called from:** `handleSpecChangesDuringTransition()` when `diffType == SAVEPOINT_REDEPLOY` - -## Behavior Details - -### Logs - -When this occurs, the operator logs: -``` -WARN: savepointRedeployNonce changed during transition from BLUE to GREEN. - Aborting current transition and will restart with new savepoint from: s3://bucket/new-savepoint -INFO: Deleted transitioning FlinkDeployment 'my-app-green' due to savepointRedeployNonce change -``` - -### State Transitions - -``` -TRANSITIONING_TO_GREEN - β†’ (nonce changed) - β†’ ACTIVE_BLUE - β†’ (next reconciliation) - β†’ TRANSITIONING_TO_GREEN (with new savepoint) -``` - -### Timing - -- Deletion: Immediate (current reconciliation) -- Restart: Next reconciliation cycle (~5 seconds with default settings) -- Total delay: ~5-10 seconds before new transition begins - -## Why This Approach? - -### User Intent -When a user changes `savepointRedeployNonce` during a transition, their intent is clear: -- "Stop what you're doing" -- "Use THIS specific savepoint instead" -- "Start the transition over with the correct state" - -### Alternative Approaches Considered - -**Option 2: Reject the Change** -- Pros: Simple, preserves in-progress work -- Cons: User has to wait and retry manually, confusing with GitOps - -**Option 3: Queue the Change** -- Pros: No transition interruption -- Cons: Complex implementation, delayed behavior (user expects immediate action) - -**Why Option 1 (Abort) is Better:** -- βœ… Clear semantics: nonce = "redeploy NOW" -- βœ… Matches user expectation -- βœ… Consistent with FlinkDeployment behavior -- βœ… No silent failures (previous behavior) - -## Testing - -### Unit Test Scenario - -**Test Name:** `testSavepointRedeployDuringTransition` - -**Steps:** -1. Create BlueGreen with BLUE running -2. Trigger transition to GREEN (change parallelism) -3. While TRANSITIONING_TO_GREEN, change savepointRedeployNonce -4. Verify GREEN is deleted -5. Verify state reverts to ACTIVE_BLUE -6. Verify next reconciliation starts new transition -7. Verify new GREEN uses new savepoint path - -### E2E Test - -**File:** `e2e-tests/test_bluegreen_savepoint_redeploy.sh` - -**Test Section:** (To be added) -```bash -# Test 4: Nonce change during transition -# 1. Start transition BLUE β†’ GREEN -# 2. Before GREEN is stable, change nonce -# 3. Verify GREEN is deleted -# 4. Verify new transition uses new savepoint -``` - -## Edge Cases - -### Case 1: Deletion Fails -**Scenario:** GREEN deployment cannot be deleted (finalizer stuck, etc.) - -**Behavior:** -- Operator logs warning -- Retries deletion on next reconciliation -- Remains in ACTIVE_BLUE state until deletion succeeds - -### Case 2: Multiple Nonce Changes -**Scenario:** User changes nonce multiple times rapidly - -**Behavior:** -- Each change triggers abort and restart -- Last nonce change wins -- May cause multiple deletion/creation cycles - -**Recommendation:** Wait for state to stabilize before changing nonce again - -### Case 3: Nonce Change Right Before Completion -**Scenario:** GREEN is about to become STABLE when nonce changes - -**Behavior:** -- GREEN is deleted (wasted work) -- Transition restarts from beginning -- New GREEN created with new savepoint - -**Note:** This is intentional - user's explicit nonce change takes precedence - -## Migration from Previous Behavior - -### Old Behavior (Before Fix) -- Nonce change during transition was **silently ignored** -- Green deployment continued with old savepoint -- User's new savepoint path was never used -- Spec marked as reconciled (appearing to work) - -### New Behavior (After Fix) -- Nonce change during transition **aborts transition** -- Green deployment is deleted -- Transition restarts with new savepoint -- User intent is honored - -### Impact -- **Breaking Change:** No (old behavior was buggy) -- **User Visible:** Yes (different logs, temporary deletion) -- **Requires Documentation Update:** Yes - -## Related Features - -- **savepointRedeployNonce**: Triggers redeploy from specific savepoint -- **initialSavepointPath**: Specifies which savepoint to restore from -- **SAVEPOINT_REDEPLOY DiffType**: Detected when nonce changes -- **Blue/Green Transitions**: Normal deployment switching mechanism - -## References - -- Implementation: `BlueGreenDeploymentService.java:440-540` -- Diff Detection: `FlinkBlueGreenDeploymentSpecDiff.java:58-89` -- Nonce Annotation: `JobSpec.java:98` -- E2E Tests: `e2e-tests/test_bluegreen_savepoint_redeploy.sh`