From 33f00578b7d0d50718ed63e9338b958d46b898f3 Mon Sep 17 00:00:00 2001
From: zanejohnson-azure <zanejohnson@microsoft.com>
Date: Fri, 12 Jun 2026 16:48:19 -0700
Subject: [PATCH] add canary in arc prod release pipeline

---
 .../ci-arc-k8s-extension-prod-release.yaml    | 546 ++++++++++++++++++
 1 file changed, 546 insertions(+)

diff --git a/.pipelines/ci-arc-k8s-extension-prod-release.yaml b/.pipelines/ci-arc-k8s-extension-prod-release.yaml
index 08470e39c..abb1154e2 100644
--- a/.pipelines/ci-arc-k8s-extension-prod-release.yaml
+++ b/.pipelines/ci-arc-k8s-extension-prod-release.yaml
@@ -63,8 +63,554 @@ extends:
     customBuildTags:
     - ES365AIMigrationTooling
     stages:
+    - stage: Stage_Canary_MCR
+      displayName: Canary MCR
+      pool:
+        name: Azure-Pipelines-CI-Test-EO
+        image: ci-1es-managed-windows-2022
+        os: windows
+      jobs:
+      - job: releaseGating
+        displayName: Release Gating
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: runCodesignValidationInjection
+          value: false
+        - name: Codeql.SkipTaskAutoInjection
+          value: true
+        - name: skipComponentGovernanceDetection
+          value: true
+        - name: skipNugetSecurityAnalysis
+          value: true
+        steps:
+        - task: 6d15af64-176c-496d-b583-fd2ae21d4df4@1
+          condition: false
+          inputs:
+            repository: none
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Branch Validation (1ES PT)
+          continueOnError: true
+          target:
+            container: host
+          env:
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+            SYSTEM_COLLECTIONURI: $(System.CollectionUri)
+            SYSTEM_TEAMPROJECT: $(System.TeamProject)
+            SYSTEM_TEAMPROJECTID: $(System.TeamProjectId)
+            BUILD_REPOSITORY_URI: $(Build.Repository.Uri)
+            BUILD_SOURCEBRANCH: $(Build.SourceBranch)
+            BUILD_REPOSITORY_NAME: $(Build.Repository.Name)
+            BUILD_REPOSITORY_ID: $(Build.Repository.ID)
+            BUILD_REPOSITORYPROVIDER: $(Build.Repository.Provider)
+            BUILD_SOURCEVERSION: $(Build.SourceVersion)
+            TASK_MODE: audit
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: release_gating.py
+      - job: approval
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: ev2Environment
+          value: Production
+        - name: Ev2MonintoringUrl
+          value: 'https://azureservicedeploy.msft.net/api/monitorrollout'
+        displayName: Approval
+        pool:
+          name: server
+        timeoutInMinutes: 7200
+        dependsOn:
+        - releaseGating
+        steps:
+        - task: ApprovalTask@1
+          inputs:
+            environment: $(ev2Environment)
+            servicetreeguid: 3170cdd2-19f0-4027-912b-1027311691a2
+      - job: Ev2_rollout_ev2_rollout
+        displayName: Agent Job - Ev2 Ev2 Rollout
+        timeoutInMinutes: '0'
+        condition: succeeded()
+        dependsOn:
+        - approval
+        variables:
+        - name: ev2Environment
+          value: Production
+        - name: RELEASE_STAGE_NAME
+          value: Canary
+        - name: Ev2MonintoringUrl
+          value: https://azureservicedeploy.msft.net/api/monitorrollout
+        - name: OneESPT.JobType
+          value: releaseJob
+          readonly: true
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: OneESPT.Workflow
+          value: ev2-classic
+          readonly: true
+        - name: runCodesignValidationInjection
+          value: false
+        - name: Codeql.SkipTaskAutoInjection
+          value: true
+        - name: skipComponentGovernanceDetection
+          value: true
+        - name: skipNugetSecurityAnalysis
+          value: true
+        - name: OneES_targetName
+          value: host
+        steps:
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Validate Hosted Pool Information (1ES PT)
+          continueOnError: false
+          target:
+            container: host
+          env:
+            HOST_ARCHITECTURE: amd64
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+            SYSTEM_DEFINITIONID: $(System.DefinitionId)
+            SYSTEM_COLLECTIONURI: $(System.CollectionUri)
+            SYSTEM_TEAMPROJECT: $(System.TeamProject)
+            SYSTEM_TEAMPROJECTID: $(System.TeamProjectId)
+            BUILD_REPOSITORY_ID: $(Build.Repository.ID)
+            BUILD_REPOSITORY_URI: $(Build.Repository.Uri)
+            PIPELINEGOVERNANCESTATUS_AUDITED: variables['PipelineGovernanceStatus_Audited']
+            PIPELINECLASSIFICATION_AUDITED: variables['PipelineClassification_Audited']
+            BUILD_REASON: $(Build.Reason)
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: validateHostedPool.ps1
+            arguments: -TargetName '' -StepTargets '' -StepsLength 0 -SkipStatelessValidation True -OS windows -IgnoreProductionPoolCheck  -IsOfficialTemplate -IsProductionReleasePipeline
+        - task: DownloadPipelineArtifact@2
+          displayName: ⏬ Pipeline Artifact Download
+          inputs:
+            buildType: specific
+            project: $(resources.pipeline._ci-arc-k8s-extension-prod-release.projectID)
+            definition: $(resources.pipeline._ci-arc-k8s-extension-prod-release.pipelineID)
+            allowFailedBuilds: false
+            buildVersionToDownload: specific
+            pipelineId: $(resources.pipeline._ci-arc-k8s-extension-prod-release.runID)
+            pipeline: _ci-arc-k8s-extension-prod-release
+            targetPath: $(Pipeline.Workspace)/ev2Artifact
+          target:
+            container: host
+        - task: AzureArtifacts.drop-validator-task.drop-validator-task.DropValidatorTask@0
+          displayName: "\U0001F6E1 Validate SBoM Manifest (1ES PT)"
+          condition: succeeded()
+          continueOnError: False
+          timeoutInMinutes: 30
+          env:
+            SBOMVALIDATOR_TEMPIGNOREMISSING: true
+          inputs:
+            BuildDropPath: $(Pipeline.Workspace)/ev2Artifact/linux-drop
+            OutputPath: $(Agent.TempDirectory)/sbom_validation_results.json
+            ValidateSignature: True
+            Verbosity: 'Verbose'
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Post-SBoM Validation (1ES PT)
+          continueOnError: true
+          target:
+            container: host
+          condition: succeeded()
+          env:
+            OutputPath: $(Agent.TempDirectory)/sbom_validation_results.json
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: post_sbom_validation.py
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Validate Source Build (1ES PT)
+          continueOnError: false
+          target:
+            container: host
+          env:
+            BuildDropPath: $(Pipeline.Workspace)/ev2Artifact/linux-drop
+            IsProduction: True
+            OneES_ArtifactType: $(DownloadPipelineArtifactResourceTypes)
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: validate_source_build.py
+        - task: securedevelopmentteam.vss-secure-development-tools.build-task-codesignvalidation.CodeSign@1
+          displayName: "\U0001F6E1 Guardian: CodeSign Validation"
+          target:
+            container: host
+          condition: and(succeeded(), ne(variables['ONEES_ENFORCED_CODESIGNVALIDATION_ENABLED'], 'false'))
+          continueOnError: true
+          timeoutInMinutes: 10
+          inputs:
+            Path: $(Pipeline.Workspace)/ev2Artifact
+            MaxThreads: $(OneES_UsableProcessorCount)
+            FailIfNoTargetsFound: false
+            ExcludePassesFromLog: False
+            Targets: f|**\*.dll;f|**\*.exe;f|**\*.sys;f|**\*.ps1;f|**\*.psm1;f|**\*.ps1xml;f|**\*.psc1;f|**\*.psd1;f|**\*.cdxml;f|**\*.vbs;f|**\*.js;f|**\*.wsf;-|.gdn\**;
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: "\U0001F6E1 Guardian: Check CodeSign Validation Results (1ES PT)"
+          continueOnError: true
+          target:
+            container: host
+          condition: and(succeeded(), ne(variables['ONEES_ENFORCED_CODESIGNVALIDATION_ENABLED'], 'false'))
+          env:
+            OneES_PipelineWorkspace: $(Pipeline.Workspace)
+            OneES_DeleteCodeSignValidationResult: True
+            OneES_CustomPolicyFile: ''
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: check_csv_results.ps1
+        - task: 6d15af64-176c-496d-b583-fd2ae21d4df4@1
+          condition: false
+          inputs:
+            repository: none
+          target:
+            container: host
+        - task: vsrm-ev2.ev2-rollout.ev2-rollout-task.Ev2RARollout@2
+          displayName: Ev2 Managed SDP - Chart Push
+          inputs:
+            EndpointProviderType: ApprovalService
+            TaskAction: RegisterAndRollout
+            UseServerMonitorTask: true
+            SkipRegistrationIfExists: True
+            ForceRegistration: true
+            ApprovalServiceEnvironment: $(ev2Environment)
+            ServiceRootPath: $(Pipeline.Workspace)/ev2Artifact/drop/build/arc-k8s-extension-Managed-SDP/ServiceGroupRoot
+            RolloutSpecPath: $(Pipeline.Workspace)/ev2Artifact/drop/build/arc-k8s-extension-Managed-SDP/ServiceGroupRoot/RolloutSpec.json
+            StageMapName: Microsoft.Azure.SDP.Standard
+            Select: regions(*)
+            ConfigurationOverrides: $(configurationOverrides)
+          env:
+            ServiceTreeGuid: 3170cdd2-19f0-4027-912b-1027311691a2
+          target:
+            container: host
+      - job: Ev2_rollout_ev2_monitoring
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: OneESPT.Workflow
+          value: ev2-classic
+          readonly: true
+        - name: ev2Environment
+          value: Production
+        - name: Ev2MonintoringUrl
+          value: 'https://azureservicedeploy.msft.net/api/monitorrollout'
+        displayName: Agent Job - Ev2 Ev2 Monitoring
+        pool:
+          name: server
+        dependsOn:
+        - Ev2_rollout_ev2_rollout
+        timeoutInMinutes: '0'
+        steps:
+        - task: vsrm-ev2.vss-server-ev2.1950188C-A844-4040-A014-A326BC8332D3.Ev2Agentless@1
+          displayName: Ev2 - Monitoring
+          inputs:
+            Ev2MonintoringUrl: $(Ev2MonintoringUrl)
+    - stage: Stage_Canary_Regions
+      displayName: Canary Regions Release
+      trigger: manual
+      pool:
+        name: Azure-Pipelines-CI-Test-EO
+        image: ci-1es-managed-windows-2022
+        os: windows
+      jobs:
+      - job: releaseGating
+        displayName: Release Gating
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: runCodesignValidationInjection
+          value: false
+        - name: Codeql.SkipTaskAutoInjection
+          value: true
+        - name: skipComponentGovernanceDetection
+          value: true
+        - name: skipNugetSecurityAnalysis
+          value: true
+        steps:
+        - task: 6d15af64-176c-496d-b583-fd2ae21d4df4@1
+          condition: false
+          inputs:
+            repository: none
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Branch Validation (1ES PT)
+          continueOnError: true
+          target:
+            container: host
+          env:
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+            SYSTEM_COLLECTIONURI: $(System.CollectionUri)
+            SYSTEM_TEAMPROJECT: $(System.TeamProject)
+            SYSTEM_TEAMPROJECTID: $(System.TeamProjectId)
+            BUILD_REPOSITORY_URI: $(Build.Repository.Uri)
+            BUILD_SOURCEBRANCH: $(Build.SourceBranch)
+            BUILD_REPOSITORY_NAME: $(Build.Repository.Name)
+            BUILD_REPOSITORY_ID: $(Build.Repository.ID)
+            BUILD_REPOSITORYPROVIDER: $(Build.Repository.Provider)
+            BUILD_SOURCEVERSION: $(Build.SourceVersion)
+            TASK_MODE: audit
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: release_gating.py
+      - job: approval
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: ev2Environment
+          value: Production
+        - name: Ev2MonintoringUrl
+          value: 'https://azureservicedeploy.msft.net/api/monitorrollout'
+        displayName: Approval
+        pool:
+          name: server
+        timeoutInMinutes: 7200
+        dependsOn:
+        - releaseGating
+        steps:
+        - task: ApprovalTask@1
+          inputs:
+            environment: $(ev2Environment)
+            servicetreeguid: 3170cdd2-19f0-4027-912b-1027311691a2
+      - job: Ev2_rollout_ev2_rollout
+        displayName: Agent job - Ev2 Ev2 Rollout
+        timeoutInMinutes: '0'
+        condition: succeeded()
+        dependsOn:
+        - approval
+        variables:
+        - name: ev2Environment
+          value: Production
+        - name: RELEASE_STAGE_NAME
+          value: CanaryStable
+        - name: Ev2MonintoringUrl
+          value: https://azureservicedeploy.msft.net/api/monitorrollout
+        - name: OneESPT.JobType
+          value: releaseJob
+          readonly: true
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: OneESPT.Workflow
+          value: ev2-classic
+          readonly: true
+        - name: runCodesignValidationInjection
+          value: false
+        - name: Codeql.SkipTaskAutoInjection
+          value: true
+        - name: skipComponentGovernanceDetection
+          value: true
+        - name: skipNugetSecurityAnalysis
+          value: true
+        - name: OneES_targetName
+          value: host
+        steps:
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Validate Hosted Pool Information (1ES PT)
+          continueOnError: false
+          target:
+            container: host
+          env:
+            HOST_ARCHITECTURE: amd64
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+            SYSTEM_DEFINITIONID: $(System.DefinitionId)
+            SYSTEM_COLLECTIONURI: $(System.CollectionUri)
+            SYSTEM_TEAMPROJECT: $(System.TeamProject)
+            SYSTEM_TEAMPROJECTID: $(System.TeamProjectId)
+            BUILD_REPOSITORY_ID: $(Build.Repository.ID)
+            BUILD_REPOSITORY_URI: $(Build.Repository.Uri)
+            PIPELINEGOVERNANCESTATUS_AUDITED: variables['PipelineGovernanceStatus_Audited']
+            PIPELINECLASSIFICATION_AUDITED: variables['PipelineClassification_Audited']
+            BUILD_REASON: $(Build.Reason)
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: validateHostedPool.ps1
+            arguments: -TargetName '' -StepTargets '' -StepsLength 0 -SkipStatelessValidation True -OS windows -IgnoreProductionPoolCheck  -IsOfficialTemplate -IsProductionReleasePipeline
+        - task: DownloadPipelineArtifact@2
+          displayName: ⏬ Pipeline Artifact Download
+          inputs:
+            buildType: specific
+            project: $(resources.pipeline._ci-arc-k8s-extension-prod-release.projectID)
+            definition: $(resources.pipeline._ci-arc-k8s-extension-prod-release.pipelineID)
+            allowFailedBuilds: false
+            buildVersionToDownload: specific
+            pipelineId: $(resources.pipeline._ci-arc-k8s-extension-prod-release.runID)
+            pipeline: _ci-arc-k8s-extension-prod-release
+            targetPath: $(Pipeline.Workspace)/ev2Artifact
+          target:
+            container: host
+        - task: AzureArtifacts.drop-validator-task.drop-validator-task.DropValidatorTask@0
+          displayName: "\U0001F6E1 Validate SBoM Manifest (1ES PT)"
+          condition: succeeded()
+          continueOnError: False
+          timeoutInMinutes: 30
+          env:
+            SBOMVALIDATOR_TEMPIGNOREMISSING: true
+          inputs:
+            BuildDropPath: $(Pipeline.Workspace)/ev2Artifact/linux-drop
+            OutputPath: $(Agent.TempDirectory)/sbom_validation_results.json
+            ValidateSignature: True
+            Verbosity: 'Verbose'
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Post-SBoM Validation (1ES PT)
+          continueOnError: true
+          target:
+            container: host
+          condition: succeeded()
+          env:
+            OutputPath: $(Agent.TempDirectory)/sbom_validation_results.json
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: post_sbom_validation.py
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: Validate Source Build (1ES PT)
+          continueOnError: false
+          target:
+            container: host
+          env:
+            BuildDropPath: $(Pipeline.Workspace)/ev2Artifact/linux-drop
+            IsProduction: True
+            OneES_ArtifactType: $(DownloadPipelineArtifactResourceTypes)
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: validate_source_build.py
+        - task: securedevelopmentteam.vss-secure-development-tools.build-task-codesignvalidation.CodeSign@1
+          displayName: "\U0001F6E1 Guardian: CodeSign Validation"
+          target:
+            container: host
+          condition: and(succeeded(), ne(variables['ONEES_ENFORCED_CODESIGNVALIDATION_ENABLED'], 'false'))
+          continueOnError: true
+          timeoutInMinutes: 10
+          inputs:
+            Path: $(Pipeline.Workspace)/ev2Artifact
+            MaxThreads: $(OneES_UsableProcessorCount)
+            FailIfNoTargetsFound: false
+            ExcludePassesFromLog: False
+            Targets: f|**\*.dll;f|**\*.exe;f|**\*.sys;f|**\*.ps1;f|**\*.psm1;f|**\*.ps1xml;f|**\*.psc1;f|**\*.psd1;f|**\*.cdxml;f|**\*.vbs;f|**\*.js;f|**\*.wsf;-|.gdn\**;
+        - task: 1ESGPTRunTask@3.0.376
+          displayName: "\U0001F6E1 Guardian: Check CodeSign Validation Results (1ES PT)"
+          continueOnError: true
+          target:
+            container: host
+          condition: and(succeeded(), ne(variables['ONEES_ENFORCED_CODESIGNVALIDATION_ENABLED'], 'false'))
+          env:
+            OneES_PipelineWorkspace: $(Pipeline.Workspace)
+            OneES_DeleteCodeSignValidationResult: True
+            OneES_CustomPolicyFile: ''
+          inputs:
+            repoId: microsoft/Docker-Provider
+            path: check_csv_results.ps1
+        - task: 6d15af64-176c-496d-b583-fd2ae21d4df4@1
+          condition: false
+          inputs:
+            repository: none
+          target:
+            container: host
+        - task: vsrm-ev2.ev2-rollout.ev2-rollout-task.Ev2RARollout@2
+          displayName: Ev2 Managed SDP - Deploy
+          inputs:
+            EndpointProviderType: ApprovalService
+            TaskAction: RegisterAndRollout
+            UseServerMonitorTask: true
+            SkipRegistrationIfExists: True
+            ForceRegistration: true
+            ApprovalServiceEnvironment: $(ev2Environment)
+            ServiceRootPath: $(Pipeline.Workspace)/ev2Artifact/drop/build/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot
+            RolloutSpecPath: $(Pipeline.Workspace)/ev2Artifact/drop/build/arc-k8s-extension-release-v2-Managed-SDP/ServiceGroupRoot/RolloutSpec.json
+            StageMapName: Microsoft.Azure.SDP.Standard
+            Select: regions(*)
+            ConfigurationOverrides: $(configurationOverrides)
+          env:
+            ServiceTreeGuid: 3170cdd2-19f0-4027-912b-1027311691a2
+          target:
+            container: host
+      - job: Ev2_rollout_ev2_monitoring
+        variables:
+        - name: OneESPT
+          value: true
+          readonly: true
+        - name: OneESPT.BuildType
+          value: Official
+          readonly: true
+        - name: OneESPT.OS
+          value: windows
+          readonly: true
+        - name: OneESPT.Workflow
+          value: ev2-classic
+          readonly: true
+        - name: ev2Environment
+          value: Production
+        - name: Ev2MonintoringUrl
+          value: 'https://azureservicedeploy.msft.net/api/monitorrollout'
+        displayName: Agent job - Ev2 Ev2 Monitoring
+        pool:
+          name: server
+        dependsOn:
+        - Ev2_rollout_ev2_rollout
+        timeoutInMinutes: '0'
+        steps:
+        - task: vsrm-ev2.vss-server-ev2.1950188C-A844-4040-A014-A326BC8332D3.Ev2Agentless@1
+          displayName: Ev2 - Monitoring
+          inputs:
+            Ev2MonintoringUrl: $(Ev2MonintoringUrl)
+    - stage: Wait_After_Canary
+      displayName: Wait after Canary Regions
+      dependsOn: Stage_Canary_Regions
+      jobs:
+      - job: WaitJob
+        displayName: Wait for Bake Time
+        timeoutInMinutes: 1600
+        pool: server
+        steps:
+        - task: Delay@1
+          inputs:
+            delayForMinutes: 1500
     - stage: Stage_1
       displayName: ci-arc-k8s-extension-all-regions-prod-release(MCR)
+      dependsOn:
+      - Wait_After_Canary
+      trigger: manual
       pool:
         name: Azure-Pipelines-CI-Test-EO
         image: ci-1es-managed-windows-2022