diff --git a/.github/workflows/chart-release.yml b/.github/workflows/chart-release.yml index 3903665..26afde0 100644 --- a/.github/workflows/chart-release.yml +++ b/.github/workflows/chart-release.yml @@ -129,53 +129,14 @@ jobs: fetch-depth: 0 fetch-tags: true - # AssumeRole on a per-service xp--gha-push role can race with - # Crossplane creating that role for the first time. Observed in - # pinpredict/trading#616: this workflow ran at 04:50:53 UTC and - # `Not authorized to perform sts:AssumeRoleWithWebIdentity` came - # back; the Crossplane composition created the role at 04:50:55, - # two seconds too late. AccessDenied is non-retryable inside - # aws-actions/configure-aws-credentials so the action's built-in - # backoff gives up after ~50s before the role exists. - # - # Three-shot pattern: try, sleep 30, retry, sleep 60, retry. Only - # the final step's failure is fatal. Zero added time on the - # steady-state path (first attempt succeeds); bounded ~90s on a - # first-colocation race. - - name: Configure AWS credentials (attempt 1) - id: aws-attempt-1 - uses: aws-actions/configure-aws-credentials@v6 - continue-on-error: true - with: - # Per-service push role when the prepare step set one - # (charts migrated to .platform/services/.yaml's - # `repositories:` block per platform-gitops#524). Empty - # `matrix.role` for un-migrated charts falls through to - # the central xp-trading-gha-push role in - # secrets.AWS_ROLE_ARN. - role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Wait for IAM role propagation (attempt 1 failed) - if: steps.aws-attempt-1.outcome == 'failure' - run: sleep 30 - - - name: Configure AWS credentials (attempt 2) - id: aws-attempt-2 - if: steps.aws-attempt-1.outcome == 'failure' - uses: aws-actions/configure-aws-credentials@v6 - continue-on-error: true - with: - role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Wait for IAM role propagation (attempt 2 failed) - if: steps.aws-attempt-2.outcome == 'failure' - run: sleep 60 - - - name: Configure AWS credentials (attempt 3, fatal) - if: steps.aws-attempt-2.outcome == 'failure' - uses: aws-actions/configure-aws-credentials@v6 + # Three-shot retry on the AWS AssumeRole to survive a + # first-colocation race with Crossplane minting the per-service + # xp--gha-push role (pinpredict/trading#616). Empty + # `matrix.role` for un-migrated charts falls back to the central + # xp-trading-gha-push role in secrets.AWS_ROLE_ARN. See the action + # for the full incident writeup. + - name: Configure AWS credentials + uses: pinpredict/.github/actions/configure-aws-with-retry@main with: role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index d03da6f..be1805a 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -59,51 +59,14 @@ jobs: fetch-depth: 0 fetch-tags: true - # AssumeRole on a per-service xp--gha-push role can race with - # Crossplane creating that role for the first time. AccessDenied - # is non-retryable inside aws-actions/configure-aws-credentials, - # so the action's built-in backoff gives up before the role - # exists. See chart-release.yml's equivalent block for the live - # incident this protects against (pinpredict/trading#616's - # 2-second race on first colocation). - # - # Three-shot pattern: try, sleep 30, retry, sleep 60, retry. Only - # the final step's failure is fatal. Zero added time on the - # steady-state path; bounded ~90s on a first-colocation race. - - name: Configure AWS credentials (attempt 1) - id: aws-attempt-1 - uses: aws-actions/configure-aws-credentials@v6 - continue-on-error: true - with: - # Per-service push role when the detect-changes step set one - # (services migrated to .platform/services/.yaml's - # `repositories:` block per platform-gitops#524). Empty - # `matrix.role` for un-migrated services falls through to - # the central xp-trading-gha-push role still held in - # secrets.AWS_ROLE_ARN. - role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Wait for IAM role propagation (attempt 1 failed) - if: steps.aws-attempt-1.outcome == 'failure' - run: sleep 30 - - - name: Configure AWS credentials (attempt 2) - id: aws-attempt-2 - if: steps.aws-attempt-1.outcome == 'failure' - uses: aws-actions/configure-aws-credentials@v6 - continue-on-error: true - with: - role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Wait for IAM role propagation (attempt 2 failed) - if: steps.aws-attempt-2.outcome == 'failure' - run: sleep 60 - - - name: Configure AWS credentials (attempt 3, fatal) - if: steps.aws-attempt-2.outcome == 'failure' - uses: aws-actions/configure-aws-credentials@v6 + # Three-shot retry on the AWS AssumeRole to survive a + # first-colocation race with Crossplane minting the per-service + # xp--gha-push role (pinpredict/trading#616). Empty + # `matrix.role` for un-migrated services falls back to the central + # xp-trading-gha-push role in secrets.AWS_ROLE_ARN. See the action + # for the full incident writeup. + - name: Configure AWS credentials + uses: pinpredict/.github/actions/configure-aws-with-retry@main with: role-to-assume: ${{ matrix.role || secrets.AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} diff --git a/README.md b/README.md index 6b47b98..a412798 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Why `.github` and not a dedicated `github-actions` repo: `.github` is *the* GitH | Action | Purpose | |---|---| | `notify-dispatch` | POSTs a signed publish notification (service, version, SHA, run URL) to Dispatch's public `/dispatch/ci` route after a successful ECR push. Called by both release workflows; warn-only on failure (the EventBridge ECR-push backstop covers a missed call). | +| `configure-aws-with-retry` | Wraps `aws-actions/configure-aws-credentials` in a three-shot retry (try / sleep 30 / retry / sleep 60 / retry) to survive a first-colocation race with Crossplane minting the per-service `xp--gha-push` role (pinpredict/trading#616). Inputs: `role-to-assume` (required), `aws-region` (default `us-east-1`). Shared by both release workflows. | | `discover-services` | Reads `.platform/services/*.yaml` and emits a docker matrix of services whose docker-relevant files changed since their last release (baseline = `refs/releases/image/` marker ref; legacy `image//*` tag fallback). Also emits `charts_changed`. | | `validate-platform-service` | Pre-merge static + render check for added/modified `.platform/services/*.yaml`. Renders each via `charts/service-template` for every env in `environments[]` with all `renderXxx` flags forced on; verifies `repositories.chart` resolves to a real `charts//Chart.yaml`. Closes the gap from platform-gitops#544 — every dis-opticodds-props-streamer failure mode would have failed CI here. | | `validate-reusable-inputs` | Cross-repo input validation for callers of `pinpredict/.github` reusable workflows. Diffs every `with:` block against the referenced workflow's `on.workflow_call.inputs` map; fails on unknown keys or missing-required keys. Closes the gap left by stock `actionlint`, which can't fetch remote reusable workflows (platform-gitops#1045). Runs automatically as a sibling job in `actionlint.yml`, so any consumer that already `uses:` that reusable workflow inherits it. | diff --git a/actions/configure-aws-with-retry/action.yml b/actions/configure-aws-with-retry/action.yml new file mode 100644 index 0000000..061adbc --- /dev/null +++ b/actions/configure-aws-with-retry/action.yml @@ -0,0 +1,68 @@ +name: Configure AWS credentials (with role-propagation retry) +description: | + Wrap aws-actions/configure-aws-credentials in a three-shot retry to + tolerate a first-colocation race with Crossplane minting the target + per-service push role. + + AssumeRole on a per-service xp--gha-push role can race with + Crossplane creating that role for the first time. Observed in + pinpredict/trading#616: the release workflow ran at 04:50:53 UTC and + `Not authorized to perform sts:AssumeRoleWithWebIdentity` came back; + the Crossplane composition created the role at 04:50:55, two seconds + too late. AccessDenied is non-retryable inside + aws-actions/configure-aws-credentials, so the action's built-in + backoff gives up after ~50s before the role exists. + + Three-shot pattern: try, sleep 30, retry, sleep 60, retry. Only the + final attempt's failure is fatal. Zero added time on the steady-state + path (first attempt succeeds); bounded ~90s on a first-colocation + race. + + Shared by chart-release.yml and docker-release.yml, which both assume + a freshly-minted per-service role. + +inputs: + role-to-assume: + description: "IAM role ARN to assume via OIDC (the per-service xp--gha-push role, or the central fallback)." + required: true + aws-region: + description: "AWS region to configure." + required: false + default: "us-east-1" + +runs: + using: composite + steps: + - name: Configure AWS credentials (attempt 1) + id: aws-attempt-1 + uses: aws-actions/configure-aws-credentials@v6 + continue-on-error: true + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }} + + - name: Wait for IAM role propagation (attempt 1 failed) + if: steps.aws-attempt-1.outcome == 'failure' + shell: bash + run: sleep 30 + + - name: Configure AWS credentials (attempt 2) + id: aws-attempt-2 + if: steps.aws-attempt-1.outcome == 'failure' + uses: aws-actions/configure-aws-credentials@v6 + continue-on-error: true + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }} + + - name: Wait for IAM role propagation (attempt 2 failed) + if: steps.aws-attempt-2.outcome == 'failure' + shell: bash + run: sleep 60 + + - name: Configure AWS credentials (attempt 3, fatal) + if: steps.aws-attempt-2.outcome == 'failure' + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }}