From 393a54910862d3d5901cab5866645ca8501126f9 Mon Sep 17 00:00:00 2001 From: laurenhitchon Date: Wed, 29 Jul 2026 18:49:31 +1000 Subject: [PATCH] feat(ci): gate v1 promotion behind a reviewed workflow and add a drift canary --- .github/workflows/promote-v1.yml | 82 +++++++++++++++++++++++++++ .github/workflows/v1-drift-canary.yml | 82 +++++++++++++++++++++++++++ MAINTENANCE.md | 32 ++++++----- 3 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/promote-v1.yml create mode 100644 .github/workflows/v1-drift-canary.yml diff --git a/.github/workflows/promote-v1.yml b/.github/workflows/promote-v1.yml new file mode 100644 index 0000000..89c13bd --- /dev/null +++ b/.github/workflows/promote-v1.yml @@ -0,0 +1,82 @@ +# Gated promotion of the mutable v1 tag — the ref every consumer stub +# executes reusable workflows from (with that repo's secrets in reach), so a +# tag move is a fleet-wide deploy and is treated like one: +# +# - refs/tags/v* is ruleset-protected; nobody force-pushes it by hand. +# - This workflow is the promotion path. It runs behind the v1-promotion +# environment (required reviewer) and pushes over RELEASE_DEPLOY_KEY, +# the ruleset's bypass actor — the same model release commits use on +# the protected main branch. +# - The previous target is recorded in the run summary; rollback is +# re-running this workflow with that SHA. +# +# The preconditions that used to be convention in MAINTENANCE.md are +# machine-enforced here: the target must be on main and its checks green. +# +# Repo-local to nswds-devops — NOT synced to consumers. +name: Promote v1 + +on: + workflow_dispatch: + inputs: + target: + description: Commit SHA on main that v1 should point to + required: true + type: string + +permissions: + contents: read + +concurrency: + group: promote-v1 + cancel-in-progress: false + +jobs: + promote: + name: promote + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: v1-promotion + steps: + - name: Checkout (deploy key) + uses: actions/checkout@v7 + with: + fetch-depth: 0 + ssh-key: ${{ secrets.RELEASE_DEPLOY_KEY }} + + - name: Validate the target commit + id: target + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET: ${{ inputs.target }} + run: | + set -euo pipefail + sha="$(git rev-parse --verify "${TARGET}^{commit}")" + if ! git merge-base --is-ancestor "$sha" origin/main; then + echo "::error::${sha} is not on main — v1 only ever points at main history" + exit 1 + fi + checks="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${sha}/check-runs?per_page=100" --jq '.check_runs')" + bad="$(jq '[.[] | select(.conclusion as $c | ["failure", "cancelled", "timed_out", "action_required"] | index($c))] | length' <<<"$checks")" + pending="$(jq '[.[] | select(.status != "completed")] | length' <<<"$checks")" + ok="$(jq '[.[] | select(.conclusion == "success")] | length' <<<"$checks")" + if [ "$bad" -gt 0 ] || [ "$pending" -gt 0 ] || [ "$ok" -eq 0 ]; then + echo "::error::check runs on ${sha}: ${ok} success, ${bad} failed, ${pending} pending — promotion needs all green" + exit 1 + fi + echo "sha=${sha}" >>"$GITHUB_OUTPUT" + + - name: Record the previous target and move the tag + env: + SHA: ${{ steps.target.outputs.sha }} + run: | + set -euo pipefail + prev="$(git ls-remote origin refs/tags/v1 | cut -f1)" + { + echo '## v1 promotion' + echo "- previous: \`${prev:-none}\`" + echo "- new: \`${SHA}\`" + echo "- rollback: re-run this workflow with target \`${prev:-none}\`" + } >>"$GITHUB_STEP_SUMMARY" + git tag -f v1 "$SHA" + git push --force origin refs/tags/v1 diff --git a/.github/workflows/v1-drift-canary.yml b/.github/workflows/v1-drift-canary.yml new file mode 100644 index 0000000..c779b67 --- /dev/null +++ b/.github/workflows/v1-drift-canary.yml @@ -0,0 +1,82 @@ +# Weekly canary for v1 promotion lag. +# +# Renovate lands dependency bumps to the reusable workflows as chore(deps) +# commits, which never cut a release — so no release exists to prompt a v1 +# move, and the fleet can silently keep executing stale CI (observed +# 2026-07-29: five action majors sat on main unpromoted, with the Release +# job green). This probe opens a single tracking issue when main carries +# week-old unpromoted changes to the executable surface +# (.github/workflows/reusable-*.yml). +# +# Repo-local to nswds-devops — NOT synced to consumers. +name: v1 drift canary + +on: + schedule: + - cron: '23 8 * * 1' # Mondays 08:23 UTC, just after the ccc canary + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Measure unpromoted reusable-workflow changes + id: drift + run: | + set -euo pipefail + if ! git rev-parse --verify --quiet v1 >/dev/null; then + echo '::error::v1 tag not found' + exit 1 + fi + count="$(git rev-list --count v1..origin/main -- '.github/workflows/reusable-*.yml')" + drifted=false + if [ "$count" -gt 0 ]; then + oldest="$(git log --reverse --format=%ct v1..origin/main -- '.github/workflows/reusable-*.yml' | head -n1)" + age_days="$(( ($(date +%s) - oldest) / 86400 ))" + if [ "$age_days" -ge 7 ]; then + drifted=true + fi + body="${RUNNER_TEMP}/v1-drift.md" + { + echo 'v1 is behind main on the executable workflow surface:' + echo + echo "- unpromoted commits touching \`reusable-*.yml\`: ${count} (oldest is ${age_days} days old)" + echo "- v1: \`$(git rev-parse v1)\` — main: \`$(git rev-parse origin/main)\`" + echo + echo '```' + git log --oneline v1..origin/main -- '.github/workflows/reusable-*.yml' + echo '```' + echo + echo 'Promote with the **Promote v1** workflow (Actions → Promote v1 → run with the target SHA).' + } >"$body" + echo "body_file=${body}" >>"$GITHUB_OUTPUT" + fi + echo "drifted=${drifted}" >>"$GITHUB_OUTPUT" + + - name: Open tracking issue when v1 has drifted + if: steps.drift.outputs.drifted == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BODY_FILE: ${{ steps.drift.outputs.body_file }} + run: | + set -euo pipefail + # Idempotent: one open tracking issue at a time, keyed on the label. + gh label create v1-drift \ + --description 'v1 lags main on reusable workflows' --color d93f0b --force + if [ "$(gh issue list --state open --label v1-drift --json number --jq 'length')" -gt 0 ]; then + echo 'Tracking issue already open; nothing to do.' + exit 0 + fi + gh issue create \ + --title 'v1 lags main on the reusable workflows — promote it' \ + --label v1-drift \ + --body-file "$BODY_FILE" diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 65792fa..2c99689 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -28,18 +28,24 @@ rather than letting prettier wrap it — an 80-wrapped line gets *unwrapped* by a width-100 config and then fails the other check. **Changing CI logic** (`reusable-*.yml`): merge to `main` as usual — nothing -reaches consumers yet, because stubs pin `@v1`. Ship it by moving the tag: - -```sh -git fetch --tags -git tag -f v1 # normally the latest release commit -git push -f origin v1 -``` - -Rules for moving v1: central CI must be green on that commit, and treat it -like a deploy — it changes CI for every repo simultaneously. Rollback is the -same command pointed at the previous commit (tags before the move: -`git rev-parse v1` and note it down). +reaches consumers yet, because stubs pin `@v1`. Ship it with the **Promote +v1** workflow (Actions → Promote v1 → run with the target SHA). Treat it +like a deploy — it changes CI for every repo simultaneously. The workflow +machine-enforces what used to be convention here: the target must be on +`main` with all checks green, the previous target is recorded in the run +summary, and the push happens over the release deploy key because +`refs/tags/v*` is ruleset-protected against manual force-push. Rollback = +re-run the workflow with the previous SHA from the last promotion's summary. + +Don't wait for a release commit to promote: Renovate's `chore(deps)` bumps +to the reusables never cut a release, so any green commit on `main` +qualifies. The **v1 drift canary** (weekly) opens a tracking issue when +unpromoted reusable-workflow changes sit on `main` for over a week. + +Emergency fallback if the promotion workflow itself is broken: temporarily +disable the tag ruleset's enforcement, push the tag, re-enable — the same +enforcement-disable two-step described for `main` in the bypass policy +section below. **Breaking CI change**: don't move v1. Tag `v2`, update `workflow-stubs/*.yml` to `@v2`, merge — the sync delivers the migration to @@ -164,7 +170,7 @@ Every entry below is something that actually happened (2026-07-15 onward). | Sync run: `[@octokit/auth-app] appId option is required` | `SYNC_APP_ID` secret missing/renamed | restore the repo secret | | Sync run: `could not read Password for 'https://***@github.com'` | App token passed as `GH_PAT` | it must go in `GH_INSTALLATION_TOKEN` | | Sync run: `ENOENT: .github/sync.yml` | driver has no checkout step | keep `actions/checkout` before the sync action | -| Consumer check: "workflow was not found" | Actions access setting reset, or the `v1` tag missing/deleted | fix the access setting; re-push the tag | +| Consumer check: "workflow was not found" | Actions access setting reset, or the `v1` tag missing/deleted | fix the access setting; re-promote via the Promote v1 workflow | | Consumer PR: "Expected — waiting for status to be reported" forever | a ruleset requires a check by its old single name | rename required context to the `job / job` form (nswds-design's "Protect main" already updated) | | `check-branch-name` red on a repo's *first* sync PR | base branch lacks the `chore/repo-sync` exemption until that PR merges | expected once; merge past it | | commitlint job: npm `EUSAGE` "can only install with an existing package-lock.json" | lockfile missing **or corrupt** — check it parses, don't trust the error text | see ONBOARDING pre-flight (a); nswds-public-sans had conflict markers committed inside it |