diff --git a/.github/workflows/chart-release.yml b/.github/workflows/chart-release.yml index 58b220f..1a1e2c3 100644 --- a/.github/workflows/chart-release.yml +++ b/.github/workflows/chart-release.yml @@ -3,12 +3,14 @@ name: Helm Chart Release # Reusable matrix-based Helm chart release. # # Auto-discovers charts under `charts/*/`. For each chart: -# - Computes the next `chart//X.Y.Z` tag. -# - Skips charts unchanged since their last release tag (unless -# workflow_dispatch forced a release). -# - Packages, pushes to ECR OCI, tags, contributes to one aggregated -# GitHub Release. Bumps past versions that already exist in ECR -# (immutable) — recovers stranded versions from prior partial runs. +# - Skips charts unchanged since their last release (baseline = +# refs/releases/chart/ marker ref, legacy chart//X.Y.Z +# tag as fallback) unless workflow_dispatch forced a release. +# - Resolves the next version from the ECR OCI repo (highest X.Y.Z +# tag + 1 — the artifact store is the version record, +# platform-gitops#1201), packages, pushes, advances the marker ref, +# mints the legacy git tag, contributes to one aggregated GitHub +# Release. # # Caller has no inputs; auto-discovery handles everything. Each repo's # `ci.yml` decides whether to call this workflow. @@ -38,7 +40,7 @@ jobs: fetch-depth: 0 fetch-tags: true - - name: Discover charts and compute candidate versions + - name: Discover charts needing release id: set run: | set -euo pipefail @@ -62,35 +64,41 @@ jobs: fi } + # Change-detection baselines live in per-chart marker refs + # (refs/releases/chart/, advanced by the package job on + # each successful push — platform-gitops#1201). Not tags, so + # checkout's fetch-tags does not bring them in. Version + # resolution happens in the package job: ECR is the version + # record and this job has no AWS credentials. + git fetch --quiet origin '+refs/releases/*:refs/releases/*' || true + for chart_dir in charts/*/; do [ -f "${chart_dir}Chart.yaml" ] || continue name=$(grep '^name:' "${chart_dir}Chart.yaml" | awk '{print $2}') - prefix="chart/${name}/" - latest=$(git tag --list "${prefix}*" --sort=-v:refname \ - | grep -E "^chart/${name}/[0-9]+\.[0-9]+\.[0-9]+$" | head -n1 || true) - - if [ -z "$latest" ]; then - candidate="0.0.1" + base="" + if git rev-parse -q --verify "refs/releases/chart/${name}" >/dev/null; then + base="refs/releases/chart/${name}" else - version="${latest#"${prefix}"}" - IFS='.' read -r major minor patch <<< "$version" - candidate="${major}.${minor}.$((patch + 1))" + # Legacy fallback until the chart has released once with + # the marker ref in place. + base=$(git tag --list "chart/${name}/*" --sort=-v:refname \ + | grep -E "^chart/${name}/[0-9]+\.[0-9]+\.[0-9]+$" | head -n1 || true) fi # Skip unchanged charts unless dispatch forced a release - if [ "$force" != "true" ] && [ -n "$latest" ]; then - changed=$(git diff --name-only "${latest}..HEAD" -- "${chart_dir}" || true) + if [ "$force" != "true" ] && [ -n "$base" ]; then + changed=$(git diff --name-only "${base}..HEAD" -- "${chart_dir}" || true) if [ -z "$changed" ]; then - echo "${name}: no changes since ${latest} — skipping" + echo "${name}: no changes since ${base} — skipping" continue fi fi - echo "${name}: candidate ${candidate}" + echo "${name}: will release" dir="${chart_dir%/}" role=$(per_service_role "$name") - entries+=("{\"name\":\"${name}\",\"dir\":\"${dir}\",\"version\":\"${candidate}\",\"role\":\"${role}\"}") + entries+=("{\"name\":\"${name}\",\"dir\":\"${dir}\",\"role\":\"${role}\"}") done if [ ${#entries[@]} -eq 0 ]; then @@ -183,32 +191,36 @@ jobs: env: NAME: ${{ matrix.name }} DIR: ${{ matrix.dir }} - CANDIDATE: ${{ matrix.version }} run: | set -euo pipefail - IFS='.' read -r major minor patch <<< "$CANDIDATE" - - # Bump past versions already in ECR (immutable). Recover - # stranded ECR pushes that lost their git tag. - while true; do - version="${major}.${minor}.${patch}" - tag="chart/${NAME}/${version}" - - if git tag --list "$tag" | grep -q .; then - echo "Git tag ${tag} already exists — bumping" - patch=$((patch + 1)) - continue - fi - if helm show chart "oci://${ECR_REGISTRY}/charts/${NAME}" --version "${version}" >/dev/null 2>&1; then - echo "::warning::Chart ${NAME}:${version} exists in ECR but no git tag — recovering" - git tag "$tag" - git push origin "$tag" || echo "::warning::Recovery tag push failed for ${tag}" - patch=$((patch + 1)) - continue - fi - - break + # The ECR OCI repo is the version record: next version is a + # patch bump of the highest X.Y.Z tag already pushed. Git tags + # are a legacy mirror, still minted below for deploy + # correlation but no longer read (platform-gitops#1201). A + # missing repo or an empty tag list both mean first release. + latest=$(aws ecr describe-images \ + --repository-name "charts/${NAME}" \ + --filter tagStatus=TAGGED \ + --query 'imageDetails[].imageTags[]' --output text 2>/dev/null \ + | tr '[:space:]' '\n' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V | tail -n1 || true) + + if [ -z "$latest" ]; then + major=0; minor=0; patch=0 + else + IFS='.' read -r major minor patch <<< "$latest" + fi + patch=$((patch + 1)) + + # ECR chart versions are immutable — probe-and-bump past any + # candidate that already exists (covers races that slip the + # workflow concurrency group and repos with gaps above the + # max semver). + while helm show chart "oci://${ECR_REGISTRY}/charts/${NAME}" \ + --version "${major}.${minor}.${patch}" >/dev/null 2>&1; do + echo "Chart ${NAME}:${major}.${minor}.${patch} exists in ECR — bumping" + patch=$((patch + 1)) done version="${major}.${minor}.${patch}" tag="chart/${NAME}/${version}" @@ -240,6 +252,13 @@ jobs: exit 1 fi + # Advance the per-chart release marker — one mutable ref, not + # a tag, so it is invisible to Kargo's tag enumeration and the + # GitHub create-webhook. The prepare job diffs against it to + # decide whether the chart needs a release. + git push --force origin "${GITHUB_SHA}:refs/releases/chart/${NAME}" \ + || echo "::warning::Marker ref push failed for refs/releases/chart/${NAME} — next run falls back to the legacy tag baseline" + mkdir -p release-info echo "${NAME} ${version}" > "release-info/${NAME}.txt" diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index ee3e4bc..fbf3cc0 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -17,11 +17,15 @@ name: Docker Release # ] # } # -# Each successful build pushes the image, creates an immutable git tag -# `image//X.Y.Z`, and contributes to a single aggregated GitHub -# Release at the end of the run. Pinpoint correlates on the per-service -# git tags via the `create` webhook, so multi-service runs map cleanly to -# per-service deploy flows. +# Version resolution reads ECR (highest X.Y.Z tag + 1) — the artifact +# store is the version record (platform-gitops#1201). Each successful +# build pushes the image, advances the `refs/releases/image/` +# marker ref (the change-detection baseline for discover-services), +# creates an immutable git tag `image//X.Y.Z`, and contributes to +# a single aggregated GitHub Release at the end of the run. Dispatch +# still correlates on the per-service git tags via the `create` webhook; +# the tags and Releases go away once Dispatch is notified directly +# (#1201 dependency 2). on: workflow_call: @@ -126,43 +130,34 @@ jobs: set -euo pipefail SHORT_SHA="${SHORT_SHA:0:7}" - # Compute candidate version from latest existing tag - prefix="image/${NAME}/" - latest_tag=$(git tag --list "${prefix}*" --sort=-v:refname \ - | grep -E "^image/${NAME}/[0-9]+\.[0-9]+\.[0-9]+$" | head -n1 || true) - - if [ -z "$latest_tag" ]; then + # The ECR repo is the version record: next version is a patch + # bump of the highest X.Y.Z tag already pushed (the - + # sibling tags are filtered out). Git tags are a legacy mirror, + # still minted below for deploy correlation but no longer read + # (platform-gitops#1201). A missing repo or an empty tag list + # both mean first release. + latest=$(aws ecr describe-images \ + --repository-name "${ECR_REPO}" \ + --filter tagStatus=TAGGED \ + --query 'imageDetails[].imageTags[]' --output text 2>/dev/null \ + | tr '[:space:]' '\n' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \ + | sort -V | tail -n1 || true) + + if [ -z "$latest" ]; then major=0; minor=0; patch=0 else - version="${latest_tag#"${prefix}"}" - IFS='.' read -r major minor patch <<< "$version" + IFS='.' read -r major minor patch <<< "$latest" fi patch=$((patch + 1)) - # Bump past any version that already exists as a git tag or in - # ECR (immutable). Recovers stranded ECR images that lost their - # git tag from a prior partial run. - while true; do - candidate="${major}.${minor}.${patch}" - tag="image/${NAME}/${candidate}" - - if git tag --list "$tag" | grep -q .; then - echo "Git tag ${tag} already exists — bumping" - patch=$((patch + 1)) - continue - fi - - if aws ecr describe-images \ - --repository-name "${ECR_REPO}" \ - --image-ids imageTag="${candidate}" >/dev/null 2>&1; then - echo "::warning::Image ${ECR_REPO}:${candidate} exists in ECR but no git tag — recovering" - git tag "$tag" - git push origin "$tag" || echo "::warning::Recovery tag push failed for ${tag}" - patch=$((patch + 1)) - continue - fi - - break + # ECR tags are immutable — probe-and-bump past any candidate + # that already exists (covers races that slip the workflow + # concurrency group and repos with gaps above the max semver). + while aws ecr describe-images \ + --repository-name "${ECR_REPO}" \ + --image-ids imageTag="${major}.${minor}.${patch}" >/dev/null 2>&1; do + echo "Image ${ECR_REPO}:${major}.${minor}.${patch} exists in ECR — bumping" + patch=$((patch + 1)) done version="${major}.${minor}.${patch}" tag="image/${NAME}/${version}" @@ -222,6 +217,13 @@ jobs: exit 1 fi + # Advance the per-service release marker — one mutable ref, + # not a tag, so it is invisible to Kargo's tag enumeration and + # the GitHub create-webhook. discover-services diffs against + # it to decide whether the service needs a rebuild. + git push --force origin "${GITHUB_SHA}:refs/releases/image/${NAME}" \ + || echo "::warning::Marker ref push failed for refs/releases/image/${NAME} — next run falls back to the legacy tag baseline" + mkdir -p release-info echo "${NAME} ${version}" > "release-info/${NAME}.txt" diff --git a/AGENTS.md b/AGENTS.md index a5236c9..e363158 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,15 +32,15 @@ Both `docker-release.yml` and `chart-release.yml` assume the deterministic role Both workflows use a three-shot retry pattern (try / sleep 30 / retry / sleep 60 / retry) on `configure-aws-credentials` to survive the race where Crossplane is still creating the per-service role on first colocation. Incident reference: pinpredict/trading#616 (2-second race). Keep the retry pattern when editing AWS auth steps. -### Image / chart tagging conventions +### Image / chart versioning and tagging conventions -- Per-service image tag: `image//X.Y.Z` (immutable git tag, pushed after successful ECR push) -- Per-chart tag: `chart//X.Y.Z` -- Per-service config tag (Kargo freight for `-config` Warehouse): `vX.Y.Z+` — semver **build metadata** form (`+`), not pre-release (`-`). The `+` form passes `semverConstraint: ">=0.0.0"` cleanly; the pre-release form would require `>=0.0.0-0`. +- **ECR is the version record** (platform-gitops#1201): both release workflows resolve the next version as the highest strict-`X.Y.Z` tag in the service's ECR repo (image or `charts/` OCI) plus one, then probe-and-bump past any existing candidate (ECR tags are immutable). Git tags are never read for versioning. +- **Release marker refs**: `refs/releases/image/` and `refs/releases/chart/` — one mutable ref per service, force-advanced to the released SHA on every successful push. They are the change-detection baseline for `discover-services` and chart-release's prepare job (legacy `image|chart//X.Y.Z` tag is the fallback until a service releases once with the marker in place). Not tags, so they don't feed Kargo's tag enumeration or the GitHub `create` webhook. Readers must fetch them explicitly (`+refs/releases/*:refs/releases/*`). +- Per-service image tag: `image//X.Y.Z` (immutable git tag, pushed after successful ECR push). **Legacy mirror** — still minted only because Dispatch correlates on the `create` webhook; goes away once Dispatch is notified directly (#1201 dependency 2). +- Per-chart tag: `chart//X.Y.Z` — same legacy status. +- Per-service config tag (Kargo freight for `-config` Warehouse): `vX.Y.Z+` — semver **build metadata** form (`+`), not pre-release (`-`). The `+` form passes `semverConstraint: ">=0.0.0"` cleanly; the pre-release form would require `>=0.0.0-0`. **This tag family stays** — it is genuine Kargo freight. -Both release workflows have a **recovery loop**: if a version exists in ECR but no matching git tag (prior partial run), they reclaim it by tagging then bumping patch. Don't simplify this away. - -Pinpoint (our deploy correlator) hooks the GitHub `create` webhook on `image//*` tags — keep the tag format stable. +Pinpoint/Dispatch (our deploy correlator) hooks the GitHub `create` webhook on `image//*` tags — keep the tag format stable until the direct CI→Dispatch notification lands. ### `tag-config.yml` → platform-gitops dispatch diff --git a/README.md b/README.md index 527c08b..d2e83cf 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Why `.github` and not a dedicated `github-actions` repo: `.github` is *the* GitH | File | Purpose | |---|---| -| `docker-release.yml` | Matrix-based image build + push to ECR; per-service `image//X.Y.Z` git tags; aggregated GitHub Release. Caller passes a `matrix` input in the standard `{include:[...]}` shape. | -| `chart-release.yml` | Auto-discovers `charts/*/`, bumps versions, packages, pushes to ECR OCI, tags `chart//X.Y.Z`. No caller inputs. | +| `docker-release.yml` | Matrix-based image build + push to ECR. Version = highest `X.Y.Z` tag in the ECR repo + 1 (ECR is the version record — platform-gitops#1201); advances the `refs/releases/image/` marker ref, mints the legacy `image//X.Y.Z` git tag (Dispatch correlation, being phased out), aggregated GitHub Release. Caller passes a `matrix` input in the standard `{include:[...]}` shape. | +| `chart-release.yml` | Auto-discovers `charts/*/`, skips charts unchanged since their `refs/releases/chart/` marker ref, resolves the next version from the ECR OCI repo, packages, pushes, advances the marker, mints the legacy `chart//X.Y.Z` tag. No caller inputs. | | `tag-config.yml` | Tags merges to main that touch `.platform/services/.yaml` with `vX.Y.Z+` (per-service Kargo `-config` Warehouse freight), then dispatches `service-config-tag` to platform-gitops so missing pointer files get seeded. | | `actionlint.yml` | Lints GitHub Actions workflow YAML with [`actionlint`](https://github.com/rhysd/actionlint) at a pinned version. Self-runs on this repo when PRs/pushes touch `.github/workflows/**` or `actions/**/action.yml`; callers reuse it via `uses: pinpredict/.github/.github/workflows/actionlint.yml@main`. | @@ -19,7 +19,7 @@ Why `.github` and not a dedicated `github-actions` repo: `.github` is *the* GitH | Action | Purpose | |---|---| -| `discover-services` | Reads `.platform/services/*.yaml` and emits a docker matrix of services whose docker-relevant files changed since their last `image//*` tag. Also emits `charts_changed`. | +| `discover-services` | Reads `.platform/services/*.yaml` and emits a docker matrix of services whose docker-relevant files changed since their last release (baseline = `refs/releases/image/` marker ref; legacy `image//*` tag fallback). Also emits `charts_changed`. | | `validate-platform-service` | Pre-merge static + render check for added/modified `.platform/services/*.yaml`. Renders each via `charts/service-template` for every env in `environments[]` with all `renderXxx` flags forced on; verifies `repositories.chart` resolves to a real `charts//Chart.yaml`. Closes the gap from platform-gitops#544 — every dis-opticodds-props-streamer failure mode would have failed CI here. | | `validate-reusable-inputs` | Cross-repo input validation for callers of `pinpredict/.github` reusable workflows. Diffs every `with:` block against the referenced workflow's `on.workflow_call.inputs` map; fails on unknown keys or missing-required keys. Closes the gap left by stock `actionlint`, which can't fetch remote reusable workflows (platform-gitops#1045). Runs automatically as a sibling job in `actionlint.yml`, so any consumer that already `uses:` that reusable workflow inherits it. | | `setup-python-uv` | Install uv + a pinned Python version + (default-on) `uv sync`. | diff --git a/actions/discover-services/action.yml b/actions/discover-services/action.yml index 6f3357a..c518e16 100644 --- a/actions/discover-services/action.yml +++ b/actions/discover-services/action.yml @@ -266,32 +266,46 @@ runs: includes="" fi + # Release baselines live in per-service marker refs + # (refs/releases/image/, advanced by docker-release on + # each successful push — platform-gitops#1201). They are not + # tags, so checkout's fetch-tags does not bring them in. + git fetch --quiet origin '+refs/releases/*:refs/releases/*' || true + entries=() for def in "${SERVICE_DEFS[@]}"; do IFS='|' read -r name ecr dockerfile target build_args source_patterns <<< "$def" - # `set -o pipefail` + `grep ... | head -n1` is SIGPIPE-prone: - # head closes the pipe after one line, grep gets SIGPIPE mid-write, - # the pipeline exits non-zero, and `set -e` aborts the whole job. - # The race fires deterministically in repos with many image tags - # (250+ in trading), where grep's stdio buffer is still full at - # the moment head closes. `|| true` swallows the pipeline status — - # an empty `latest_tag` is handled by the `[ -z ... ]` block below. - latest_tag=$(git tag --list "image/${name}/*" --sort=-v:refname \ - | grep -E "^image/${name}/[0-9]+\.[0-9]+\.[0-9]+$" | head -n1 || true) - - if [ -z "$latest_tag" ]; then - echo "${name}: no previous tags — first release" + base="" + if git rev-parse -q --verify "refs/releases/image/${name}" >/dev/null; then + base="refs/releases/image/${name}" + else + # Legacy fallback until the service has released once with + # the marker ref in place. + # + # `set -o pipefail` + `grep ... | head -n1` is SIGPIPE-prone: + # head closes the pipe after one line, grep gets SIGPIPE mid-write, + # the pipeline exits non-zero, and `set -e` aborts the whole job. + # The race fires deterministically in repos with many image tags + # (250+ in trading), where grep's stdio buffer is still full at + # the moment head closes. `|| true` swallows the pipeline status — + # an empty `base` is handled by the `[ -z ... ]` block below. + base=$(git tag --list "image/${name}/*" --sort=-v:refname \ + | grep -E "^image/${name}/[0-9]+\.[0-9]+\.[0-9]+$" | head -n1 || true) + fi + + if [ -z "$base" ]; then + echo "${name}: no release baseline — first release" entries+=("$(json_entry "$name" "$ecr" "$dockerfile" "$target" "$build_args")") continue fi if [ -n "$includes" ]; then - docker_changed=$(git diff --name-only "${latest_tag}..HEAD" \ + docker_changed=$(git diff --name-only "${base}..HEAD" \ | grep -E "^($(echo "$includes" | paste -sd'|' -))" || true) else # No .dockerignore allow-list — treat everything as docker-relevant. - docker_changed=$(git diff --name-only "${latest_tag}..HEAD") + docker_changed=$(git diff --name-only "${base}..HEAD") fi [ -z "$docker_changed" ] && { echo "${name}: no docker-relevant changes — skipping"; continue; }