diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46d2a356..97d54e2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,6 +218,9 @@ jobs: with: tool: protoc + - name: Setup sccache + uses: ./.github/actions/setup-sccache + # `cargo fmt --all --check` and the same `--all-targets` clippy the local # push gate runs, plus the two boundary guards that otherwise gate nothing. # The clippy feature set mirrors the test-doc workspace check so the @@ -230,6 +233,13 @@ jobs: # shellcheck disable=SC2086 cargo clippy --workspace --all-targets --locked ${LASH_CI_FEATURES} -- -D warnings + # Functional only: exercise every runtime and Lashlang measurement path + # cheaply on each push/PR. Full budget enforcement belongs to release.yml. + - name: Run performance harness smoke + run: | + python3 scripts/profile_runtime.py --profile quick --out .benchmarks/perf-smoke/runtime.json + python3 scripts/profile_lashlang.py --debug --iterations 10 --profile-iterations 10 --out .benchmarks/perf-smoke/lashlang.json + - name: Check core/UI boundary run: bash scripts/check-core-ui-boundary.sh @@ -258,7 +268,7 @@ jobs: - shard: fault-matrix runner: blacksmith-16vcpu-ubuntu-2404 - shard: sim-unit-perf-guards - runner: ubuntu-latest + runner: blacksmith-16vcpu-ubuntu-2404 - shard: sim-generated runner: blacksmith-16vcpu-ubuntu-2404 - shard: minimizer-fixtures @@ -275,22 +285,13 @@ jobs: uses: dtolnay/rust-toolchain@stable - - name: Restore cargo cache (Blacksmith) - if: matrix.runner != 'ubuntu-latest' + - name: Restore cargo cache uses: useblacksmith/rust-cache@v3.0.1 with: cache-bin: false shared-key: linux-debug save-if: false - - name: Restore cargo cache (GitHub) - if: matrix.runner == 'ubuntu-latest' - uses: Swatinem/rust-cache@v2 - with: - cache-bin: false - shared-key: linux-debug - save-if: false - - name: Install protoc and nextest uses: taiki-e/install-action@v2 with: @@ -504,12 +505,11 @@ jobs: - name: Restore cargo cache - uses: Swatinem/rust-cache@v2 + uses: useblacksmith/rust-cache@v3.0.1 with: cache-bin: false # Sole writer of the linux release-profile cache; release.yml's - # x86_64 build restores it (then stamps the release version, so only - # the workspace crates rebuild — the cached deps stay warm). + # perf jobs restore it, so release-profile dependencies stay warm. shared-key: linux-release - name: Install protoc @@ -526,7 +526,7 @@ jobs: link-with-mold: "false" - name: Build release workspace - run: cargo build --locked --release --workspace --target x86_64-unknown-linux-gnu + run: cargo build --locked --release --workspace - name: Report sccache stats if: always() diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 4db28825..90e03d9c 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -1,14 +1,10 @@ name: Performance -# The heavyweight runtime performance checks deliberately do not run per push (see -# ci.yml); this nightly run is the sole enforcement point for the budgets in -# scripts/perf_guard_budgets.json — without it the budget system gates -# nothing. +# The release workflow enforces these full profiles before publishing. This +# workflow keeps the same budget-enforcing check available on demand. on: workflow_dispatch: - schedule: - - cron: "17 3 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,18 +19,21 @@ env: jobs: perf-guard-full: name: Perf Guard Full - runs-on: ubuntu-latest + runs-on: blacksmith-16vcpu-ubuntu-2404 steps: - name: Check out repository uses: actions/checkout@v7 + - name: Reclaim runner disk + run: bash scripts/ci-reclaim-disk.sh + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - name: Restore cargo cache - uses: Swatinem/rust-cache@v2 + uses: useblacksmith/rust-cache@v3.0.1 with: cache-bin: false # Restore-only against the release cache ci.yml warms on main. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e34fdf5..e60142af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ on: type: string permissions: - contents: write + contents: read actions: read concurrency: @@ -52,7 +52,7 @@ jobs: runs_file="$(mktemp)" gh run list --workflow ci.yml --commit "${sha}" --limit 20 \ - --json conclusion,event,headSha,status > "${runs_file}" + --json conclusion,databaseId,displayTitle,event,headSha,status,url,workflowName > "${runs_file}" python3 - "${sha}" "${runs_file}" <<'PY' import json import pathlib @@ -60,14 +60,33 @@ jobs: sha = sys.argv[1] runs = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8")) - if not any( - run.get("headSha") == sha - and run.get("event") == "push" - and run.get("status") == "completed" - and run.get("conclusion") == "success" + matching = [ + run for run in runs + if run.get("headSha") == sha + and run.get("event") in ("push", "workflow_dispatch") + ] + if not matching: + raise SystemExit( + f"release refused: no main CI run found for target {sha}" + ) + run = matching[0] + if ( + run.get("status") != "completed" + or run.get("conclusion") != "success" ): - raise SystemExit(f"no successful main CI push run found for {sha}") + state = ( + run.get("conclusion") + if run.get("status") == "completed" + else run.get("status") + ) + raise SystemExit( + "release refused: target " + f"{sha} has no successful main CI run; latest matching " + f"{run.get('workflowName', 'CI')} run {run.get('databaseId')} " + f"({run.get('displayTitle', 'untitled')}) is {state}: " + f"{run.get('url', 'URL unavailable')}" + ) PY git checkout --detach "${sha}" @@ -80,33 +99,63 @@ jobs: echo "Computed release tag ${tag} already exists on another commit." >&2 exit 1 fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag "${tag}" "${sha}" - git push origin "${tag}" fi echo "sha=${sha}" >> "${GITHUB_OUTPUT}" echo "tag=${tag}" >> "${GITHUB_OUTPUT}" - echo "Preparing ${tag} from ${sha}." >> "${GITHUB_STEP_SUMMARY}" + echo "Validating ${sha} for ${tag}." >> "${GITHUB_STEP_SUMMARY}" validate-release-ref: name: Validate release ref - runs-on: ubuntu-latest + runs-on: blacksmith-16vcpu-ubuntu-2404 needs: prepare-release steps: - name: Check out repository uses: actions/checkout@v7 with: - ref: ${{ needs.prepare-release.outputs.release_tag }} + ref: ${{ needs.prepare-release.outputs.release_sha }} + + - name: Reclaim runner disk + run: bash scripts/ci-reclaim-disk.sh - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Restore cargo cache + uses: useblacksmith/rust-cache@v3.0.1 + with: + cache-bin: false + # Restore-only against the release cache ci.yml warms on main. + shared-key: linux-release + save-if: false + + - name: Install protoc + uses: taiki-e/install-action@v2 + with: + tool: protoc + - name: Validate locked workspace metadata run: cargo metadata --format-version 1 --locked >/dev/null + - name: Run full runtime performance checks + run: | + python3 scripts/profile_runtime.py --profile full --release --enforce-budgets --out .benchmarks/perf-guard/runtime.json + python3 scripts/profile_lashlang.py --iterations 2500 --profile-iterations 2500 --enforce-budgets --out .benchmarks/perf-guard/lashlang.json + + - name: Publish performance summaries + run: | + python3 scripts/perfreport.py .benchmarks/perf-guard/runtime.json >> "$GITHUB_STEP_SUMMARY" + python3 scripts/perfreport.py .benchmarks/perf-guard/lashlang.json >> "$GITHUB_STEP_SUMMARY" + + - name: Upload performance guard artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: release-perf-guard-${{ needs.prepare-release.outputs.release_tag }}-attempt-${{ github.run_attempt }} + path: .benchmarks/perf-guard/* + if-no-files-found: warn + publish-crates: name: Publish crates to crates.io runs-on: ubuntu-latest @@ -121,7 +170,26 @@ jobs: - name: Check out repository uses: actions/checkout@v7 with: - ref: ${{ needs.prepare-release.outputs.release_tag }} + ref: ${{ needs.prepare-release.outputs.release_sha }} + + - name: Validate immutable release source + env: + RELEASE_SHA: ${{ needs.prepare-release.outputs.release_sha }} + RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }} + run: | + set -euo pipefail + head_sha="$(git rev-parse HEAD)" + if [ "${head_sha}" != "${RELEASE_SHA}" ]; then + echo "Release checkout ${head_sha} does not match validated SHA ${RELEASE_SHA}." >&2 + exit 1 + fi + + git fetch --force origin --tags + tag_sha="$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}^{commit}" || true)" + if [ -n "${tag_sha}" ] && [ "${tag_sha}" != "${RELEASE_SHA}" ]; then + echo "Release tag ${RELEASE_TAG} points to ${tag_sha}, not validated SHA ${RELEASE_SHA}." >&2 + exit 1 + fi - name: Check out release tooling uses: actions/checkout@v7 @@ -138,10 +206,10 @@ jobs: # publish=false crates are skipped; versions already visible on crates.io # are skipped, so reruns resume after partial publication. # - # --version stamps the real release version into the tag checkout's + # --version stamps the real release version into the validated checkout's # manifests + lockfile before reading the graph and publishing: `main` # carries the 0.0.0-dev placeholder, so this is what makes the published - # crates pin the real version. The stamp runs the tag checkout's own + # crates pin the real version. The stamp runs the validated checkout's own # scripts/release_version.py (the publisher script itself is the pinned # `.release-tools` copy). # @@ -169,26 +237,54 @@ jobs: name: Publish SDK release runs-on: ubuntu-latest needs: [prepare-release, publish-crates] + permissions: + contents: write + actions: write env: + RELEASE_SHA: ${{ needs.prepare-release.outputs.release_sha }} RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }} steps: - name: Check out repository uses: actions/checkout@v7 with: - ref: ${{ needs.prepare-release.outputs.release_tag }} + ref: ${{ needs.prepare-release.outputs.release_sha }} # Full history + tags: release_notes.py walks previous-tag..tag to # assemble the curated release body. fetch-depth: 0 fetch-tags: true - # Curated notes from `Release-Notes:` commit sections become the release # body; the auto-generated commit list is appended below. Written # outside dist/ so it is not uploaded as an asset. prepare-release has # already required curated notes before creating a new tag. - name: Collect release notes - run: python3 scripts/release_notes.py collect --end "${RELEASE_TAG}" --out "${{ runner.temp }}/RELEASE_NOTES.md" + run: python3 scripts/release_notes.py collect --end "${RELEASE_SHA}" --out "${{ runner.temp }}/RELEASE_NOTES.md" + + - name: Publish validated release tag + run: | + set -euo pipefail + head_sha="$(git rev-parse HEAD)" + if [ "${head_sha}" != "${RELEASE_SHA}" ]; then + echo "Release checkout ${head_sha} does not match validated SHA ${RELEASE_SHA}." >&2 + exit 1 + fi + + git fetch --force origin main --tags + tag_sha="$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}^{commit}" || true)" + if [ -n "${tag_sha}" ]; then + if [ "${tag_sha}" != "${RELEASE_SHA}" ]; then + echo "Release tag ${RELEASE_TAG} points to ${tag_sha}, not validated SHA ${RELEASE_SHA}." >&2 + exit 1 + fi + echo "Release tag ${RELEASE_TAG} already points to ${RELEASE_SHA}." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag "${RELEASE_TAG}" "${RELEASE_SHA}" + git push origin "refs/tags/${RELEASE_TAG}" - name: Publish release uses: softprops/action-gh-release@v3 @@ -196,3 +292,76 @@ jobs: tag_name: ${{ env.RELEASE_TAG }} body_path: ${{ runner.temp }}/RELEASE_NOTES.md generate_release_notes: true + + # The release is already public at this point. Keep the checked-in docs + # pin synchronized without allowing a transient main-branch race (or any + # other docs-bump failure) to turn a successful publish red. + - name: Stamp published version into docs + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + trap 'echo "::warning title=Docs pin bump failed::The release was published, but its automated docs pin bump failed. Update the docs pin manually."' ERR + + version="${RELEASE_TAG#v}" + git fetch --force origin main --tags + latest_tag="$(git tag --list 'v*' --sort=-v:refname | sed -n '1p')" + if [ "${latest_tag}" != "${RELEASE_TAG}" ]; then + echo "Skipping docs pin for superseded ${RELEASE_TAG}; newer release tag ${latest_tag} already exists." + echo "Skipped docs pin for superseded ${RELEASE_TAG}; newer release ${latest_tag} exists." >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + git checkout -B release-docs-pin origin/main + python3 scripts/release_version.py stamp-docs "${version}" + python3 scripts/lint_docs.py + + if [ -z "$(git status --short)" ]; then + echo "Docs already reference ${version}; no commit needed." + echo "Docs already reference ${version}; no commit was created." >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit \ + -m "docs: stamp release ${version}" \ + -m "Release-Notes: Internal: Stamp documentation version pins for ${version}." + + max_attempts=3 + attempt=1 + while true; do + echo "Pushing docs pin to main (attempt ${attempt}/${max_attempts})." + if git push origin HEAD:main; then + echo "Stamped docs for ${version} on main." >> "${GITHUB_STEP_SUMMARY}" + git fetch --force origin main + if [ "$(git rev-parse HEAD)" = "$(git rev-parse origin/main)" ]; then + if gh workflow run ci.yml --ref main; then + echo "Dispatched CI for the docs pin commit." >> "${GITHUB_STEP_SUMMARY}" + else + echo "::warning title=Docs pin CI dispatch failed::The docs pin reached main, but its CI run could not be dispatched." + echo "WARNING: CI was not dispatched for the docs pin commit." >> "${GITHUB_STEP_SUMMARY}" + fi + else + echo "A newer main commit superseded the docs pin head and owns CI validation." >> "${GITHUB_STEP_SUMMARY}" + fi + exit 0 + fi + + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "::warning title=Docs pin push exhausted::The release was published, but the docs pin could not be pushed to main after ${max_attempts} attempts." + echo "WARNING: docs pin for ${version} was not pushed after ${max_attempts} attempts; update it manually." >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + git fetch origin main + if ! git rebase origin/main; then + git rebase --abort || true + echo "::warning title=Docs pin rebase failed::The release was published, but the docs pin commit conflicted with main." + echo "WARNING: docs pin for ${version} conflicted with main; update it manually." >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + attempt=$((attempt + 1)) + done diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index cba3a4ee..bba9039a 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -31,7 +31,8 @@ dependency order and waits for crates.io visibility between crates. There is no version-bump commit and no second CI pass. `main` always carries the `0.0.0-dev` placeholder in every workspace manifest; the release version is -computed at cut time and stamped into an ephemeral tag checkout at packaging +computed at cut time and stamped into an immutable checkout of the validated +release SHA at packaging time (`scripts/release_version.py stamp`, `scripts/publish_workspace.py --version`), so `main` never records a released version. @@ -41,18 +42,21 @@ time (`scripts/release_version.py stamp`, `scripts/publish_workspace.py `Release` workflow. `release_sha` may name any commit on `main`; leaving it blank selects the current `main` head. 3. The workflow proves that the selected commit is on `main` and has a - successful main-push CI run, then requires curated notes for the unreleased - range. `scripts/release_version.py print-next` computes the next version from - `[workspace.metadata.release].channel` and the existing `v*` tags. The - workflow tags that exact commit and is idempotent when retried. -4. `release.yml` validates `cargo metadata --locked`, then: + successful main CI run, then requires curated notes for the unreleased range. + `scripts/release_version.py print-next` computes the next version from + `[workspace.metadata.release].channel` and the existing `v*` tags. +4. `release.yml` validates `cargo metadata --locked` and runs both full, + budget-enforcing performance profiles on the validated SHA. If either gate + fails, nothing is published and no tag is created. Then: - `publish-crates` runs `python3 .release-tools/scripts/publish_workspace.py --version `, which stamps the manifests + lockfile and publishes every crate in topological dependency layers (crates in a layer publish concurrently, one crates.io visibility wait per layer). Already-published versions are skipped, so a failed run can be re-run to resume. - - the `publish` job builds the SDK GitHub release with the curated notes. + - the `publish` job tags the validated SHA and builds the SDK GitHub release + with the curated notes. A retry verifies an existing tag still points to + that SHA. The independent `lash-cli` Release workflow owns binary artifacts, checksums, the installer, and `lash --version` stamping. @@ -77,8 +81,8 @@ value with local `v*` tags and fails when a newer release tag exists. If the matching tag is unavailable in an offline or shallow checkout, the checked-in value is the fallback, so docs lint never needs network access. -Every release is followed by a mechanical docs-pin bump PR that updates the -display snippets and fallback together: +After a release is published, the release workflow runs the same mechanical +docs-pin update against current `main`: ```bash python3 scripts/release_version.py stamp-docs X.Y.Z @@ -87,10 +91,16 @@ python3 scripts/lint_docs.py `stamp-docs` is deliberately separate from the ephemeral manifest stamp used by the release workflow: it changes checked-in documentation, not release -artifacts. The `test-doc` CI job fetches release tags, so full-clone docs lint -intentionally goes red after a release and stays red until that mechanical PR -lands. Tagless local and offline checkouts continue to use -`docs/released-version.txt` as their fallback authority. +artifacts. The workflow lints the result, then commits it directly to `main` +after publishing. An already-current pin creates no commit, and a rerun of an +older release cannot replace a newer release pin. A concurrent main update +triggers a bounded rebase-and-push retry; exhaustion is reported loudly but +cannot make an otherwise successful release fail. Because GitHub suppresses +push-triggered workflows for commits made with `GITHUB_TOKEN`, the release +workflow explicitly dispatches CI for the new main head after pushing the pin. +A newer racing main commit owns its normal push validation instead. Tagless +local and offline checkouts continue to use `docs/released-version.txt` as +their fallback authority. ## Release notes (required) @@ -114,8 +124,10 @@ a section, the release stops without publishing. The publish job collects the same range's sections (oldest first) into the GitHub release body; the auto-generated commit list is appended below. The previous tag is resolved by graph ancestry (`git describe`), not version sorting, so tags from unrelated -history lines are ignored. The flow authors no synthetic commits, so every -commit in range is a real change eligible to contribute notes. +history lines are ignored. The flow's post-release `docs: stamp release` +commit appears in the next range and carries its required categorized trailer, +but the collector excludes that mechanical note so it cannot satisfy the next +release gate by itself. Every other commit remains eligible to contribute. ## Docs code snippets diff --git a/scripts/profile_runtime.py b/scripts/profile_runtime.py index 092f0d14..9923fa16 100644 --- a/scripts/profile_runtime.py +++ b/scripts/profile_runtime.py @@ -55,7 +55,7 @@ def parse_args() -> argparse.Namespace: "--profile", choices=sorted(PROFILE_DEFAULTS), default="full", - help="Benchmark size preset: quick for push CI, full for manual/nightly runs.", + help="Benchmark size preset: quick for push CI, full for release/manual runs.", ) parser.add_argument("--runs", type=int, help="Measured runs. Defaults from --profile.") parser.add_argument("--warmups", type=int, help="Warm-up runs. Defaults from --profile.") diff --git a/scripts/release_notes.py b/scripts/release_notes.py index 43936930..3c0dd69d 100644 --- a/scripts/release_notes.py +++ b/scripts/release_notes.py @@ -18,10 +18,14 @@ The release pipeline uses two entry points: - the manually dispatched release workflow runs `collect --require` for the - selected green main commit before it creates a tag. - - the publish job runs `collect --end --out ` and feeds the file - to the GitHub release body (the auto-generated commit list is appended - below it). + selected green main commit before anything is published. + - the publish job runs `collect --end --out ` and feeds the file + to the GitHub release body before tagging that SHA (the auto-generated + commit list is appended below it). + +The automated post-release `docs: stamp release ` commit carries the +required categorized trailer for repository history, but is excluded from +collection so it cannot satisfy the next release's curated-notes gate by itself. Uses only the Python standard library, like the sibling release scripts. """ @@ -39,6 +43,8 @@ MARKER_RE = re.compile(r"^Release-Notes:(.*)$") SQUASH_SUBJECT_RE = re.compile(r"^\*\s") RECORD_SEPARATOR = "\x1e" +FIELD_SEPARATOR = "\x1f" +AUTOMATED_DOCS_STAMP_SUBJECT_RE = re.compile(r"^docs: stamp release \S+$") CATEGORIES = ("Breaking", "Added", "Fixed", "Changed", "Internal") CATEGORY_COLON_RE = re.compile( r"^(Breaking|Added|Fixed|Changed|Internal):\s*(.*)$", re.IGNORECASE @@ -148,14 +154,14 @@ def render_notes(notes: list[str]) -> str: return "\n\n".join(sections).strip() -def commit_bodies(range_spec: str) -> list[str]: - """Return commit bodies in chronological order for an arbitrary range.""" +def commit_messages(range_spec: str) -> list[tuple[str, str]]: + """Return (subject, body) pairs in chronological order for a git range.""" result = subprocess.run( [ "git", "log", "--reverse", - f"--format=%B{RECORD_SEPARATOR}", + f"--format=%s{FIELD_SEPARATOR}%B{RECORD_SEPARATOR}", range_spec, ], cwd=ROOT, @@ -163,17 +169,33 @@ def commit_bodies(range_spec: str) -> list[str]: capture_output=True, text=True, ) - return [ - record.strip("\n") - for record in result.stdout.split(RECORD_SEPARATOR) - if record.strip() - ] + messages: list[tuple[str, str]] = [] + for record in result.stdout.split(RECORD_SEPARATOR): + if not record.strip(): + continue + subject, separator, body = record.partition(FIELD_SEPARATOR) + if not separator: + raise ValueError("git log record is missing its subject/body separator") + messages.append((subject.strip(), body.strip("\n"))) + return messages + + +def commit_bodies(range_spec: str) -> list[str]: + """Return commit bodies in chronological order for an arbitrary range.""" + return [body for _, body in commit_messages(range_spec)] + + +def is_automated_docs_stamp(subject: str) -> bool: + """Whether a commit is the release workflow's mechanical docs-pin update.""" + return AUTOMATED_DOCS_STAMP_SUBJECT_RE.fullmatch(subject) is not None def collect_notes_for_range(range_spec: str) -> list[str]: """Collect all notes from an arbitrary git range, oldest first.""" notes: list[str] = [] - for body in commit_bodies(range_spec): + for subject, body in commit_messages(range_spec): + if is_automated_docs_stamp(subject): + continue notes.extend(extract_notes(body)) return notes diff --git a/scripts/release_version.py b/scripts/release_version.py index 9282b28f..80f5a275 100644 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -14,7 +14,7 @@ stamp Rewrite the workspace manifests + lockfile to (used by the release workflow's ephemeral checkout). stamp-docs Rewrite the checked-in doc install snippets to - (a maintainer convenience; CI no longer runs it). + (used by release automation and maintainers). """ from __future__ import annotations @@ -249,8 +249,8 @@ def update_doc_example_versions(version: str) -> None: The docs are static HTML/Markdown rather than a templated site build. Only lines that mention Lash crates are rewritten; unrelated dependency versions - in examples stay untouched. CI no longer runs this (there is no release - commit); it stays as a maintainer convenience for refreshing the display + in examples stay untouched. Release automation runs this for its + post-publish commit, and maintainers can run it when refreshing the display snippets in a normal PR. """ simple_dep_re = re.compile( diff --git a/scripts/test_confidence_gate_ci_contract.py b/scripts/test_confidence_gate_ci_contract.py index 81757cba..40c148bc 100644 --- a/scripts/test_confidence_gate_ci_contract.py +++ b/scripts/test_confidence_gate_ci_contract.py @@ -15,6 +15,7 @@ CONFIDENCE_WORKFLOW = ROOT / ".github" / "workflows" / "confidence.yml" PERF_WORKFLOW = ROOT / ".github" / "workflows" / "perf.yml" RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +RELEASE_NOTES = ROOT / "scripts" / "release_notes.py" GATE = ROOT / "scripts" / "confidence-gate.sh" QUARANTINE_CHECK = ROOT / "scripts" / "check_test_quarantines.py" CARGO_TOML = ROOT / "Cargo.toml" @@ -35,6 +36,9 @@ VALIDATE_QUARANTINE_MANIFEST = runpy.run_path(str(QUARANTINE_CHECK))[ "validate_manifest" ] +IS_AUTOMATED_DOCS_STAMP = runpy.run_path(str(RELEASE_NOTES))[ + "is_automated_docs_stamp" +] def shell_int_constant(script: str, name: str) -> int: @@ -194,6 +198,25 @@ def test_lint_job_checks_and_previews_pr_release_notes(self) -> None: self.assertIn("python3 scripts/release_notes.py check-pr", lint) self.assertIn('--summary "$GITHUB_STEP_SUMMARY"', lint) + def test_lint_job_runs_functional_perf_smoke_without_budgets(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + lint = workflow_job_block(workflow, "lint") + + self.assertIn("runs-on: blacksmith-8vcpu-ubuntu-2404", lint) + self.assertIn( + "profile_runtime.py --profile quick " + "--out .benchmarks/perf-smoke/runtime.json", + lint, + ) + self.assertIn( + "profile_lashlang.py --debug --iterations 10 " + "--profile-iterations 10 --out .benchmarks/perf-smoke/lashlang.json", + lint, + ) + smoke = lint[lint.index("- name: Run performance harness smoke") :] + smoke = smoke[: smoke.index("- name: Check core/UI boundary")] + self.assertNotIn("--enforce-budgets", smoke) + def test_workflow_graph_example_is_in_functional_matrix(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") justfile = JUSTFILE.read_text(encoding="utf-8") @@ -293,27 +316,88 @@ def test_release_is_manual_and_requires_a_green_main_commit(self) -> None: 'requested="${REQUESTED_SHA:-origin/main}"', 'git merge-base --is-ancestor "${sha}" origin/main', 'gh run list --workflow ci.yml --commit "${sha}"', - 'run.get("event") == "push"', - 'run.get("conclusion") == "success"', + 'run.get("event") in ("push", "workflow_dispatch")', + "run = matching[0]", + 'run.get("conclusion") != "success"', + "release refused: target ", + "run.get('databaseId')", + "run.get('url', 'URL unavailable')", "release_notes.py collect --require", "release_version.py print-next", - 'git tag "${tag}" "${sha}"', + 'git tag "${RELEASE_TAG}" "${RELEASE_SHA}"', ] for snippet in required_snippets: self.assertIn(snippet, workflow) self.assertNotIn("\n push:\n", workflow) + self.assertLess( + workflow.index( + "profile_runtime.py --profile full --release --enforce-budgets" + ), + workflow.index('git tag "${RELEASE_TAG}" "${RELEASE_SHA}"'), + ) def test_runtime_release_publishes_sdk_without_host_assets(self) -> None: workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") publish = workflow_job_block(workflow, "publish") publish_crates = workflow_job_block(workflow, "publish-crates") + validate_release = workflow_job_block(workflow, "validate-release-ref") self.assertNotIn("build-release-assets", workflow) - self.assertNotIn("perf-guard-full", workflow) self.assertNotIn("install_lash.sh", workflow) self.assertIn("needs: [prepare-release, publish-crates]", publish) self.assertIn("needs: [prepare-release, validate-release-ref]", publish_crates) + self.assertIn("runs-on: blacksmith-16vcpu-ubuntu-2404", validate_release) + self.assertIn( + "ref: ${{ needs.prepare-release.outputs.release_sha }}", validate_release + ) + self.assertIn( + "ref: ${{ needs.prepare-release.outputs.release_sha }}", publish_crates + ) + self.assertIn("ref: ${{ needs.prepare-release.outputs.release_sha }}", publish) + self.assertIn('head_sha="$(git rev-parse HEAD)"', publish_crates) + self.assertIn('head_sha="$(git rev-parse HEAD)"', publish) + self.assertIn( + "profile_runtime.py --profile full --release --enforce-budgets", + validate_release, + ) + self.assertIn( + "profile_lashlang.py --iterations 2500 --profile-iterations 2500 " + "--enforce-budgets", + validate_release, + ) + + def test_full_perf_is_release_gated_and_only_manually_dispatchable(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + perf = PERF_WORKFLOW.read_text(encoding="utf-8") + release = RELEASE_WORKFLOW.read_text(encoding="utf-8") + release_cache = workflow_job_block(workflow, "linux-release-cache") + + self.assertIn("workflow_dispatch:", perf) + self.assertNotIn("schedule:", perf) + self.assertIn("runs-on: blacksmith-16vcpu-ubuntu-2404", perf) + self.assertIn("useblacksmith/rust-cache@v3.0.1", release_cache) + self.assertIn("cargo build --locked --release --workspace", release_cache) + self.assertNotIn("--target x86_64-unknown-linux-gnu", release_cache) + for command in ( + "profile_runtime.py --profile full --release --enforce-budgets", + "profile_lashlang.py --iterations 2500 --profile-iterations 2500 " + "--enforce-budgets", + ): + self.assertIn(command, perf) + self.assertIn(command, release) + + def test_all_confidence_fast_shards_use_blacksmith(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + confidence_fast = workflow_job_block(workflow, "confidence-fast") + + self.assertIn( + "- shard: sim-unit-perf-guards\n" + " runner: blacksmith-16vcpu-ubuntu-2404", + confidence_fast, + ) + self.assertNotIn("ubuntu-latest", confidence_fast) + self.assertNotIn("Restore cargo cache (GitHub)", confidence_fast) def test_broad_lane_is_manual_or_scheduled_confidence_not_ci_cd(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") @@ -462,13 +546,14 @@ def test_property_and_await_cancel_evidence_pinned_in_fast_gate(self) -> None: for snippet in required_snippets: self.assertIn(snippet, gate) - def test_publish_time_version_injection_has_no_bump_commit_or_second_pass(self) -> None: + def test_publish_time_version_injection_has_only_post_release_docs_commit(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") release = RELEASE_WORKFLOW.read_text(encoding="utf-8") cargo = CARGO_TOML.read_text(encoding="utf-8") - # The bump commit and pass-1/pass-2 re-run chain are gone. A green main - # push only validates; a manual release stamps an ephemeral checkout. + # The manifest bump commit and pass-1/pass-2 re-run chain are gone. A + # green main push only validates; a manual release stamps an ephemeral + # checkout, then updates only the checked-in docs pin after publishing. self.assertNotIn("release_version.py set", workflow) self.assertNotIn("Commit release version", workflow) self.assertNotIn("Dispatch validation pass", workflow) @@ -493,6 +578,24 @@ def test_publish_time_version_injection_has_no_bump_commit_or_second_pass(self) # The publisher stamps the ephemeral checkout before packaging crates. # Host-application binary stamping belongs to lash-cli's release. self.assertIn("publish_workspace.py --version", release) + self.assertIn('release_version.py stamp-docs "${version}"', release) + self.assertIn("git push origin HEAD:main", release) + self.assertIn("git rebase origin/main", release) + self.assertIn("continue-on-error: true", release) + self.assertIn( + "Release-Notes: Internal: Stamp documentation version pins", release + ) + self.assertIn( + "Skipping docs pin for superseded ${RELEASE_TAG}", release + ) + self.assertIn( + "git tag --list 'v*' --sort=-v:refname", release + ) + self.assertIn("python3 scripts/lint_docs.py", release) + self.assertIn("gh workflow run ci.yml --ref main", release) + self.assertIn("permissions:\n contents: read\n actions: read", release) + publish = workflow_job_block(release, "publish") + self.assertIn("permissions:\n contents: write\n actions: write", publish) def test_release_notes_are_gated_only_when_a_manual_release_is_cut(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") @@ -503,6 +606,13 @@ def test_release_notes_are_gated_only_when_a_manual_release_is_cut(self) -> None prepare_release = workflow_job_block(release, "prepare-release") self.assertIn("release_notes.py collect --require", prepare_release) + def test_automated_docs_stamp_cannot_satisfy_next_release_notes_gate(self) -> None: + release_notes = RELEASE_NOTES.read_text(encoding="utf-8") + + self.assertTrue(IS_AUTOMATED_DOCS_STAMP("docs: stamp release 0.1.0-alpha.113")) + self.assertFalse(IS_AUTOMATED_DOCS_STAMP("docs: explain release 0.1.0-alpha.113")) + self.assertIn("if is_automated_docs_stamp(subject):", release_notes) + def test_workspace_tests_are_sharded_off_the_critical_path(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") @@ -519,7 +629,13 @@ def test_heavy_compile_jobs_route_through_sccache(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") release = RELEASE_WORKFLOW.read_text(encoding="utf-8") - for job_id in ("test-doc", "test-shard", "confidence-fast", "linux-release-cache"): + for job_id in ( + "test-doc", + "test-shard", + "lint", + "confidence-fast", + "linux-release-cache", + ): block = workflow_job_block(workflow, job_id) self.assertIn("./.github/actions/setup-sccache", block) self.assertNotIn("cargo build", release)