From d55e9355b9aee415fc9552f98932739da723e292 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 13:09:43 +0800 Subject: [PATCH 1/7] Update release process sanity guidance --- docs/release-process.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/release-process.md b/docs/release-process.md index c269903b..1e1d1ca2 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -144,11 +144,17 @@ cargo package --manifest-path sdk/rust/macros/Cargo.toml --allow-dirty cargo package --manifest-path sdk/rust/Cargo.toml --allow-dirty --features macros ``` +If `flame-rs` depends on a `flame-rs-macros` version that has not been +published yet, the final `flame-rs` package verification will fail while +resolving registry dependencies. In that case, publish and verify +`flame-rs-macros` first, then rerun the full `flame-rs` package command before +publishing `flame-rs`. + Python package verification: ```shell cd sdk/python -uv run -n pytest tests/test_runner_e2e.py tests/test_runner.py -q +uv run -n --extra dev pytest tests/test_runner_e2e.py tests/test_runner.py -q uv run -n python -c 'import flamepy; print(flamepy.__version__)' uv build --out-dir /tmp/flamepy-${PYTHON_VERSION}-dist cd - @@ -250,8 +256,16 @@ Podman prerequisites: ```shell podman info podman login --get-login docker.io || podman login docker.io +podman run --rm --platform linux/amd64 docker.io/library/rust:1.95 rustc -vV ``` +If the amd64 Rust smoke test fails under emulation, do not publish a stable +arm64-only tag by default. Use a Podman farm or remote Podman connection with a +native amd64 builder, or document the Docker release as blocked. If the release +owner explicitly narrows the Docker scope to an arm64-first publish, push only +the versioned arm64 tags, leave `latest` untouched, and record the missing +amd64/multi-arch artifacts as a release gap. + With Podman, build both platforms into each manifest: ```shell @@ -366,6 +380,28 @@ make release-sanity Set `RELEASE_SANITY_COMPOSE_DOWN=0` only when you need to inspect the compose cluster after a failed run. +The compose smoke uses the TLS settings in `ci/flame-cluster.yaml` and +`ci/flame.yaml`. If `ci/certs` does not already contain release-test +certificates, generate them before running the compose sanity check: + +```shell +ci/generate-certs.sh --output ci/certs \ + --san-list localhost,127.0.0.1,flame-session-manager,flame-object-cache \ + --ip-range 172.20.0.0/24 +``` + +For an explicitly scoped arm64-first Docker release, keep the compose smoke +versioned-tag-only and set the expected platform list: + +```shell +RELEASE_SANITY_EXPECTED_PLATFORMS=linux/arm64 \ +RELEASE_SANITY_LOCAL_CHECKS=0 \ +RELEASE_SANITY_PACKAGE_CHECKS=0 \ +RELEASE_SANITY_REMOTE_CHECKS=1 \ +RELEASE_SANITY_COMPOSE_E2E=1 \ +make release-sanity +``` + If Docker Hub times out while pulling base images, retry the base image pull for the affected platform before rebuilding. Use the matching tool for the selected build path: From 2ee4b79ae79b692122ac7fd900ac1a4516d437e3 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 15:58:51 +0800 Subject: [PATCH 2/7] Address release sanity review feedback --- ci/release/sanity.sh | 2 +- docs/release-process.md | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ci/release/sanity.sh b/ci/release/sanity.sh index b2f111d6..72695a60 100755 --- a/ci/release/sanity.sh +++ b/ci/release/sanity.sh @@ -260,7 +260,7 @@ run_package_checks() { log "uv run Python SDK runner tests" ( cd "$ROOT_DIR/sdk/python" - uv run -n pytest tests/test_runner_e2e.py tests/test_runner.py -q + uv run -n --extra dev pytest tests/test_runner_e2e.py tests/test_runner.py -q ) log "uv run Python SDK version import" diff --git a/docs/release-process.md b/docs/release-process.md index 1e1d1ca2..530d88b9 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -16,6 +16,7 @@ export STDNG_VERSION=0.1.8 export DOCKER_TAG="${RELEASE_TAG}" export RELEASE_BRANCH=release-0.6 export IMAGE_REGISTRY=docker.io/xflops +export RUST_BUILDER_IMAGE=docker.io/library/rust:1.95 export RELEASE_NOTES_FILE=/tmp/flame-${RELEASE_TAG}-notes.md ``` @@ -251,12 +252,16 @@ only after the versioned tag has been pushed and verified. Use either Podman or Docker Buildx. Both paths must publish a multi-arch tag that contains `linux/amd64` and `linux/arm64`. +Set `RUST_BUILDER_IMAGE` to the Rust builder image used by the release +Dockerfiles. Do not use `rust:latest` for release validation because it can drift +from the image build inputs. + Podman prerequisites: ```shell podman info podman login --get-login docker.io || podman login docker.io -podman run --rm --platform linux/amd64 docker.io/library/rust:1.95 rustc -vV +podman run --rm --platform linux/amd64 "${RUST_BUILDER_IMAGE}" rustc -vV ``` If the amd64 Rust smoke test fails under emulation, do not publish a stable @@ -407,12 +412,12 @@ the affected platform before rebuilding. Use the matching tool for the selected build path: ```shell -podman pull --platform linux/amd64 docker.io/library/rust:1.95 -podman pull --platform linux/arm64 docker.io/library/rust:1.95 +podman pull --platform linux/amd64 "${RUST_BUILDER_IMAGE}" +podman pull --platform linux/arm64 "${RUST_BUILDER_IMAGE}" podman pull --platform linux/amd64 docker.io/library/ubuntu:24.04 podman pull --platform linux/arm64 docker.io/library/ubuntu:24.04 -docker pull --platform linux/amd64 docker.io/library/rust:1.95 -docker pull --platform linux/arm64 docker.io/library/rust:1.95 +docker pull --platform linux/amd64 "${RUST_BUILDER_IMAGE}" +docker pull --platform linux/arm64 "${RUST_BUILDER_IMAGE}" docker pull --platform linux/amd64 docker.io/library/ubuntu:24.04 docker pull --platform linux/arm64 docker.io/library/ubuntu:24.04 ``` From ba7692971bf246ba3ddd9802e7174e5fa580ce4a Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 16:24:43 +0800 Subject: [PATCH 3/7] Add release image make targets --- Makefile | 76 ++++++++++++++++++++--- docs/release-process.md | 132 ++++++++-------------------------------- 2 files changed, 96 insertions(+), 112 deletions(-) diff --git a/Makefile b/Makefile index 42888177..5626fd23 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,7 @@ -# Detect OS and set container runtime -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin) - CONTAINER_RUNTIME ?= podman -else - CONTAINER_RUNTIME ?= docker -endif +# Detect a Docker-compatible container CLI. +DETECTED_CONTAINER_CLI := $(shell if command -v podman >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; else echo docker; fi) +CONTAINER_CLI ?= $(DETECTED_CONTAINER_CLI) +CONTAINER_RUNTIME ?= $(CONTAINER_CLI) # Docker image configuration DOCKER_REGISTRY ?= xflops @@ -22,6 +19,13 @@ FSM_DOCKERFILE = docker/Dockerfile.fsm FEM_DOCKERFILE = docker/Dockerfile.fem CONSOLE_DOCKERFILE = docker/Dockerfile.console +# Release image configuration +IMAGE_REGISTRY ?= docker.io/$(DOCKER_REGISTRY) +DOCKER_TAG ?= $(RELEASE_TAG) +RELEASE_IMAGE_PLATFORMS ?= linux/amd64,linux/arm64 +RUST_BUILDER_IMAGE ?= docker.io/library/rust:1.95 +UBUNTU_BASE_IMAGE ?= docker.io/library/ubuntu:24.04 + # Installation configuration INSTALL_PREFIX ?= /tmp/flame-dev FLAME_ENDPOINT ?= http://127.0.0.1:8080 @@ -171,6 +175,64 @@ docker-release: init docker-build docker-push ## Build and push all images for r release-sanity: ## Run non-publishing release sanity checks ci/release/sanity.sh +.PHONY: release-images release-images-build release-images-inspect release-images-push release-images-pull-bases release-images-check-cli release-images-login release-images-verify require-release-image-tag + +require-release-image-tag: + @test -n "$(DOCKER_TAG)" || (echo "DOCKER_TAG must be set, for example DOCKER_TAG=v0.6.0" >&2; exit 1) + +release-images-check-cli: ## Check the detected container CLI and amd64 Rust builder image + $(CONTAINER_CLI) info + $(CONTAINER_CLI) run --rm --platform linux/amd64 "$(RUST_BUILDER_IMAGE)" rustc -vV + +release-images-login: ## Log in to Docker Hub with the detected container CLI + $(CONTAINER_CLI) login docker.io + +release-images-build: require-release-image-tag ## Build local multi-arch release image manifests + @set -eu; \ + platforms=$$(printf '%s' "$(RELEASE_IMAGE_PLATFORMS)" | tr ',' ' '); \ + build_image() { \ + image="$$1"; \ + dockerfile="$$2"; \ + for platform in $$platforms; do \ + echo "$(CONTAINER_CLI) build --platform $$platform --manifest $(IMAGE_REGISTRY)/$$image:$(DOCKER_TAG) -f $$dockerfile ."; \ + $(CONTAINER_CLI) build --platform "$$platform" --manifest "$(IMAGE_REGISTRY)/$$image:$(DOCKER_TAG)" -f "$$dockerfile" .; \ + done; \ + }; \ + build_image flame-session-manager docker/Dockerfile.fsm; \ + build_image flame-object-cache docker/Dockerfile.foc; \ + build_image flame-executor-manager docker/Dockerfile.fem; \ + build_image flame-console docker/Dockerfile.console + +release-images-inspect: require-release-image-tag ## Inspect local release image manifests + $(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" + +release-images-push: require-release-image-tag ## Push release manifest lists + $(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" + +release-images: require-release-image-tag ## Build, inspect, and push release image manifests + $(MAKE) release-images-build + $(MAKE) release-images-inspect + $(MAKE) release-images-push + +release-images-verify: require-release-image-tag ## Verify remote release image manifests + $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" + $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" + +release-images-pull-bases: ## Pull release base images with the detected container CLI + @set -eu; \ + for platform in $$(printf '%s' "$(RELEASE_IMAGE_PLATFORMS)" | tr ',' ' '); do \ + $(CONTAINER_CLI) pull --platform "$$platform" "$(RUST_BUILDER_IMAGE)"; \ + $(CONTAINER_CLI) pull --platform "$$platform" "$(UBUNTU_BASE_IMAGE)"; \ + done + ci-image: update_protos ## Build images for CI (without version tags) $(CONTAINER_RUNTIME) build -t $(FSM_IMAGE) -f $(FSM_DOCKERFILE) . $(CONTAINER_RUNTIME) build -t $(FEM_IMAGE) -f $(FEM_DOCKERFILE) . diff --git a/docs/release-process.md b/docs/release-process.md index 530d88b9..db42fadd 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -16,7 +16,11 @@ export STDNG_VERSION=0.1.8 export DOCKER_TAG="${RELEASE_TAG}" export RELEASE_BRANCH=release-0.6 export IMAGE_REGISTRY=docker.io/xflops +export RELEASE_IMAGE_PLATFORMS=linux/amd64,linux/arm64 export RUST_BUILDER_IMAGE=docker.io/library/rust:1.95 +export UBUNTU_BASE_IMAGE=docker.io/library/ubuntu:24.04 +# Optional override; Makefile auto-detects podman first, then docker. +# export CONTAINER_CLI=podman export RELEASE_NOTES_FILE=/tmp/flame-${RELEASE_TAG}-notes.md ``` @@ -46,10 +50,7 @@ Confirm container image credentials with the tool selected for the image publish step: ```shell -# Podman path: -podman login --get-login docker.io || podman login docker.io -# Docker path: -docker login docker.io +make release-images-login ``` Required permissions: @@ -249,19 +250,21 @@ Release Docker images as multi-arch manifest tags for `linux/amd64` and Do not move `latest` for release candidates. For stable releases, move `latest` only after the versioned tag has been pushed and verified. -Use either Podman or Docker Buildx. Both paths must publish a multi-arch tag -that contains `linux/amd64` and `linux/arm64`. +Build release images with manifest lists. The Makefile detects a +Docker-compatible `CONTAINER_CLI` from the host, preferring `podman` when it is +installed and falling back to `docker`. Set `CONTAINER_CLI=docker` or +`CONTAINER_CLI=podman` when you need to override that choice. The selected CLI +must support `build --manifest` and `manifest inspect/push` for the release +image targets. Set `RUST_BUILDER_IMAGE` to the Rust builder image used by the release Dockerfiles. Do not use `rust:latest` for release validation because it can drift from the image build inputs. -Podman prerequisites: +Container CLI prerequisites: ```shell -podman info -podman login --get-login docker.io || podman login docker.io -podman run --rm --platform linux/amd64 "${RUST_BUILDER_IMAGE}" rustc -vV +make release-images-check-cli ``` If the amd64 Rust smoke test fails under emulation, do not publish a stable @@ -269,106 +272,32 @@ arm64-only tag by default. Use a Podman farm or remote Podman connection with a native amd64 builder, or document the Docker release as blocked. If the release owner explicitly narrows the Docker scope to an arm64-first publish, push only the versioned arm64 tags, leave `latest` untouched, and record the missing -amd64/multi-arch artifacts as a release gap. +amd64/multi-arch artifacts as a release gap. Set +`RELEASE_IMAGE_PLATFORMS=linux/arm64` before running the image Make targets for +that scoped build. -With Podman, build both platforms into each manifest: +Build both platforms into local manifests, inspect them, and push the manifest +lists: ```shell -podman build --platform linux/amd64 \ - --manifest "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fsm . -podman build --platform linux/arm64 \ - --manifest "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fsm . - -podman build --platform linux/amd64 \ - --manifest "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" \ - -f docker/Dockerfile.foc . -podman build --platform linux/arm64 \ - --manifest "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" \ - -f docker/Dockerfile.foc . - -podman build --platform linux/amd64 \ - --manifest "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fem . -podman build --platform linux/arm64 \ - --manifest "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fem . - -podman build --platform linux/amd64 \ - --manifest "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" \ - -f docker/Dockerfile.console . -podman build --platform linux/arm64 \ - --manifest "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" \ - -f docker/Dockerfile.console . +make release-images ``` -Inspect local Podman manifests before pushing: +To split the image path into smaller steps, run: ```shell -podman manifest inspect "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" -podman manifest inspect "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" -podman manifest inspect "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" -podman manifest inspect "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" +make release-images-build +make release-images-inspect +make release-images-push ``` -Push the Podman manifest lists: +Verify the registry exposes both architectures: ```shell -podman manifest push "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" \ - "docker://${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" -podman manifest push "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" \ - "docker://${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" -podman manifest push "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" \ - "docker://${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" -podman manifest push "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" \ - "docker://${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" +make release-images-verify ``` -Docker Buildx prerequisites: - -```shell -docker info -docker login docker.io -docker buildx ls -``` - -With Docker Buildx, build and push both platforms directly: - -```shell -docker buildx build --platform linux/amd64,linux/arm64 \ - -t "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fsm --push . -docker buildx build --platform linux/amd64,linux/arm64 \ - -t "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" \ - -f docker/Dockerfile.foc --push . -docker buildx build --platform linux/amd64,linux/arm64 \ - -t "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" \ - -f docker/Dockerfile.fem --push . -docker buildx build --platform linux/amd64,linux/arm64 \ - -t "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" \ - -f docker/Dockerfile.console --push . -``` - -Verify the registry exposes both architectures with Podman: - -```shell -podman manifest inspect "docker://${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" -podman manifest inspect "docker://${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" -podman manifest inspect "docker://${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" -podman manifest inspect "docker://${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" -``` - -Verify the registry exposes both architectures with Docker: - -```shell -docker buildx imagetools inspect "${IMAGE_REGISTRY}/flame-session-manager:${DOCKER_TAG}" -docker buildx imagetools inspect "${IMAGE_REGISTRY}/flame-object-cache:${DOCKER_TAG}" -docker buildx imagetools inspect "${IMAGE_REGISTRY}/flame-executor-manager:${DOCKER_TAG}" -docker buildx imagetools inspect "${IMAGE_REGISTRY}/flame-console:${DOCKER_TAG}" -``` - -After the Docker tags and PyPI package are published, run the Docker Compose +After the image tags and PyPI package are published, run the Docker Compose release smoke check. It pulls the target image tag, starts a compose cluster, and runs `python -m flamepy.runner.e2e --tasks 1 --json` from a clean Python image that installs `flamepy==${PYTHON_VERSION}` from PyPI instead of using the SDK @@ -412,14 +341,7 @@ the affected platform before rebuilding. Use the matching tool for the selected build path: ```shell -podman pull --platform linux/amd64 "${RUST_BUILDER_IMAGE}" -podman pull --platform linux/arm64 "${RUST_BUILDER_IMAGE}" -podman pull --platform linux/amd64 docker.io/library/ubuntu:24.04 -podman pull --platform linux/arm64 docker.io/library/ubuntu:24.04 -docker pull --platform linux/amd64 "${RUST_BUILDER_IMAGE}" -docker pull --platform linux/arm64 "${RUST_BUILDER_IMAGE}" -docker pull --platform linux/amd64 docker.io/library/ubuntu:24.04 -docker pull --platform linux/arm64 docker.io/library/ubuntu:24.04 +make release-images-pull-bases ``` ## Kubernetes And Helm Verification From edda34e291a9a028457e3070a6328265a775d008 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 16:54:54 +0800 Subject: [PATCH 4/7] Fix Makefile help target coverage --- Makefile | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5626fd23..630508a1 100644 --- a/Makefile +++ b/Makefile @@ -34,11 +34,25 @@ E2E_SYSTEM_PROFILE ?= all E2E_SYSTEM_PYTEST_ARGS ?= # Default target -.PHONY: help build build-release docker-build docker-push docker-release docker-clean release-sanity update_protos init sdk-go-build sdk-go-test sdk-go-clean e2e e2e-py e2e-py-docker e2e-py-local e2e-py-system-docker e2e-py-system-local e2e-py-system-stress e2e-py-system-longevity e2e-py-system-runner e2e-local e2e-rs format format-rust format-python install install-dev uninstall uninstall-dev start-services stop-services +.PHONY: help build build-release init update_protos +.PHONY: install install-dev uninstall uninstall-dev start-services stop-services +.PHONY: sdk-python sdk-python-generate sdk-python-test sdk-python-clean +.PHONY: format format-rust format-python format-e2e +.PHONY: e2e e2e-local e2e-py e2e-py-docker e2e-py-local e2e-rs +.PHONY: e2e-py-system-docker e2e-py-system-local e2e-py-system-stress +.PHONY: e2e-py-system-longevity e2e-py-system-runner +.PHONY: docker-build docker-build-fsm docker-build-fem docker-build-console +.PHONY: docker-push docker-push-fsm docker-push-fem docker-push-console +.PHONY: docker-release release-sanity ci-image +.PHONY: release-images release-images-build release-images-inspect release-images-push +.PHONY: release-images-pull-bases release-images-check-cli release-images-login +.PHONY: release-images-verify require-release-image-tag +.PHONY: docker-clean docker-clean-all docker-run-fsm docker-run-fem docker-run-console +.PHONY: docker-images docker-logs docker-release-legacy help: ## Show this help message @echo "Available targets:" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + @grep -hE '^[[:alnum:]_.-]+:.*## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*## "}; {printf "\033[36m%-32s\033[0m %s\n", $$1, $$2}' build: update_protos ## Build the Rust project cargo build @@ -175,8 +189,6 @@ docker-release: init docker-build docker-push ## Build and push all images for r release-sanity: ## Run non-publishing release sanity checks ci/release/sanity.sh -.PHONY: release-images release-images-build release-images-inspect release-images-push release-images-pull-bases release-images-check-cli release-images-login release-images-verify require-release-image-tag - require-release-image-tag: @test -n "$(DOCKER_TAG)" || (echo "DOCKER_TAG must be set, for example DOCKER_TAG=v0.6.0" >&2; exit 1) From 08b242654f411bf0644de7f7bb555b6996b6549a Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 17:54:38 +0800 Subject: [PATCH 5/7] Verify release image platforms --- Makefile | 17 +++++++---- ci/release/check-image-platforms.py | 46 +++++++++++++++++++++++++++++ ci/release/sanity.sh | 35 ++-------------------- docs/release-process.md | 3 +- 4 files changed, 62 insertions(+), 39 deletions(-) create mode 100644 ci/release/check-image-platforms.py diff --git a/Makefile b/Makefile index 630508a1..43b3a64a 100644 --- a/Makefile +++ b/Makefile @@ -232,11 +232,18 @@ release-images: require-release-image-tag ## Build, inspect, and push release im $(MAKE) release-images-inspect $(MAKE) release-images-push -release-images-verify: require-release-image-tag ## Verify remote release image manifests - $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" - $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" - $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" - $(CONTAINER_CLI) manifest inspect "docker://$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" +release-images-verify: require-release-image-tag ## Verify remote release image manifests include expected platforms + @set -eu; \ + expected_platforms="$(RELEASE_IMAGE_PLATFORMS)"; \ + check_image() { \ + image="$$1"; \ + echo "$(CONTAINER_CLI) manifest inspect $$image"; \ + $(CONTAINER_CLI) manifest inspect "$$image" | python3 ci/release/check-image-platforms.py "$$image" "$$expected_platforms"; \ + }; \ + check_image "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)"; \ + check_image "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)"; \ + check_image "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)"; \ + check_image "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" release-images-pull-bases: ## Pull release base images with the detected container CLI @set -eu; \ diff --git a/ci/release/check-image-platforms.py b/ci/release/check-image-platforms.py new file mode 100644 index 00000000..b7fadc3c --- /dev/null +++ b/ci/release/check-image-platforms.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import json +import sys + + +def add_platform(platforms, os_name, architecture): + if not os_name or not architecture: + return + if os_name == "unknown" or architecture == "unknown": + return + platforms.add(f"{os_name}/{architecture}") + + +def manifest_platforms(data): + if isinstance(data, list): + data = data[0] if data else {} + + platforms = set() + for manifest in data.get("manifests", []): + platform = manifest.get("platform", {}) + add_platform(platforms, platform.get("os"), platform.get("architecture")) + + add_platform(platforms, data.get("os"), data.get("architecture")) + add_platform(platforms, data.get("Os"), data.get("Architecture")) + return platforms + + +def main(): + if len(sys.argv) != 3: + raise SystemExit("usage: check-image-platforms.py IMAGE EXPECTED_PLATFORMS") + + image = sys.argv[1] + expected = set(sys.argv[2].replace(",", " ").split()) + data = json.load(sys.stdin) + platforms = manifest_platforms(data) + missing = expected - platforms + + print(f"{image}: platforms={','.join(sorted(platforms)) or 'unknown'}") + if missing: + raise SystemExit( + f"{image}: missing expected platforms {','.join(sorted(missing))}" + ) + + +if __name__ == "__main__": + main() diff --git a/ci/release/sanity.sh b/ci/release/sanity.sh index 72695a60..5f52676f 100755 --- a/ci/release/sanity.sh +++ b/ci/release/sanity.sh @@ -301,39 +301,8 @@ check_manifest_platforms() { local image="$1" local manifest_file="$2" - python3 - "$image" "$manifest_file" "$EXPECTED_PLATFORMS" <<'PY' -import json -import sys - -image = sys.argv[1] -manifest_file = sys.argv[2] -expected = set(sys.argv[3].split()) - -with open(manifest_file) as f: - data = json.load(f) - -if isinstance(data, list): - data = data[0] if data else {} - -platforms = set() -for manifest in data.get("manifests", []): - platform = manifest.get("platform", {}) - os_name = platform.get("os") - arch = platform.get("architecture") - if os_name and arch: - platforms.add(f"{os_name}/{arch}") - -if not platforms and data.get("os") and data.get("architecture"): - platforms.add(f"{data['os']}/{data['architecture']}") - -if not platforms and data.get("Os") and data.get("Architecture"): - platforms.add(f"{data['Os']}/{data['Architecture']}") - -missing = expected - platforms -print(f"{image}: platforms={','.join(sorted(platforms)) or 'unknown'}") -if missing: - raise SystemExit(f"{image}: missing expected platforms {','.join(sorted(missing))}") -PY + python3 "$ROOT_DIR/ci/release/check-image-platforms.py" \ + "$image" "$EXPECTED_PLATFORMS" <"$manifest_file" } inspect_image_manifest() { diff --git a/docs/release-process.md b/docs/release-process.md index db42fadd..19220032 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -291,7 +291,8 @@ make release-images-inspect make release-images-push ``` -Verify the registry exposes both architectures: +Verify the registry exposes every platform listed in `RELEASE_IMAGE_PLATFORMS` +for all four images: ```shell make release-images-verify From 6cc0d282d4ee0fd4a4cf9e46cd4df4998d6ebe6d Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 18:50:04 +0800 Subject: [PATCH 6/7] Prefer usable container CLI in Makefile --- Makefile | 4 ++-- docs/release-process.md | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 43b3a64a..9bde9381 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# Detect a Docker-compatible container CLI. -DETECTED_CONTAINER_CLI := $(shell if command -v podman >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; else echo docker; fi) +# Detect a usable Docker-compatible container CLI. +DETECTED_CONTAINER_CLI := $(shell if command -v podman >/dev/null 2>&1 && podman info >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi) CONTAINER_CLI ?= $(DETECTED_CONTAINER_CLI) CONTAINER_RUNTIME ?= $(CONTAINER_CLI) diff --git a/docs/release-process.md b/docs/release-process.md index 19220032..e29f5238 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -19,7 +19,7 @@ export IMAGE_REGISTRY=docker.io/xflops export RELEASE_IMAGE_PLATFORMS=linux/amd64,linux/arm64 export RUST_BUILDER_IMAGE=docker.io/library/rust:1.95 export UBUNTU_BASE_IMAGE=docker.io/library/ubuntu:24.04 -# Optional override; Makefile auto-detects podman first, then docker. +# Optional override; Makefile auto-detects usable podman first, then docker. # export CONTAINER_CLI=podman export RELEASE_NOTES_FILE=/tmp/flame-${RELEASE_TAG}-notes.md ``` @@ -251,11 +251,11 @@ Do not move `latest` for release candidates. For stable releases, move `latest` only after the versioned tag has been pushed and verified. Build release images with manifest lists. The Makefile detects a -Docker-compatible `CONTAINER_CLI` from the host, preferring `podman` when it is -installed and falling back to `docker`. Set `CONTAINER_CLI=docker` or -`CONTAINER_CLI=podman` when you need to override that choice. The selected CLI -must support `build --manifest` and `manifest inspect/push` for the release -image targets. +Docker-compatible `CONTAINER_CLI` from the host, preferring `podman` when +`podman info` succeeds and falling back to `docker` when Docker is installed. +Set `CONTAINER_CLI=docker` or `CONTAINER_CLI=podman` when you need to override +that choice. The selected CLI must support `build --manifest` and +`manifest inspect/push` for the release image targets. Set `RUST_BUILDER_IMAGE` to the Rust builder image used by the release Dockerfiles. Do not use `rust:latest` for release validation because it can drift From 5b7e0de10c22463f79013bd56db622f21720aeb8 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Thu, 4 Jun 2026 19:29:09 +0800 Subject: [PATCH 7/7] Prefer Docker for usable container CLI --- Makefile | 58 ++++++++++++++++++++++------------------- docs/release-process.md | 10 +++---- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 9bde9381..0bbcc3eb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ # Detect a usable Docker-compatible container CLI. -DETECTED_CONTAINER_CLI := $(shell if command -v podman >/dev/null 2>&1 && podman info >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi) +DETECTED_CONTAINER_CLI := $(shell if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1 && podman info >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi) +ifdef CONTAINER_RUNTIME +CONTAINER_CLI ?= $(CONTAINER_RUNTIME) +else CONTAINER_CLI ?= $(DETECTED_CONTAINER_CLI) +endif CONTAINER_RUNTIME ?= $(CONTAINER_CLI) # Docker image configuration @@ -128,13 +132,13 @@ e2e-py: ## Run Python E2E tests (use e2e-py-docker for docker compose or e2e-py- @echo "Use 'make e2e-py-docker' for docker compose tests or 'make e2e-py-local' for local cluster tests" e2e-py-docker: ## Run Python E2E tests with docker compose - $(CONTAINER_RUNTIME) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 ." + $(CONTAINER_CLI) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 ." e2e-py-local: ## Run Python E2E tests against local cluster (requires flamepy installed via pip) cd e2e && PYTHONPATH="$(CURDIR)/e2e/src:$$PYTHONPATH" FLAME_ENDPOINT=$(FLAME_ENDPOINT) pytest -vv --durations=0 . e2e-py-system-docker: ## Run opt-in Python system tests with docker compose (E2E_SYSTEM_PROFILE=all|stress|longevity|runner) - $(CONTAINER_RUNTIME) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS)" + $(CONTAINER_CLI) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS)" e2e-py-system-local: ## Run opt-in Python system tests against local cluster (E2E_SYSTEM_PROFILE=all|stress|longevity|runner) cd e2e && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH="$(CURDIR)/e2e/src:$$PYTHONPATH" FLAME_ENDPOINT=$(FLAME_ENDPOINT) pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS) @@ -157,29 +161,29 @@ e2e-local: e2e-py-local e2e-rs ## Run all E2E tests against local cluster # Docker build targets docker-build-fsm: update_protos ## Build session manager Docker image - $(CONTAINER_RUNTIME) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) . - $(CONTAINER_RUNTIME) tag $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest + $(CONTAINER_CLI) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) . + $(CONTAINER_CLI) tag $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest docker-build-fem: update_protos ## Build executor manager Docker image - $(CONTAINER_RUNTIME) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) . - $(CONTAINER_RUNTIME) tag $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest + $(CONTAINER_CLI) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) . + $(CONTAINER_CLI) tag $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest docker-build-console: update_protos ## Build console Docker image - $(CONTAINER_RUNTIME) build -t $(CONSOLE_IMAGE):$(CONSOLE_TAG) -f $(CONSOLE_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(CONSOLE_IMAGE):$(CONSOLE_TAG) -f $(CONSOLE_DOCKERFILE) . docker-build: docker-build-fsm docker-build-fem docker-build-console ## Build all Docker images # Docker push targets docker-push-fsm: docker-build-fsm ## Push session manager Docker image - $(CONTAINER_RUNTIME) push $(FSM_IMAGE):$(FSM_TAG) - $(CONTAINER_RUNTIME) push $(FSM_IMAGE):latest + $(CONTAINER_CLI) push $(FSM_IMAGE):$(FSM_TAG) + $(CONTAINER_CLI) push $(FSM_IMAGE):latest docker-push-fem: docker-build-fem ## Push executor manager Docker image - $(CONTAINER_RUNTIME) push $(FEM_IMAGE):$(FEM_TAG) - $(CONTAINER_RUNTIME) push $(FEM_IMAGE):latest + $(CONTAINER_CLI) push $(FEM_IMAGE):$(FEM_TAG) + $(CONTAINER_CLI) push $(FEM_IMAGE):latest docker-push-console: docker-build-console ## Push console Docker image - $(CONTAINER_RUNTIME) push $(CONSOLE_IMAGE):$(CONSOLE_TAG) + $(CONTAINER_CLI) push $(CONSOLE_IMAGE):$(CONSOLE_TAG) docker-push: docker-push-fsm docker-push-fem docker-push-console ## Push all Docker images @@ -253,37 +257,37 @@ release-images-pull-bases: ## Pull release base images with the detected contain done ci-image: update_protos ## Build images for CI (without version tags) - $(CONTAINER_RUNTIME) build -t $(FSM_IMAGE) -f $(FSM_DOCKERFILE) . - $(CONTAINER_RUNTIME) build -t $(FEM_IMAGE) -f $(FEM_DOCKERFILE) . - $(CONTAINER_RUNTIME) build -t $(CONSOLE_IMAGE) -f $(CONSOLE_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(FSM_IMAGE) -f $(FSM_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(FEM_IMAGE) -f $(FEM_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(CONSOLE_IMAGE) -f $(CONSOLE_DOCKERFILE) . # Cleanup targets docker-clean: ## Remove all flame Docker images - $(CONTAINER_RUNTIME) rmi $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest 2>/dev/null || true - $(CONTAINER_RUNTIME) rmi $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest 2>/dev/null || true - $(CONTAINER_RUNTIME) rmi $(CONSOLE_IMAGE):$(CONSOLE_TAG) 2>/dev/null || true + $(CONTAINER_CLI) rmi $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest 2>/dev/null || true + $(CONTAINER_CLI) rmi $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest 2>/dev/null || true + $(CONTAINER_CLI) rmi $(CONSOLE_IMAGE):$(CONSOLE_TAG) 2>/dev/null || true docker-clean-all: ## Remove all Docker images and containers (use with caution) - $(CONTAINER_RUNTIME) system prune -a -f + $(CONTAINER_CLI) system prune -a -f # Development targets docker-run-fsm: docker-build-fsm ## Run session manager container - $(CONTAINER_RUNTIME) run --rm -it $(FSM_IMAGE):latest + $(CONTAINER_CLI) run --rm -it $(FSM_IMAGE):latest docker-run-fem: docker-build-fem ## Run executor manager container - $(CONTAINER_RUNTIME) run --rm -it $(FEM_IMAGE):latest + $(CONTAINER_CLI) run --rm -it $(FEM_IMAGE):latest docker-run-console: docker-build-console ## Run console container - $(CONTAINER_RUNTIME) run --rm -it $(CONSOLE_IMAGE):latest + $(CONTAINER_CLI) run --rm -it $(CONSOLE_IMAGE):latest # Utility targets docker-images: ## List all flame Docker images - $(CONTAINER_RUNTIME) images | grep $(DOCKER_REGISTRY)/flame + $(CONTAINER_CLI) images | grep $(DOCKER_REGISTRY)/flame docker-logs: ## Show logs for running flame containers - $(CONTAINER_RUNTIME) ps | grep flame | awk '{print $$1}' | xargs -I {} $(CONTAINER_RUNTIME) logs {} + $(CONTAINER_CLI) ps | grep flame | awk '{print $$1}' | xargs -I {} $(CONTAINER_CLI) logs {} # Legacy targets for backward compatibility docker-release-legacy: init ## Legacy release target (original implementation) - $(CONTAINER_RUNTIME) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) . - $(CONTAINER_RUNTIME) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) . + $(CONTAINER_CLI) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) . diff --git a/docs/release-process.md b/docs/release-process.md index e29f5238..a555b25e 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -251,11 +251,11 @@ Do not move `latest` for release candidates. For stable releases, move `latest` only after the versioned tag has been pushed and verified. Build release images with manifest lists. The Makefile detects a -Docker-compatible `CONTAINER_CLI` from the host, preferring `podman` when -`podman info` succeeds and falling back to `docker` when Docker is installed. -Set `CONTAINER_CLI=docker` or `CONTAINER_CLI=podman` when you need to override -that choice. The selected CLI must support `build --manifest` and -`manifest inspect/push` for the release image targets. +Docker-compatible `CONTAINER_CLI` from the host, preferring a usable Docker +daemon and falling back to Podman when Docker is unavailable. Set +`CONTAINER_CLI=docker` or `CONTAINER_CLI=podman` when you need to override that +choice. The selected CLI must support `build --manifest` and `manifest inspect/push` +for the release image targets. Set `RUST_BUILDER_IMAGE` to the Rust builder image used by the release Dockerfiles. Do not use `rust:latest` for release validation because it can drift