Skip to content

[codex] Update release process sanity guidance #493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k82cn merged 7 commits into from
Jun 4, 2026
Merged

[codex] Update release process sanity guidance #493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d55e935
Update release process sanity guidance
k82cn Jun 4, 2026
2ee4b79
Address release sanity review feedback
k82cn Jun 4, 2026
ba76929
Add release image make targets
k82cn Jun 4, 2026
edda34e
Fix Makefile help target coverage
k82cn Jun 4, 2026
08b2426
Verify release image platforms
k82cn Jun 4, 2026
6cc0d28
Prefer usable container CLI in Makefile
k82cn Jun 4, 2026
5b7e0de
Prefer Docker for usable container CLI
k82cn Jun 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 118 additions & 33 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# Detect OS and set container runtime
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Darwin)
CONTAINER_RUNTIME ?= podman
# Detect a usable Docker-compatible container CLI.
DETECTED_CONTAINER_CLI := $(shell if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1 && podman info >/dev/null 2>&1; then echo podman; elif command -v docker >/dev/null 2>&1; then echo docker; elif command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)
ifdef CONTAINER_RUNTIME
CONTAINER_CLI ?= $(CONTAINER_RUNTIME)
else
CONTAINER_RUNTIME ?= docker
CONTAINER_CLI ?= $(DETECTED_CONTAINER_CLI)
endif
CONTAINER_RUNTIME ?= $(CONTAINER_CLI)

# Docker image configuration
DOCKER_REGISTRY ?= xflops
Expand All @@ -22,6 +23,13 @@ FSM_DOCKERFILE = docker/Dockerfile.fsm
FEM_DOCKERFILE = docker/Dockerfile.fem
CONSOLE_DOCKERFILE = docker/Dockerfile.console

# Release image configuration
IMAGE_REGISTRY ?= docker.io/$(DOCKER_REGISTRY)
DOCKER_TAG ?= $(RELEASE_TAG)
RELEASE_IMAGE_PLATFORMS ?= linux/amd64,linux/arm64
RUST_BUILDER_IMAGE ?= docker.io/library/rust:1.95
UBUNTU_BASE_IMAGE ?= docker.io/library/ubuntu:24.04

# Installation configuration
INSTALL_PREFIX ?= /tmp/flame-dev
FLAME_ENDPOINT ?= http://127.0.0.1:8080
Expand All @@ -30,11 +38,25 @@ E2E_SYSTEM_PROFILE ?= all
E2E_SYSTEM_PYTEST_ARGS ?=

# Default target
.PHONY: help build build-release docker-build docker-push docker-release docker-clean release-sanity update_protos init sdk-go-build sdk-go-test sdk-go-clean e2e e2e-py e2e-py-docker e2e-py-local e2e-py-system-docker e2e-py-system-local e2e-py-system-stress e2e-py-system-longevity e2e-py-system-runner e2e-local e2e-rs format format-rust format-python install install-dev uninstall uninstall-dev start-services stop-services
.PHONY: help build build-release init update_protos
.PHONY: install install-dev uninstall uninstall-dev start-services stop-services
.PHONY: sdk-python sdk-python-generate sdk-python-test sdk-python-clean
.PHONY: format format-rust format-python format-e2e
.PHONY: e2e e2e-local e2e-py e2e-py-docker e2e-py-local e2e-rs
.PHONY: e2e-py-system-docker e2e-py-system-local e2e-py-system-stress
.PHONY: e2e-py-system-longevity e2e-py-system-runner
.PHONY: docker-build docker-build-fsm docker-build-fem docker-build-console
.PHONY: docker-push docker-push-fsm docker-push-fem docker-push-console
.PHONY: docker-release release-sanity ci-image
.PHONY: release-images release-images-build release-images-inspect release-images-push
.PHONY: release-images-pull-bases release-images-check-cli release-images-login
.PHONY: release-images-verify require-release-image-tag
.PHONY: docker-clean docker-clean-all docker-run-fsm docker-run-fem docker-run-console
.PHONY: docker-images docker-logs docker-release-legacy

help: ## Show this help message
@echo "Available targets:"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
@grep -hE '^[[:alnum:]_.-]+:.*## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*## "}; {printf "\033[36m%-32s\033[0m %s\n", $$1, $$2}'

build: update_protos ## Build the Rust project
cargo build
Expand Down Expand Up @@ -110,13 +132,13 @@ e2e-py: ## Run Python E2E tests (use e2e-py-docker for docker compose or e2e-py-
@echo "Use 'make e2e-py-docker' for docker compose tests or 'make e2e-py-local' for local cluster tests"

e2e-py-docker: ## Run Python E2E tests with docker compose
$(CONTAINER_RUNTIME) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 ."
$(CONTAINER_CLI) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 ."

e2e-py-local: ## Run Python E2E tests against local cluster (requires flamepy installed via pip)
cd e2e && PYTHONPATH="$(CURDIR)/e2e/src:$$PYTHONPATH" FLAME_ENDPOINT=$(FLAME_ENDPOINT) pytest -vv --durations=0 .

e2e-py-system-docker: ## Run opt-in Python system tests with docker compose (E2E_SYSTEM_PROFILE=all|stress|longevity|runner)
$(CONTAINER_RUNTIME) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS)"
$(CONTAINER_CLI) compose exec -w /opt/e2e flame-console bash -c "source /usr/local/flame/sbin/flmenv.sh && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH=/opt/e2e/src:\$$PYTHONPATH python3 -m pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS)"

e2e-py-system-local: ## Run opt-in Python system tests against local cluster (E2E_SYSTEM_PROFILE=all|stress|longevity|runner)
cd e2e && FLAME_E2E_SYSTEM_TESTS=$(E2E_SYSTEM_PROFILE) PYTHONPATH="$(CURDIR)/e2e/src:$$PYTHONPATH" FLAME_ENDPOINT=$(FLAME_ENDPOINT) pytest -vv --durations=0 tests/test_system.py $(E2E_SYSTEM_PYTEST_ARGS)
Expand All @@ -139,29 +161,29 @@ e2e-local: e2e-py-local e2e-rs ## Run all E2E tests against local cluster

# Docker build targets
docker-build-fsm: update_protos ## Build session manager Docker image
$(CONTAINER_RUNTIME) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) .
$(CONTAINER_RUNTIME) tag $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest
$(CONTAINER_CLI) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) .
$(CONTAINER_CLI) tag $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest

docker-build-fem: update_protos ## Build executor manager Docker image
$(CONTAINER_RUNTIME) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) .
$(CONTAINER_RUNTIME) tag $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest
$(CONTAINER_CLI) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) .
$(CONTAINER_CLI) tag $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest

docker-build-console: update_protos ## Build console Docker image
$(CONTAINER_RUNTIME) build -t $(CONSOLE_IMAGE):$(CONSOLE_TAG) -f $(CONSOLE_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(CONSOLE_IMAGE):$(CONSOLE_TAG) -f $(CONSOLE_DOCKERFILE) .

docker-build: docker-build-fsm docker-build-fem docker-build-console ## Build all Docker images

# Docker push targets
docker-push-fsm: docker-build-fsm ## Push session manager Docker image
$(CONTAINER_RUNTIME) push $(FSM_IMAGE):$(FSM_TAG)
$(CONTAINER_RUNTIME) push $(FSM_IMAGE):latest
$(CONTAINER_CLI) push $(FSM_IMAGE):$(FSM_TAG)
$(CONTAINER_CLI) push $(FSM_IMAGE):latest

docker-push-fem: docker-build-fem ## Push executor manager Docker image
$(CONTAINER_RUNTIME) push $(FEM_IMAGE):$(FEM_TAG)
$(CONTAINER_RUNTIME) push $(FEM_IMAGE):latest
$(CONTAINER_CLI) push $(FEM_IMAGE):$(FEM_TAG)
$(CONTAINER_CLI) push $(FEM_IMAGE):latest

docker-push-console: docker-build-console ## Push console Docker image
$(CONTAINER_RUNTIME) push $(CONSOLE_IMAGE):$(CONSOLE_TAG)
$(CONTAINER_CLI) push $(CONSOLE_IMAGE):$(CONSOLE_TAG)

docker-push: docker-push-fsm docker-push-fem docker-push-console ## Push all Docker images

Expand All @@ -171,38 +193,101 @@ docker-release: init docker-build docker-push ## Build and push all images for r
release-sanity: ## Run non-publishing release sanity checks
ci/release/sanity.sh

require-release-image-tag:
@test -n "$(DOCKER_TAG)" || (echo "DOCKER_TAG must be set, for example DOCKER_TAG=v0.6.0" >&2; exit 1)

release-images-check-cli: ## Check the detected container CLI and amd64 Rust builder image
$(CONTAINER_CLI) info
$(CONTAINER_CLI) run --rm --platform linux/amd64 "$(RUST_BUILDER_IMAGE)" rustc -vV

release-images-login: ## Log in to Docker Hub with the detected container CLI
$(CONTAINER_CLI) login docker.io

release-images-build: require-release-image-tag ## Build local multi-arch release image manifests
@set -eu; \
platforms=$$(printf '%s' "$(RELEASE_IMAGE_PLATFORMS)" | tr ',' ' '); \
build_image() { \
image="$$1"; \
dockerfile="$$2"; \
for platform in $$platforms; do \
echo "$(CONTAINER_CLI) build --platform $$platform --manifest $(IMAGE_REGISTRY)/$$image:$(DOCKER_TAG) -f $$dockerfile ."; \
$(CONTAINER_CLI) build --platform "$$platform" --manifest "$(IMAGE_REGISTRY)/$$image:$(DOCKER_TAG)" -f "$$dockerfile" .; \
done; \
}; \
build_image flame-session-manager docker/Dockerfile.fsm; \
build_image flame-object-cache docker/Dockerfile.foc; \
build_image flame-executor-manager docker/Dockerfile.fem; \
build_image flame-console docker/Dockerfile.console

release-images-inspect: require-release-image-tag ## Inspect local release image manifests
$(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest inspect "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)"

release-images-push: require-release-image-tag ## Push release manifest lists
$(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)"
$(CONTAINER_CLI) manifest push "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)" "docker://$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)"

release-images: require-release-image-tag ## Build, inspect, and push release image manifests
$(MAKE) release-images-build
$(MAKE) release-images-inspect
$(MAKE) release-images-push

release-images-verify: require-release-image-tag ## Verify remote release image manifests include expected platforms
@set -eu; \
expected_platforms="$(RELEASE_IMAGE_PLATFORMS)"; \
check_image() { \
image="$$1"; \
echo "$(CONTAINER_CLI) manifest inspect $$image"; \
$(CONTAINER_CLI) manifest inspect "$$image" | python3 ci/release/check-image-platforms.py "$$image" "$$expected_platforms"; \
}; \
check_image "$(IMAGE_REGISTRY)/flame-session-manager:$(DOCKER_TAG)"; \
check_image "$(IMAGE_REGISTRY)/flame-object-cache:$(DOCKER_TAG)"; \
check_image "$(IMAGE_REGISTRY)/flame-executor-manager:$(DOCKER_TAG)"; \
check_image "$(IMAGE_REGISTRY)/flame-console:$(DOCKER_TAG)"

release-images-pull-bases: ## Pull release base images with the detected container CLI
@set -eu; \
for platform in $$(printf '%s' "$(RELEASE_IMAGE_PLATFORMS)" | tr ',' ' '); do \
$(CONTAINER_CLI) pull --platform "$$platform" "$(RUST_BUILDER_IMAGE)"; \
$(CONTAINER_CLI) pull --platform "$$platform" "$(UBUNTU_BASE_IMAGE)"; \
done

ci-image: update_protos ## Build images for CI (without version tags)
$(CONTAINER_RUNTIME) build -t $(FSM_IMAGE) -f $(FSM_DOCKERFILE) .
$(CONTAINER_RUNTIME) build -t $(FEM_IMAGE) -f $(FEM_DOCKERFILE) .
$(CONTAINER_RUNTIME) build -t $(CONSOLE_IMAGE) -f $(CONSOLE_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(FSM_IMAGE) -f $(FSM_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(FEM_IMAGE) -f $(FEM_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(CONSOLE_IMAGE) -f $(CONSOLE_DOCKERFILE) .

# Cleanup targets
docker-clean: ## Remove all flame Docker images
$(CONTAINER_RUNTIME) rmi $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest 2>/dev/null || true
$(CONTAINER_RUNTIME) rmi $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest 2>/dev/null || true
$(CONTAINER_RUNTIME) rmi $(CONSOLE_IMAGE):$(CONSOLE_TAG) 2>/dev/null || true
$(CONTAINER_CLI) rmi $(FSM_IMAGE):$(FSM_TAG) $(FSM_IMAGE):latest 2>/dev/null || true
$(CONTAINER_CLI) rmi $(FEM_IMAGE):$(FEM_TAG) $(FEM_IMAGE):latest 2>/dev/null || true
$(CONTAINER_CLI) rmi $(CONSOLE_IMAGE):$(CONSOLE_TAG) 2>/dev/null || true

docker-clean-all: ## Remove all Docker images and containers (use with caution)
$(CONTAINER_RUNTIME) system prune -a -f
$(CONTAINER_CLI) system prune -a -f

# Development targets
docker-run-fsm: docker-build-fsm ## Run session manager container
$(CONTAINER_RUNTIME) run --rm -it $(FSM_IMAGE):latest
$(CONTAINER_CLI) run --rm -it $(FSM_IMAGE):latest

docker-run-fem: docker-build-fem ## Run executor manager container
$(CONTAINER_RUNTIME) run --rm -it $(FEM_IMAGE):latest
$(CONTAINER_CLI) run --rm -it $(FEM_IMAGE):latest

docker-run-console: docker-build-console ## Run console container
$(CONTAINER_RUNTIME) run --rm -it $(CONSOLE_IMAGE):latest
$(CONTAINER_CLI) run --rm -it $(CONSOLE_IMAGE):latest

# Utility targets
docker-images: ## List all flame Docker images
$(CONTAINER_RUNTIME) images | grep $(DOCKER_REGISTRY)/flame
$(CONTAINER_CLI) images | grep $(DOCKER_REGISTRY)/flame

docker-logs: ## Show logs for running flame containers
$(CONTAINER_RUNTIME) ps | grep flame | awk '{print $$1}' | xargs -I {} $(CONTAINER_RUNTIME) logs {}
$(CONTAINER_CLI) ps | grep flame | awk '{print $$1}' | xargs -I {} $(CONTAINER_CLI) logs {}

# Legacy targets for backward compatibility
docker-release-legacy: init ## Legacy release target (original implementation)
$(CONTAINER_RUNTIME) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) .
$(CONTAINER_RUNTIME) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(FSM_IMAGE):$(FSM_TAG) -f $(FSM_DOCKERFILE) .
$(CONTAINER_CLI) build -t $(FEM_IMAGE):$(FEM_TAG) -f $(FEM_DOCKERFILE) .
46 changes: 46 additions & 0 deletions ci/release/check-image-platforms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
import json
import sys


def add_platform(platforms, os_name, architecture):
if not os_name or not architecture:
return
if os_name == "unknown" or architecture == "unknown":
return
platforms.add(f"{os_name}/{architecture}")


def manifest_platforms(data):
if isinstance(data, list):
data = data[0] if data else {}

platforms = set()
for manifest in data.get("manifests", []):
platform = manifest.get("platform", {})
add_platform(platforms, platform.get("os"), platform.get("architecture"))

add_platform(platforms, data.get("os"), data.get("architecture"))
add_platform(platforms, data.get("Os"), data.get("Architecture"))
return platforms


def main():
if len(sys.argv) != 3:
raise SystemExit("usage: check-image-platforms.py IMAGE EXPECTED_PLATFORMS")

image = sys.argv[1]
expected = set(sys.argv[2].replace(",", " ").split())
data = json.load(sys.stdin)
platforms = manifest_platforms(data)
missing = expected - platforms

print(f"{image}: platforms={','.join(sorted(platforms)) or 'unknown'}")
if missing:
raise SystemExit(
f"{image}: missing expected platforms {','.join(sorted(missing))}"
)


if __name__ == "__main__":
main()
37 changes: 3 additions & 34 deletions ci/release/sanity.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ run_package_checks() {
log "uv run Python SDK runner tests"
(
cd "$ROOT_DIR/sdk/python"
uv run -n pytest tests/test_runner_e2e.py tests/test_runner.py -q
uv run -n --extra dev pytest tests/test_runner_e2e.py tests/test_runner.py -q
)

log "uv run Python SDK version import"
Expand Down Expand Up @@ -301,39 +301,8 @@ check_manifest_platforms() {
local image="$1"
local manifest_file="$2"

python3 - "$image" "$manifest_file" "$EXPECTED_PLATFORMS" <<'PY'
import json
import sys

image = sys.argv[1]
manifest_file = sys.argv[2]
expected = set(sys.argv[3].split())

with open(manifest_file) as f:
data = json.load(f)

if isinstance(data, list):
data = data[0] if data else {}

platforms = set()
for manifest in data.get("manifests", []):
platform = manifest.get("platform", {})
os_name = platform.get("os")
arch = platform.get("architecture")
if os_name and arch:
platforms.add(f"{os_name}/{arch}")

if not platforms and data.get("os") and data.get("architecture"):
platforms.add(f"{data['os']}/{data['architecture']}")

if not platforms and data.get("Os") and data.get("Architecture"):
platforms.add(f"{data['Os']}/{data['Architecture']}")

missing = expected - platforms
print(f"{image}: platforms={','.join(sorted(platforms)) or 'unknown'}")
if missing:
raise SystemExit(f"{image}: missing expected platforms {','.join(sorted(missing))}")
PY
python3 "$ROOT_DIR/ci/release/check-image-platforms.py" \
"$image" "$EXPECTED_PLATFORMS" <"$manifest_file"
}

inspect_image_manifest() {
Expand Down
Loading
Loading