From 0203903672874cee85dbba613cd94654d08fe9b8 Mon Sep 17 00:00:00 2001 From: Martin Rowe Date: Sat, 9 May 2026 18:29:46 +1000 Subject: [PATCH] feat: arm64 controller Builds the controller/manager container for both amd64 and arm64, in a single multi-platform image. This allows control plane nodes running arm64/aarch64 CPUs to be able to manage clusters with AMD GPUs. Not changes: - Build host still has to be amd64/x86_64 - GPU node still has to be amd64/x86_64 - e2e still only tests amd64/x86_64 Benefits: - Heterogeneous/multi-arch clusters do not need dedicated amd64/x86_64 control plane nodes just to control AMD GPU scheduling. Resolves #331 --- .gitignore | 3 ++- Dockerfile | 6 ++++-- Dockerfile.build | 10 ++++++++-- Makefile | 9 +++++---- internal/utils_container/Dockerfile | 5 +++-- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index b51fefb8..ed144c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /bin/ -/manager +/manager* *.tgz /*gpu-operator*.tgz /helm-charts-k8s/charts/*.tgz @@ -26,6 +26,7 @@ tests/pytests/* .cline_storage .vscode +.idea branch_policy.yml tools-internal/* CLAUDE.md diff --git a/Dockerfile b/Dockerfile index 6362149f..d72c11a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,18 +36,20 @@ RUN cd helm-charts-k8s/charts && \ tar -xvzf node-feature-discovery-chart-0.18.3.tgz ARG TARGET +ARG TARGETARCH # Build RUN git config --global --add safe.directory ${PWD} && make ${TARGET} -RUN curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && \ +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" && \ chmod +x ./kubectl FROM ${OPERATOR_CONTROLLER_BASE_IMAGE} ARG TARGET +ARG TARGETARCH -COPY --from=builder /opt/app-root/src/${TARGET} /usr/local/bin/manager +COPY --from=builder /opt/app-root/src/${TARGET}.${TARGETARCH} /usr/local/bin/manager COPY --from=builder /opt/app-root/src/kubectl /usr/local/bin/kubectl COPY --from=builder /opt/app-root/src/LICENSE /licenses/LICENSE COPY --from=builder /opt/app-root/src/helm-charts-k8s/crds/deviceconfig-crd.yaml \ diff --git a/Dockerfile.build b/Dockerfile.build index 6e46fe21..603d7746 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -24,6 +24,8 @@ RUN apt-get update -y && \ protobuf-compiler \ locales \ ca-certificates \ + qemu-user-static \ + libc6-arm64-cross \ sudo && \ apt-get clean && rm -rf /var/lib/apt/lists/* @@ -68,12 +70,16 @@ RUN curl -o /usr/local/bin/kubectl -LO 'https://dl.k8s.io/release/v1.30.4/bin/li ARG INSECURE_REGISTRY RUN echo "INSECURE_REGISTRY is: $INSECURE_REGISTRY" && \ - if [ -n "$INSECURE_REGISTRY" ]; then \ mkdir -p /etc/docker && \ + if [ -n "$INSECURE_REGISTRY" ]; then \ echo "{ \ - \"insecure-registries\": [\"$INSECURE_REGISTRY\"] \ + \"insecure-registries\": [\"$INSECURE_REGISTRY\"], \ + \"features\": {\"containerd-snapshotter\": true} \ }" > /etc/docker/daemon.json; \ else \ + echo "{ \ + \"features\": {\"containerd-snapshotter\": true} \ + }" > /etc/docker/daemon.json; \ echo "INSECURE_REGISTRY is not set"; \ fi diff --git a/Makefile b/Makefile index 9b3685c0..a61dbf56 100644 --- a/Makefile +++ b/Makefile @@ -339,11 +339,12 @@ docs-lint: ## Run docs Markdown lint + spelling (full ROCm-style docs lint). ##@ Build manager: $(shell find -name "*.go") go.mod go.sum ## Build manager binary. - go build -ldflags="-X main.Version=$(PROJECT_VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTag=$(HOURLY_TAG_LABEL)" -o $@ ./cmd + GOOS=linux GOARCH=amd64 go build -ldflags="-X main.Version=$(PROJECT_VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTag=$(HOURLY_TAG_LABEL)" -o $@.amd64 ./cmd + GOOS=linux GOARCH=arm64 go build -ldflags="-X main.Version=$(PROJECT_VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTag=$(HOURLY_TAG_LABEL)" -o $@.arm64 ./cmd .PHONY: docker-build docker-build: ## Build docker image with the manager. - DOCKER_BUILDKIT=1 docker build -t $(IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) --build-arg TARGET=manager --build-arg GOLANG_BASE_IMG=$(GOLANG_BASE_IMG) --build-arg OPERATOR_CONTROLLER_BASE_IMAGE=$(OPERATOR_CONTROLLER_BASE_IMAGE) . + DOCKER_BUILDKIT=1 docker build --platform linux/amd64,linux/arm64 -t $(IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) --build-arg TARGET=manager --build-arg GOLANG_BASE_IMG=$(GOLANG_BASE_IMG) --build-arg OPERATOR_CONTROLLER_BASE_IMAGE=$(OPERATOR_CONTROLLER_BASE_IMAGE) . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -355,7 +356,7 @@ docker-save: ## Save the container image with the manager. .PHONY: docker-build-utils docker-build-utils: ## Build docker image for utils container. - DOCKER_BUILDKIT=1 docker build -t $(UTILS_IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f internal/utils_container/Dockerfile . + DOCKER_BUILDKIT=1 docker build --platform linux/amd64,linux/arm64 -t $(UTILS_IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f internal/utils_container/Dockerfile . .PHONY: docker-push-utils docker-push-utils: ## Push docker image for utils container. @@ -427,7 +428,7 @@ bundle-build: operator-sdk manifests kustomize ## OpenShift Build OLM bundle. KUBECTL_CMD=${KUBECTL_CMD} ./hack/generate-bundle cp $(shell pwd)/hack/openshift-patch/olm-bundle-patch/*.yaml $(shell pwd)/bundle/manifests/ ${OPERATOR_SDK} bundle validate ./bundle - docker build --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f bundle.Dockerfile -t $(BUNDLE_IMG) . + DOCKER_BUILDKIT=1 docker build --platform linux/amd64,linux/arm64 --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f bundle.Dockerfile -t $(BUNDLE_IMG) . .PHONY: dep-docs dep-docs: diff --git a/internal/utils_container/Dockerfile b/internal/utils_container/Dockerfile index 1849f6b2..75b589be 100644 --- a/internal/utils_container/Dockerfile +++ b/internal/utils_container/Dockerfile @@ -1,4 +1,5 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7 +ARG TARGETARCH LABEL name="amd-gpu-operator-utils" LABEL maintainer="sriram.ravishankar@amd.com,yan.sun3@amd.com" @@ -17,8 +18,8 @@ ADD LICENSE /licenses/LICENSE # Install kubectl and oc RUN mkdir -p /oc && cd /oc && \ - curl -SsLO 'https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux-amd64-rhel9.tar.gz' && \ - tar -xzf openshift-client-linux-amd64-rhel9.tar.gz -C /oc && \ + curl -SsLO "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux-${TARGETARCH}-rhel9.tar.gz" && \ + tar -xzf "openshift-client-linux-${TARGETARCH}-rhel9.tar.gz" -C /oc && \ cp ./kubectl /usr/local/bin && chmod +x /usr/local/bin/kubectl && \ cp ./oc /usr/local/bin && chmod +x /usr/local/bin/oc && \ rm -rf /oc