From 2553f3bc5c31de471d11c098a7f990ede5504beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Tue, 6 Dec 2022 14:37:48 +0100 Subject: [PATCH 1/8] First attempt at buildcache --- spack/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/spack/Dockerfile b/spack/Dockerfile index 9f736f4..34634cc 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -34,6 +34,13 @@ RUN curl -L https://github.com/arcaneframework/spack_recipes/archive/refs/heads/ RUN source /spack/share/spack/setup-env.sh && \ spack repo add --scope site /recipes +ADD buildcache /home/user/buildcache + + +RUN source /spack/share/spack/setup-env.sh && \ + spack mirror add cache /home/user/buildcache && \ + spack buildcache update-index -d /home/user/buildcache + # Define and compile an alien environment RUN source /spack/share/spack/setup-env.sh && \ spack env create alien --without-view && \ @@ -41,6 +48,11 @@ RUN source /spack/share/spack/setup-env.sh && \ spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ spack concretize -f && spack install --fail-fast +RUN source /spack/share/spack/setup-env.sh && \ + spack env create alien --without-view && \ + spack env activate alien && \ + spack buildcache create -a -d /home/user/bcache + # # Define and compile an arcane environment # RUN source /spack/share/spack/setup-env.sh && \ # spack env create arcane --without-view && \ From 1774861b1c91d7c3607d0bc82ea00987019a5cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Wed, 7 Dec 2022 10:06:48 +0100 Subject: [PATCH 2/8] Import signing keys for buildcache --- spack/Dockerfile | 12 +++++++++++- spack/import_signing_key.sh | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 spack/import_signing_key.sh diff --git a/spack/Dockerfile b/spack/Dockerfile index 34634cc..3ecc62f 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -48,8 +48,17 @@ RUN source /spack/share/spack/setup-env.sh && \ spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ spack concretize -f && spack install --fail-fast +USER root + +RUN DEBIAN_FRONTEND=noninteractive apt update && DEBIAN_FRONTEND=noninteractive apt -y install file + +USER user + +ADD import_signing_key.sh /tmp + +RUN --mount=type=secret,id=signingkey,uid=$UID /tmp/import_signing_key.sh /run/secrets/signingkey + RUN source /spack/share/spack/setup-env.sh && \ - spack env create alien --without-view && \ spack env activate alien && \ spack buildcache create -a -d /home/user/bcache @@ -71,4 +80,5 @@ RUN source /spack/share/spack/setup-env.sh && \ RUN source /spack/share/spack/setup-env.sh && \ spack clean -a +USER root RUN rm -rf /tmp/* || true diff --git a/spack/import_signing_key.sh b/spack/import_signing_key.sh new file mode 100755 index 0000000..572b9ec --- /dev/null +++ b/spack/import_signing_key.sh @@ -0,0 +1,13 @@ +#! /usr/bin/bash + +SPACKHOME=/spack +export GNUPGHOME=${SPACKHOME}/opt/spack/gpg + +. ${SPACKHOME}/share/spack/setup-env.sh +spack gpg init +spack gpg list 2> /dev/null + +gpg --import "$1" +for fpr in $(gpg --no-tty --list-keys --with-colons | awk -F: '/fpr:/ {print $10}' | sort -u); do + echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $fpr trust; +done From 163d52a266a26988aac89a5a489dabee9c19a370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Wed, 7 Dec 2022 10:44:24 +0100 Subject: [PATCH 3/8] Rely on run cache mount --- spack/Dockerfile | 27 ++++++++++----------------- spack/install_spack_deps.sh | 12 +++++++----- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/spack/Dockerfile b/spack/Dockerfile index 3ecc62f..e90b742 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -1,4 +1,5 @@ -ARG BASEIMAGE=ecpe4s/rhel8-runner-x86_64 +# syntax=docker/dockerfile:1 +ARG BASEIMAGE=ubuntu:22.04 FROM $BASEIMAGE @@ -15,7 +16,8 @@ RUN mkdir -p /spack /recipes /home/user && useradd spack && chown -R user: /spac ADD install_spack_deps.sh /tmp/ -RUN bash /tmp/install_spack_deps.sh +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked bash /tmp/install_spack_deps.sh USER user @@ -34,33 +36,24 @@ RUN curl -L https://github.com/arcaneframework/spack_recipes/archive/refs/heads/ RUN source /spack/share/spack/setup-env.sh && \ spack repo add --scope site /recipes -ADD buildcache /home/user/buildcache +ADD import_signing_key.sh /tmp +RUN --mount=type=secret,id=signingkey,uid=1000 /tmp/import_signing_key.sh /run/secrets/signingkey -RUN source /spack/share/spack/setup-env.sh && \ +RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ spack mirror add cache /home/user/buildcache && \ spack buildcache update-index -d /home/user/buildcache # Define and compile an alien environment -RUN source /spack/share/spack/setup-env.sh && \ +RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ spack env create alien --without-view && \ spack env activate alien && \ spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ spack concretize -f && spack install --fail-fast -USER root - -RUN DEBIAN_FRONTEND=noninteractive apt update && DEBIAN_FRONTEND=noninteractive apt -y install file - -USER user - -ADD import_signing_key.sh /tmp - -RUN --mount=type=secret,id=signingkey,uid=$UID /tmp/import_signing_key.sh /run/secrets/signingkey - -RUN source /spack/share/spack/setup-env.sh && \ +RUN --mount=type=cache,target=/home/user/outcache,source=outcache source /spack/share/spack/setup-env.sh && \ spack env activate alien && \ - spack buildcache create -a -d /home/user/bcache + spack buildcache create -a -d /home/user/outcache # # Define and compile an arcane environment # RUN source /spack/share/spack/setup-env.sh && \ diff --git a/spack/install_spack_deps.sh b/spack/install_spack_deps.sh index 2fd96e2..7eb829e 100755 --- a/spack/install_spack_deps.sh +++ b/spack/install_spack_deps.sh @@ -1,13 +1,15 @@ #! /usr/bin/bash +# debian like install_apt() { -export DEBIAN_FRONTEND=noninteractive -apt update -apt install -y build-essential ca-certificates coreutils curl environment-modules gfortran git gpg lsb-release python3 python3-distutils python3-venv unzip zip curl -apt clean -rm -rf /var/lib/apt/lists/* + export DEBIAN_FRONTEND=noninteractive + apt update + apt install -y build-essential ca-certificates coreutils curl environment-modules gfortran git gpg lsb-release python3 python3-distutils python3-venv unzip zip curl file + # apt clean + # rm -rf /var/lib/apt/lists/* } +# RedHat like install_dnf() { dnf -y update From 6cf880e903d6c758ffeb902d11b32616e838a36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Wed, 7 Dec 2022 14:40:16 +0100 Subject: [PATCH 4/8] Build and run container as root --- spack/Dockerfile | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/spack/Dockerfile b/spack/Dockerfile index e90b742..ffe2dbc 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -6,43 +6,33 @@ FROM $BASEIMAGE ARG SPACK_VERSION=0.19.0 ARG RECIPES_BRANCH=main -ARG UID=1000 -ARG GID=100 - -RUN ( ( getent group $GID ) || groupadd --gid $GID user ) \ - &&( ( getent passwd $UID ) || useradd --gid $GID --uid $UID user ) - -RUN mkdir -p /spack /recipes /home/user && useradd spack && chown -R user: /spack /recipes /home/user - ADD install_spack_deps.sh /tmp/ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked bash /tmp/install_spack_deps.sh -USER user - SHELL ["/bin/bash", "-c"] # Download last release of spack -RUN curl -L https://github.com/spack/spack/releases/download/v${SPACK_VERSION}/spack-${SPACK_VERSION}.tar.gz | tar xz -C /spack --strip-components=1 && \ +RUN mkdir -p /spack && \ + curl -L https://github.com/spack/spack/releases/download/v${SPACK_VERSION}/spack-${SPACK_VERSION}.tar.gz | tar xz -C /spack --strip-components=1 && \ source /spack/share/spack/setup-env.sh && \ spack config --scope site add 'packages:all:target:[x86_64]' && \ spack compiler find # Get arcane recipes -RUN curl -L https://github.com/arcaneframework/spack_recipes/archive/refs/heads/${RECIPES_BRANCH}.tar.gz | tar xz -C /recipes --strip-components 1 - -# setup our recipes -RUN source /spack/share/spack/setup-env.sh && \ +RUN mkdir -p recipes && \ + curl -L https://github.com/arcaneframework/spack_recipes/archive/refs/heads/${RECIPES_BRANCH}.tar.gz | tar xz -C /recipes --strip-components 1 && \ + source /spack/share/spack/setup-env.sh && \ spack repo add --scope site /recipes ADD import_signing_key.sh /tmp RUN --mount=type=secret,id=signingkey,uid=1000 /tmp/import_signing_key.sh /run/secrets/signingkey -RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ - spack mirror add cache /home/user/buildcache && \ - spack buildcache update-index -d /home/user/buildcache +RUN --mount=type=cache,target=/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ + spack mirror add cache /buildcache && \ + spack buildcache update-index -d /buildcache # Define and compile an alien environment RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ @@ -51,9 +41,10 @@ RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /sp spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ spack concretize -f && spack install --fail-fast -RUN --mount=type=cache,target=/home/user/outcache,source=outcache source /spack/share/spack/setup-env.sh && \ +RUN --mount=type=cache,target=/outcache,source=outcache source /spack/share/spack/setup-env.sh && \ + ls -ld /outcache && \ spack env activate alien && \ - spack buildcache create -a -d /home/user/outcache + spack buildcache create -a -d /outcache # # Define and compile an arcane environment # RUN source /spack/share/spack/setup-env.sh && \ From 833b91d374dfeb050bb4d7f6adda584597db1e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Wed, 7 Dec 2022 17:29:04 +0100 Subject: [PATCH 5/8] Multi stage docker file --- spack/Dockerfile | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/spack/Dockerfile b/spack/Dockerfile index ffe2dbc..45c9bec 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -1,7 +1,7 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1.3 ARG BASEIMAGE=ubuntu:22.04 -FROM $BASEIMAGE +FROM $BASEIMAGE AS base ARG SPACK_VERSION=0.19.0 ARG RECIPES_BRANCH=main @@ -30,21 +30,31 @@ ADD import_signing_key.sh /tmp RUN --mount=type=secret,id=signingkey,uid=1000 /tmp/import_signing_key.sh /run/secrets/signingkey -RUN --mount=type=cache,target=/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ +FROM base AS builder + +RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh && \ spack mirror add cache /buildcache && \ - spack buildcache update-index -d /buildcache + spack buildcache update-index -d /buildcache && \ + spack buildcache list --allarch + +# Populate cache with build tools +# Uninstall to avoid bad reuse +RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh && \ + spack env create tools --without-view && \ + spack env activate tools && \ + spack add cmake autotools && \ + spack concretize -f && spack install --fail-fast && \ + spack buildcache create -a -m cache && \ + spack uninstall --all # Define and compile an alien environment -RUN --mount=type=cache,target=/home/user/buildcache,source=buildcache source /spack/share/spack/setup-env.sh && \ +RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh && \ + spack buildcache update-index -d /buildcache && \ spack env create alien --without-view && \ spack env activate alien && \ spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ - spack concretize -f && spack install --fail-fast - -RUN --mount=type=cache,target=/outcache,source=outcache source /spack/share/spack/setup-env.sh && \ - ls -ld /outcache && \ - spack env activate alien && \ - spack buildcache create -a -d /outcache + spack concretize -f && spack install --fail-fast && \ + spack buildcache create -a -m cache # # Define and compile an arcane environment # RUN source /spack/share/spack/setup-env.sh && \ @@ -61,8 +71,8 @@ RUN --mount=type=cache,target=/outcache,source=outcache source /spack/share/spac # spack add arcane ~wrapper~monoembed && \ # spack concretize -f && spack install --fail-fast -RUN source /spack/share/spack/setup-env.sh && \ - spack clean -a +# RUN source /spack/share/spack/setup-env.sh && \ +# spack clean -a -USER root -RUN rm -rf /tmp/* || true +FROM base AS final +COPY --from=builder /spack/opt/ /spack/ From 1e3aa81210e78f3575d5836103f3af57f8bd1862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Wed, 7 Dec 2022 21:52:46 +0100 Subject: [PATCH 6/8] Smaller image size - strip all binaries - no explicit instantiation for Trilinos --- spack/Dockerfile | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/spack/Dockerfile b/spack/Dockerfile index 45c9bec..03253e0 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -42,19 +42,19 @@ RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh && \ spack env create tools --without-view && \ spack env activate tools && \ - spack add cmake autotools && \ - spack concretize -f && spack install --fail-fast && \ - spack buildcache create -a -m cache && \ - spack uninstall --all + spack add cmake meson m4 pkgconf autoconf automake libtool && \ + spack concretize -f && (spack install --fail-fast ; \ + spack buildcache create -a -m cache) && \ + spack uninstall -y --all # Define and compile an alien environment RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh && \ spack buildcache update-index -d /buildcache && \ spack env create alien --without-view && \ spack env activate alien && \ - spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp && \ - spack concretize -f && spack install --fail-fast && \ - spack buildcache create -a -m cache + spack add alien +hypre+petsc+xml+hdf5+move+ref ginkgo=omp trilinos=omp ^trilinos~explicit_template_instantiation && \ + spack concretize -f && (spack install --fail-fast; \ + spack buildcache create -a -m cache) # # Define and compile an arcane environment # RUN source /spack/share/spack/setup-env.sh && \ @@ -74,5 +74,12 @@ RUN --mount=type=cache,target=/buildcache source /spack/share/spack/setup-env.sh # RUN source /spack/share/spack/setup-env.sh && \ # spack clean -a +# Strip all the binaries +RUN find -L /spack/opt/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + FROM base AS final COPY --from=builder /spack/opt/ /spack/ From 4fe6feb833ad7179c3afc3a8a43a6d7f180188f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Thu, 8 Dec 2022 15:52:59 +0100 Subject: [PATCH 7/8] [ci] docker with buildkit --- .github/workflows/docker-publish.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8e3d4dc..4a14f57 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -51,7 +51,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 + + + - uses: docker/setup-buildx-action@v1 # Login against a Docker registry except on PR # https://github.com/docker/login-action @@ -77,7 +80,7 @@ jobs: tags: | type=raw, ${{ github.event.inputs.tag_name }} type=raw, ${{ github.event.inputs.tag_name }}-{{date 'YYYYMMDD'}} - + # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - name: Build and push Docker image From bc21963983f54755b7700309918342d2ac8d028c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Chevalier?= Date: Thu, 8 Dec 2022 16:57:06 +0100 Subject: [PATCH 8/8] [spack] Use signing key for buildcache --- .github/workflows/docker-publish.yml | 2 ++ spack/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4a14f57..5a41746 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -94,3 +94,5 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + secrets: | + "signingkey=${{ secrets.BUILDCACHE_PRIVATE_KEY }}" diff --git a/spack/Dockerfile b/spack/Dockerfile index 03253e0..77a53ac 100644 --- a/spack/Dockerfile +++ b/spack/Dockerfile @@ -28,7 +28,7 @@ RUN mkdir -p recipes && \ ADD import_signing_key.sh /tmp -RUN --mount=type=secret,id=signingkey,uid=1000 /tmp/import_signing_key.sh /run/secrets/signingkey +RUN --mount=type=secret,id=signingkey /tmp/import_signing_key.sh /run/secrets/signingkey FROM base AS builder