Plat 1278 flux spike

.github/workflows/push-flux-components.yaml

@@ -0,0 +1,121 @@
+---
+name: Publish artifacts
+
+on: # yamllint disable-line rule:truthy
+  # Publish artifacts on every push
+  push:
+  # Also allow publication to be done via a workflow call
+  # In this case, the chart version is returned as an output
+  workflow_call:
+    inputs:
+      ref:
+        type: string
+        description: The ref to build.
+        required: true
+
+jobs:
+  push_flux_artifacts:
+    name: Build and push images
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        base:
+          - base
+        overlay:
+          - alerts
+        customer:
+          - customer
+        include:
+          - base: base
+            overlay: ""
+            customer: ""
+      fail-fast: false
+    permissions:
+      contents: read
+      id-token: write # needed for signing the images with GitHub OIDC Token
+      packages: write # required for pushing container images
+      security-events: write # required for pushing SARIF files
+    steps:
+      - name: Generate environment name
+        id: environment
+        run: |
+          echo name="${BASE}""${OVERLAY:+-$OVERLAY}${CUSTOMER:+-$CUSTOMER}" >> "$GITHUB_OUTPUT"
+        env:
+          BASE: ${{ matrix.base }}
+          OVERLAY: ${{ matrix.overlay }}
+          CUSTOMER: ${{ matrix.customer }}
+
+      - name: Check out the repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+          fetch-depth: 0
+          persist-credentials: true
+
+      - name: Generate Flux artifact manifests
+        shell: bash
+        run: |
+          python -m pip install -r requirements.txt
+          python flux-components/tools/main.py "${OVERLAY}" "${CUSTOMER}"
+        env:
+          OVERLAY: ${{ matrix.overlay }}
+          CUSTOMER: ${{ matrix.customer }}
+
+      - name: Load super-linter configuration
+        # Use grep inverse matching to exclude eventual comments in the .env file
+        # because the GitHub Actions command to set environment variables doesn't
+        # support comments.
+        # yamllint disable-line rule:line-length
+        # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-environment-variable
+        run: grep -v '^#' super-linter.env >> "$GITHUB_ENV"
+        if: always()
+
+      - name: Run super-linter for flux components
+        uses: super-linter/super-linter/slim@9e863354e3ff62e0727d37183162c4a88873df41 # v8.6.0
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          KUBERNETES_KUBECONFORM_OPTIONS: "--ignore-missing-schemas"
+          FILTER_REGEX_INCLUDE: flux-components/*
+          FILTER_REGEX_EXCLUDE: (^|/)encrypted/|crds.yaml
+          VALIDATE_SHELL_SHFMT: false
+          VALIDATE_SPELL_CODESPELL: false
+
+      - name: Install Flux CLI
+        shell: bash
+        run: |
+          wget -q https://github.com/fluxcd/flux2/releases/download/v2.9.1/flux_2.9.1_linux_amd64.tar.gz \
+            -O tarball.tar.gz
+          tar -xzvf tarball.tar.gz
+          rm tarball.tar.gz
+          mkdir -p /home/runner/.local/bin
+          mv flux /home/runner/.local/bin/flux
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Calculate metadata for image
+        id: image-meta
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
+        with:
+          images: ghcr.io/azimuth-cloud/azimuth-config/flux/${{ steps.environment.outputs.name }}
+          # Produce the branch name or tag and the SHA as tags
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+
+      - name: Build and push artifact
+        shell: bash
+        run: |
+          echo "${TAG}"
+          ./flux-components/tools/push-artifact.sh \
+            ./flux-components/working/artifact \
+            ghcr.io/azimuth-cloud/azimuth-config/flux/"${ENV}" \
+            "${TAG}"
+        env:
+          TAG: ${{ steps.image-meta.outputs.tag-names }}
+          ENV: ${{ steps.environment.outputs.name }}

.gitignore

@@ -4,6 +4,7 @@
 .venv
 .work
 .python-version
+__pycache__
 /clouds.yaml*
 tilt-settings.yaml
 # Ignore local mkdocs builds
@@ -15,3 +16,5 @@ environments/standalone/inventory/group_vars/all/secrets.yml
 # Ignore other types of secrets in CI environments
 .github/environments/**/env.secret
 .github/environments/**/clouds.yaml
+# Ignore temporary working flux artifacts with overrides
+flux-components/working

README.md

@@ -10,6 +10,8 @@ This repository is designed to be forked for a specific site and is structured i
 "environments", allowing common configuration to be shared but overridden where required
 using a layered approach.
 
+To use the recommeded git hooks run `./bin/init-githooks.sh`.
+
 ## Documentation
 
 Documentation on deploying Azimuth can be found at <https://azimuth-config.readthedocs.io>.

bin/init-githooks.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+ln "${ROOT_DIR}"/bin/pre-commit-hook.sh "${ROOT_DIR}"/.git/hooks/pre-commit

bin/pre-commit-hook.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+while IFS= read -r file; do
+    # Skip deleted files
+    [[ -f "$file" ]] || continue
+
+    if grep -q "^kind: Secret$" "$file" && ! grep -q "BEGIN AGE ENCRYPTED FILE" "$file"; then
+        echo "ERROR: $file is an unencrypted Secret"
+        exit 1
+    fi
+done < <(git diff --cached --name-only --diff-filter=ACM -- flux-components)
+
+exit 0

flux-components/README.md

@@ -0,0 +1,193 @@
+# Customising Kubernetes Deployments using Flux
+
+## Glossary
+
+**Component**: The broadest view of a piece of functionality in the deployed
+cluster, covers the whole stack. So the kube-prometheus-stack component is
+the ability to monitor and alert, the component can be disabled, configured
+and installed.
+
+**Deployment**: The set of YAML that describes a component and how its applied
+to a cluster, the raw manifests any flux options and deployed Kubernetes
+objects in the final cluster (`HelmRelease`, `ConfigMaps` etc). A deployment
+can be deployed, disabled, deleted, reconciled and configured.
+
+**Manifests**: The YAML files that are bundled as Flux artifacts and moved in
+and out of registries. Manifests are static and can be edited or updated.
+
+## Assumptions and Rules
+
+- There is one HelmRelease per deployment directory contained in the `manifest.yaml`
+- Any other YAML files in the deployment directory will be treated as Kubernetes
+  manifest and the controller will attempt to reconcile them to the cluster
+- Secrets should be stored in an `encrypted` directory in their `age` key encrypted form
+- There is nothing stopping the same object being defined twice ie in two different
+  manifest YAML files, this would be a bad thing, don't do it
+
+## Deployments
+
+Various helm chart deployments are made into the Azimuth management cluster
+using Flux:
+
+- `kube-prometheus-stack` - the monitoring stack, includes Prometheus, Grafana
+  and AlertManager
+
+The canonical names (in `monospace`) are important and used as keys to customise
+each deployment.
+Configuration extends as far as supplying arbitrary chart values to each
+deployment and allowing different environments to specify overriding or extra
+chart values.
+
+### The base deployment
+
+In the `base/` directory there is a directory per deployment containing the flat
+default manifests.
+When defining a configuration for an environment the base is copied over verbatim
+and any changes applied to it.
+
+### Environments
+
+Any objects added to the final Flux artifact from the environment have a label
+applied to them `azimuth/override-source:` that describes the environment and
+deployment (or directory) they are sourced from.
+
+#### Overriding or adding Helm chart values
+
+In each environment there is an `overrides.yaml` file describing helm chart
+overrides for the deployments.
+It is a list of overrides where each follows the following form:
+
+```YAML
+- deployment_canonical_name: # (string)
+    configMapName:  # name of the map to create in Kube (string)
+    forceReload: # whether the helm release should be immediately reconciled when this changes (optional bool, default False)
+    disabled: # whether the deployment should be disabled (optional bool, default False)
+    values: # dict of chart values to apply to the release
+```
+
+eg.
+
+```YAML
+- kube-prometheus-stack:
+    configMapName: prom-stack-values
+    values:
+      fullnameOverride: foo
+```
+
+The canonical name must match the directory name in the `base/` configuration.
+
+The `configMapName` is used to name the created config map and to update the
+`HelmRelease` list of `valuesFrom`, the name must be unique in the same
+namespace as the `HelmRelease` to prevent clobbering.
+
+If set the `forceReload` flag means that if a config map changes then flux will
+immediately reconcile any `HelmRelease` objects using it.
+Otherwise the reconciliation will occur on the `interval` defined in the release
+itself.
+
+If users want to specify multiple overrides per deployment they can and the
+precedence is that the last specified values wins:
+
+```YAML
+- kube-prometheus-stack:
+    configMapName: prom-stack-values
+    values:
+      fullnameOverride: foo
+- kube-prometheus-stack:
+    configMapName: prom-stack-values-two
+    values:
+      fullnameOverride: bar
+```
+
+In the above case both config maps will be created and the release object will
+contain:
+
+```YAML
+...
+  valuesFrom:
+    - kind: ConfigMap
+      name: prom-stack-values-default
+    - kind: ConfigMap
+      name: prom-stack-values
+    - kind: ConfigMap
+      name: prom-stack-values-two
+...
+```
+
+The final value of `fullnameOverride: bar` will win.
+
+#### Creating whole Kubernetes objects
+
+As well as configuring Helm charts an environment can specify whole Kubernetes
+objects as raw YAML manifests.
+The manifests should be placed under `<environment>/manifests/`, if an object
+relates to a named deployment in the base we recommend placing it in a
+directory of the same name (ie `environment/nanifests/<deployment>`).
+This is so the automatically applied metadata is consistent.
+All raw `.yaml` manifests are rolled in to the Flux artifact so they do not need
+to belong to a parent named deployment and users can create their own directory
+structure.
+If a manifest is not under a named deployment the label will contain the name of
+the top-level directory below `<environment>/manifests/`.
+
+NB: Raw manifests have no validation applied, the namespace must be correctly
+set etc.
+
+### Using kustomize for dynamic values
+
+Flux has a feature called [PostBuild](https://fluxcd.io/flux/components/kustomize/kustomizations/#post-build-variable-substitution)
+that can be used to inject dynamic config into Flux controlled objects.
+It behaves a little like Bash variable substitution where `${VAR}` references in
+the manifests are substituted for the value of `VAR` in a config map called
+`cluster-info` located in the `flux-system` namespace.
+
+This method can be used to insert the cluster domain at runtime as the domain
+is generated dynamically during deployment.
+There is a task in the Ansible playbook that bootstraps the Flux operator that
+creates the config map.
+
+It is possible to use more config maps as source and restrict them to certain
+manifests.
+Currently the Flux deployment system is set up to use on cluster-wide
+kustomization object.
+If users want to put other kustomization objects in their manifests they can but
+there can be issues where if an object is watching any encrypted secret it needs
+access to the SOPS key so be careful with the manifest path it controls.
+
+### Creating and pushing Flux artifacts
+
+To create the final manifests with the override objects and updated
+`HelmReleases` use the Python script in `tools/`.
+The script requires the name of an en environment (base is allowed and should
+reproduce the default config).
+The script copies the base config to a working directory, creates any extra
+config maps required for overrides, updates `HelmRelease` objects to use the
+overrides and copies any encrypted secrets from the named environment.
+It can optionally take arguments to set the base and working directories.
+The working and base paths should be defined relative to the Git repository root.
+
+```bash
+python tools/main.py <environment_name> [--working=<path_to_output_manifests>] [--base=<path_to_base_config>]
+```
+
+To push the manifests to an OCI registry as a usable Flux artifact use `tools/push-artifact.sh`
+
+```bash
+push-artifact.sh [-h] [-d] [-c <registry_credentials>] <path_to_artifact_directory> <artifact_url> [<artifact_tag>]
+```
+
+`-d` will perform a dry run and print out the push command but not execute.
+
+All files and directories under the given artifact directory will be packaged
+into a tar and pushed to the given URL (if the URL doesn't start with `oci://`
+it is automatically added).
+If no artifact tag is supplied then `latest` is used.
+
+#### Notes about pushing to your own GHCR
+
+To push to the GCHR under your user profile you can generate a GitHub token
+and pass a creds option like `<username>:<token>` eg:
+
+```bash
+push-artifact.sh -c irt-shpc:${TOKEN} working/artifact oci://ghcr.io/irt-shpc/flux-test my-tag
+```