diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..18224b0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +{ + "name": "Kubebuilder DevContainer", + "image": "golang:1.26", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "moby": false, + "dockerDefaultAddressPool": "base=172.30.0.0/16,size=24" + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "upgradePackages": true + } + }, + + "runArgs": ["--privileged", "--init"], + + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-azuretools.vscode-docker" + ] + } + }, + + "remoteEnv": { + "GO111MODULE": "on" + }, + + "onCreateCommand": "bash .devcontainer/post-install.sh" +} diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh new file mode 100644 index 0000000..6d75a49 --- /dev/null +++ b/.devcontainer/post-install.sh @@ -0,0 +1,153 @@ +#!/bin/bash +set -euo pipefail + +echo "====================================" +echo "Kubebuilder DevContainer Setup" +echo "====================================" + +# Verify running as root (required for installing to /usr/local/bin and /etc) +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: This script must be run as root" + exit 1 +fi + +echo "" +echo "Detecting system architecture..." +# Detect architecture using uname +MACHINE=$(uname -m) +case "${MACHINE}" in + x86_64) + ARCH="amd64" + ;; + aarch64|arm64) + ARCH="arm64" + ;; + *) + echo "WARNING: Unsupported architecture ${MACHINE}, defaulting to amd64" + ARCH="amd64" + ;; +esac +echo "Architecture: ${ARCH}" + +echo "" +echo "------------------------------------" +echo "Setting up bash completion..." +echo "------------------------------------" + +BASH_COMPLETIONS_DIR="/usr/share/bash-completion/completions" + +# Enable bash-completion in root's .bashrc (devcontainer runs as root) +if ! grep -q "source /usr/share/bash-completion/bash_completion" ~/.bashrc 2>/dev/null; then + echo 'source /usr/share/bash-completion/bash_completion' >> ~/.bashrc + echo "Added bash-completion to .bashrc" +fi + +echo "" +echo "------------------------------------" +echo "Installing development tools..." +echo "------------------------------------" + +# Install kind +if ! command -v kind &> /dev/null; then + echo "Installing kind..." + curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/latest/kind-linux-${ARCH}" + chmod +x /usr/local/bin/kind + echo "kind installed successfully" +fi + +# Generate kind bash completion +if command -v kind &> /dev/null; then + if kind completion bash > "${BASH_COMPLETIONS_DIR}/kind" 2>/dev/null; then + echo "kind completion installed" + else + echo "WARNING: Failed to generate kind completion" + fi +fi + +# Install kubebuilder +if ! command -v kubebuilder &> /dev/null; then + echo "Installing kubebuilder..." + curl -Lo /usr/local/bin/kubebuilder "https://go.kubebuilder.io/dl/latest/linux/${ARCH}" + chmod +x /usr/local/bin/kubebuilder + echo "kubebuilder installed successfully" +fi + +# Generate kubebuilder bash completion +if command -v kubebuilder &> /dev/null; then + if kubebuilder completion bash > "${BASH_COMPLETIONS_DIR}/kubebuilder" 2>/dev/null; then + echo "kubebuilder completion installed" + else + echo "WARNING: Failed to generate kubebuilder completion" + fi +fi + +# Install kubectl +if ! command -v kubectl &> /dev/null; then + echo "Installing kubectl..." + KUBECTL_VERSION=$(curl -Ls https://dl.k8s.io/release/stable.txt) + curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" + chmod +x /usr/local/bin/kubectl + echo "kubectl installed successfully" +fi + +# Generate kubectl bash completion +if command -v kubectl &> /dev/null; then + if kubectl completion bash > "${BASH_COMPLETIONS_DIR}/kubectl" 2>/dev/null; then + echo "kubectl completion installed" + else + echo "WARNING: Failed to generate kubectl completion" + fi +fi + +# Generate Docker bash completion +if command -v docker &> /dev/null; then + if docker completion bash > "${BASH_COMPLETIONS_DIR}/docker" 2>/dev/null; then + echo "docker completion installed" + else + echo "WARNING: Failed to generate docker completion" + fi +fi + +echo "" +echo "------------------------------------" +echo "Configuring Docker environment..." +echo "------------------------------------" + +# Wait for Docker to be ready +echo "Waiting for Docker to be ready..." +for i in {1..30}; do + if docker info >/dev/null 2>&1; then + echo "Docker is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "WARNING: Docker not ready after 30s" + fi + sleep 1 +done + +# Create kind network (ignore if already exists) +if ! docker network inspect kind >/dev/null 2>&1; then + if docker network create kind >/dev/null 2>&1; then + echo "Created kind network" + else + echo "WARNING: Failed to create kind network (may already exist)" + fi +fi + +echo "" +echo "------------------------------------" +echo "Verifying installations..." +echo "------------------------------------" +kind version +kubebuilder version +kubectl version --client +docker --version +go version + +echo "" +echo "====================================" +echo "DevContainer ready!" +echo "====================================" +echo "All development tools installed successfully." +echo "You can now start building Kubernetes operators." diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index d61b0cd..0000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -# Exclude the dockerfile itself -Dockerfile - -# Exclude Python caches -**/__pycache__ -*.egg-info diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 53634b1..96f4e85 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,4 @@ version: 2 -updates: updates: # Using dependabot for all GHA violates pinning policy - package-ecosystem: github-actions @@ -19,18 +18,18 @@ updates: - dependency-name: "actions/*" - dependency-name: "github-actions/*" - # Automatically propose PRs for Python dependencies - - package-ecosystem: pip + # Automatically propose PRs for Go module dependencies + - package-ecosystem: gomod directory: "/" schedule: - # Check for new versions daily - interval: daily + # Check for new versions weekly + interval: weekly groups: - pip-updates: + go-updates: patterns: ["*"] labels: - automation - - pip-update + - go-update commit-message: - # Prefix all commit messages with "pip: " - prefix: "pip" + # Prefix all commit messages with "go: " + prefix: "go" diff --git a/.github/workflows/build-push-artifacts.yaml b/.github/workflows/build-push-artifacts.yaml index 9687e84..e3af823 100644 --- a/.github/workflows/build-push-artifacts.yaml +++ b/.github/workflows/build-push-artifacts.yaml @@ -23,7 +23,7 @@ on: jobs: build_push_images: - name: Build and push images + name: Build and push images (Nix) runs-on: ubuntu-latest permissions: contents: read @@ -36,6 +36,9 @@ jobs: with: ref: ${{ inputs.ref }} + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@07ebb8d2749aa2fac2baae7d1cfa011e4acfd8d1 # v5 + - name: Login to GitHub Container Registry uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: @@ -47,27 +50,63 @@ jobs: id: semver uses: azimuth-cloud/github-actions/semver@master - - name: Calculate metadata for image - id: image-meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 - with: - images: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack - # Produce the branch name or tag and the SHA as tags - tags: | - type=ref,event=branch - type=ref,event=tag - type=raw,value=${{ steps.semver.outputs.short-sha }} + - name: Compute image tags + id: tags + run: | + # All final tags for the manifest list (space-separated) + TAGS="" + REF="${{ github.ref }}" + SHA="${{ steps.semver.outputs.short-sha }}" + IMAGE="ghcr.io/azimuth-cloud/cluster-api-janitor-openstack" + if [[ "$REF" == refs/tags/* ]]; then + TAG="${REF#refs/tags/}" + TAGS="$IMAGE:$TAG $IMAGE:$SHA" + elif [[ "$REF" == refs/heads/* ]]; then + BRANCH="${REF#refs/heads/}" + TAGS="$IMAGE:$BRANCH $IMAGE:$SHA" + else + TAGS="$IMAGE:$SHA" + fi + echo "tags=$TAGS" >> "$GITHUB_OUTPUT" + echo "sha_tag=$IMAGE:$SHA" >> "$GITHUB_OUTPUT" + echo "image=$IMAGE" >> "$GITHUB_OUTPUT" + + - name: Build amd64 image + run: nix-build nix -A image -o image-amd64.tar.gz + + - name: Build arm64 image (cross-compiled) + run: nix-build nix -A image-arm64 -o image-arm64.tar.gz + + - name: Push arch-specific images and create manifest list + env: + SHA_TAG: ${{ steps.tags.outputs.sha_tag }} + IMAGE: ${{ steps.tags.outputs.image }} + TAGS: ${{ steps.tags.outputs.tags }} + run: | + # Push each arch image under a unique tag used only to assemble the manifest. + skopeo copy \ + docker-archive:image-amd64.tar.gz \ + "docker://${SHA_TAG}-amd64" + skopeo copy \ + docker-archive:image-arm64.tar.gz \ + "docker://${SHA_TAG}-arm64" + + # For each final tag, create and push a multi-arch manifest list. + for TAG in $TAGS; do + docker manifest create "$TAG" \ + --amend "${SHA_TAG}-amd64" \ + --amend "${SHA_TAG}-arm64" + docker manifest push "$TAG" + done + + - name: Generate SBOM (CycloneDX) + run: nix-build nix -A sbom -o sbom.cdx.json - - name: Build and push image - uses: azimuth-cloud/github-actions/docker-multiarch-build-push@master + - name: Upload SBOM as artifact + uses: actions/upload-artifact@v4 with: - cache-key: cluster-api-janitor-openstack - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.image-meta.outputs.tags }} - labels: ${{ steps.image-meta.outputs.labels }} - github-token: ${{ secrets.GITHUB_TOKEN }} + name: sbom-${{ steps.semver.outputs.short-sha }} + path: sbom.cdx.json build_push_chart: name: Build and push Helm chart diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4c44777 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: CI + +on: + push: + pull_request: + +permissions: {} + +jobs: + nix: + name: fmt + vet + test (Nix) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@07ebb8d2749aa2fac2baae7d1cfa011e4acfd8d1 # v5 + + - name: Run fmt + vet + unit tests + run: nix-build nix -A tests diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d80461b..de16282 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -25,21 +25,34 @@ concurrency: # TODO(mkjpryor): Change this in the future to use the CAPI management only variation ##### jobs: - # Run the unit tests on every PR, even from external repos - unit_tests: - uses: ./.github/workflows/tox.yaml - with: - ref: ${{ github.event.pull_request.head.sha }} - # Run the chart linting on every PR, even from external repos lint: uses: ./.github/workflows/helm-lint.yaml with: ref: ${{ github.event.pull_request.head.sha }} + # Detect which parts of the PR changed, to decide whether the (expensive) + # Azimuth integration test job needs to run. + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + go_tests: ${{ steps.filter.outputs.go_tests }} + chart: ${{ steps.filter.outputs.chart }} + steps: + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: filter + with: + filters: | + go_tests: + - '**/*_test.go' + chart: + - 'chart/**' + # This job exists so that PRs from outside the main repo are rejected fail_on_remote: - needs: [unit_tests, lint] + needs: [lint] runs-on: ubuntu-latest steps: - name: PR must be from a branch in the azimuth-cloud/cluster-api-janitor-openstack repo @@ -58,7 +71,8 @@ jobs: security-events: write # required for pushing SARIF files run_azimuth_tests: - needs: [publish_artifacts] + needs: [publish_artifacts, changes] + if: needs.changes.outputs.go_tests == 'true' || needs.changes.outputs.chart == 'true' runs-on: ubuntu-latest steps: # Check out the configuration repository diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml deleted file mode 100644 index db2ae36..0000000 --- a/.github/workflows/tox.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Tox unit tests - -on: - workflow_call: - inputs: - ref: - type: string - description: The ref to build. - required: true - -jobs: - build: - name: Tox unit tests and linting - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.12','3.13'] - - steps: - - name: Check out the repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.ref || github.ref }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install tox - - - name: Test with tox - run: tox - - - name: Generate coverage reports - run: tox -e cover - - - name: Archive code coverage results - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: "code-coverage-report-${{ matrix.python-version }}" - path: cover/ diff --git a/.gitignore b/.gitignore index bf551e2..fa018a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ -__pycache__ -*.egg-info -.python-version +# Go +/bin/ +*.test +*.out + +# Helm chart/charts/* chart/Chart.lock -# ignore unit test stuff -.stestr/* -.tox/* -.coverage + +# Nix +result +result-* +sbom.cdx.json +*.tar.gz diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..806c7a3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + # config/ contains kubebuilder-scaffolded multi-document manifests + # (e.g. config/manager/manager.yaml: Namespace + Deployment). + args: [--allow-multiple-documents] + exclude: ^chart/templates/ + - id: check-added-large-files + - id: check-merge-conflict + + - repo: local + hooks: + - id: go-fmt + name: go fmt + description: Fail if any Go file is not gofmt-formatted (mirrors nix -A tests). + entry: bash -c 'out=$(gofmt -l "$@"); if [ -n "$out" ]; then echo "$out"; exit 1; fi' -- + language: system + types: [go] + + - id: go-vet + name: go vet + description: Run go vet ./... (mirrors nix -A tests / make vet). + entry: bash -c 'go vet ./...' + language: system + types: [go] + pass_filenames: false + + - id: go-test + name: go test + description: Run the unit test suite, excluding e2e (mirrors nix -A tests / make test). + entry: bash -c 'go test $(go list ./... | grep -v /test/e2e)' + language: system + types: [go] + pass_filenames: false + + - id: helm-lint + name: helm lint chart + description: Lint the Helm chart (mirrors the CI helm-lint workflow). + entry: helm lint chart --strict + language: system + files: ^chart/ + pass_filenames: false diff --git a/.stestr.conf b/.stestr.conf deleted file mode 100644 index dd7e683..0000000 --- a/.stestr.conf +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULT] -test_path=./capi_janitor/tests -top_dir=./ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8d76c16 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,322 @@ +# cluster-api-janitor-openstack - AI Agent Guide + +## Project Structure + +**Single-group layout (default):** +``` +cmd/main.go Manager entry (registers controllers/webhooks) +api//*_types.go CRD schemas (+kubebuilder markers) +api//zz_generated.* Auto-generated (DO NOT EDIT) +internal/controller/* Reconciliation logic +internal/webhook/* Validation/defaulting (if present) +config/crd/bases/* Generated CRDs (DO NOT EDIT) +config/rbac/role.yaml Generated RBAC (DO NOT EDIT) +config/samples/* Example CRs (edit these) +Makefile Build/test/deploy commands +PROJECT Kubebuilder metadata Auto-generated (DO NOT EDIT) +``` + +**Multi-group layout** (for projects with multiple API groups): +``` +api///*_types.go CRD schemas by group +internal/controller//* Controllers by group +internal/webhook///* Webhooks by group and version (if present) +``` + +Multi-group layout organizes APIs by group name (e.g., `batch`, `apps`). Check the `PROJECT` file for `multigroup: true`. + +**To convert to multi-group layout:** +1. Run: `kubebuilder edit --multigroup=true` +2. Move APIs: `mkdir -p api/ && mv api/ api//` +3. Move controllers: `mkdir -p internal/controller/ && mv internal/controller/*.go internal/controller//` +4. Move webhooks (if present): `mkdir -p internal/webhook/ && mv internal/webhook/ internal/webhook//` +5. Update import paths in all files +6. Fix `path` in `PROJECT` file for each resource +7. Update test suite CRD paths (add one more `..` to relative paths) + +## Critical Rules + +### Never Edit These (Auto-Generated) +- `config/crd/bases/*.yaml` - from `make manifests` +- `config/rbac/role.yaml` - from `make manifests` +- `config/webhook/manifests.yaml` - from `make manifests` +- `**/zz_generated.*.go` - from `make generate` +- `PROJECT` - from `kubebuilder [OPTIONS]` + +### Never Remove Scaffold Markers +Do NOT delete `// +kubebuilder:scaffold:*` comments. CLI injects code at these markers. + +### Keep Project Structure +Do not move files around. The CLI expects files in specific locations. + +### Always Use CLI Commands +Always use `kubebuilder create api` and `kubebuilder create webhook` to scaffold. Do NOT create files manually. + +### E2E Tests Require an Isolated Kind Cluster +The e2e tests are designed to validate the solution in an isolated environment (similar to GitHub Actions CI). +Ensure you run them against a dedicated [Kind](https://kind.sigs.k8s.io/) cluster (not your “real” dev/prod cluster). + +## After Making Changes + +**After editing `*_types.go` or markers:** +``` +make manifests # Regenerate CRDs/RBAC from markers +make generate # Regenerate DeepCopy methods +``` + +**After editing `*.go` files:** +``` +make lint-fix # Auto-fix code style +make test # Run unit tests +``` + +## CLI Commands Cheat Sheet + +### Create API (your own types) +```bash +kubebuilder create api --group --version --kind +``` + +### Deploy Image Plugin (scaffold to deploy/manage ANY container image) + +Generate a controller that deploys and manages a container image (nginx, redis, memcached, your app, etc.): + +```bash +# Example: deploying memcached +kubebuilder create api --group example.com --version v1alpha1 --kind Memcached \ + --image=memcached:alpine \ + --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Scaffolds good-practice code: reconciliation logic, status conditions, finalizers, RBAC. Use as a reference implementation. + + +### Create Webhooks +```bash +# Validation + defaulting +kubebuilder create webhook --group --version --kind \ + --defaulting --programmatic-validation + +# Conversion webhook (for multi-version APIs) +kubebuilder create webhook --group --version v1 --kind \ + --conversion --spoke v2 +``` + +### Controller for Core Kubernetes Types +```bash +# Watch Pods +kubebuilder create api --group core --version v1 --kind Pod \ + --controller=true --resource=false + +# Watch Deployments +kubebuilder create api --group apps --version v1 --kind Deployment \ + --controller=true --resource=false +``` + +### Controller for External Types (e.g., from other operators) + +Watch resources from external APIs (cert-manager, Argo CD, Istio, etc.): + +```bash +# Example: watching cert-manager Certificate resources +kubebuilder create api \ + --group cert-manager --version v1 --kind Certificate \ + --controller=true --resource=false \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +**Note:** Use `--external-api-module=@` only if you need a specific version. Otherwise, omit `@` to use what's in go.mod. + +### Webhook for External Types + +```bash +# Example: validating external resources +kubebuilder create webhook \ + --group cert-manager --version v1 --kind Issuer \ + --defaulting \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +## Testing & Development + +```bash +make test # Run unit tests (uses envtest: real K8s API + etcd) +make run # Run locally (uses current kubeconfig context) +``` + +Tests use **Ginkgo + Gomega** (BDD style). Check `suite_test.go` for setup. + +## Deployment Workflow + +```bash +# 1. Regenerate manifests +make manifests generate + +# 2. Build & deploy +export IMG=/:tag +nix-build nix -A image -o image.tar.gz # produces an OCI archive, see nix/default.nix +skopeo copy docker-archive:image.tar.gz "docker://$IMG" # or: kind load image-archive image.tar.gz --name +make deploy IMG=$IMG + +# 3. Test +kubectl apply -k config/samples/ + +# 4. Debug +kubectl logs -n -system deployment/-controller-manager -c manager -f +``` + +### API Design + +**Key markers for** `api//*_types.go`: + +```go +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.conditions[?(@.type=='Ready')].status" + +// On fields: +// +kubebuilder:validation:Required +// +kubebuilder:validation:Minimum=1 +// +kubebuilder:validation:MaxLength=100 +// +kubebuilder:validation:Pattern="^[a-z]+$" +// +kubebuilder:default="value" +``` + +- **Use** `metav1.Condition` for status (not custom string fields) +- **Use predefined types**: `metav1.Time` instead of `string` for dates +- **Follow K8s API conventions**: Standard field names (`spec`, `status`, `metadata`) + +### Controller Design + +**RBAC markers in** `internal/controller/*_controller.go`: + +```go +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/finalizers,verbs=update +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +``` + +**Implementation rules:** +- **Idempotent reconciliation**: Safe to run multiple times +- **Re-fetch before updates**: `r.Get(ctx, req.NamespacedName, obj)` before `r.Update` to avoid conflicts +- **Structured logging**: `log := log.FromContext(ctx); log.Info("msg", "key", val)` +- **Owner references**: Enable automatic garbage collection (`SetControllerReference`) +- **Watch secondary resources**: Use `.Owns()` or `.Watches()`, not just `RequeueAfter` +- **Finalizers**: Clean up external resources (buckets, VMs, DNS entries) + +### Logging + +**Follow Kubernetes logging message style guidelines:** + +- Start from a capital letter +- Do not end the message with a period +- Active voice: subject present (`"Deployment could not create Pod"`) or omitted (`"Could not create Pod"`) +- Past tense: `"Could not delete Pod"` not `"Cannot delete Pod"` +- Specify object type: `"Deleted Pod"` not `"Deleted"` +- Balanced key-value pairs + +```go +log.Info("Starting reconciliation") +log.Info("Created Deployment", "name", deploy.Name) +log.Error(err, "Failed to create Pod", "name", name) +``` + +**Reference:** https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines + +### Webhooks +- **Create all types together**: `--defaulting --programmatic-validation --conversion` +- **When`--force`is used**: Backup custom logic first, then restore after scaffolding +- **For multi-version APIs**: Use hub-and-spoke pattern (`--conversion --spoke v2`) + - Hub version: Usually oldest stable version (v1) + - Spoke versions: Newer versions that convert to/from hub (v2, v3) + - Example: `--group crew --version v1 --kind Captain --conversion --spoke v2` (v1 is hub, v2 is spoke) + +### Learning from Examples + +The **deploy-image plugin** scaffolds a complete controller following good practices. Use it as a reference implementation: + +```bash +kubebuilder create api --group example --version v1alpha1 --kind MyApp \ + --image= --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Generated code includes: status conditions (`metav1.Condition`), finalizers, owner references, events, idempotent reconciliation. + +## Distribution Options + +### Option 1: YAML Bundle (Kustomize) + +```bash +# Generate dist/install.yaml from Kustomize manifests +make build-installer IMG=/:tag +``` + +**Key points:** +- The `dist/install.yaml` is generated from Kustomize manifests (CRDs, RBAC, Deployment) +- Commit this file to your repository for easy distribution +- Users only need `kubectl` to install (no additional tools required) + +**Example:** Users install with a single command: +```bash +kubectl apply -f https://raw.githubusercontent.com////dist/install.yaml +``` + +### Option 2: Helm Chart + +```bash +kubebuilder edit --plugins=helm/v2-alpha # Generates dist/chart/ (default) +kubebuilder edit --plugins=helm/v2-alpha --output-dir=charts # Generates charts/chart/ +``` + +**For development:** +```bash +make helm-deploy IMG=/: # Deploy manager via Helm +make helm-deploy IMG=$IMG HELM_EXTRA_ARGS="--set ..." # Deploy with custom values +make helm-status # Show release status +make helm-uninstall # Remove release +make helm-history # View release history +make helm-rollback # Rollback to previous version +``` + +**For end users/production:** +```bash +helm install my-release .//chart/ --namespace --create-namespace +``` + +**Important:** If you add webhooks or modify manifests after initial chart generation: +1. Backup any customizations in `/chart/values.yaml` and `/chart/manager/manager.yaml` +2. Re-run: `kubebuilder edit --plugins=helm/v2-alpha --force` (use same `--output-dir` if customized) +3. Manually restore your custom values from the backup + +### Publish Container Image + +```bash +export IMG=/: +nix-build nix -A image -o image.tar.gz # see nix/default.nix +skopeo copy docker-archive:image.tar.gz "docker://$IMG" +``` + +## References + +### Essential Reading +- **Kubebuilder Book**: https://book.kubebuilder.io (comprehensive guide) +- **controller-runtime FAQ**: https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md (common patterns and questions) +- **Good Practices**: https://book.kubebuilder.io/reference/good-practices.html (why reconciliation is idempotent, status conditions, etc.) +- **Logging Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines (message style, verbosity levels) + +### API Design & Implementation +- **API Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md +- **Operator Pattern**: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ +- **Markers Reference**: https://book.kubebuilder.io/reference/markers.html + +### Tools & Libraries +- **controller-runtime**: https://github.com/kubernetes-sigs/controller-runtime +- **controller-tools**: https://github.com/kubernetes-sigs/controller-tools +- **Kubebuilder Repo**: https://github.com/kubernetes-sigs/kubebuilder diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8e3c729..0000000 --- a/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM ubuntu:24.04 AS python-builder - -RUN apt-get update && \ - apt-get install -y python3 python3-venv - -RUN python3 -m venv /venv && \ - /venv/bin/pip install -U pip setuptools - -COPY requirements.txt /app/requirements.txt -RUN /venv/bin/pip install --requirement /app/requirements.txt - -COPY . /app -RUN /venv/bin/pip install /app - - -FROM ubuntu:24.04 - -# Don't buffer stdout and stderr as it breaks realtime logging -ENV PYTHONUNBUFFERED=1 - -# Create the user that will be used to run the app -ENV APP_UID=1001 -ENV APP_GID=1001 -ENV APP_USER=app -ENV APP_GROUP=app -RUN groupadd --gid $APP_GID $APP_GROUP && \ - useradd \ - --no-create-home \ - --no-user-group \ - --gid $APP_GID \ - --shell /sbin/nologin \ - --uid $APP_UID \ - $APP_USER - -RUN apt-get update && \ - apt-get install -y ca-certificates python3 tini && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=python-builder /venv /venv - -USER $APP_UID -CMD ["/venv/bin/kopf", "run", "--module", "capi_janitor.openstack.operator", "--all-namespaces", "--verbose"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..014e246 --- /dev/null +++ b/Makefile @@ -0,0 +1,202 @@ +# Image URL used to set the manager image in generated/deployed manifests +# (the image itself is built via Nix, see nix/default.nix). +IMG ?= controller:latest +# YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. +YEAR ?= $(shell date +%Y) + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + "$(CONTROLLER_GEN)" rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt",year=$(YEAR) paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# kubectl kuberc is disabled by default for test isolation; enable with: +# - KUBECTL_KUBERC=true +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= cluster-api-janitor-openstack-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) + +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./cmd/main.go + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default > dist/install.yaml + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" apply -f -; else echo "No CRDs to install; skipping."; fi + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p "$(LOCALBIN)" + +## Tool Binaries +KUBECTL ?= kubectl +KIND ?= kind +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.8.1 +CONTROLLER_TOOLS_VERSION ?= v0.21.0 + +#ENVTEST_VERSION is the controller-runtime version to use for setup-envtest, derived from go.mod +ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_VERSION manually (controller-runtime replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v") + +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @"$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f "$(1)" ;\ +GOBIN="$(LOCALBIN)" go install $${package} ;\ +mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ +} ;\ +ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" +endef + +define gomodver +$(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) +endef diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..b273a2e --- /dev/null +++ b/PROJECT @@ -0,0 +1,11 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +cliVersion: 4.15.0 +domain: capi.stackhpc.com +layout: +- go.kubebuilder.io/v4 +projectName: cluster-api-janitor-openstack +repo: github.com/azimuth-cloud/cluster-api-janitor-openstack +version: "3" diff --git a/README.md b/README.md index 801a63a..eee45b0 100644 --- a/README.md +++ b/README.md @@ -3,128 +3,220 @@ `cluster-api-janitor-openstack` is a Kubernetes operator that cleans up resources created in [OpenStack](https://www.openstack.org/) by the [OpenStack Cloud Controller Manager (OCCM)](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md) -and +and the [Cinder CSI plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) -for Kubernetes clusters created using the -[Cluster API OpenStack infrastructure provider](https://github.com/kubernetes-sigs/cluster-api-provider-openstack). +for Kubernetes clusters created with the +[Cluster API OpenStack infrastructure provider (CAPO)](https://github.com/kubernetes-sigs/cluster-api-provider-openstack). -## Installation +The operator watches `OpenStackCluster` resources and, upon deletion, removes any +dangling OpenStack resources (floating IPs, load balancers, security groups, Cinder +volumes and snapshots, and the application credential) that would otherwise be left +behind after the CAPI cluster is gone. + +## Requirements + +| Tool | Minimum version | +|---|---| +| Go | 1.26 | +| Kubernetes | 1.29 | +| CAPO | 0.14 | +| Helm | 3.x | + +## How it works + +1. When an `OpenStackCluster` is created, the operator adds its finalizer + (`janitor.capi.stackhpc.com`) to the resource. +2. When the `OpenStackCluster` is marked for deletion (`deletionTimestamp` set), + the operator authenticates to OpenStack using the credential referenced by + `spec.identityRef` and deletes all resources tagged with the cluster name. +3. The cluster name is taken from the `cluster.x-k8s.io/cluster-name` label if + present, falling back to `metadata.name`. +4. Once the purge succeeds the finalizer is removed and the `OpenStackCluster` can + be fully deleted. +5. If the purge fails, the operator sets a retry annotation + (`janitor.capi.stackhpc.com/retry`) and returns without error, triggering a + re-reconcile. + +> **Why a finalizer instead of a post-delete job?** +> +> Some OCCM-created load balancers hold references to the cluster network, which +> prevents the Cluster API OpenStack provider from deleting that network. Running +> cleanup *before* the network is torn down (but *after* all machines are gone) +> avoids this deadlock and eliminates any race with a still-running OCCM. + +## Resources cleaned up + +| OpenStack service | Resources | +|---|---| +| Neutron | Floating IPs associated with `LoadBalancer` services | +| Octavia | Load balancers with name prefix `kube_service__` | +| Neutron | Security groups matching the OCCM naming convention | +| Cinder | Volumes provisioned by the Cinder CSI (configurable — see below) | +| Cinder | Snapshots of those volumes | +| Keystone | The application credential used by the cluster (if authorised) | + +## Configuration + +### Volume deletion policy + +Cinder volumes are deleted by default. This can be changed at two levels: + +**Operator-wide default** (via Helm): + +```sh +helm upgrade ... --set defaultVolumesPolicy=keep +``` + +**Per-cluster override** (annotation on `OpenStackCluster`): + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackCluster +metadata: + name: my-cluster + annotations: + janitor.capi.stackhpc.com/volumes-policy: "keep" # or "delete" +``` + +> Any value other than `delete` means volumes will be kept. + +**Per-volume override** (set directly on the OpenStack volume): + +```sh +openstack volume set --property janitor.capi.azimuth-cloud.com/keep=true +``` + +Any value other than `true` results in the volume being deleted. + +### Retry delay + +When cleanup fails, the operator waits before re-queuing. The default delay is +60 seconds and can be changed via Helm: -`cluster-api-janitor-openstack` can be installed using [Helm](https://helm.sh): +```sh +helm upgrade ... --set retryDefaultDelay=120 +``` + +### Environment variables + +| Variable | Default | Description | +|---|---|---| +| `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` | `delete` | Operator-wide volume policy | +| `CAPI_JANITOR_RETRY_DEFAULT_DELAY` | `60` | Retry delay in seconds | + +## Installation ```sh helm repo add \ cluster-api-janitor-openstack \ https://azimuth-cloud.github.io/cluster-api-janitor-openstack -# Use the latest version from the main branch helm upgrade \ cluster-api-janitor-openstack \ cluster-api-janitor-openstack/cluster-api-janitor-openstack \ --install ``` -## Tox for unittests and linting +## Development -We use tox to run unit tests and linters across the code. -To run all the checks, including efforts to automatically -fix linting issues, please run: +### Build ```sh -tox +go build ./... ``` -You can run individual unit tests by running: +### Run tests ```sh -tox -e py3 -- +go test ./... ``` -Note, failures on your initial tox run may be automatically -fixed, where possible. So your second tox run may pass. -This way we can run the default tox target in CI. - -## Configuration +The test suite covers 108 unit tests across 4 packages using only the standard +`testing` package and `controller-runtime`'s fake client — no external cluster +required. -`cluster-api-janitor-openstack` will always clean up -[Octavia loadbalancers](https://docs.openstack.org/octavia/latest/), and associated -[floating IPs](https://docs.openstack.org/neutron/latest/), that are created by -the OCCM for `LoadBalancer` services on Cluster API clusters. +### Lint and format -By default, [Cinder volumes](https://docs.openstack.org/cinder/latest/) created by the -Cinder CSI plugin for `PersistentVolumeClaim`s are also cleaned up. However this behaviour -carries a risk of deleting important data, so can be customised in two ways. +```sh +go fmt ./... +go vet ./... +``` -The operator default can be changed to `keep`, meaning that volumes provisioned by the -Cinder CSI plugin will be kept unless overridden by the cluster: +### Makefile targets ```sh -helm upgrade ... --set defaultVolumesPolicy=keep +make help # list all targets +make generate # regenerate DeepCopy methods +make manifests # regenerate CRD/RBAC YAML +make fmt # go fmt +make vet # go vet +make test # go test (excludes e2e) +make build # go build ./cmd/main.go ``` -Regardless of the operator default, individual `OpenStackCluster`s can also be annotated -to indicate whether volumes for that cluster should be kept or removed: +## Building the OCI image -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 -kind: OpenStackCluster -metadata: - name: my-cluster - annotations: - janitor.capi.stackhpc.com/volumes-policy: "keep|delete" -``` +### Nix (reproducible, multi-arch + SBOM) + +CI uses `nix-build` for reproducible builds. The `tests` derivation runs +`go fmt`, `go vet`, and the full unit-test suite inside the Nix sandbox — no +external toolchain needed: -> **NOTE**: Any value other than `delete` means volumes will be kept. +```sh +# CI check: go fmt + go vet + 108 unit tests +nix-build nix -A tests -### User-configurable behaviour +# Build the manager binary only +nix-build nix -A manager -Annotations on the Kubernetes resources are only available to administrators with -access to the Cluster API management cluster's Kubernetes API; therefore, the Janitor -also provides an alternative user-facing mechanism for marking volumes which should -not be deleted during cluster clean up. This is done by adding a property to the -OpenStack volume using: +# Build the amd64 OCI image +nix-build nix -A image -``` -openstack volume set --property janitor.capi.azimuth-cloud.com/keep='true' +# Build the arm64 OCI image (cross-compiled from amd64) +nix-build nix -A image-arm64 + +# Generate the CycloneDX SBOM +nix-build nix -A sbom ``` -Any value other than 'true' will result in the volume being deleted when the workload -cluster is deleted. +> **`nix/nixpkgs.nix`** pins `nixos-26.05` (Go 1.26+). +> **`vendorHash`** in `nix/default.nix` is set to `sha256-5p5z+fzRkBk6rIb3DWwA3jsF4MdMVAwKHz7xza09fCc=` +> (run `nix-build nix -A manager` after any `go.mod` change — the build will +> fail and print the new hash to substitute). -## How it works +**Binary linkage note**: on macOS the local build produces a darwin/arm64 Mach-O +(dynamically linked against system libraries — normal for Go on Darwin). The +arm64 image produced via `pkgsCross.aarch64-multiplatform` contains a Linux ELF +dynamically linked against glibc; `buildLayeredImage` automatically includes the +full Nix closure (glibc and its dependencies) so the image is self-contained and +runs correctly in Kubernetes. -`cluster-api-janitor-openstack` watches for `OpenStackCluster`s being created and adds its -own finalizer to them. This prevents the `OpenStackCluster`, and hence the corresponding -Cluster API `Cluster`, from being removed until the finalizer is removed. - -`cluster-api-janitor-openstack` then waits for the `OpenStackCluster` to be deleted -(specifically, it waits for the `deletionTimestamp` to be set, indicating that a deletion -has been requested), at which point it uses the credential from -`OpenStackCluster.spec.identityRef` to remove any dangling resources that were created by -the OCCM or Cinder CSI with the same cluster name as the cluster being deleted. -The cluster name is determined by the `cluster.x-k8s.io/cluster-name` label on the -OpenStackCluster resource, if present. -If the label is not set, the name of the OpenStackCluster resource (`metadata.name`) is -used instead. -Once all the resources have been deleted, the finalizer is removed. - -> **WARNING** -> -> The cluster name of the OCCM and Cinder CSI **must** be set to the `metadata.name` -> of the OpenStackCluster resource, or to the value of the `cluster.x-k8s.io/cluster-name` -> label if it is present on the OpenStackCluster resource. -> -> For instance, the `openstack-cluster` chart from the -> [capi-helm-charts](https://github.com/azimuth-cloud/capi-helm-charts) ensures that this happens -> automatically and sets the OpenStackCluster's `metadata.name` for OCCM and Cinder CSI. - -The advantage of this approach vs. a task that runs before the cluster deletion is started -is that the external resource deletion happens _after_ all the machines have been deleted, -meaning that there is no chance of racing with the OCCM and/or Cinder CSI still running on -the cluster that may continue to try and replace resources that are cleaned up. - -It is not possible to run this cleanup as a post cluster deletion task, because some of the -resources created by the OCCM may actually block cluster deletion completely. For example, -a load-balancer created by the OCCM for a `LoadBalancer` service maintains a port on the cluster -network, meaning that the network cannot be cleaned up by the Cluster API OpenStack provider -and preventing deletion of the cluster. +## Observability + +### Prometheus metrics + +| Metric | Labels | Description | +|---|---|---| +| `capi_janitor_cleanups_total` | `result="success\|failure"` | Total cleanup attempts | + +### Kubernetes events + +| Reason | Type | Emitted when | +|---|---|---| +| `CleanupSucceeded` | Normal | OpenStack purge completed successfully | +| `CleanupFailed` | Warning | OpenStack purge returned an error | + +## Project layout + +``` +cmd/ # Operator entry point +internal/ + controller/ # Reconciler, metrics, config + openstack/ # Authentication, resource discovery & deletion +chart/ # Helm chart + templates/ + tests/ # helm-unittest tests +nix/ # Reproducible OCI build + SBOM (no Flake) +config/ # Kustomize bases (RBAC, manager, Prometheus) +test/e2e/ # End-to-end test suite (Ginkgo) +``` diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..92625fd --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,733 @@ +# Go Rewrite + +## Context + +The current project is written in Python (asyncio + kopf + easykube + httpx). It is a Kubernetes operator that cleans up OpenStack resources left behind by OCCM and the Cinder CSI when Cluster API clusters are deleted. + +The rewrite follows TDD: Gherkin tests are written first, then the implementations. + +Scaffolding tool: **kubebuilder**. + +--- + +## Audit of the Existing Python Code + +### Main Modules + +| File | Role | +|---|---| +| `capi_janitor/openstack/openstack.py` | OpenStack client: authentication, service catalog, paginated REST resources | +| `capi_janitor/openstack/operator.py` | Operator logic: kopf handlers, resource filters, OpenStack purge | + +### Covered Features + +**OpenStack Authentication** +- Only `v3applicationcredential` +- X-Auth-Token management (refresh with asyncio mutex) +- Custom CA certificate support (cacert from K8s secret) +- Service catalog filtered by interface (public/internal/admin) and region + +**Resource Filtering** +- Floating IPs: description `"Floating IP for Kubernetes external service … from cluster "` +- Octavia Load Balancers: name `kube_service__*` +- Security Groups: description `"Security Group for Service LoadBalancer in cluster "` +- Cinder Volumes: metadata `cinder.csi.openstack.org/cluster == `, unless property `janitor.capi.azimuth-cloud.com/keep == true` +- Cinder Snapshots: same cluster metadata + +**Deletion Policy** +- Volumes: configurable via env var `CAPI_JANITOR_DEFAULT_VOLUMES_POLICY` (default `delete`) and annotation `janitor.capi.stackhpc.com/volumes-policy` per cluster +- Application Credential: deleted if annotation `janitor.capi.stackhpc.com/credential-policy: delete` on the secret AND it is the last finalizer + +**Kubernetes Lifecycle** +- Finalizer `janitor.capi.stackhpc.com` on `OpenStackCluster` +- Cluster name: label `cluster.x-k8s.io/cluster-name` takes priority, otherwise `metadata.name` +- Retry via random annotation `janitor.capi.stackhpc.com/retry` (triggers a new event) +- Configurable backoff `CAPI_JANITOR_RETRY_DEFAULT_DELAY` (default 60s) + +**Error Handling** +- HTTP 400/409 during deletion: silent retry +- HTTP 404 during catalog fetch: authentication considered failed (no fatal error) +- HTTP 422 during finalizer patch: kopf `TemporaryError` +- Catalog error `volumev3` → fallback to `block-storage` + +### Existing Tests + +| File | What is tested | +|---|---| +| `test_openstack.py` | Successful auth, 404, missing interface, missing region, multiple services | +| `test_operator.py` | FIP/LB/SG/volume/snapshot filtering; `empty()`; `try_delete()`; event handler (add finalizer, skip, purge); auth error in purge | + +**Notable gap**: `test_purge_openstack_resources_success` is commented out (mock complexity). + +### Helm Chart + +- `ClusterRole`: namespaces (list/watch), events (create), secrets (get/delete), openstackclusters (list/get/watch/patch), CRDs (list/get/watch) +- Value `defaultVolumesPolicy: delete` +- Image: `ghcr.io/azimuth-cloud/cluster-api-janitor-openstack` + +### Pending PRs to Integrate + +| PR | Title | Impact | +|---|---|---| +| #261 | Fix leaving Azimuth cluster loadbalancers behind | Adds detection of Azimuth LBs (`kube_service__` + LBs named differently by Azimuth) | + +--- + +## Agile Roadmap + +--- + +### Epic 1 — OpenStack Authentication + +#### US1.1 — Authentication via Application Credential v3 + +```gherkin +Feature: OpenStack Authentication via Application Credential + In order to access OpenStack APIs + As an operator + I want to authenticate using a v3 Application Credential + + Scenario: Successful authentication + Given a clouds.yaml with auth_type "v3applicationcredential" + And a valid application_credential_id and application_credential_secret + When the operator initialises the OpenStack connection + Then an X-Auth-Token is obtained from Keystone + And the service catalog is loaded + + Scenario: Token refresh on expiry + Given an expired X-Auth-Token + When the operator makes an API call + Then a new token is requested from Keystone + And the original call is replayed with the new token + + Scenario: Authentication with unsupported type + Given a clouds.yaml with auth_type "password" + When the operator attempts to create a Cloud client + Then an UnsupportedAuthenticationError is raised +``` + +#### US1.2 — Service Catalog Filtering by Interface and Region + +```gherkin +Feature: OpenStack Service Catalog + Scenario: Endpoint selected by configured interface + Given a catalog with "public" and "internal" endpoints + And the configured interface is "public" + When the catalog is loaded + Then only "public" endpoints are retained + + Scenario: Endpoint selected by configured region + Given a catalog with endpoints for "RegionOne" and "RegionTwo" + And the configured region is "RegionOne" + When the catalog is loaded + Then only "RegionOne" endpoints are retained + + Scenario: No region configured + Given a catalog with endpoints in multiple regions + And no region is configured + When the catalog is loaded + Then the first endpoint matching the interface is retained for each service +``` + +#### US1.3 — Revoked or Invalid Credential Handling + +```gherkin +Feature: Invalid OpenStack Credential + Scenario: Application credential deleted before purge + Given an OpenStack cluster being deleted + And the application credential has already been deleted + When the operator attempts to authenticate + Then is_authenticated returns false + And if include_appcred is true, a warning is emitted and the purge stops cleanly + And if include_appcred is false, an AuthenticationError is raised + + Scenario: Catalog returns 404 + Given a valid Keystone URL but the catalog returns 404 + When the operator loads the catalog + Then is_authenticated returns false + And no fatal error is raised +``` + +#### US1.4 — Custom CA Certificate Support + +```gherkin +Feature: Custom CA Certificate + Scenario: CA provided in the Kubernetes secret + Given a Kubernetes secret containing a "cacert" entry + When the operator initialises the TLS transport + Then the CA is loaded into the SSL context + And HTTPS calls to OpenStack use this CA for verification + + Scenario: No CA provided + Given a Kubernetes secret without a "cacert" entry + When the operator initialises the TLS transport + Then the system CA is used for TLS verification +``` + +--- + +### Epic 2 — Floating IP Cleanup + +#### US2.1 — Identify Floating IPs of a Cluster + +```gherkin +Feature: Identifying Floating IPs of a Cluster + Scenario: FIP belonging to the cluster + Given a list of OpenStack Floating IPs + And a FIP with description "Floating IP for Kubernetes external service from cluster mycluster" + When the FIPs of cluster "mycluster" are listed + Then this FIP is included in the result + + Scenario: FIP from another cluster + Given a FIP with description "Floating IP for Kubernetes external service from cluster othercluster" + When the FIPs of cluster "mycluster" are listed + Then this FIP is excluded from the result + + Scenario: FIP without a Kubernetes description + Given a FIP with description "Some other description" + When the FIPs of cluster "mycluster" are listed + Then this FIP is excluded from the result +``` + +#### US2.2 — Delete Floating IPs + +```gherkin +Feature: Floating IP Deletion + Scenario: Successful deletion + Given a FIP belonging to cluster "mycluster" + When the FIP purge is triggered + Then the FIP is deleted via the Neutron API + And an INFO log is emitted + + Scenario: HTTP 400 error during deletion + Given a FIP deletion returns HTTP 400 + When the purge attempts to delete the FIP + Then a warning is emitted + And deletion continues for other FIPs + And check_fips is true to trigger a verification + + Scenario: HTTP 500 error during deletion + Given a FIP deletion returns HTTP 500 + When the purge attempts to delete the FIP + Then an exception is propagated + + Scenario: FIP deletion confirmed via polling + Given a FIP still appears in a first verification listing (OpenStack PENDING_DELETE) + When the purge polls again shortly after + And the FIP has disappeared + Then no error is raised + + Scenario: FIP still present after exhausting verification attempts + Given a FIP still appears in every verification listing + When the purge exhausts its polling attempts + Then an error is returned mentioning the cluster name +``` + +> Deletion verification for FIPs, LBs, security groups, volumes and snapshots +> (Epics 2 to 6) shares a common polling mechanism: verify immediately after +> issuing the deletes, then retry a bounded number of times with a fixed +> delay between attempts if the resource is still listed. This absorbs +> OpenStack's eventual consistency (`PENDING_DELETE` states) without +> incurring a wait when nothing needs one. + +--- + +### Epic 3 — Octavia Load Balancer Cleanup + +#### US3.1 — Identify Kubernetes Load Balancers of a Cluster + +```gherkin +Feature: Identifying Kubernetes Load Balancers + Scenario: LB belonging to the cluster + Given an LB with name "kube_service_mycluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is included in the result + + Scenario: LB from another cluster + Given an LB with name "kube_service_othercluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is excluded from the result + + Scenario: LB without kube_service prefix + Given an LB with name "fake_service_mycluster_api" + When the LBs of cluster "mycluster" are listed + Then this LB is excluded from the result +``` + +#### US3.2 — Identify Azimuth Load Balancers (PR #261) + +```gherkin +Feature: Identifying Azimuth Load Balancers + Scenario: Azimuth LB belonging to the cluster + Given an Azimuth LB identifiable as belonging to cluster "mycluster" + When the LBs of cluster "mycluster" are listed + Then this Azimuth LB is included in the result + + Scenario: HTTP error during LB listing + Given the Octavia API returns an HTTP error during listing + When the LBs of cluster "mycluster" are listed + Then an ERROR log is emitted with the HTTP code + And no exception is propagated + And a warning indicates that LBs may remain + + Scenario: HTTP error while verifying LB deletion after polling + Given LBs were deleted for cluster "mycluster" + And the Octavia API returns an HTTP error while verifying their deletion + When the verification polling exhausts its attempts due to the error + Then an ERROR log is emitted + And no exception is propagated +``` + +> Unlike the other resource types, LB verification failures stay non-fatal +> (logged only) in both cases above — before and after the deletes are +> issued — since Octavia listing is known to be slower/less reliable than +> Neutron or Cinder (see PR #261). + +#### US3.3 — Delete Load Balancers with Cascade + +```gherkin +Feature: Cascaded Load Balancer Deletion + Scenario: Successful deletion with cascade + Given an LB belonging to cluster "mycluster" + When the LB purge is triggered + Then the LB is deleted with the cascade=true parameter + And associated Octavia resources (listeners, pools, members) are deleted +``` + +--- + +### Epic 4 — Security Group Cleanup + +#### US4.1 — Identify Security Groups of a Cluster + +```gherkin +Feature: Identifying Security Groups of a Cluster + Scenario: SG belonging to the cluster + Given an SG with description "Security Group for Service LoadBalancer in cluster mycluster" + When the SGs of cluster "mycluster" are listed + Then this SG is included in the result + + Scenario: SG from another cluster + Given an SG with description "Security Group for Service LoadBalancer in cluster othercluster" + When the SGs of cluster "mycluster" are listed + Then this SG is excluded from the result +``` + +#### US4.2 — Delete Security Groups + +```gherkin +Feature: Security Group Deletion + Scenario: Successful deletion + Given an SG belonging to cluster "mycluster" + When the SG purge is triggered + Then the SG is deleted via the Neutron API + + Scenario: SG still in use (HTTP 409) + Given an SG deletion returns HTTP 409 + When the purge attempts to delete the SG + Then a warning is emitted + And check_secgroups is true for a later verification +``` + +--- + +### Epic 5 — Cinder Volume Management + +#### US5.1 — Identify Volumes of a Cluster + +```gherkin +Feature: Identifying Cinder Volumes of a Cluster + Scenario: Volume belonging to the cluster without keep flag + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + And the property "janitor.capi.azimuth-cloud.com/keep" is absent or != "true" + When the volumes of cluster "mycluster" are listed + Then this volume is included in the result + + Scenario: Volume flagged keep by the user + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + And the property "janitor.capi.azimuth-cloud.com/keep" = "true" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result + + Scenario: Volume from another cluster + Given a volume with metadata "cinder.csi.openstack.org/cluster" = "othercluster" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result + + Scenario: Volume without CSI metadata + Given a volume without metadata "cinder.csi.openstack.org/cluster" + When the volumes of cluster "mycluster" are listed + Then this volume is excluded from the result +``` + +#### US5.2 — Volume Deletion Policy + +```gherkin +Feature: Volume Deletion Policy + Scenario: Global policy "delete" (default) + Given the environment variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY is not set + When a cluster is deleted without a volumes annotation + Then the cluster's volumes are deleted + + Scenario: Global policy "keep" + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + When a cluster is deleted without a volumes annotation + Then the cluster's volumes are kept + + Scenario: Annotation "delete" on the cluster (overrides global keep) + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + And the annotation "janitor.capi.stackhpc.com/volumes-policy" = "delete" on the OpenStackCluster + When the cluster is deleted + Then the cluster's volumes are deleted + + Scenario: Annotation "keep" on the cluster (overrides global delete) + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "delete" + And the annotation "janitor.capi.stackhpc.com/volumes-policy" = "keep" on the OpenStackCluster + When the cluster is deleted + Then the cluster's volumes are kept +``` + +--- + +### Epic 6 — Cinder Snapshot Management + +#### US6.1 — Identify and Delete Snapshots of a Cluster + +```gherkin +Feature: Cinder Snapshots of a Cluster + Scenario: Snapshot belonging to the cluster + Given a snapshot with metadata "cinder.csi.openstack.org/cluster" = "mycluster" + When the snapshots of cluster "mycluster" are listed + Then this snapshot is included in the result + + Scenario: Snapshot from another cluster + Given a snapshot with metadata "cinder.csi.openstack.org/cluster" = "othercluster" + When the snapshots of cluster "mycluster" are listed + Then this snapshot is excluded from the result + + Scenario: Snapshots deleted before volumes + Given snapshots and volumes belonging to cluster "mycluster" + When the purge is triggered with include_volumes = true + Then snapshots are deleted first + And volumes are deleted afterwards +``` + +--- + +### Epic 7 — Application Credential Management + +#### US7.1 — Delete the OpenStack Application Credential + +```gherkin +Feature: Application Credential Deletion + Scenario: Deletion authorised (last finalizer) + Given the annotation "janitor.capi.stackhpc.com/credential-policy" = "delete" on the secret + And the operator's finalizer is the only finalizer present + When the purge of OpenStack resources is complete + Then the Application Credential is deleted via the Identity API + And the Kubernetes secret containing clouds.yaml is deleted + + Scenario: Other finalizers still present + Given the annotation "credential-policy" = "delete" on the secret + And other finalizers are still present on the OpenStackCluster + When the purge is complete + Then the Application Credential is not deleted + And a FinalizerStillPresentError is raised to trigger a retry + + Scenario: Application Credential cannot be deleted (403) + Given the Application Credential is restricted (no unrestricted flag) + When the appcred deletion is attempted + Then a warning is emitted + And the Kubernetes secret deletion proceeds anyway +``` + +--- + +### Epic 8 — Kubernetes Lifecycle (Finalizer Pattern) + +#### US8.1 — Add a Finalizer on Creation + +```gherkin +Feature: Adding the Janitor Finalizer to OpenStackCluster + Scenario: Cluster without deletionTimestamp and without janitor finalizer + Given an OpenStackCluster without deletionTimestamp + And without finalizer "janitor.capi.stackhpc.com" + When an event is received for this cluster + Then the finalizer "janitor.capi.stackhpc.com" is added via patch + And an INFO log confirms the addition + + Scenario: Cluster with finalizer already present + Given an OpenStackCluster without deletionTimestamp + And with the finalizer "janitor.capi.stackhpc.com" already present + When an event is received + Then no patch is made +``` + +#### US8.2 — Cluster Name from Label or metadata.name + +```gherkin +Feature: Cluster Name Resolution + Scenario: Label cluster.x-k8s.io/cluster-name present + Given an OpenStackCluster with label "cluster.x-k8s.io/cluster-name" = "myapp" + And metadata.name = "myapp-openstack" + When the operator resolves the cluster name for cleanup + Then the name "myapp" is used + + Scenario: Label absent + Given an OpenStackCluster without label "cluster.x-k8s.io/cluster-name" + And metadata.name = "mycluster" + When the operator resolves the cluster name + Then the name "mycluster" is used +``` + +#### US8.3 — Remove the Finalizer after Successful Cleanup + +```gherkin +Feature: Finalizer Removal after Purge + Scenario: Successful purge + Given an OpenStackCluster being deleted + And all OpenStack resources have been deleted + When the purge completes without error + Then the finalizer "janitor.capi.stackhpc.com" is removed via patch + And an INFO log confirms the finalizer removal + + Scenario: Finalizer absent at removal time + Given an OpenStackCluster with deletionTimestamp + And without the finalizer "janitor.capi.stackhpc.com" + When an event is received + Then no purge is triggered + And an INFO log indicates the finalizer is absent +``` + +#### US8.4 — Retry Mechanism via Annotation + +```gherkin +Feature: Retry via Random Annotation + Scenario: Transient error during purge + Given a purge that fails with a ResourcesStillPresentError + When the operator handles the error + Then after a backoff delay (5s for ResourcesStillPresent) + And a random annotation "janitor.capi.stackhpc.com/retry" is set on the OpenStackCluster + And a new event is triggered to replay the purge + + Scenario: Unknown error during purge + Given a purge that fails with an unclassified exception + When the operator handles the error + Then the delay is CAPI_JANITOR_RETRY_DEFAULT_DELAY (default 60s) + And the exception is logged with a stack trace + + Scenario: Resource deleted between the error and the retry + Given the OpenStackCluster is deleted during backoff + When the operator attempts to annotate the resource + Then the 404 ApiError is ignored +``` + +--- + +### Epic 9 — Operator Configuration + +#### US9.1 — Configuration via Environment Variables + +```gherkin +Feature: Configuration via Environment Variables + Scenario: Default volumes policy configured + Given CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" + When the operator starts + Then the default policy for all clusters is "keep" + + Scenario: Configurable retry delay + Given CAPI_JANITOR_RETRY_DEFAULT_DELAY = "120" + When an unclassified error occurs + Then the retry delay is 120 seconds +``` + +--- + +### Epic 10 — Packaging and Deployment + +#### US10.1 — Secure Image (Nix Build) + +```gherkin +Feature: Secure OCI Image for the Go Operator + Scenario: Reproducible Nix build + Given the Go operator source code + When `nix-build nix -A image` is run + Then the manager binary is built with buildGoModule and CGO_ENABLED=0 + And the image contains only the Nix closure required to run the binary + + Scenario: Image security + Given the built image + Then the process runs as non-root (UID 65532) + And the root filesystem is read-only + And all Linux capabilities are dropped +``` + +#### US10.2 — Helm Chart + +```gherkin +Feature: Deployment via Helm Chart + Scenario: Installation with default values + Given the cluster-api-janitor-openstack Helm chart + When helm install is executed + Then a Deployment, ServiceAccount, ClusterRole, and ClusterRoleBinding are created + And the default volumes policy is "delete" + And the default retry delay is 60 seconds + + Scenario: Override volumes policy + Given helm install with --set defaultVolumesPolicy=keep + When the chart is deployed + Then the variable CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" is injected into the pod + + Scenario: Health probes active + Given the deployed Deployment + Then a livenessProbe on /healthz:8081 is configured + And a readinessProbe on /readyz:8081 is configured + + Scenario: Complete RBAC + Given the deployed ClusterRole + Then the "update" verb is present on openstackclusters + (required for r.Update during finalizer management) +``` + +#### US10.3 — OCI Build via Nix (without Flake) and SBOM + +```gherkin +Feature: Reproducible OCI Build via Nix and SBOM Generation + Scenario: Build the amd64 image with Nix + Given the nix/default.nix file + When nix-build nix -A image is executed + Then an amd64 OCI image is produced (dockerTools.buildLayeredImage) + And the image runs as User 65532:65532 + + Scenario: Build the arm64 image by cross-compilation + Given the nix/default.nix file + When nix-build nix -A image-arm64 is executed + Then an arm64 OCI image is produced via pkgsCross.aarch64-multiplatform + And both images are combined into a multi-arch manifest via skopeo + docker manifest + + Scenario: CycloneDX SBOM generation + Given the compiled Go binary + When nix-build nix -A sbom is executed + Then an sbom.cdx.json file in CycloneDX format is produced + And it lists all Go modules (extracted from the buildinfo embedded in the binary) + And it is uploaded as an artefact of the GitHub Actions workflow +``` + +--- + +### Epic 11 — Observability + +#### US11.1 — Prometheus Metrics + +```gherkin +Feature: Prometheus Metrics + Scenario: Successful purge → success counter incremented + Given a cluster being deleted + When the OpenStack purge succeeds + Then capi_janitor_cleanups_total{result="success"} is incremented by 1 + + Scenario: Failed purge → failure counter incremented + Given a cluster being deleted + When the OpenStack purge fails + Then capi_janitor_cleanups_total{result="failure"} is incremented by 1 +``` + +> Implemented: `CounterVec` exposed via `ctrlmetrics.Registry` (port 8080/metrics). +> `Metrics *Metrics` field is injectable on the reconciler (DI for tests). + +#### US11.2 — Kubernetes Events + +```gherkin +Feature: Kubernetes Events on OpenStackCluster + Scenario: Successful purge → Normal "CleanupSucceeded" event + Given a cluster being deleted + When the OpenStack purge succeeds + Then a Normal event with reason "CleanupSucceeded" is emitted on the OpenStackCluster + + Scenario: Failed purge → Warning "CleanupFailed" event + Given a cluster being deleted + When the OpenStack purge fails + Then a Warning event with reason "CleanupFailed" and the error message is emitted +``` + +> Implemented: `record.EventRecorder` injectable on the reconciler; `SetupWithManager` +> auto-initialises via `mgr.GetEventRecorderFor("capi-janitor")` when nil. + +--- + +### Epic 12 — Robustness + +#### US12.1 — HTTP Client Timeout + +```gherkin +Feature: HTTP Timeout on the OpenStack Client + Scenario: Context cancelled before the call + Given an already-cancelled context + When Authenticate is called + Then an error is returned immediately + + Scenario: Safety net on the http.Client + Given no context with a deadline provided by the caller + Then the http.Client has a Timeout of 30 seconds + (prevents calls from blocking indefinitely when OpenStack is unreachable) +``` + +#### US12.2 — Cinder Service Legacy Aliases + +```gherkin +Feature: Cinder Service Detection with Aliases + Scenario: Catalog with "volumev3" (standard >= Stein) + Given an OpenStack catalog with service type "volumev3" + When the operator looks up the Cinder client + Then the "volumev3" client is used + + Scenario: Catalog with "block-storage" only + Given an OpenStack catalog without "volumev3" but with "block-storage" + When the operator looks up the Cinder client + Then the "block-storage" client is used + + Scenario: Catalog with "volume" only (legacy alias < Stein) + Given an OpenStack catalog without "volumev3" or "block-storage" but with "volume" + When the operator looks up the Cinder client + Then the "volume" client is used + + Scenario: Catalog without a Cinder service + Given a catalog without "volumev3", "block-storage", or "volume" + When the operator looks up the Cinder client + Then a CatalogError is raised with the appropriate message +``` + +--- + +## Actions + +1. [x] Audit the existing code +2. [x] Write the agile roadmap (this document) +3. [x] Scaffold the Go project with kubebuilder +4. [x] Write Go tests (TDD) for each user story +5. [x] Implement the features in Go (108 tests) +6. [x] Migrate the Helm chart for the Go image +7. [x] Implement epics 11 (observability) and 12 (robustness) +8. [x] OCI build via Nix (without Flake) + CycloneDX SBOM (US10.3 — outside initial plan) + +## Final Result + +| Layer | Key Files | +|---|---| +| OpenStack client | `internal/openstack/cloud.go`, `resources.go`, `purge.go` | +| Controller | `internal/controller/openstackcluster_controller.go`, `metrics.go` | +| Config | `internal/controller/config.go` (env vars) | +| Tests | 108 tests (4 packages) | +| Packaging | `nix/default.nix`, `nix/nixpkgs.nix` | +| Helm | `chart/` — Deployment, ClusterRole, RBAC, health probes | +| CI | `.github/workflows/build-push-artifacts.yaml` (Nix + skopeo + SBOM) | + +## Implementation Order + +``` +Epic 1 (Auth) → Epic 2 (FIPs) → Epic 3 (LBs + PR #261) +→ Epic 4 (SGs) → Epic 5 (Volumes) → Epic 6 (Snapshots) +→ Epic 7 (AppCreds) → Epic 8 (Lifecycle K8s) → Epic 9 (Config) +→ Epic 10 (Packaging + Nix/SBOM) → Epic 11 (Observability) +→ Epic 12 (Robustness) +``` diff --git a/capi_janitor/__init__.py b/capi_janitor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/openstack/__init__.py b/capi_janitor/openstack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/openstack/openstack.py b/capi_janitor/openstack/openstack.py deleted file mode 100644 index 4744aff..0000000 --- a/capi_janitor/openstack/openstack.py +++ /dev/null @@ -1,249 +0,0 @@ -import asyncio -import contextlib -import re -import urllib.parse - -import httpx -from easykube import rest - - -class UnsupportedAuthenticationError(Exception): - """Raised when an unsupported authentication method is used.""" - - def __init__(self, auth_type): - super().__init__(f"unsupported authentication type: {auth_type}") - - -class AuthenticationError(Exception): - """Raised when an unknown authentication error is encountered.""" - - def __init__(self, user): - super().__init__(f"failed to authenticate as user: {user}") - - -class CatalogError(Exception): - """Raised when an unknown catalog service type is requested.""" - - def __init__(self, name): - super().__init__(f"service type {name} not found in OpenStack service catalog") - - -class Auth(httpx.Auth): - """Authenticator class for OpenStack connections.""" - - def __init__( - self, auth_url, application_credential_id, application_credential_secret - ): - self.url = auth_url - self._application_credential_id = application_credential_id - self._application_credential_secret = application_credential_secret - self._token = None - self._user_id = None - self._lock = asyncio.Lock() - - @contextlib.asynccontextmanager - async def _refresh_token(self): - """Context manager to ensure only one request at a time - - triggers a token refresh. - """ - token = self._token - async with self._lock: - # Only yield to the wrapped block if the token has not changed - # in the time it took to acquire the lock - if token == self._token: - yield - - def _build_token_request(self): - return httpx.Request( - "POST", - f"{self.url}/v3/auth/tokens", - json={ - "auth": { - "identity": { - "methods": ["application_credential"], - "application_credential": { - "id": self._application_credential_id, - "secret": self._application_credential_secret, - }, - }, - }, - }, - ) - - def _handle_token_response(self, response): - response.raise_for_status() - self._token = response.headers["X-Subject-Token"] - self._user_id = response.json()["token"]["user"]["id"] - - async def async_auth_flow(self, request): - if self._token is None: - async with self._refresh_token(): - response = yield self._build_token_request() - await response.aread() - self._handle_token_response(response) - request.headers["X-Auth-Token"] = self._token - response = yield request - - -class Resource(rest.Resource): - """Base resource for OpenStack APIs.""" - - def __init__(self, client, name, prefix=None, plural_name=None, singular_name=None): - super().__init__(client, name, prefix) - # Some resources support a /detail endpoint - # In this case, we just want to use the name up to the slash as the plural name - self._plural_name = plural_name or self._name.split("/")[0].replace("-", "_") - # If no singular name is given, assume the name ends in 's' - self._singular_name = singular_name or self._plural_name[:-1] - - @property - def singular_name(self): - return self._singular_name - - def _extract_list(self, response): - # Some resources support a /detail endpoint - # In this case, we just want to use the name up to the slash - return response.json()[self._plural_name] - - def _extract_next_page(self, response): - next_url = next( - ( - link["href"] - for link in response.json().get(f"{self._plural_name}_links", []) - if link["rel"] == "next" - ), - None, - ) - # Sometimes, the returned URLs have http where they should have https - # To mitigate this, we split the URL and return the path and params separately - url = urllib.parse.urlsplit(next_url) - params = urllib.parse.parse_qs(url.query) - return url.path, params - - def _extract_one(self, response): - content_type = response.headers.get("content-type") - if content_type == "application/json": - return response.json()[self._singular_name] - else: - return super()._extract_one(response) - - -class Client(rest.AsyncClient): - """Client for OpenStack APIs.""" - - def __init__(self, /, base_url, prefix=None, **kwargs): - # Extract the path part of the base_url - url = urllib.parse.urlsplit(base_url) - # Initialise the client with the scheme/host - super().__init__(base_url=f"{url.scheme}://{url.netloc}", timeout=60, **kwargs) - # If another prefix is not given, use the path from the base URL as the prefix, - # otherwise combine the prefixes and remove duplicated path sections. - # This ensures things like pagination work nicely without duplicating the prefix - if prefix: - self._prefix = "/".join( - [url.path.rstrip("/"), prefix.lstrip("/").lstrip(url.path)] - ) - else: - self._prefix = url.path - - def __aenter__(self): - # Prevent individual clients from being used in a context manager - raise RuntimeError("clients must be used via a cloud object") - - def resource(self, name, prefix=None, plural_name=None, singular_name=None): - # If an additional prefix is given, combine it with the existing prefix - if prefix: - prefix = "/".join([self._prefix.rstrip("/"), prefix.lstrip("/")]) - else: - prefix = self._prefix - return Resource(self, name, prefix, plural_name, singular_name) - - -class Cloud: - """Object for interacting with OpenStack clouds.""" - - def __init__(self, auth, transport, interface, region=None): - self._auth = auth - self._transport = transport - self._interface = interface - self._endpoints = {} - self._region = region - # A map of api name to client - self._clients = {} - - async def __aenter__(self): - await self._transport.__aenter__() - # Once the transport has been initialised, we can initialise the endpoints - client = Client( - base_url=self._auth.url, auth=self._auth, transport=self._transport - ) - try: - # Use the full auth URL to avoid losing any path prefix (e.g. - # /identity) that Client.__init__ strips from the httpx base_url - # into _prefix, which raw client.get() calls do not prepend. - response = await client.get(f"{self._auth.url}/v3/auth/catalog") - except httpx.HTTPStatusError as exc: - # If the auth fails, we just have an empty app catalog - if exc.response.status_code == 404: - return self - else: - raise - self._endpoints = {} - for entry in response.json()["catalog"]: - for ep in entry.get("endpoints", []): - if ep.get("interface") == self._interface and ( - not self._region or ep.get("region_id") == self._region - ): - self._endpoints[entry["type"]] = ep["url"] - break - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - await self._transport.__aexit__(exc_type, exc_value, traceback) - - @property - def is_authenticated(self): - """Returns True if the cloud is authenticated, False otherwise.""" - return bool(self._endpoints) - - @property - def current_user_id(self): - """The ID of the current user.""" - return self._auth._user_id - - @property - def apis(self): - """The APIs supported by the cloud.""" - return list(self._endpoints.keys()) - - def api_client(self, name, prefix=None): - """Returns a client for the named API.""" - if name not in self._clients: - self._clients[name] = Client( - base_url=self._endpoints[name], - prefix=prefix, - auth=self._auth, - transport=self._transport, - ) - return self._clients[name] - - @classmethod - def from_clouds(cls, clouds, cloud, cacert): - config = clouds["clouds"][cloud] - if config["auth_type"] != "v3applicationcredential": - raise UnsupportedAuthenticationError(config["auth_type"]) - auth_url = re.sub("/v3/?$", "", config["auth"]["auth_url"]) - auth = Auth( - auth_url, - config["auth"]["application_credential_id"], - config["auth"]["application_credential_secret"], - ) - region = config.get("region_name") - # Create a default context using the verification from the config - context = httpx.create_ssl_context(verify=config.get("verify", True)) - # If a cacert was given, load it into the context - if cacert is not None: - context.load_verify_locations(cadata=cacert) - transport = httpx.AsyncHTTPTransport(verify=context) - return cls(auth, transport, config.get("interface", "public"), region) diff --git a/capi_janitor/openstack/operator.py b/capi_janitor/openstack/operator.py deleted file mode 100644 index 1197550..0000000 --- a/capi_janitor/openstack/operator.py +++ /dev/null @@ -1,496 +0,0 @@ -import asyncio -import base64 -import functools -import os -import random -import string - -import easykube -import httpx -import kopf -import yaml - -from . import openstack - -ekconfig: easykube.Configuration -ekclient: easykube.AsyncClient - -CAPO_API_GROUP = "infrastructure.cluster.x-k8s.io" -FINALIZER = "janitor.capi.stackhpc.com" - -VOLUMES_ANNOTATION = "janitor.capi.stackhpc.com/volumes-policy" -VOLUMES_ANNOTATION_DELETE = "delete" -VOLUMES_ANNOTATION_DEFAULT = os.environ.get( - "CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", VOLUMES_ANNOTATION_DELETE -) - -CREDENTIAL_ANNOTATION = "janitor.capi.stackhpc.com/credential-policy" -CREDENTIAL_ANNOTATION_DELETE = "delete" - -RETRY_ANNOTATION = "janitor.capi.stackhpc.com/retry" -RETRY_DEFAULT_DELAY = int(os.environ.get("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "60")) - -# The property on the OpenStack volume resource which, if set to 'true', -# will instruct the Janitor to ignore this volume when cleaning up cluster -# resources. -OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY = "janitor.capi.azimuth-cloud.com/keep" - - -@kopf.on.startup() -async def on_startup(**kwargs): - global ekconfig - ekconfig = easykube.Configuration.from_environment() - global ekclient - ekclient = ekconfig.async_client() - - -@kopf.on.cleanup() -async def on_cleanup(**kwargs): - """Runs on operator shutdown.""" - # Make sure that the easykube client is shut down properly - await ekclient.aclose() - - -class FinalizerStillPresentError(Exception): - """Raised when a finalizer from another controller is preventing us from - - deleting an appcred. - """ - - def __init__(self, finalizer, cluster): - super().__init__(f"finalizer '{finalizer}' still present for cluster {cluster}") - - -class ResourcesStillPresentError(Exception): - """Raised when cluster resources are still present even after being deleted, - - e.g. while waiting for deletion. - """ - - def __init__(self, resource, cluster): - super().__init__(f"{resource} still present for cluster {cluster}") - - -async def fips_for_cluster(resource, cluster): - """Async iterator for FIPs belonging to the specified cluster.""" - async for fip in resource.list(): - if not fip.description.startswith( - "Floating IP for Kubernetes external service" - ): - continue - if not fip.description.endswith(f"from cluster {cluster}"): - continue - yield fip - - -async def lbs_for_cluster(resource, cluster): - """Async iterator for loadbalancers belonging to the specified cluster.""" - async for lb in resource.list(): - if lb.name.startswith(f"kube_service_{cluster}_"): - yield lb - - -async def secgroups_for_cluster(resource, cluster): - """Async iterator for security groups belonging to the specified cluster.""" - async for sg in resource.list(): - if not sg.description.startswith("Security Group for"): - continue - if not sg.description.endswith(f"Service LoadBalancer in cluster {cluster}"): - continue - yield sg - - -async def filtered_volumes_for_cluster(resource, cluster): - """Async iterator for volumes belonging to the specified cluster.""" - async for vol in resource.list(): - # CSI Cinder sets metadata on the volumes that we can look for - owner = vol.metadata.get("cinder.csi.openstack.org/cluster") - # Skip volumes with the keep property set to true - if ( - owner - and owner == cluster - and vol.metadata.get(OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY) != "true" - ): - yield vol - - -async def snapshots_for_cluster(resource, cluster): - """Async iterator for snapshots belonging to the specified cluster.""" - async for snapshot in resource.list(): - # CSI Cinder sets metadata on the volumes that we can look for - owner = snapshot.metadata.get("cinder.csi.openstack.org/cluster") - if owner and owner == cluster: - yield snapshot - - -async def empty(async_iterator): - """Returns True if the given async iterator is empty, False otherwise.""" - try: - _ = await async_iterator.__anext__() - except StopAsyncIteration: - return True - else: - return False - - -async def try_delete(logger, resource, instances, **kwargs): - """Tries to delete the specified instances, catches 400 & 409 exceptions for retry. - - It returns a boolean indicating whether a check is required for the resource. - """ - check_required = False - async for instance in instances: - check_required = True - try: - await resource.delete(instance.id, **kwargs) - except httpx.HTTPStatusError as exc: - if exc.response.status_code in {400, 409}: - logger.warn( - f"got status code {exc.response.status_code} when trying to delete " - f"{resource.singular_name} with ID {instance.id} - will retry" - ) - else: - raise - return check_required - - -async def purge_openstack_resources( # noqa: C901 - logger, - clouds, - cloud_name, - cacert, - name, - include_volumes, - include_loadbalancers, - include_appcred, -): - """Cleans up the OpenStack resources created by the OCCM and CSI for a cluster.""" - # Use the credential to delete external resources as required - async with openstack.Cloud.from_clouds(clouds, cloud_name, cacert) as cloud: - if not cloud.is_authenticated: - if include_appcred: - # If the session is not authenticated then we've already - # cleaned up and deleted the app cred. - logger.warn("application credential has been deleted") - else: - # Raise an error and skip removing the finalizer to block cluster - # deletion to avoid leaking resources. - raise openstack.AuthenticationError(cloud.current_user_id) - return - - # Release any floating IPs associated with loadbalancer services for the cluster - networkapi = cloud.api_client("network", "/v2.0/") - fips = networkapi.resource("floatingips") - check_fips = await try_delete(logger, fips, fips_for_cluster(fips, name)) - logger.info("deleted floating IPs for LoadBalancer services") - - # Delete any loadbalancers associated with loadbalancer services for the cluster - lbapi = cloud.api_client("load-balancer", "/v2/lbaas/") - loadbalancers = lbapi.resource("loadbalancers") - check_lbs = False - if include_loadbalancers: - check_lbs = await try_delete( - logger, - loadbalancers, - lbs_for_cluster(loadbalancers, name), - cascade="true", - ) - logger.info("deleted load balancers for LoadBalancer services") - - # Delete security groups associated with loadbalancer services for the cluster - secgroups = networkapi.resource("security-groups") - check_secgroups = await try_delete( - logger, secgroups, secgroups_for_cluster(secgroups, name) - ) - logger.info("deleted security groups for LoadBalancer services") - - # Delete volumes and snapshots associated with PVCs, unless requested - # otherwise via the annotation - # NOTE(sd109): DevStack uses 'block-storage' as the Cinder service - # type in the catalog which is technically correct and volumev3 is just - # a historically valid alias, see: - # - https://docs.openstack.org/keystone/latest/contributor/service-catalog.html - # - https://service-types.openstack.org/service-types.json - # TODO(sd109): Make use of https://opendev.org/openstack/os-service-types to - # improve service type alias handling? - try: - volumeapi = cloud.api_client("volumev3") - except KeyError: - try: - volumeapi = cloud.api_client("block-storage") - except KeyError: - raise openstack.CatalogError("volumev3 or block-storage") - - snapshots_detail = volumeapi.resource("snapshots/detail") - snapshots = volumeapi.resource("snapshots") - check_snapshots = False - volumes_detail = volumeapi.resource("volumes/detail") - volumes = volumeapi.resource("volumes") - check_volumes = False - if include_volumes: - check_snapshots = await try_delete( - logger, snapshots, snapshots_for_cluster(snapshots_detail, name) - ) - logger.info("deleted snapshots for persistent volume claims") - check_volumes = await try_delete( - logger, volumes, filtered_volumes_for_cluster(volumes_detail, name) - ) - logger.info("deleted volumes for persistent volume claims") - - # Check that the resources have actually been deleted - if check_fips and not await empty(fips_for_cluster(fips, name)): - raise ResourcesStillPresentError("floatingips", name) - if check_lbs and not await empty(lbs_for_cluster(loadbalancers, name)): - raise ResourcesStillPresentError("loadbalancers", name) - if check_secgroups and not await empty(secgroups_for_cluster(secgroups, name)): - raise ResourcesStillPresentError("security-groups", name) - if check_volumes and not await empty( - filtered_volumes_for_cluster(volumes_detail, name) - ): - raise ResourcesStillPresentError("volumes", name) - if check_snapshots and not await empty( - snapshots_for_cluster(snapshots_detail, name) - ): - raise ResourcesStillPresentError("snapshots", name) - - # Now we have finished deleting resources, try to delete the appcred itself - # This requires an appcred to be unrestricted - # If it is not, we proceed but emit a warning - if include_appcred: - identityapi = cloud.api_client("identity", "v3") - appcreds = identityapi.resource( - "application_credentials", - # appcreds are user-namespaced - prefix=f"users/{cloud.current_user_id}", - ) - appcred_id = clouds["clouds"]["openstack"]["auth"][ - "application_credential_id" - ] - try: - await appcreds.delete(appcred_id) - except httpx.HTTPStatusError as exc: - if exc.response.status_code == 403: - logger.warn("unable to delete application credential for cluster") - else: - raise - logger.info("deleted application credential for cluster") - - -async def patch_finalizers(resource, name, namespace, finalizers): - """Patches the finalizers of a resource. - - If the resource does not exist anymore, that is classed as a success. - """ - try: - await resource.patch( - name, {"metadata": {"finalizers": finalizers}}, namespace=namespace - ) - except easykube.ApiError as exc: - # Patching the finalizers can result in a 422 if we are deleting and CAPO - # has removed its finalizer while we were working - if exc.status_code == 422: - raise kopf.TemporaryError("error patching finalizers", delay=1) - elif exc.status_code != 404: - raise - - -def retry_event(handler): - """Decorator for retrying events on Kubernetes objects. - - Instead of retrying within the handler, potentially on stale data, the object is - annotated with the number of times it has been retried. This triggers a new event - for the retry which contains up-to-date data. - - As recommended in the kopf docs, handlers should be idempotent as this mechanism - may result in the handler being called twice if an update happens between a failure - and the associated retry. - """ - - @functools.wraps(handler) - async def wrapper(**kwargs): - body = kwargs["body"] - resource = await ekclient.api(body["apiVersion"]).resource(body["kind"]) - try: - return await handler(**kwargs) - except Exception as exc: - if isinstance(exc, kopf.TemporaryError): - kwargs["logger"].warn(str(exc)) - backoff = exc.delay - elif isinstance( - exc, FinalizerStillPresentError | ResourcesStillPresentError - ): - kwargs["logger"].warn(str(exc)) - backoff = 5 - else: - kwargs["logger"].exception(str(exc)) - # Calculate the backoff - backoff = RETRY_DEFAULT_DELAY - # Wait for the backoff before annotating the resource - if backoff is None: - backoff = RETRY_DEFAULT_DELAY - await asyncio.sleep(backoff) - # Annotate the object with a random value to trigger another event - try: - await resource.patch( - kwargs["name"], - { - "metadata": { - "annotations": { - RETRY_ANNOTATION: "".join( - random.choices( - string.ascii_lowercase + string.digits, k=8 - ) - ), - } - } - }, - namespace=kwargs["namespace"], - ) - except easykube.ApiError as exc: - if exc.status_code != 404: - raise - - return wrapper - - -@kopf.on.event(CAPO_API_GROUP, "openstackclusters") -@retry_event -async def on_openstackcluster_event( - name, namespace, meta, labels, spec, status, logger, **kwargs -): - await _on_openstackcluster_event_impl( - name, namespace, meta, labels, spec, status, logger, **kwargs - ) - - -async def _on_openstackcluster_event_impl( - name, namespace, meta, labels, spec, status, logger, **kwargs -): - """Executes whenever an event occurs for an OpenStack cluster.""" - # Get the resource for manipulating OpenStackClusters at the preferred version - openstackclusters = await _get_os_cluster_client() - - # Use the value of the `cluster.x-k8s.io/cluster-name` label as the cluster name - # if it exists, otherwise, fall back to the OpenStackCluster resource name. - clustername = labels.get("cluster.x-k8s.io/cluster-name", name) - logger.debug(f"cluster name that will be used for cleanup: '{clustername}'") - - finalizers = meta.get("finalizers", []) - # We add a custom finalizer to OpenStack cluster objects to - # prevent them from being deleted until we have acted - if not meta.get("deletionTimestamp"): - if FINALIZER not in finalizers: - await patch_finalizers( - openstackclusters, - name, - namespace, - [*finalizers, FINALIZER], - ) - logger.info("added janitor finalizer to cluster") - return - - # NOTE: If we get to here, the cluster is deleting - - # If our finalizer is not present, we don't do anything - if FINALIZER not in finalizers: - logger.info("janitor finalizer not present, skipping cleanup") - return - - # Get the cloud credential from the cluster and use it to delete dangling - # resources created by OpenStack integrations on the cluster - clouds_secret = await _get_clouds_secret( - spec["identityRef"]["name"], namespace=namespace - ) - if clouds_secret is None: - # TODO(johngarbutt): fail better when secret not found? - logger.error(f"clouds.yaml not found for: {clustername}") - - else: - clouds = yaml.safe_load(base64.b64decode(clouds_secret.data["clouds.yaml"])) - if "cacert" in clouds_secret.data: - cacert = base64.b64decode(clouds_secret.data["cacert"]).decode() - else: - cacert = None - # The cloud name comes from spec.identityRef.cloudName from v1beta1 onwards - # Prior to that, it comes from spec.cloudName - cloud_name = spec["identityRef"].get( - "cloudName", spec.get("cloudName", "openstack") - ) - # The value of this annotation on the cluster decides whether to delete volumes - volumes_annotation_value = meta.get("annotations", {}).get( - VOLUMES_ANNOTATION, VOLUMES_ANNOTATION_DEFAULT - ) - # We want to remove the appcred iff: - # - # 1. The annotation on the secret is present and says delete - # 2. Our finalizer is the last finalizer - # - # This is because we need to allow CAPO to finish its work before we remove the - # appcred + secret, but we do need to act to remove other resources that might - # block CAPO from completing - credential_annotation_value = clouds_secret.metadata.get("annotations", {}).get( - CREDENTIAL_ANNOTATION - ) - remove_appcred = credential_annotation_value == CREDENTIAL_ANNOTATION_DELETE - - # Handle the case where the API server load balancer is enabled but was - # never successfully created (possibly due to LB permissions error) - # meaning that there cannot be any other LBs to remove - # (since API server was never online due to missing load balancer) - remove_loadbalancers = ( - # Default to false since this is default CAPO behaviour if - # ApiServerLoadBalancer field is omitted from OpenStackClusterSpec - # https://github.com/kubernetes-sigs/cluster-api-provider-openstack/blob/57ae27ee114bda3606d92163397697b640272673/api/v1beta1/openstackcluster_types.go#L99-L101 - spec.get("apiServerLoadBalancer", {}).get("enabled", False) - and status.get("apiServerLoadBalancer", {}).get("id", "") != "" - ) - - await purge_openstack_resources( - logger, - clouds, - cloud_name, - cacert, - clustername, - volumes_annotation_value == VOLUMES_ANNOTATION_DELETE, - remove_loadbalancers, - remove_appcred and len(finalizers) == 1, - ) - - # If we get to here, OpenStack resources have been successfully deleted - # So we can remove the appcred secret if we are the last actor - if remove_appcred and len(finalizers) == 1: - await _delete_secret(clouds_secret.metadata["name"], namespace) - logger.info("cloud credential secret deleted") - elif remove_appcred: - # If the annotation says delete but other controllers are still acting, - # go round again - raise FinalizerStillPresentError( - next(f for f in finalizers if f != FINALIZER), name - ) - - # If we get to here, we can remove the finalizer - await patch_finalizers( - openstackclusters, name, namespace, [f for f in finalizers if f != FINALIZER] - ) - logger.info("removed janitor finalizer from cluster") - - -async def _delete_secret(name, namespace): - secrets = await ekclient.api("v1").resource("secrets") - await secrets.delete(name, namespace=namespace) - - -async def _get_os_cluster_client(): - capoapi = await ekclient.api_preferred_version(CAPO_API_GROUP) - openstackclusters = await capoapi.resource("openstackclusters") - return openstackclusters - - -async def _get_clouds_secret(secret_name, namespace): - secrets = await ekclient.api("v1").resource("secrets") - try: - return await secrets.fetch(secret_name, namespace=namespace) - except easykube.ApiError as exc: - if exc.status_code != 404: - raise - # TODO(johngarbutt): fail better when not found? diff --git a/capi_janitor/tests/__init__.py b/capi_janitor/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/tests/openstack/__init__.py b/capi_janitor/tests/openstack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/capi_janitor/tests/openstack/test_openstack.py b/capi_janitor/tests/openstack/test_openstack.py deleted file mode 100644 index 444303c..0000000 --- a/capi_janitor/tests/openstack/test_openstack.py +++ /dev/null @@ -1,280 +0,0 @@ -import unittest -from unittest.mock import AsyncMock, MagicMock, patch - -from capi_janitor.openstack.openstack import AuthenticationError, Cloud - - -class TestCloudAsyncContext(unittest.IsolatedAsyncioTestCase): - # Set up common variables for all tests - async def asyncSetUp(self): - # Auth is mocked to simulate authentication - self.auth = MagicMock() - self.auth.url = "https://example.com:5000/identity" - # Transport is awaited so can be Async Mocked - self.transport = AsyncMock() - # Interface & Region can be fixed for the tests - self.interface = "public" - self.region = "region1" - # Create a Cloud instance with the mocked auth and transport - self.cloud = Cloud(self.auth, self.transport, self.interface, self.region) - - # Test the __aenter__ method for auth success and general functionality - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_successful_authentication(self, mock_client): - # Patched client to simulate a successful authentication - mock_client_instance = AsyncMock() - # Return mock for the client - mock_client.return_value = mock_client_instance - # Mock the get method to return a simple successful response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - # Mock the base_url for the client - mock_client_instance._base_url = "https://compute.example.com" - - # Assert return values - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertEqual( - cloud.api_client("compute")._base_url, "https://compute.example.com" - ) - # Full URL must be used so the /identity path prefix is preserved. - # Client.__init__ strips the path into _prefix (not used by raw - # client.get()), so a relative "/v3/auth/catalog" would resolve - # against base_url "https://example.com:5000" and lose /identity. - mock_client_instance.get.assert_called_once_with( - "https://example.com:5000/identity/v3/auth/catalog" - ) - - # Test the __aenter__ method for auth failure - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_authentication_failure(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Simulate an auth error with a named user - mock_client_instance.get.side_effect = AuthenticationError("test_user") - - with self.assertRaises(AuthenticationError) as context: - async with self.cloud: - pass - # Assert that the AuthenticationError is raised with the correct message - self.assertEqual( - str(context.exception), "failed to authenticate as user: test_user" - ) - - # Test the __aenter__ method for 404 error - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_auth_404_error(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Simulate a 404 error - mock_client_instance.get.side_effect = MagicMock( - response=MagicMock(status_code=404) - ) - - # Assert auth failed and no endpoints are returned - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - # Test the __aenter__ method for no matching interface - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_matching_interface(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # No matching interface in the response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_matching_region_id(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # No matching region_id in the response - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region2", - "url": "https://compute.example.com", - } - ], - } - ] - } - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - self.assertEqual(cloud.apis, []) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_filter_endpoints(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return multiple endpoints, one matching, one not - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertNotIn("network", cloud.apis) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_multiple_services(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return multiple services, some matching, some not - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "storage", - "endpoints": [ - { - "interface": "internal", - "region_id": "region1", - "url": "https://storage.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertNotIn("storage", cloud.apis) - self.assertIn("network", cloud.apis) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_empty_endpoint_list(self, mock_client): - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - mock_client_instance.get.return_value.json = MagicMock( - return_value={"catalog": [{"type": "compute", "endpoints": []}]} - ) - - async with self.cloud as cloud: - self.assertFalse(cloud.is_authenticated) - - @patch("capi_janitor.openstack.openstack.Client") - async def test_cloud_no_region_specified(self, mock_client): - # Set up the cloud instance without a region - self.cloud = Cloud(self.auth, self.transport, self.interface, region=None) - mock_client_instance = AsyncMock() - mock_client.return_value = mock_client_instance - # Return endpoints with different region_ids - mock_client_instance.get.return_value.json = MagicMock( - return_value={ - "catalog": [ - { - "type": "compute", - "endpoints": [ - { - "interface": "public", - "region_id": "region1", - "url": "https://compute.example.com", - } - ], - }, - { - "type": "network", - "endpoints": [ - { - "interface": "public", - "region_id": "region2", - "url": "https://network.example.com", - } - ], - }, - ] - } - ) - - async with self.cloud as cloud: - self.assertTrue(cloud.is_authenticated) - self.assertIn("compute", cloud.apis) - self.assertIn("network", cloud.apis) diff --git a/capi_janitor/tests/openstack/test_operator.py b/capi_janitor/tests/openstack/test_operator.py deleted file mode 100644 index 5792d8d..0000000 --- a/capi_janitor/tests/openstack/test_operator.py +++ /dev/null @@ -1,475 +0,0 @@ -import base64 -import unittest -from unittest import mock - -import easykube -import httpx -import yaml - -from capi_janitor.openstack import openstack, operator -from capi_janitor.openstack.operator import OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY - - -# Helper to create an async iterable -class AsyncIterList: - def __init__(self, items): - self.items = items - self.kwargs = None - - async def list(self, **kwargs): - self.kwargs = kwargs - for item in self.items: - yield item - - def __aiter__(self): - self._iter = iter(self.items) - return self - - async def __anext__(self): - try: - return next(self._iter) - except StopIteration: - raise StopAsyncIteration - - -class TestOperator(unittest.IsolatedAsyncioTestCase): - async def test_operator(self): - mock_easykube = mock.AsyncMock(spec=easykube.AsyncClient) - operator.ekclient = mock_easykube - - await operator.on_cleanup() # type: ignore - - mock_easykube.aclose.assert_awaited_once_with() - - async def test_fips_for_cluster(self): - prefix = "Floating IP for Kubernetes external service from cluster" - fips = [ - mock.Mock(description=f"{prefix} mycluster"), - mock.Mock(description=f"{prefix} asdf"), - mock.Mock(description="Some other description"), - mock.Mock(description=f"{prefix} mycluster"), - ] - resource_mock = AsyncIterList(fips) - - result = [ - fip async for fip in operator.fips_for_cluster(resource_mock, "mycluster") - ] - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], fips[0]) - self.assertEqual(result[1], fips[3]) - self.assertEqual(resource_mock.kwargs, {}) - - async def test_lbs_for_cluster(self): - lbs = [ - mock.Mock(name="lb0"), - mock.Mock(name="lb1"), - mock.Mock(name="lb2"), - mock.Mock(name="lb3"), - ] - # can't pass name into mock.Mock() to do this - lbs[0].name = "kube_service_mycluster_api" - lbs[1].name = "kube_service_othercluster_api" - lbs[2].name = "fake_service_mycluster_api" - lbs[3].name = "kube_service_mycluster_ui" - resource_mock = AsyncIterList(lbs) - - result = [ - lb async for lb in operator.lbs_for_cluster(resource_mock, "mycluster") - ] - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], lbs[0]) - self.assertEqual(result[1], lbs[3]) - - async def test_secgroups_for_cluster(self): - prefix = "Security Group for Service LoadBalancer in cluster" - secgroups = [ - mock.Mock(description=f"{prefix} mycluster"), - mock.Mock(description=f"{prefix} othercluster"), - mock.Mock(description="Other description"), - mock.Mock(description=f"{prefix} mycluster"), - ] - resource_mock = AsyncIterList(secgroups) - - result = [] - async for sg in operator.secgroups_for_cluster(resource_mock, "mycluster"): - result.append(sg) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], secgroups[0]) - self.assertEqual(result[1], secgroups[3]) - - async def test_filtered_volumes_for_cluster(self): - volumes = [ - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "false", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "true", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "othercluster", - OPENSTACK_USER_VOLUMES_RECLAIM_PROPERTY: "false", - } - ), - mock.Mock( - metadata={ - "cinder.csi.openstack.org/cluster": "mycluster", - } - ), - mock.Mock(metadata={"other_key": "value"}), - ] - resource_mock = AsyncIterList(volumes) - - result = [] - async for vol in operator.filtered_volumes_for_cluster( - resource_mock, "mycluster" - ): - result.append(vol) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], volumes[0]) - self.assertEqual(result[1], volumes[3]) - - async def test_snapshots_for_cluster(self): - snapshots = [ - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "mycluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "othercluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "mycluster"}), - mock.Mock(metadata={"cinder.csi.openstack.org/cluster": "othercluster"}), - # Volumes with invalid metadata - mock.Mock(metadata={"another_key": "value"}), - ] - resource_mock = AsyncIterList(snapshots) - - result = [] - async for snap in operator.snapshots_for_cluster(resource_mock, "mycluster"): - result.append(snap) - - self.assertEqual(len(result), 2) - self.assertEqual(result[0], snapshots[0]) - self.assertEqual(result[1], snapshots[2]) - - async def test_empty_iterator_returns_true(self): - async_iter = AsyncIterList([]).__aiter__() - result = await operator.empty(async_iter) - self.assertTrue(result) - - async def test_non_empty_iterator_returns_false(self): - async_iter = AsyncIterList([1, 2, 3]).__aiter__() - result = await operator.empty(async_iter) - self.assertFalse(result) - - async def test_try_delete_success(self): - instances = [mock.Mock(id=1), mock.Mock(id=2)] - resource = mock.Mock() - resource.delete = mock.AsyncMock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - result = await operator.try_delete(logger, resource, AsyncIterList(instances)) - - self.assertTrue(result) - resource.delete.assert_any_await(1) - resource.delete.assert_any_await(2) - logger.warn.assert_not_called() - - async def test_try_delete_with_400_and_409_errors(self): - instances = [mock.Mock(id=1), mock.Mock(id=2)] - resource = mock.Mock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - resource.delete = mock.AsyncMock( - side_effect=[ - httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=400) - ), - httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=409) - ), - ] - ) - result = await operator.try_delete(logger, resource, AsyncIterList(instances)) - - self.assertTrue(result) - self.assertEqual(resource.delete.await_count, 2) - self.assertEqual(logger.warn.call_count, 2) - - async def test_delete_with_unexpected_error_raises(self): - instances = [mock.Mock(id=1)] - resource = mock.Mock() - resource.singular_name = "test_resource" - logger = mock.Mock() - - resource.delete = mock.AsyncMock( - side_effect=httpx.HTTPStatusError( - "error", request=mock.Mock(), response=mock.Mock(status_code=500) - ) - ) - with self.assertRaises(httpx.HTTPStatusError): - await operator.try_delete(logger, resource, AsyncIterList(instances)) - - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_adds_finalizers( - self, mock_get_client, mock_patch_finalizers - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="default", - meta={}, - labels={}, - spec={}, - status={}, - logger=logger, - body={}, - ) - - mock_patch_finalizers.assert_awaited_once_with( - "mock_client", "mycluster", "default", ["janitor.capi.stackhpc.com"] - ) - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls([mock.call("added janitor finalizer to cluster")]) - - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_skip_no_finalizers( - self, mock_get_client, mock_patch_finalizers - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="default", - meta={"deletionTimestamp": "2023-10-01T00:00:00Z"}, - labels={}, - spec={}, - status={}, - logger=logger, - body={}, - ) - - mock_patch_finalizers.assert_not_awaited() - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls( - [mock.call("janitor finalizer not present, skipping cleanup")] - ) - - @mock.patch.object(operator, "_delete_secret") - @mock.patch.object(operator, "_get_clouds_secret") - @mock.patch.object(operator, "purge_openstack_resources") - @mock.patch.object(operator, "patch_finalizers") - @mock.patch.object(operator, "_get_os_cluster_client") - async def test_on_openstackcluster_event_calls_purge( - self, - mock_get_client, - mock_patch_finalizers, - mock_purge, - mock_clouds_secret, - mock_delete_secret, - ): - logger = mock.Mock() - mock_get_client.return_value = "mock_client" - mock_secret = mock.Mock() - mock_clouds_secret.return_value = mock_secret - clouds_yaml_data = { - "openstack": { - "auth": { - "auth_url": "https://example.com:5000/v3", - "username": "user", - "password": "pass", - "project_name": "project", - "user_domain_name": "Default", - "project_domain_name": "Default", - } - } - } - mock_secret.data = { - "clouds.yaml": base64.b64encode(yaml.dump(clouds_yaml_data).encode("utf-8")) - } - mock_secret.metadata = { - "annotations": {"janitor.capi.stackhpc.com/credential-policy": "delete"}, - "name": "appcred42", - } - - await operator._on_openstackcluster_event_impl( - name="mycluster", - namespace="namespace1", - meta={ - "deletionTimestamp": "2023-10-01T00:00:00Z", - "finalizers": ["janitor.capi.stackhpc.com"], - "annotations": { - "janitor.capi.stackhpc.com/volumes-policy": "delete", - }, - }, - labels={}, - spec={"identityRef": {"name": "appcred42"}}, - status={}, - logger=logger, - body={}, - ) - - mock_purge.assert_awaited_once_with( - logger, - clouds_yaml_data, - "openstack", - None, - "mycluster", - True, - False, - True, - ) - mock_delete_secret.assert_awaited_once_with("appcred42", "namespace1") - mock_patch_finalizers.assert_awaited_once_with( - "mock_client", "mycluster", "namespace1", [] - ) - logger.debug.assert_has_calls( - [mock.call("cluster name that will be used for cleanup: 'mycluster'")] - ) - logger.info.assert_has_calls( - [ - mock.call("cloud credential secret deleted"), - mock.call("removed janitor finalizer from cluster"), - ] - ) - - @mock.patch.object(openstack.Cloud, "from_clouds") - async def test_purge_openstack_resources_raises(self, mock_from_clouds): - mock_networkapi = mock.AsyncMock() - mock_networkapi.resource.side_effect = lambda resource: resource - - mock_cloud = mock.AsyncMock() - mock_cloud.__aenter__.return_value = mock_cloud - mock_cloud.is_authenticated = False - mock_cloud.current_user_id = "user" - mock_cloud.api_client.return_value = mock_networkapi - - mock_from_clouds.return_value = mock_cloud - - logger = mock.Mock() - clouds_yaml_data = { - "clouds": { - "openstack": { - "auth": { - "auth_url": "https://example.com:5000/v3", - "application_credential_id": "user", - "application_credential_secret": "pass", - }, - "region_name": "RegionOne", - "interface": "public", - "identity_api_version": 3, - "auth_type": "v3applicationcredential", - } - } - } - with self.assertRaises(openstack.AuthenticationError) as e: - await operator.purge_openstack_resources( - logger, - clouds_yaml_data, - "openstack", - None, - "mycluster", - True, - True, - False, - ) - self.assertEqual( - str(e.exception), - "failed to authenticate as user: user", - ) - - # @mock.patch.object(openstack.Cloud, "from_clouds") - # async def test_purge_openstack_resources_success(self, mock_from_clouds): - # # Mocking the cloud object and API clients - # mock_cloud = mock.AsyncMock() - # mock_networkapi = mock.AsyncMock() - # mock_lbapi = mock.AsyncMock() - # mock_volumeapi = mock.AsyncMock() - # mock_identityapi = mock.AsyncMock() - - # # Mock the __aenter__ to return the mock_cloud - # mock_cloud.__aenter__.return_value = mock_cloud - # mock_cloud.is_authenticated = True - # mock_cloud.current_user_id = "user" - # mock_cloud.api_client.return_value = mock_networkapi - - # # Return mock clients for different services when requested - # mock_from_clouds.return_value = mock_cloud - - # # Mocking the resources for Network, Load Balancer, and Volume APIs - # mock_networkapi.resource.side_effect = lambda resource: { - # "floatingips": mock.AsyncMock(), - # "security-groups": mock.AsyncMock(), - # }.get(resource, None) - - # mock_lbapi.resource.side_effect = lambda resource: { - # "loadbalancers": mock.AsyncMock(), - # }.get(resource, None) - - # mock_volumeapi.resource.side_effect = lambda resource: { - # "snapshots/detail": mock.AsyncMock(), - # "snapshots": mock.AsyncMock(), - # "volumes/detail": mock.AsyncMock(), - # "volumes": mock.AsyncMock(), - # }.get(resource, None) - - # mock_identityapi.resource.side_effect = lambda resource: { - # "application_credentials": mock.AsyncMock(), - # }.get(resource, None) - - # # Mock logger - # logger = mock.Mock() - - # clouds_yaml_data = { - # "clouds": { - # "openstack": { - # "auth": { - # "auth_url": "https://example.com:5000/v3", - # "application_credential_id": "user", - # "application_credential_secret": "pass", - # }, - # "region_name": "RegionOne", - # "interface": "public", - # "identity_api_version": 3, - # "auth_type": "v3applicationcredential", - # } - # } - # } - - # # Simulate the purge_openstack_resources method behavior - # await operator.purge_openstack_resources( - # logger, - # clouds_yaml_data, # Pass the mock cloud config - # "openstack", - # None, - # "mycluster", - # True, - # False, - # ) - - # # Add assertions here based on expected behavior - # # Example: Check that the resources were interacted with - # mock_networkapi.resource.assert_any_call("floatingips") - # mock_lbapi.resource.assert_any_call("loadbalancers") - # mock_volumeapi.resource.assert_any_call("snapshots") - # mock_volumeapi.resource.assert_any_call("volumes") - - # # Example: Validate if appcred deletion was attempted - # mock_identityapi.resource.assert_any_call("application_credentials") diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index b54b30b..759adb9 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -4,7 +4,6 @@ metadata: name: {{ include "cluster-api-janitor-openstack.fullname" . }} labels: {{ include "cluster-api-janitor-openstack.labels" . | nindent 4 }} rules: - # Required for kopf to watch resources properly - apiGroups: - "" resources: @@ -12,7 +11,6 @@ rules: verbs: - list - watch - # Required for kopf to produce events properly - apiGroups: - "" - events.k8s.io @@ -20,7 +18,6 @@ rules: - events verbs: - create - # We need access to OpenStackClusters and the cloud credential secrets - apiGroups: - "" resources: @@ -37,7 +34,7 @@ rules: - get - watch - patch - # Required to prevent erroneous error logs during kopf startup + - update - apiGroups: - apiextensions.k8s.io resources: diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 9e84cc1..a7a18f8 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -27,6 +27,20 @@ spec: env: - name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY value: {{ .Values.defaultVolumesPolicy }} + - name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: {{ .Values.retryDefaultDelay | quote }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 resources: {{ toYaml .Values.resources | nindent 12 }} volumeMounts: - name: tmp diff --git a/chart/tests/__snapshot__/snapshot_test.yaml.snap b/chart/tests/__snapshot__/snapshot_test.yaml.snap index 59b69c4..2abd006 100644 --- a/chart/tests/__snapshot__/snapshot_test.yaml.snap +++ b/chart/tests/__snapshot__/snapshot_test.yaml.snap @@ -41,6 +41,7 @@ templated manifests should match snapshot: - get - watch - patch + - update - apiGroups: - apiextensions.k8s.io resources: @@ -97,9 +98,23 @@ templated manifests should match snapshot: - env: - name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY value: delete + - name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "60" image: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack:main imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 name: operator + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 resources: {} securityContext: allowPrivilegeEscalation: false diff --git a/chart/tests/packaging_test.yaml b/chart/tests/packaging_test.yaml new file mode 100644 index 0000000..f67edc2 --- /dev/null +++ b/chart/tests/packaging_test.yaml @@ -0,0 +1,98 @@ +# Epic 10 — US10.1 / US10.2 packaging scenarios +# Run with: docker run -i --rm -v $(pwd):/apps helmunittest/helm-unittest chart +suite: Packaging and deployment tests + +tests: + # US10.1 — Image sécurisée + - it: should run pods as non-root (US10.1) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + + - it: should use a read-only root filesystem (US10.1) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + + - it: should drop all Linux capabilities (US10.1) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.drop + content: ALL + + # US10.2 — Helm chart + - it: should inject CAPI_JANITOR_DEFAULT_VOLUMES_POLICY with default value "delete" (US10.2) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY + value: delete + + - it: should inject overridden defaultVolumesPolicy "keep" (US10.2) + template: templates/deployment.yaml + set: + defaultVolumesPolicy: keep + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY + value: keep + + - it: should inject CAPI_JANITOR_RETRY_DEFAULT_DELAY with default value "60" (US10.2) + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "60" + + - it: should inject overridden retryDefaultDelay (US10.2) + template: templates/deployment.yaml + set: + retryDefaultDelay: 120 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CAPI_JANITOR_RETRY_DEFAULT_DELAY + value: "120" + + - it: should have a liveness probe at /healthz (US10.2) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /healthz + + - it: should have a readiness probe at /readyz (US10.2) + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /readyz + + - it: should grant update verb on openstackclusters (US10.2) + template: templates/clusterrole.yaml + asserts: + - contains: + path: rules + content: + apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - openstackclusters + verbs: + - list + - get + - watch + - patch + - update diff --git a/chart/values.yaml b/chart/values.yaml index bcb4d49..238092c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,6 +1,9 @@ # The default volumes policy for the operator defaultVolumesPolicy: delete +# The default retry delay in seconds after a failed cleanup attempt +retryDefaultDelay: 60 + # The image to use for the operator image: repository: ghcr.io/azimuth-cloud/cluster-api-janitor-openstack diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..33d347a --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,205 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + "flag" + "os" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(infrav1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// nolint:gocyclo +func main() { + var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("Disabling HTTP/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + webhookServerOptions := webhook.Options{ + TLSOpts: webhookTLSOpts, + } + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + webhookServerOptions.CertDir = webhookCertPath + webhookServerOptions.CertName = webhookCertName + webhookServerOptions.KeyName = webhookCertKey + } + + webhookServer := webhook.NewServer(webhookServerOptions) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + metricsServerOptions.CertDir = metricsCertPath + metricsServerOptions.CertName = metricsCertName + metricsServerOptions.KeyName = metricsCertKey + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "4e3e11ff.capi.stackhpc.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "Failed to start manager") + os.Exit(1) + } + + if err := (&controller.OpenStackClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + DefaultVolumesPolicy: controller.DefaultVolumesFromEnv(), + RetryDefaultDelay: controller.RetryDelayFromEnv(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to set up controller", "controller", "OpenStackCluster") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "Failed to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "Failed to set up ready check") + os.Exit(1) + } + + setupLog.Info("Starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "Failed to run manager") + os.Exit(1) + } +} diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..d975015 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..a30e017 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,234 @@ +# Adds namespace to all resources. +namespace: cluster-api-janitor-openstack-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: cluster-api-janitor-openstack- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +#- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- path: manager_webhook_patch.yaml +# target: +# kind: Deployment + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true + +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml new file mode 100644 index 0000000..2aaef65 --- /dev/null +++ b/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml new file mode 100644 index 0000000..6c012aa --- /dev/null +++ b/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..5c5f0b8 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..5635130 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,103 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build a multi-arch manager image with `nix-build nix -A image-arm64` + # (see nix/default.nix). + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect + - --health-probe-bind-address=:8081 + image: controller:latest + name: manager + ports: + - containerPort: 8081 + name: health + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + volumeMounts: [] + volumes: [] + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..1f18b6b --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..ec0fb5e --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..fdc5481 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..9580214 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,27 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: cluster-api-janitor-openstack diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..5619aa0 --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,20 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..501a507 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..2f83d3b --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000..32d2e4e --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000..e775d67 --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..b286585 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - delete + - get +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - openstackclusters + verbs: + - get + - list + - patch + - update + - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..f75a3e8 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..aad2a02 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: cluster-api-janitor-openstack + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..52079b8 --- /dev/null +++ b/go.mod @@ -0,0 +1,105 @@ +module github.com/azimuth-cloud/cluster-api-janitor-openstack + +go 1.26.0 + +require ( + github.com/go-logr/logr v1.4.3 + github.com/onsi/ginkgo/v2 v2.28.2 + github.com/onsi/gomega v1.42.0 + github.com/prometheus/client_golang v1.23.2 + k8s.io/api v0.36.0 + k8s.io/apimachinery v0.36.0 + k8s.io/client-go v0.36.0 + sigs.k8s.io/cluster-api-provider-openstack v0.14.6 + sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gophercloud/gophercloud/v2 v2.10.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.41.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiserver v0.36.0 // indirect + k8s.io/component-base v0.36.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.0 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/cluster-api v1.12.8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..da655db --- /dev/null +++ b/go.sum @@ -0,0 +1,265 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gophercloud/gophercloud/v2 v2.10.0 h1:NRadC0aHNvy4iMoFXj5AFiPmut/Sj3hAPAo9B59VMGc= +github.com/gophercloud/gophercloud/v2 v2.10.0/go.mod h1:Ki/ILhYZr/5EPebrPL9Ej+tUg4lqx71/YH2JWVeU+Qk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps= +github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= +github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= +k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= +k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= +k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= +k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= +k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= +k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= +k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= +k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= +k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= +k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= +k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= +k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/cluster-api v1.12.8 h1:37SLcQRG9EMhmsZJwyEx8pNBkmY1Xhog53slhDy44m4= +sigs.k8s.io/cluster-api v1.12.8/go.mod h1:Xz6YnayDc2+/3OA1i6wlXrhmErV1KKJmyfs7JzMDn+k= +sigs.k8s.io/cluster-api-provider-openstack v0.14.6 h1:yOln3Hd6JOFe2mkOGckFdWwFNhJdcNWrlfgwMdxmzto= +sigs.k8s.io/cluster-api-provider-openstack v0.14.6/go.mod h1:A5d/QLwDgQ9vOAD6/z6EKN23+4GXZK4E8vmh7Jlfsm0= +sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= +sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..70b4dea --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/internal/controller/config_test.go b/internal/controller/config_test.go new file mode 100644 index 0000000..b859edc --- /dev/null +++ b/internal/controller/config_test.go @@ -0,0 +1,146 @@ +package controller_test + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── US9.1: Configuration via environment variables ──────────────────────── + +// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY not set → "delete" +func TestDefaultVolumesFromEnv_DefaultsToDelete(t *testing.T) { + t.Setenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", "") + if got := controller.DefaultVolumesFromEnv(); got != controller.PolicyDelete { + t.Errorf("expected %q, got %q", controller.PolicyDelete, got) + } +} + +// Scenario: CAPI_JANITOR_DEFAULT_VOLUMES_POLICY = "keep" → "keep" +func TestDefaultVolumesFromEnv_ReadsKeep(t *testing.T) { + t.Setenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY", "keep") + if got := controller.DefaultVolumesFromEnv(); got != "keep" { + t.Errorf("expected %q, got %q", "keep", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY not set → 60s (default) +func TestRetryDelayFromEnv_DefaultsTo60(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "") + if got := controller.RetryDelayFromEnv(); got != 60 { + t.Errorf("expected 60, got %d", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY = "120" → 120 +func TestRetryDelayFromEnv_ReadsConfiguredDelay(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "120") + if got := controller.RetryDelayFromEnv(); got != 120 { + t.Errorf("expected 120, got %d", got) + } +} + +// Scenario: CAPI_JANITOR_RETRY_DEFAULT_DELAY invalide → 60 (fallback) +func TestRetryDelayFromEnv_InvalidValue_FallsBackToDefault(t *testing.T) { + t.Setenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY", "not-a-number") + if got := controller.RetryDelayFromEnv(); got != 60 { + t.Errorf("expected 60 for invalid value, got %d", got) + } +} + +// ── Volume policy via the reconciler ─────────────────────────────────────── + +func withVolumePolicy(policy string) func(*infrav1.OpenStackCluster) { + return func(c *infrav1.OpenStackCluster) { + if c.Annotations == nil { + c.Annotations = make(map[string]string) + } + c.Annotations[controller.VolumesPolicyAnnotation] = policy + } +} + +func reconcileForVolumesCapture(t *testing.T, defaultPolicy string, clusterOpts ...func(*infrav1.OpenStackCluster)) openstack.PurgeOptions { + t.Helper() + opts := append([]func(*infrav1.OpenStackCluster){withFinalizer, withDeletionTimestamp}, clusterOpts...) + cluster := newCluster("mycluster", "default", opts...) + secret := newSecret("cloud-credentials", "default") + + var captured openstack.PurgeOptions + r, _ := newReconciler(func(_ context.Context, o openstack.PurgeOptions) error { + captured = o + return nil + }, cluster, secret) + r.DefaultVolumesPolicy = defaultPolicy + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + return captured +} + +// Scenario: Global policy "delete" → volumes included in the purge +func TestReconcile_VolumesPolicy_IncludesVolumes_WhenDelete(t *testing.T) { + opts := reconcileForVolumesCapture(t, controller.PolicyDelete) + if !opts.IncludeVolumes { + t.Error("expected IncludeVolumes=true for policy 'delete'") + } +} + +// Scenario: Global policy "keep" → volumes excluded from the purge +func TestReconcile_VolumesPolicy_ExcludesVolumes_WhenKeep(t *testing.T) { + opts := reconcileForVolumesCapture(t, "keep") + if opts.IncludeVolumes { + t.Error("expected IncludeVolumes=false for policy 'keep'") + } +} + +// Scenario: Annotation "delete" on the cluster (overrides global keep) +func TestReconcile_VolumesPolicy_AnnotationDeleteOverridesKeepGlobal(t *testing.T) { + opts := reconcileForVolumesCapture(t, "keep", withVolumePolicy(controller.PolicyDelete)) + if !opts.IncludeVolumes { + t.Error("expected IncludeVolumes=true when annotation 'delete' overrides global 'keep'") + } +} + +// Scenario: Annotation "keep" on the cluster (overrides global delete) +func TestReconcile_VolumesPolicy_AnnotationKeepOverridesDeleteGlobal(t *testing.T) { + opts := reconcileForVolumesCapture(t, controller.PolicyDelete, withVolumePolicy("keep")) + if opts.IncludeVolumes { + t.Error("expected IncludeVolumes=false when annotation 'keep' overrides global 'delete'") + } +} + +// ── Configurable retry delay ─────────────────────────────────────────────── + +// Scenario: RetryDefaultDelay = 120 → sleep called with 120 seconds +func TestReconcile_RetryDelay_UsesConfiguredDelay(t *testing.T) { + cluster := newCluster("mycluster", "default", + withFinalizer, + func(c *infrav1.OpenStackCluster) { + now := metav1.Now() + c.DeletionTimestamp = &now + }, + ) + secret := newSecret("cloud-credentials", "default") + + var sleptFor time.Duration + r, _ := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + return context.DeadlineExceeded // simulate purge failure + }, cluster, secret) + r.RetryDefaultDelay = 120 + r.SleepFunc = func(d time.Duration) { sleptFor = d } + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if sleptFor != 120*time.Second { + t.Errorf("expected sleep of 120s, got %v", sleptFor) + } +} diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go new file mode 100644 index 0000000..f7a3d91 --- /dev/null +++ b/internal/controller/metrics.go @@ -0,0 +1,21 @@ +package controller + +import "github.com/prometheus/client_golang/prometheus" + +// Metrics holds the Prometheus counters for the janitor controller. +type Metrics struct { + CleanupsTotal *prometheus.CounterVec +} + +// NewMetrics creates and registers the janitor metrics with reg. +func NewMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + CleanupsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "capi_janitor", + Name: "cleanups_total", + Help: "Total number of OpenStack cluster cleanups, labelled by result (success|failure).", + }, []string{"result"}), + } + reg.MustRegister(m.CleanupsTotal) + return m +} diff --git a/internal/controller/observability_test.go b/internal/controller/observability_test.go new file mode 100644 index 0000000..66b90a6 --- /dev/null +++ b/internal/controller/observability_test.go @@ -0,0 +1,122 @@ +package controller_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +func newObsReconciler( + reg *prometheus.Registry, + purgeFunc func(context.Context, openstack.PurgeOptions) error, + objs ...client.Object, +) (*controller.OpenStackClusterReconciler, *record.FakeRecorder) { + r, _ := newReconciler(purgeFunc, objs...) + r.Metrics = controller.NewMetrics(reg) + rec := record.NewFakeRecorder(10) + r.Recorder = rec + return r, rec +} + +// ── US11.1: Prometheus Metrics ──────────────────────────────────────────── + +// Scenario: successful cleanup → capi_janitor_cleanups_total{result="success"} += 1 +func TestMetrics_IncrementsSuccess_OnCleanup(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, _ := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := testutil.ToFloat64(r.Metrics.CleanupsTotal.WithLabelValues("success")); got != 1 { + t.Errorf("expected cleanupsTotal{result=success}=1, got %v", got) + } +} + +// Scenario: failed purge → capi_janitor_cleanups_total{result="failure"} += 1 +func TestMetrics_IncrementsFailure_OnPurgeError(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, _ := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return errors.New("purge failed") }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got := testutil.ToFloat64(r.Metrics.CleanupsTotal.WithLabelValues("failure")); got != 1 { + t.Errorf("expected cleanupsTotal{result=failure}=1, got %v", got) + } +} + +// ── US11.2: Kubernetes Events ─────────────────────────────────────────────── + +// Scenario: successful cleanup → Normal "CleanupSucceeded" event +func TestEvents_EmitsNormal_OnCleanupSuccess(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, rec := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + select { + case event := <-rec.Events: + if !strings.Contains(event, "Normal") || !strings.Contains(event, "CleanupSucceeded") { + t.Errorf("expected Normal/CleanupSucceeded event, got: %q", event) + } + default: + t.Error("no event was recorded") + } +} + +// Scenario: failed purge → Warning "CleanupFailed" event +func TestEvents_EmitsWarning_OnPurgeFailure(t *testing.T) { + cluster := newCluster("c", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + reg := prometheus.NewRegistry() + r, rec := newObsReconciler(reg, + func(_ context.Context, _ openstack.PurgeOptions) error { return errors.New("purge failed") }, + cluster, secret, + ) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("c", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + select { + case event := <-rec.Events: + if !strings.Contains(event, "Warning") || !strings.Contains(event, "CleanupFailed") { + t.Errorf("expected Warning/CleanupFailed event, got: %q", event) + } + default: + t.Error("no event was recorded") + } +} diff --git a/internal/controller/openstackcluster_controller.go b/internal/controller/openstackcluster_controller.go new file mode 100644 index 0000000..1a6045c --- /dev/null +++ b/internal/controller/openstackcluster_controller.go @@ -0,0 +1,321 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/base64" + "fmt" + "math/rand" + "os" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +const ( + Finalizer = "janitor.capi.stackhpc.com" + + VolumesPolicyAnnotation = "janitor.capi.stackhpc.com/volumes-policy" + CredentialPolicyAnnotation = "janitor.capi.stackhpc.com/credential-policy" + RetryAnnotation = "janitor.capi.stackhpc.com/retry" + ClusterNameLabel = "cluster.x-k8s.io/cluster-name" + + PolicyDelete = "delete" + + defaultRetryDelay = 60 // seconds +) + +// OpenStackClusterReconciler reconciles OpenStackCluster objects from CAPO. +type OpenStackClusterReconciler struct { + client.Client + Scheme *runtime.Scheme + DefaultVolumesPolicy string + RetryDefaultDelay int + Metrics *Metrics + Recorder record.EventRecorder + // PurgeFunc is called to clean up OpenStack resources; defaults to openstack.PurgeResources. + PurgeFunc func(context.Context, openstack.PurgeOptions) error + // SleepFunc is called instead of time.Sleep; defaults to time.Sleep. + SleepFunc func(time.Duration) +} + +func (r *OpenStackClusterReconciler) purge(ctx context.Context, opts openstack.PurgeOptions) error { + if r.PurgeFunc != nil { + return r.PurgeFunc(ctx, opts) + } + return openstack.PurgeResources(ctx, opts) +} + +func (r *OpenStackClusterReconciler) sleep(d time.Duration) { + if r.SleepFunc != nil { + r.SleepFunc(d) + } else { + time.Sleep(d) + } +} + +func (r *OpenStackClusterReconciler) incMetric(result string) { + if r.Metrics != nil { + r.Metrics.CleanupsTotal.WithLabelValues(result).Inc() + } +} + +func (r *OpenStackClusterReconciler) recordEvent(obj client.Object, eventType, reason, msg string) { + if r.Recorder != nil { + r.Recorder.Event(obj, eventType, reason, msg) + } +} + +//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=openstackclusters,verbs=get;list;watch;patch;update +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;delete +//+kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch +//+kubebuilder:rbac:groups="",resources=events,verbs=create +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch + +func (r *OpenStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var cluster infrav1.OpenStackCluster + if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + clusterName := clusterNameFor(&cluster) + logger = logger.WithValues("clusterName", clusterName) + logger.V(1).Info("reconciling OpenStackCluster") + + // Not deleting: ensure our finalizer is present. + if cluster.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(&cluster, Finalizer) { + controllerutil.AddFinalizer(&cluster, Finalizer) + if err := r.Update(ctx, &cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("adding finalizer: %w", err) + } + logger.Info("added janitor finalizer to cluster") + } + return ctrl.Result{}, nil + } + + // Deleting: only act if our finalizer is present. + if !controllerutil.ContainsFinalizer(&cluster, Finalizer) { + logger.Info("janitor finalizer not present, skipping cleanup") + return ctrl.Result{}, nil + } + + // Fetch the cloud credential secret. + secret, err := r.getSecret(ctx, cluster.Spec.IdentityRef.Name, req.Namespace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("fetching identity secret: %w", err) + } + if secret == nil { + logger.Error(nil, "clouds.yaml secret not found", "secretName", cluster.Spec.IdentityRef.Name) + return ctrl.Result{}, nil + } + + cloudsYAML := decodeSecretField(secret.Data["clouds.yaml"]) + cacert := decodeSecretField(secret.Data["cacert"]) + + cloudName := cluster.Spec.IdentityRef.CloudName + if cloudName == "" { + cloudName = "openstack" + } + + includeVolumes := r.volumesPolicyFor(&cluster) == PolicyDelete + + credentialPolicy := secret.Annotations[CredentialPolicyAnnotation] + includeAppcred := credentialPolicy == PolicyDelete && len(cluster.Finalizers) == 1 + + purgeErr := r.purge(ctx, openstack.PurgeOptions{ + CloudsYAML: cloudsYAML, + CloudName: cloudName, + CACert: cacert, + ClusterName: clusterName, + IncludeVolumes: includeVolumes, + IncludeAppcred: includeAppcred, + Logger: logger, + }) + if purgeErr != nil { + r.incMetric("failure") + r.recordEvent(&cluster, corev1.EventTypeWarning, "CleanupFailed", purgeErr.Error()) + logger.Error(purgeErr, "purge failed, will retry") + r.sleep(r.retryDelay()) + if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + r.incMetric("success") + r.recordEvent(&cluster, corev1.EventTypeNormal, "CleanupSucceeded", "OpenStack resources cleaned up successfully") + + // Delete appcred secret if this is the last finalizer and policy says so. + if credentialPolicy == PolicyDelete { + if len(cluster.Finalizers) == 1 { + if err := r.deleteSecret(ctx, secret.Name, req.Namespace); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("deleting credential secret: %w", err) + } + logger.Info("cloud credential secret deleted") + } else { + // Other finalizers still present; trigger a retry when they are removed. + other := otherFinalizer(cluster.Finalizers, Finalizer) + logger.Info("waiting for other finalizer before deleting appcred", "otherFinalizer", other) + r.sleep(5 * time.Second) + if err := r.annotateRetry(ctx, &cluster); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + // Remove our finalizer. + controllerutil.RemoveFinalizer(&cluster, Finalizer) + if err := r.Update(ctx, &cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("removing finalizer: %w", err) + } + logger.Info("removed janitor finalizer from cluster") + return ctrl.Result{}, nil +} + +// SetupWithManager registers the reconciler with the controller manager. +func (r *OpenStackClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorderFor("capi-janitor") + } + if r.Metrics == nil { + r.Metrics = NewMetrics(ctrlmetrics.Registry) + } + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.OpenStackCluster{}). + Complete(r) +} + +// clusterNameFor returns the cluster name to use for resource cleanup. +// It prefers the cluster.x-k8s.io/cluster-name label over metadata.name. +func clusterNameFor(cluster *infrav1.OpenStackCluster) string { + if name, ok := cluster.Labels[ClusterNameLabel]; ok { + return name + } + return cluster.Name +} + +func (r *OpenStackClusterReconciler) volumesPolicyFor(cluster *infrav1.OpenStackCluster) string { + if ann, ok := cluster.Annotations[VolumesPolicyAnnotation]; ok { + return ann + } + if r.DefaultVolumesPolicy != "" { + return r.DefaultVolumesPolicy + } + return PolicyDelete +} + +func (r *OpenStackClusterReconciler) retryDelay() time.Duration { + d := r.RetryDefaultDelay + if d <= 0 { + d = defaultRetryDelay + } + return time.Duration(d) * time.Second +} + +func (r *OpenStackClusterReconciler) annotateRetry(ctx context.Context, cluster *infrav1.OpenStackCluster) error { + patch := client.MergeFrom(cluster.DeepCopy()) + if cluster.Annotations == nil { + cluster.Annotations = make(map[string]string) + } + cluster.Annotations[RetryAnnotation] = randString(8) + return r.Patch(ctx, cluster, patch) +} + +func (r *OpenStackClusterReconciler) getSecret(ctx context.Context, name, namespace string) (*corev1.Secret, error) { + var secret corev1.Secret + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, &secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return &secret, nil +} + +func (r *OpenStackClusterReconciler) deleteSecret(ctx context.Context, name, namespace string) error { + var secret corev1.Secret + secret.Name = name + secret.Namespace = namespace + return r.Delete(ctx, &secret) +} + +// decodeSecretField base64-decodes a secret data field if needed. +// Kubernetes stores secret data already base64-decoded in the Go API. +func decodeSecretField(data []byte) string { + if len(data) == 0 { + return "" + } + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + return string(data) // already raw bytes from Kubernetes API + } + return string(decoded) +} + +func randString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} + +func otherFinalizer(finalizers []string, skip string) string { + for _, f := range finalizers { + if f != skip { + return f + } + } + return "" +} + +// DefaultVolumesFromEnv reads CAPI_JANITOR_DEFAULT_VOLUMES_POLICY from environment. +func DefaultVolumesFromEnv() string { + if v := os.Getenv("CAPI_JANITOR_DEFAULT_VOLUMES_POLICY"); v != "" { + return v + } + return PolicyDelete +} + +// RetryDelayFromEnv reads CAPI_JANITOR_RETRY_DEFAULT_DELAY from environment. +func RetryDelayFromEnv() int { + if v := os.Getenv("CAPI_JANITOR_RETRY_DEFAULT_DELAY"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return defaultRetryDelay +} diff --git a/internal/controller/openstackcluster_controller_test.go b/internal/controller/openstackcluster_controller_test.go new file mode 100644 index 0000000..13a3de6 --- /dev/null +++ b/internal/controller/openstackcluster_controller_test.go @@ -0,0 +1,285 @@ +package controller_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/controller" + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +var testScheme *runtime.Scheme + +func init() { + testScheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(testScheme) + _ = infrav1.AddToScheme(testScheme) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func reconcileRequest(name, namespace string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Name: name, Namespace: namespace}} +} + +func newReconciler(purgeFunc func(context.Context, openstack.PurgeOptions) error, objs ...client.Object) (*controller.OpenStackClusterReconciler, client.Client) { + c := fake.NewClientBuilder().WithScheme(testScheme).WithObjects(objs...).Build() + r := &controller.OpenStackClusterReconciler{ + Client: c, + Scheme: testScheme, + PurgeFunc: purgeFunc, + SleepFunc: func(time.Duration) {}, // no-op: avoid real sleeps in tests + } + return r, c +} + +func newCluster(name, namespace string, opts ...func(*infrav1.OpenStackCluster)) *infrav1.OpenStackCluster { + c := &infrav1.OpenStackCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: infrav1.OpenStackClusterSpec{ + IdentityRef: infrav1.OpenStackIdentityReference{ + Name: "cloud-credentials", + CloudName: "openstack", + }, + }, + } + for _, opt := range opts { + opt(c) + } + return c +} + +func withFinalizer(c *infrav1.OpenStackCluster) { + controllerutil.AddFinalizer(c, controller.Finalizer) +} + +func withDeletionTimestamp(c *infrav1.OpenStackCluster) { + now := metav1.Now() + c.DeletionTimestamp = &now +} + +func withClusterLabel(value string) func(*infrav1.OpenStackCluster) { + return func(c *infrav1.OpenStackCluster) { + if c.Labels == nil { + c.Labels = make(map[string]string) + } + c.Labels[controller.ClusterNameLabel] = value + } +} + +func newSecret(name, namespace string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string][]byte{"clouds.yaml": []byte("clouds: {}")}, + } +} + +func getClusterOrNil(t *testing.T, c client.Client, name, namespace string) *infrav1.OpenStackCluster { + t.Helper() + var cluster infrav1.OpenStackCluster + err := c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &cluster) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + t.Fatalf("getting cluster: %v", err) + } + return &cluster +} + +// ── US8.1: Add a finalizer ─────────────────────────────────────────────────── + +// Scenario: Cluster without deletionTimestamp and without finalizer → finalizer added +func TestReconcile_AddsFinalizer_WhenNotPresent(t *testing.T) { + cluster := newCluster("mycluster", "default") + r, c := newReconciler(nil, cluster) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if !controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to be added, got finalizers: %v", controller.Finalizer, got.Finalizers) + } +} + +// Scenario: Cluster with finalizer already present → no state change +func TestReconcile_FinalizerAlreadyPresent_Idempotent(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer) + r, c := newReconciler(nil, cluster) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if !controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to still be present", controller.Finalizer) + } +} + +// ── US8.2: Cluster name from label or metadata.name ────────────────────────── + +// Scenario: Label cluster.x-k8s.io/cluster-name present → label name used +func TestReconcile_ClusterName_FromLabel(t *testing.T) { + cluster := newCluster("mycluster-openstack", "default", + withFinalizer, + withDeletionTimestamp, + withClusterLabel("mycluster"), + ) + secret := newSecret("cloud-credentials", "default") + + var capturedName string + r, _ := newReconciler(func(_ context.Context, opts openstack.PurgeOptions) error { + capturedName = opts.ClusterName + return nil + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster-openstack", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capturedName != "mycluster" { + t.Errorf("expected ClusterName %q (from label), got %q", "mycluster", capturedName) + } +} + +// Scenario: Label absent → metadata.name used +func TestReconcile_ClusterName_FallsBackToMetadataName(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + var capturedName string + r, _ := newReconciler(func(_ context.Context, opts openstack.PurgeOptions) error { + capturedName = opts.ClusterName + return nil + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capturedName != "mycluster" { + t.Errorf("expected ClusterName %q (from metadata.name), got %q", "mycluster", capturedName) + } +} + +// ── US8.3: Remove the finalizer after successful cleanup ───────────────────── + +// Scenario: Successful purge → finalizer removed +func TestReconcile_RemovesFinalizer_AfterSuccessfulPurge(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { return nil }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // After removing the last finalizer with DeletionTimestamp set, the fake client may GC the object. + got := getClusterOrNil(t, c, "mycluster", "default") + if got != nil && controllerutil.ContainsFinalizer(got, controller.Finalizer) { + t.Errorf("expected finalizer %q to be removed after purge", controller.Finalizer) + } +} + +// Scenario: Finalizer absent at deletion time → no purge +func TestReconcile_SkipsCleanup_WhenFinalizerAbsent(t *testing.T) { + // Use a non-janitor finalizer to keep the cluster alive in the fake client + // when we call Delete (which sets DeletionTimestamp instead of deleting immediately). + cluster := newCluster("mycluster", "default") + controllerutil.AddFinalizer(cluster, "other.finalizer.example.com") + + purgeCalled := false + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + purgeCalled = true + return nil + }, cluster) + + // Trigger deletion — fake client sets DeletionTimestamp (kept alive by other finalizer). + if err := c.Delete(context.Background(), cluster); err != nil { + t.Fatalf("marking cluster for deletion: %v", err) + } + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if purgeCalled { + t.Error("expected purge NOT to be called when janitor finalizer is absent") + } +} + +// ── US8.4: Retry mechanism via annotation ──────────────────────────────────── + +// Scenario: Error during purge → retry annotation set, Reconcile returns nil +func TestReconcile_AnnotatesRetry_OnPurgeError(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(func(_ context.Context, _ openstack.PurgeOptions) error { + return errors.New("purge failed") + }, cluster, secret) + + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("expected nil (retry handled internally), got: %v", err) + } + + got := getClusterOrNil(t, c, "mycluster", "default") + if got == nil { + t.Fatal("cluster not found after reconcile") + } + if got.Annotations[controller.RetryAnnotation] == "" { + t.Errorf("expected retry annotation %q to be set after purge error", controller.RetryAnnotation) + } +} + +// Scenario: Cluster deleted between purge error and retry annotation → NotFound ignored +func TestReconcile_IgnoresNotFound_WhenClusterDeletedDuringRetry(t *testing.T) { + cluster := newCluster("mycluster", "default", withFinalizer, withDeletionTimestamp) + secret := newSecret("cloud-credentials", "default") + + r, c := newReconciler(nil, cluster, secret) + + // PurgeFunc deletes the cluster mid-flight to simulate it disappearing during cleanup. + r.PurgeFunc = func(ctx context.Context, _ openstack.PurgeOptions) error { + // Remove finalizer first so the fake client actually deletes on Delete call. + var cl infrav1.OpenStackCluster + _ = c.Get(ctx, types.NamespacedName{Name: "mycluster", Namespace: "default"}, &cl) + controllerutil.RemoveFinalizer(&cl, controller.Finalizer) + _ = c.Update(ctx, &cl) + _ = c.Delete(ctx, &cl) + return fmt.Errorf("cleanup failed") + } + + // annotateRetry will see NotFound — must be ignored, not propagated. + if _, err := r.Reconcile(context.Background(), reconcileRequest("mycluster", "default")); err != nil { + t.Fatalf("expected no error when cluster deleted during retry annotation, got: %v", err) + } +} diff --git a/internal/openstack/cloud.go b/internal/openstack/cloud.go new file mode 100644 index 0000000..76ee009 --- /dev/null +++ b/internal/openstack/cloud.go @@ -0,0 +1,452 @@ +// Package openstack provides OpenStack client utilities and resource cleanup +// logic for the cluster-api-janitor-openstack operator. +package openstack + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "sigs.k8s.io/yaml" +) + +const ( + // KeepProperty is the OpenStack volume metadata key that marks a volume as user-kept. + KeepProperty = "janitor.capi.azimuth-cloud.com/keep" + + // authTypeAppCred is the clouds.yaml auth_type for application credentials. + authTypeAppCred = "v3applicationcredential" + // authTypePassword is the clouds.yaml auth_type for username/password auth. + authTypePassword = "v3password" +) + +// AuthenticationError is returned when OpenStack authentication fails. +type AuthenticationError struct { + UserID string +} + +func (e *AuthenticationError) Error() string { + return fmt.Sprintf("failed to authenticate as user: %s", e.UserID) +} + +// UnsupportedAuthTypeError is returned when clouds.yaml uses an unsupported auth type. +type UnsupportedAuthTypeError struct { + AuthType string +} + +func (e *UnsupportedAuthTypeError) Error() string { + return fmt.Sprintf("unsupported authentication type: %s", e.AuthType) +} + +// CatalogError is returned when a required service is absent from the OpenStack catalog. +type CatalogError struct { + ServiceType string +} + +func (e *CatalogError) Error() string { + return fmt.Sprintf("service type %s not found in OpenStack service catalog", e.ServiceType) +} + +// cloudsFile represents a minimal clouds.yaml structure. +type cloudsFile struct { + Clouds map[string]cloudEntry `yaml:"clouds" json:"clouds"` +} + +type cloudEntry struct { + AuthType string `yaml:"auth_type" json:"auth_type"` + Auth authBlock `yaml:"auth" json:"auth"` + Region string `yaml:"region_name" json:"region_name"` + Interface string `yaml:"interface" json:"interface"` +} + +type authBlock struct { + AuthURL string `yaml:"auth_url" json:"auth_url"` + ApplicationCredentialID string `yaml:"application_credential_id" json:"application_credential_id"` + ApplicationCredentialSecret string `yaml:"application_credential_secret" json:"application_credential_secret"` + + // Username/password (v3password) fields. + Username string `yaml:"username" json:"username"` + UserID string `yaml:"user_id" json:"user_id"` + Password string `yaml:"password" json:"password"` + ProjectID string `yaml:"project_id" json:"project_id"` + ProjectName string `yaml:"project_name" json:"project_name"` + UserDomainName string `yaml:"user_domain_name" json:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id" json:"user_domain_id"` + ProjectDomainName string `yaml:"project_domain_name" json:"project_domain_name"` + ProjectDomainID string `yaml:"project_domain_id" json:"project_domain_id"` + DomainName string `yaml:"domain_name" json:"domain_name"` + DomainID string `yaml:"domain_id" json:"domain_id"` +} + +// parseCloudsYAML parses a clouds.yaml string into a cloudsFile. +func parseCloudsYAML(data string) (*cloudsFile, error) { + var cf cloudsFile + if err := yaml.Unmarshal([]byte(data), &cf); err != nil { + return nil, fmt.Errorf("parsing clouds.yaml: %w", err) + } + return &cf, nil +} + +// authURLBase strips any trailing /v3 path segment. +var v3Suffix = regexp.MustCompile(`/v3/?$`) + +func authURLBase(raw string) string { + return v3Suffix.ReplaceAllString(raw, "") +} + +// Session holds an authenticated OpenStack session with discovered endpoints. +type Session struct { + token string + userID string + endpoints map[string]string + httpClient *http.Client + authenticated bool + // SleepFunc is called instead of time.Sleep for polling waits. + // A nil value defaults to time.Sleep. + SleepFunc func(time.Duration) +} + +// sleep calls SleepFunc if set, otherwise time.Sleep. +func (s *Session) sleep(d time.Duration) { + if s.SleepFunc != nil { + s.SleepFunc(d) + } else { + time.Sleep(d) + } +} + +// IsAuthenticated reports whether the session has a valid token and catalog. +func (s *Session) IsAuthenticated() bool { return s.authenticated } + +// UserID returns the ID of the authenticated user. +func (s *Session) UserID() string { return s.userID } + +// HasEndpoint reports whether the session has a discovered endpoint for the given service type. +func (s *Session) HasEndpoint(serviceType string) bool { + _, ok := s.endpoints[serviceType] + return ok +} + +// Authenticate performs token authentication against Keystone and discovers +// the service catalog, returning a ready-to-use Session. +func Authenticate(ctx context.Context, cloudsYAML, cloudName, cacert string) (*Session, error) { + cf, err := parseCloudsYAML(cloudsYAML) + if err != nil { + return nil, err + } + entry, ok := cf.Clouds[cloudName] + if !ok { + return nil, fmt.Errorf("cloud %q not found in clouds.yaml", cloudName) + } + authType, err := resolveAuthType(entry) + if err != nil { + return nil, err + } + + tlsCfg := &tls.Config{} //nolint:gosec + if cacert != "" { + if err := loadCACert(tlsCfg, cacert); err != nil { + return nil, err + } + } + hc := &http.Client{ + Transport: &http.Transport{TLSClientConfig: tlsCfg}, + Timeout: 30 * time.Second, + } + + iface := entry.Interface + if iface == "" { + iface = "public" + } + baseURL := authURLBase(entry.Auth.AuthURL) + + s := &Session{httpClient: hc} + if err := s.getToken(ctx, baseURL, authType, entry.Auth); err != nil { + if isHTTP(err, http.StatusNotFound) { + return s, nil // deleted appcred case: unauthenticated, no fatal error + } + return nil, err + } + if err := s.loadCatalog(ctx, baseURL, iface, entry.Region); err != nil { + return nil, err + } + return s, nil +} + +// resolveAuthType determines the effective auth type for a cloud entry. +// It honours an explicit auth_type, and otherwise infers it from the auth +// block (application credential fields imply v3applicationcredential, a +// username/user_id implies v3password) to match common clouds.yaml usage +// where auth_type is omitted. +func resolveAuthType(entry cloudEntry) (string, error) { + switch entry.AuthType { + case authTypeAppCred, authTypePassword: + return entry.AuthType, nil + case "", "password": + // Infer from the auth block for empty or ambiguous "password" values. + if entry.Auth.ApplicationCredentialID != "" || entry.Auth.ApplicationCredentialSecret != "" { + return authTypeAppCred, nil + } + if entry.Auth.Username != "" || entry.Auth.UserID != "" { + return authTypePassword, nil + } + return "", &UnsupportedAuthTypeError{AuthType: entry.AuthType} + default: + return "", &UnsupportedAuthTypeError{AuthType: entry.AuthType} + } +} + +// passwordScope builds the Keystone auth scope from the auth block, preferring +// project scoping and falling back to domain scoping when set. +func passwordScope(auth authBlock) map[string]any { + if auth.ProjectID != "" || auth.ProjectName != "" { + project := map[string]any{} + if auth.ProjectID != "" { + project["id"] = auth.ProjectID + } else { + project["name"] = auth.ProjectName + domain := map[string]any{} + switch { + case auth.ProjectDomainID != "": + domain["id"] = auth.ProjectDomainID + case auth.ProjectDomainName != "": + domain["name"] = auth.ProjectDomainName + case auth.UserDomainID != "": + domain["id"] = auth.UserDomainID + case auth.UserDomainName != "": + domain["name"] = auth.UserDomainName + } + if len(domain) > 0 { + project["domain"] = domain + } + } + return map[string]any{"project": project} + } + if auth.DomainID != "" { + return map[string]any{"domain": map[string]any{"id": auth.DomainID}} + } + if auth.DomainName != "" { + return map[string]any{"domain": map[string]any{"name": auth.DomainName}} + } + return nil +} + +// passwordTokenBody builds the Keystone token request body for v3password auth. +func passwordTokenBody(auth authBlock) map[string]any { + user := map[string]any{"password": auth.Password} + if auth.UserID != "" { + user["id"] = auth.UserID + } else { + user["name"] = auth.Username + domain := map[string]any{} + if auth.UserDomainID != "" { + domain["id"] = auth.UserDomainID + } else if auth.UserDomainName != "" { + domain["name"] = auth.UserDomainName + } + if len(domain) > 0 { + user["domain"] = domain + } + } + identity := map[string]any{ + "methods": []string{"password"}, + "password": map[string]any{"user": user}, + } + authReq := map[string]any{"identity": identity} + if scope := passwordScope(auth); scope != nil { + authReq["scope"] = scope + } + return map[string]any{"auth": authReq} +} + +// appCredTokenBody builds the Keystone token request body for application credential auth. +func appCredTokenBody(auth authBlock) map[string]any { + return map[string]any{ + "auth": map[string]any{ + "identity": map[string]any{ + "methods": []string{"application_credential"}, + "application_credential": map[string]any{ + "id": auth.ApplicationCredentialID, + "secret": auth.ApplicationCredentialSecret, + }, + }, + }, + } +} + +func (s *Session) getToken(ctx context.Context, baseURL, authType string, auth authBlock) error { + var payload map[string]any + if authType == authTypePassword { + payload = passwordTokenBody(auth) + } else { + payload = appCredTokenBody(auth) + } + body, _ := json.Marshal(payload) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + baseURL+"/v3/auth/tokens", strings.NewReader(string(body))) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + s.token = resp.Header.Get("X-Subject-Token") + var result struct { + Token struct { + User struct { + ID string `json:"id"` + } `json:"user"` + } `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + s.userID = result.Token.User.ID + return nil +} + +func (s *Session) loadCatalog(ctx context.Context, baseURL, iface, region string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + baseURL+"/v3/auth/catalog", nil) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil + } + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + var catalog struct { + Catalog []struct { + Type string `json:"type"` + Endpoints []struct { + Interface string `json:"interface"` + RegionID string `json:"region_id"` + URL string `json:"url"` + } `json:"endpoints"` + } `json:"catalog"` + } + if err := json.NewDecoder(resp.Body).Decode(&catalog); err != nil { + return err + } + s.endpoints = make(map[string]string) + for _, entry := range catalog.Catalog { + for _, ep := range entry.Endpoints { + if ep.Interface != iface { + continue + } + if region != "" && ep.RegionID != region { + continue + } + s.endpoints[entry.Type] = ep.URL + break + } + } + if len(s.endpoints) > 0 { + s.authenticated = true + } + return nil +} + +func (s *Session) endpointFor(serviceTypes ...string) (string, error) { + for _, st := range serviceTypes { + if u, ok := s.endpoints[st]; ok { + return u, nil + } + } + return "", &CatalogError{ServiceType: strings.Join(serviceTypes, " or ")} +} + +// doGet issues an authenticated GET and returns the response body. +func (s *Session) doGet(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return nil, &httpStatusError{code: resp.StatusCode} + } + return io.ReadAll(resp.Body) +} + +// doDelete issues an authenticated DELETE request. +func (s *Session) doDelete(ctx context.Context, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return err + } + req.Header.Set("X-Auth-Token", s.token) + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil + } + if resp.StatusCode >= 400 { + return &httpStatusError{code: resp.StatusCode} + } + return nil +} + +type httpStatusError struct{ code int } + +func (e *httpStatusError) Error() string { return fmt.Sprintf("HTTP %d", e.code) } +func (e *httpStatusError) StatusCode() int { return e.code } + +func isHTTP(err error, code int) bool { + var he *httpStatusError + if errors.As(err, &he) { + return he.code == code + } + return false +} + +// isTransient returns true for HTTP 400 and 409 (conflict/bad-request = retry). +func isTransient(err error) bool { + var he *httpStatusError + if errors.As(err, &he) { + return he.code == http.StatusBadRequest || he.code == http.StatusConflict + } + return false +} + +// AppCredentialID extracts the application_credential_id from a clouds.yaml string. +func AppCredentialID(cloudsYAML, cloudName string) (string, error) { + cf, err := parseCloudsYAML(cloudsYAML) + if err != nil { + return "", err + } + entry, ok := cf.Clouds[cloudName] + if !ok { + return "", fmt.Errorf("cloud %q not found", cloudName) + } + return entry.Auth.ApplicationCredentialID, nil +} diff --git a/internal/openstack/cloud_test.go b/internal/openstack/cloud_test.go new file mode 100644 index 0000000..9992b41 --- /dev/null +++ b/internal/openstack/cloud_test.go @@ -0,0 +1,716 @@ +package openstack_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// keystoneServer is a configurable mock Keystone server for unit tests. +type keystoneServer struct { + *httptest.Server + tokenStatus int + tokenUserID string + catalogStatus int + catalog map[string]any + lastTokenBody map[string]any +} + +func newKeystoneServer(t *testing.T) *keystoneServer { + t.Helper() + ks := &keystoneServer{ + tokenStatus: http.StatusCreated, + tokenUserID: "test-user-id", + catalogStatus: http.StatusOK, + catalog: map[string]any{"catalog": []any{}}, + } + mux := http.NewServeMux() + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + var reqBody map[string]any + if err := json.NewDecoder(r.Body).Decode(&reqBody); err == nil { + ks.lastTokenBody = reqBody + } + if ks.tokenStatus >= 400 { + w.WriteHeader(ks.tokenStatus) + return + } + w.Header().Set("X-Subject-Token", "test-token-abc123") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(ks.tokenStatus) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{ + "user": map[string]any{"id": ks.tokenUserID}, + }, + }) + }) + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + if ks.catalogStatus == http.StatusNotFound { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(ks.catalogStatus) + _ = json.NewEncoder(w).Encode(ks.catalog) + }) + ks.Server = httptest.NewServer(mux) + t.Cleanup(ks.Close) + return ks +} + +// newTLSKeystoneServer creates a TLS-enabled mock Keystone server. +func newTLSKeystoneServer(t *testing.T) *keystoneServer { + t.Helper() + ks := &keystoneServer{ + tokenStatus: http.StatusCreated, + tokenUserID: "test-user-id", + catalogStatus: http.StatusOK, + catalog: map[string]any{"catalog": []any{}}, + } + mux := http.NewServeMux() + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Subject-Token", "test-token-tls") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{ + "user": map[string]any{"id": ks.tokenUserID}, + }, + }) + }) + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(ks.catalog) + }) + ks.Server = httptest.NewTLSServer(mux) + t.Cleanup(ks.Close) + return ks +} + +func buildCloudsYAML(authURL, authType string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: %s + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + region_name: RegionOne + interface: public +`, authType, authURL) +} + +func buildCloudsYAMLWithInterface(authURL, iface string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + interface: %s +`, authURL, iface) +} + +func buildCloudsYAMLWithRegion(authURL, region string) string { + if region == "" { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + interface: public +`, authURL) + } + return fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: appcred-abc123 + application_credential_secret: super-secret + region_name: %s + interface: public +`, authURL, region) +} + +// catalogWith builds a catalog payload with one service and given endpoints. +func catalogWith(entries ...map[string]any) map[string]any { + return map[string]any{"catalog": entries} +} + +func serviceEntry(svcType string, endpoints ...map[string]any) map[string]any { + return map[string]any{ + "type": svcType, + "endpoints": endpoints, + } +} + +func endpoint(iface, regionID, url string) map[string]any { + return map[string]any{ + "interface": iface, + "region_id": regionID, + "url": url, + } +} + +// ── US1.1: Authentication via Application Credential v3 ──────────────────── + +// Scenario: Successful authentication +// Given a clouds.yaml with auth_type "v3applicationcredential" +// And a valid application_credential_id and application_credential_secret +// When the operator initialises the OpenStack connection +// Then an X-Auth-Token is obtained from Keystone +// And the service catalog is loaded +func TestAuthenticate_SuccessfulAuthentication(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + if session.UserID() != "test-user-id" { + t.Errorf("expected userID %q, got %q", "test-user-id", session.UserID()) + } +} + +// Scenario: Authentication with unsupported type +// Given a clouds.yaml with an unsupported auth_type "token" +// When the operator attempts to create a Cloud client +// Then an UnsupportedAuthTypeError is raised +func TestAuthenticate_UnsupportedAuthType(t *testing.T) { + ks := newKeystoneServer(t) + clouds := buildCloudsYAML(ks.URL, "token") + + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatal("expected UnsupportedAuthTypeError, got nil") + } + var target *openstack.UnsupportedAuthTypeError + if !errorAs(err, &target) { + t.Fatalf("expected *UnsupportedAuthTypeError, got %T: %v", err, err) + } + if target.AuthType != "token" { + t.Errorf("expected AuthType %q, got %q", "token", target.AuthType) + } +} + +// ── Password (v3password) authentication ────────────────────────────────── + +func buildPasswordCloudsYAML(authURL, authType string) string { + return fmt.Sprintf(` +clouds: + openstack: + auth_type: %s + auth: + auth_url: %s + username: alice + password: s3cret + project_id: proj-123 + user_domain_name: Default + region_name: RegionOne + interface: public +`, authType, authURL) +} + +// Scenario: Successful authentication via username/password +func TestAuthenticate_Password_Successful(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildPasswordCloudsYAML(ks.URL, "v3password") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + + // Verify the request used the password method with a project scope. + auth, _ := ks.lastTokenBody["auth"].(map[string]any) + identity, _ := auth["identity"].(map[string]any) + methods, _ := identity["methods"].([]any) + if len(methods) != 1 || methods[0] != "password" { + t.Errorf("expected password method, got %v", methods) + } + pw, _ := identity["password"].(map[string]any) + user, _ := pw["user"].(map[string]any) + if user["name"] != "alice" || user["password"] != "s3cret" { + t.Errorf("unexpected user block: %v", user) + } + scope, _ := auth["scope"].(map[string]any) + project, _ := scope["project"].(map[string]any) + if project["id"] != "proj-123" { + t.Errorf("expected project id proj-123, got %v", project["id"]) + } +} + +// Scenario: auth_type omitted but username present is inferred as v3password +func TestAuthenticate_Password_InferredFromUsername(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildPasswordCloudsYAML(ks.URL, "") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected session to be authenticated") + } + auth, _ := ks.lastTokenBody["auth"].(map[string]any) + identity, _ := auth["identity"].(map[string]any) + methods, _ := identity["methods"].([]any) + if len(methods) != 1 || methods[0] != "password" { + t.Errorf("expected inferred password method, got %v", methods) + } +} + +// Scenario: Cloud missing in clouds.yaml +func TestAuthenticate_CloudNotFound(t *testing.T) { + ks := newKeystoneServer(t) + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + + _, err := openstack.Authenticate(context.Background(), clouds, "nonexistent-cloud", "") + + if err == nil { + t.Fatal("expected error for missing cloud, got nil") + } +} + +// Scenario: Invalid clouds.yaml (malformed YAML) +func TestAuthenticate_InvalidCloudsYAML(t *testing.T) { + _, err := openstack.Authenticate(context.Background(), "not: valid: yaml: :", "openstack", "") + + if err == nil { + t.Fatal("expected parse error, got nil") + } +} + +// ── US1.3: Revoked or invalid credential handling ─────────────────────────── + +// Scenario: Application credential deleted before purge (token request → 404) +// Given a cluster being deleted +// And the application credential has already been deleted +// When the operator attempts to authenticate +// Then is_authenticated returns false +// And no fatal error is raised +func TestAuthenticate_DeletedApplicationCredential_Returns404(t *testing.T) { + ks := newKeystoneServer(t) + ks.tokenStatus = http.StatusNotFound + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error for deleted appcred (404), got: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated when appcred is deleted") + } +} + +// Scenario: Token request fails with a non-404 error +// When the operator attempts to authenticate +// Then the error is propagated +func TestAuthenticate_TokenRequestFails_NonFatal(t *testing.T) { + for _, code := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusInternalServerError} { + code := code + t.Run(fmt.Sprintf("HTTP_%d", code), func(t *testing.T) { + ks := newKeystoneServer(t) + ks.tokenStatus = code + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatalf("expected error for HTTP %d, got nil", code) + } + }) + } +} + +// Scenario: Catalog returns 404 +// Given a valid Keystone URL but the catalog returns 404 +// When the operator loads the catalog +// Then is_authenticated returns false +// And no fatal error is raised +func TestAuthenticate_CatalogReturns404(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalogStatus = http.StatusNotFound + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("expected no error when catalog returns 404, got: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated when catalog returns 404") + } +} + +// ── US1.2: Catalog filtering by interface and region ──────────────────────── + +// Scenario: Endpoint selected by configured interface +// Given a catalog with "public" and "internal" endpoints +// And the configured interface is "public" +// When the catalog is loaded +// Then only "public" endpoints are retained +func TestAuthenticate_FiltersByInterface_Public(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-public.example.com"), + endpoint("internal", "RegionOne", "http://compute-internal.example.com"), + ), + serviceEntry("network", + endpoint("internal", "RegionOne", "http://network-internal.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithInterface(ks.URL, "public") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected session to be authenticated") + } + // compute endpoint should exist (public) + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be present") + } + // network endpoint should NOT exist (only internal) + if session.HasEndpoint("network") { + t.Error("expected network endpoint to be absent (only internal available)") + } +} + +// Scenario: Endpoint selected by configured interface (internal) +func TestAuthenticate_FiltersByInterface_Internal(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-public.example.com"), + ), + serviceEntry("network", + endpoint("internal", "RegionOne", "http://network-internal.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithInterface(ks.URL, "internal") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be absent (only public available)") + } + if !session.HasEndpoint("network") { + t.Error("expected network endpoint to be present (internal)") + } +} + +// Scenario: Endpoint selected by configured region +// Given a catalog with endpoints for "RegionOne" and "RegionTwo" +// And the configured region is "RegionOne" +// When the catalog is loaded +// Then only "RegionOne" endpoints are retained +func TestAuthenticate_FiltersByRegion(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-region1.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionTwo", "http://network-region2.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithRegion(ks.URL, "RegionOne") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint (RegionOne) to be present") + } + if session.HasEndpoint("network") { + t.Error("expected network endpoint (RegionTwo) to be absent when region is RegionOne") + } +} + +// Scenario: No region configured +// Given a catalog with endpoints in multiple regions +// And no region is configured +// When the catalog is loaded +// Then the first endpoint matching the interface is retained for each service +func TestAuthenticate_NoRegionFilter_AcceptsAllRegions(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute-region1.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionTwo", "http://network-region2.example.com"), + ), + ) + + clouds := buildCloudsYAMLWithRegion(ks.URL, "") // no region + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !session.HasEndpoint("compute") { + t.Error("expected compute endpoint to be present when no region filter") + } + if !session.HasEndpoint("network") { + t.Error("expected network endpoint to be present when no region filter") + } +} + +// Scenario: Catalog with multiple services +func TestAuthenticate_MultipleServicesInCatalog(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + serviceEntry("identity", + endpoint("public", "RegionOne", "http://identity.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, svc := range []string{"compute", "network", "identity"} { + if !session.HasEndpoint(svc) { + t.Errorf("expected endpoint %q to be present", svc) + } + } +} + +// Scenario: Empty catalog → unauthenticated +func TestAuthenticate_EmptyCatalog_NotAuthenticated(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith() // no services + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if session.IsAuthenticated() { + t.Error("expected session to be unauthenticated with empty catalog") + } +} + +// ── US1.4: Custom CA certificate support ──────────────────────────────────── + +// Scenario: CA provided in the Kubernetes secret +// Given a Kubernetes secret containing a "cacert" entry +// When the operator initialises the TLS transport +// Then the CA is loaded into the SSL context +// And HTTPS calls to OpenStack use this CA for verification +func TestAuthenticate_WithCustomCACert(t *testing.T) { + ks := newTLSKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + // Extract the server's self-signed certificate as PEM. + tlsConfig := ks.Server.TLS + rawCert := tlsConfig.Certificates[0].Certificate[0] + x509Cert, err := x509.ParseCertificate(rawCert) + if err != nil { + t.Fatalf("parsing server cert: %v", err) + } + certPEM := string(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: x509Cert.Raw, + })) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", certPEM) + + if err != nil { + t.Fatalf("expected no error with custom CA, got: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected authenticated session with valid custom CA cert") + } +} + +// Scenario: No CA provided → system CA used (plain HTTP connection must work) +func TestAuthenticate_NoCACert_HTTPServerWorks(t *testing.T) { + ks := newKeystoneServer(t) // plain HTTP + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err != nil { + t.Fatalf("unexpected error without CA cert: %v", err) + } + if !session.IsAuthenticated() { + t.Error("expected authenticated session") + } +} + +// Scenario: CA provided but invalid → TLS must fail +func TestAuthenticate_InvalidCACert_TLSFails(t *testing.T) { + ks := newTLSKeystoneServer(t) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + // Pass a wrong/empty CA cert → TLS verification should fail. + _, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + + if err == nil { + t.Fatal("expected TLS error without correct CA cert, got nil") + } +} + +// ── AppCredentialID ───────────────────────────────────────────────────────── + +// Scenario: Extracting application credential ID from clouds.yaml +func TestAppCredentialID_Found(t *testing.T) { + clouds := ` +clouds: + mycloud: + auth_type: v3applicationcredential + auth: + auth_url: https://keystone.example.com/v3 + application_credential_id: my-appcred-id-xyz + application_credential_secret: secret123 +` + id, err := openstack.AppCredentialID(clouds, "mycloud") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if id != "my-appcred-id-xyz" { + t.Errorf("expected %q, got %q", "my-appcred-id-xyz", id) + } +} + +// Scenario: Cloud absent → error +func TestAppCredentialID_CloudNotFound(t *testing.T) { + clouds := ` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: https://keystone.example.com/v3 + application_credential_id: some-id + application_credential_secret: secret +` + _, err := openstack.AppCredentialID(clouds, "nonexistent") + if err == nil { + t.Fatal("expected error for missing cloud, got nil") + } +} + +// ── Error types ───────────────────────────────────────────────────────────── + +func TestAuthenticationError_Message(t *testing.T) { + err := &openstack.AuthenticationError{UserID: "user-abc"} + expected := "failed to authenticate as user: user-abc" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +func TestUnsupportedAuthTypeError_Message(t *testing.T) { + err := &openstack.UnsupportedAuthTypeError{AuthType: "password"} + expected := "unsupported authentication type: password" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +func TestCatalogError_Message(t *testing.T) { + err := &openstack.CatalogError{ServiceType: "volumev3"} + expected := "service type volumev3 not found in OpenStack service catalog" + if err.Error() != expected { + t.Errorf("expected %q, got %q", expected, err.Error()) + } +} + +// ── TLS verification ──────────────────────────────────────────────────────── + +// certPoolFromPEM builds an x509.CertPool from a PEM string (helper for assertions). +func certPoolFromPEM(pemData string) *x509.CertPool { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(pemData)) + return pool +} + +// tlsFromServer extracts TLS config from a test server (for CA assertions). +func tlsFromServer(s *httptest.Server) *tls.Config { + return s.TLS +} + +// errorAs is a typed helper to avoid importing errors package in test helpers. +func errorAs[T error](err error, target *T) bool { + if t, ok := err.(T); ok { + *target = t + return true + } + return false +} diff --git a/internal/openstack/purge.go b/internal/openstack/purge.go new file mode 100644 index 0000000..35c4449 --- /dev/null +++ b/internal/openstack/purge.go @@ -0,0 +1,68 @@ +package openstack + +import ( + "context" + + "github.com/go-logr/logr" +) + +// PurgeOptions holds parameters for cleaning up OpenStack resources +// associated with a deleted Cluster API cluster. +type PurgeOptions struct { + // CloudsYAML is the decoded content of the clouds.yaml credential file. + CloudsYAML string + // CloudName is the entry name within clouds.yaml to use. + CloudName string + // CACert is an optional PEM-encoded CA certificate for TLS verification. + CACert string + // ClusterName is the CAPI cluster name used to identify owned resources. + ClusterName string + // IncludeVolumes controls whether Cinder volumes and snapshots are deleted. + IncludeVolumes bool + // IncludeAppcred controls whether the OpenStack application credential is deleted. + IncludeAppcred bool + // Logger receives structured log messages during cleanup. + Logger logr.Logger +} + +// PurgeResources removes all OpenStack resources (FIPs, load balancers, +// security groups, volumes, snapshots, and optionally the application +// credential) created by OCCM/CSI for the given cluster. +func PurgeResources(ctx context.Context, opts PurgeOptions) error { + session, err := Authenticate(ctx, opts.CloudsYAML, opts.CloudName, opts.CACert) + if err != nil { + return err + } + + if !session.IsAuthenticated() { + if opts.IncludeAppcred { + opts.Logger.Info("application credential has been deleted, skipping cleanup") + return nil + } + return &AuthenticationError{UserID: session.UserID()} + } + + if err := session.DeleteFloatingIPs(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteLoadBalancers(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteSecurityGroups(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if opts.IncludeVolumes { + if err := session.DeleteSnapshots(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + if err := session.DeleteVolumes(ctx, opts.Logger, opts.ClusterName); err != nil { + return err + } + } + if opts.IncludeAppcred { + if err := session.DeleteAppCredential(ctx, opts.Logger, opts.CloudsYAML, opts.CloudName); err != nil { + return err + } + } + return nil +} diff --git a/internal/openstack/resources.go b/internal/openstack/resources.go new file mode 100644 index 0000000..d549c93 --- /dev/null +++ b/internal/openstack/resources.go @@ -0,0 +1,417 @@ +package openstack + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/go-logr/logr" +) + +// DeleteFloatingIPs removes FIPs whose description matches the cluster. +// Expected: "Floating IP for Kubernetes external service from cluster " +func (s *Session) DeleteFloatingIPs(ctx context.Context, logger logr.Logger, cluster string) error { + networkURL, err := s.endpointFor("network") + if err != nil { + return err + } + prefix := "Floating IP for Kubernetes" + suffix := "from cluster " + cluster + + type fip struct { + ID string `json:"id"` + Description string `json:"description"` + } + list, err := listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") + if err != nil { + return err + } + deleted := false + for _, f := range list { + if strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) { + // Mark found before attempting deletion so verification always runs, + // even when the delete returns a transient error. + deleted = true + logger.Info("deleting floating IP", "id", f.ID) + if err := s.doDelete(ctx, networkURL+"/v2.0/floatingips/"+f.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting floating IP, will verify", "id", f.ID, "error", err) + } else { + return err + } + } + } + } + if deleted { + listFIPs := func() ([]fip, error) { + return listPages[fip](ctx, s, networkURL+"/v2.0/floatingips", "floatingips") + } + matchFIP := func(f fip) bool { + return strings.HasPrefix(f.Description, prefix) && strings.HasSuffix(f.Description, suffix) + } + if err := waitForDeletion(ctx, s, logger, listFIPs, matchFIP, "floating IPs for cluster "+cluster); err != nil { + return err + } + } + logger.Info("deleted floating IPs for LoadBalancer services") + return nil +} + +// DeleteLoadBalancers removes Octavia LBs whose name starts with kube_service__ +// or follows the Azimuth naming convention. +func (s *Session) DeleteLoadBalancers(ctx context.Context, logger logr.Logger, cluster string) error { + lbURL, err := s.endpointFor("load-balancer") + if err != nil { + logger.Info("load-balancer service not found in catalog, skipping LB cleanup") + return nil + } + + type lb struct { + ID string `json:"id"` + Name string `json:"name"` + } + list, err := listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") + if err != nil { + logger.Error(err, "failed to list load balancers, some may remain") + return nil + } + kubePrefix := "kube_service_" + cluster + "_" + deleted := false + for _, l := range list { + if strings.HasPrefix(l.Name, kubePrefix) { + deleted = true + logger.Info("deleting load balancer", "id", l.ID, "name", l.Name) + target := lbURL + "/v2/lbaas/loadbalancers/" + l.ID + "?cascade=true" + if err := s.doDelete(ctx, target); err != nil { + if isTransient(err) { + logger.Info("transient error deleting load balancer, will verify", "id", l.ID, "error", err) + } else { + return err + } + } + } + } + if deleted { + listLBs := func() ([]lb, error) { + return listPages[lb](ctx, s, lbURL+"/v2/lbaas/loadbalancers", "loadbalancers") + } + matchLB := func(l lb) bool { + return strings.HasPrefix(l.Name, kubePrefix) + } + if err := waitForDeletion(ctx, s, logger, listLBs, matchLB, "load balancers for cluster "+cluster); err != nil { + var stillPresent *stillPresentError + if errors.As(err, &stillPresent) { + return err + } + logger.Error(err, "failed to verify LB deletion after polling") + } + } + logger.Info("deleted load balancers for LoadBalancer services") + return nil +} + +// DeleteSecurityGroups removes SGs whose description matches the cluster LB pattern. +// Expected: "Security Group for Service LoadBalancer in cluster " +func (s *Session) DeleteSecurityGroups(ctx context.Context, logger logr.Logger, cluster string) error { + networkURL, err := s.endpointFor("network") + if err != nil { + return err + } + sgSuffix := "Service LoadBalancer in cluster " + cluster + + type sg struct { + ID string `json:"id"` + Description string `json:"description"` + } + list, err := listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") + if err != nil { + return err + } + deleted := false + for _, g := range list { + if strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) { + deleted = true + logger.Info("deleting security group", "id", g.ID) + if err := s.doDelete(ctx, networkURL+"/v2.0/security-groups/"+g.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting security group, will verify", "id", g.ID, "error", err) + } else { + return err + } + } + } + } + if deleted { + listSGs := func() ([]sg, error) { + return listPages[sg](ctx, s, networkURL+"/v2.0/security-groups", "security_groups") + } + matchSG := func(g sg) bool { + return strings.HasPrefix(g.Description, "Security Group for") && strings.HasSuffix(g.Description, sgSuffix) + } + if err := waitForDeletion(ctx, s, logger, listSGs, matchSG, "security groups for cluster "+cluster); err != nil { + return err + } + } + logger.Info("deleted security groups for LoadBalancer services") + return nil +} + +// volumeItem represents a Cinder volume or snapshot with its metadata. +type volumeItem struct { + ID string `json:"id"` + Metadata map[string]string `json:"metadata"` +} + +// DeleteSnapshots removes Cinder snapshots tagged with the cluster CSI metadata. +func (s *Session) DeleteSnapshots(ctx context.Context, logger logr.Logger, cluster string) error { + cinderURL, err := s.cinderEndpoint() + if err != nil { + return err + } + list, err := s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") + if err != nil { + return err + } + deleted := false + for _, snap := range list { + if snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster { + deleted = true + logger.Info("deleting snapshot", "id", snap.ID) + if err := s.doDelete(ctx, cinderURL+"/snapshots/"+snap.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting snapshot, will verify", "id", snap.ID, "error", err) + } else { + return err + } + } + } + } + if deleted { + listSnaps := func() ([]volumeItem, error) { + return s.listVolumeItems(ctx, cinderURL+"/snapshots/detail", "snapshots") + } + matchSnap := func(snap volumeItem) bool { + return snap.Metadata["cinder.csi.openstack.org/cluster"] == cluster + } + if err := waitForDeletion(ctx, s, logger, listSnaps, matchSnap, "snapshots for cluster "+cluster); err != nil { + return err + } + } + logger.Info("deleted snapshots for persistent volume claims") + return nil +} + +// DeleteVolumes removes Cinder volumes tagged with the cluster CSI metadata, +// unless the user has set the keep property to "true". +func (s *Session) DeleteVolumes(ctx context.Context, logger logr.Logger, cluster string) error { + cinderURL, err := s.cinderEndpoint() + if err != nil { + return err + } + list, err := s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") + if err != nil { + return err + } + deleted := false + for _, vol := range list { + if vol.Metadata["cinder.csi.openstack.org/cluster"] != cluster { + continue + } + if vol.Metadata[KeepProperty] == "true" { + continue + } + deleted = true + logger.Info("deleting volume", "id", vol.ID) + if err := s.doDelete(ctx, cinderURL+"/volumes/"+vol.ID); err != nil { + if isTransient(err) { + logger.Info("transient error deleting volume, will verify", "id", vol.ID, "error", err) + } else { + return err + } + } + } + if deleted { + listVols := func() ([]volumeItem, error) { + return s.listVolumeItems(ctx, cinderURL+"/volumes/detail", "volumes") + } + matchVol := func(vol volumeItem) bool { + return vol.Metadata["cinder.csi.openstack.org/cluster"] == cluster && + vol.Metadata[KeepProperty] != "true" + } + if err := waitForDeletion(ctx, s, logger, listVols, matchVol, "volumes for cluster "+cluster); err != nil { + return err + } + } + logger.Info("deleted volumes for persistent volume claims") + return nil +} + +// DeleteAppCredential removes the OpenStack application credential for the cluster. +func (s *Session) DeleteAppCredential(ctx context.Context, logger logr.Logger, cloudsYAML, cloudName string) error { + identityURL, err := s.endpointFor("identity") + if err != nil { + return err + } + appcredID, err := AppCredentialID(cloudsYAML, cloudName) + if err != nil { + return err + } + if appcredID == "" { + // Password (v3password) auth has no application credential to delete. + logger.Info("no application credential in use, skipping deletion") + return nil + } + target := strings.TrimRight(identityURL, "/") + "/v3/users/" + s.userID + "/application_credentials/" + appcredID + req, err := newDeleteRequest(ctx, target, s.token) + if err != nil { + return err + } + resp, err := s.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNoContent, http.StatusOK, http.StatusNotFound: + logger.Info("deleted application credential for cluster") + return nil + case http.StatusForbidden: + logger.Info("unable to delete application credential (restricted), skipping") + return nil + default: + return fmt.Errorf("deleting application credential: HTTP %d", resp.StatusCode) + } +} + +func (s *Session) cinderEndpoint() (string, error) { + return s.endpointFor("volumev3", "block-storage", "volume") +} + +func (s *Session) listVolumeItems(ctx context.Context, endpoint, key string) ([]volumeItem, error) { + body, err := s.doGet(ctx, endpoint) + if err != nil { + return nil, err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var items []volumeItem + if err := json.Unmarshal(raw[key], &items); err != nil { + return nil, err + } + return items, nil +} + +func newDeleteRequest(ctx context.Context, target, token string) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, target, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Auth-Token", token) + return req, nil +} + +// stillPresentError indicates that a resource was still present after +// waitForDeletion exhausted its polling attempts, as opposed to an error +// from the underlying listFunc itself. Callers use errors.As to tell the two +// apart, since some resource types treat listing failures as non-fatal. +type stillPresentError struct{ desc string } + +func (e *stillPresentError) Error() string { return e.desc + " still present" } + +// waitForDeletion polls a list function until no items match the predicate, +// or the maximum number of attempts is exceeded. This avoids failing +// immediately when OpenStack has accepted a DELETE but the resource has +// not yet been removed from list results. +func waitForDeletion[T any](ctx context.Context, s *Session, logger logr.Logger, listFunc func() ([]T, error), matchFunc func(T) bool, desc string) error { + const ( + maxPollAttempts = 6 + pollInterval = 5 * time.Second + ) + for attempt := 0; attempt < maxPollAttempts; attempt++ { + if attempt > 0 { + s.sleep(pollInterval) + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + items, err := listFunc() + if err != nil { + return err + } + remaining := 0 + for _, item := range items { + if matchFunc(item) { + remaining++ + } + } + if remaining == 0 { + return nil + } + logger.Info(desc+" still present, retrying", "remaining", remaining, "attempt", attempt+1, "maxAttempts", maxPollAttempts) + } + return &stillPresentError{desc: desc} +} + +// listPages fetches all pages of a paginated OpenStack list endpoint and +// decodes items into T using the given JSON key. +func listPages[T any](ctx context.Context, s *Session, endpoint, key string) ([]T, error) { + var results []T + next := endpoint + for next != "" { + body, err := s.doGet(ctx, next) + if err != nil { + return nil, err + } + var page map[string]json.RawMessage + if err := json.Unmarshal(body, &page); err != nil { + return nil, err + } + var items []T + if err := json.Unmarshal(page[key], &items); err != nil { + return nil, err + } + results = append(results, items...) + next = nextPageURL(body, key) + } + return results, nil +} + +func nextPageURL(body []byte, key string) string { + linksKey := key + "_links" + var page map[string]json.RawMessage + if err := json.Unmarshal(body, &page); err != nil { + return "" + } + raw, ok := page[linksKey] + if !ok { + return "" + } + var links []struct { + Rel string `json:"rel"` + Href string `json:"href"` + } + if err := json.Unmarshal(raw, &links); err != nil { + return "" + } + for _, l := range links { + if l.Rel == "next" { + u, err := url.Parse(l.Href) + if err != nil { + return l.Href + } + // OpenStack sometimes returns http where https is required. + u.Scheme = "https" + return u.String() + } + } + return "" +} diff --git a/internal/openstack/resources_test.go b/internal/openstack/resources_test.go new file mode 100644 index 0000000..3d507aa --- /dev/null +++ b/internal/openstack/resources_test.go @@ -0,0 +1,2051 @@ +package openstack_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── Mock server ─────────────────────────────────────────────────────────────── + +// fipRecord represents a Neutron floating IP in list responses. +type fipRecord struct { + ID string `json:"id"` + Description string `json:"description"` +} + +// networkTestServer is a mock OpenStack server that handles Keystone auth + +// a Neutron-like API on the same HTTP test server. The catalog advertises the +// server's own URL as the "network" endpoint, so Sessions authenticate and +// resolve resource URLs against this single server. +type networkTestServer struct { + *httptest.Server + mu sync.Mutex + fipLists [][]fipRecord // sequence of list responses; last entry is reused + fipGetCount int // total number of GET /v2.0/floatingips calls + // Per-FIP DELETE response status (default 204 NoContent). + deleteStatus map[string]int + // IDs deleted in call order. + deletedFIPs []string +} + +func newNetworkTestServer(t *testing.T) *networkTestServer { + t.Helper() + srv := &networkTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + // Keystone: token endpoint + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-network-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Keystone: service catalog — self-referential: "network" endpoint = this server + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "network", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Neutron: list floating IPs (exact path — no trailing slash to avoid redirect) + mux.HandleFunc("/v2.0/floatingips", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.fipGetCount++ + idx := srv.fipGetCount - 1 + var list []fipRecord + if len(srv.fipLists) > 0 { + if idx < len(srv.fipLists) { + list = srv.fipLists[idx] + } else { + list = srv.fipLists[len(srv.fipLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"floatingips": list}) + }) + + // Neutron: delete floating IP by ID (subtree pattern handles /v2.0/floatingips/{id}) + mux.HandleFunc("/v2.0/floatingips/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2.0/floatingips/") + srv.mu.Lock() + srv.deletedFIPs = append(srv.deletedFIPs, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +// authenticate creates an authenticated Session against this server. +func (srv *networkTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with network endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +// fipDesc returns the description that OCCM writes for a service FIP. +func fipDesc(cluster string) string { + return fmt.Sprintf("Floating IP for Kubernetes external service from cluster %s", cluster) +} + +// ── LB mock server ─────────────────────────────────────────────────────────── + +type lbRecord struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// lbTestServer is a mock OpenStack server that handles Keystone auth + +// an Octavia-like API on the same HTTP test server. The catalog advertises the +// server's own URL as the "load-balancer" endpoint. +type lbTestServer struct { + *httptest.Server + mu sync.Mutex + lbLists [][]lbRecord + lbGetCount int + listStatusOverride int // if non-zero, GET /v2/lbaas/loadbalancers returns this status + listFailFrom int // 1-based GET call number from which listStatusOverride applies; 0 = from the first call + deleteStatus map[string]int + deletedLBs []string + deleteCascade []string // cascade query param value per DELETE call +} + +func newLBTestServer(t *testing.T) *lbTestServer { + t.Helper() + srv := &lbTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-lb-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "load-balancer" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "load-balancer", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/v2/lbaas/loadbalancers", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + override := srv.listStatusOverride + failFrom := srv.listFailFrom + srv.lbGetCount++ + callNum := srv.lbGetCount + idx := callNum - 1 + var list []lbRecord + if len(srv.lbLists) > 0 { + if idx < len(srv.lbLists) { + list = srv.lbLists[idx] + } else { + list = srv.lbLists[len(srv.lbLists)-1] + } + } + srv.mu.Unlock() + if override != 0 && (failFrom == 0 || callNum >= failFrom) { + w.WriteHeader(override) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"loadbalancers": list}) + }) + + mux.HandleFunc("/v2/lbaas/loadbalancers/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2/lbaas/loadbalancers/") + cascade := r.URL.Query().Get("cascade") + srv.mu.Lock() + srv.deletedLBs = append(srv.deletedLBs, id) + srv.deleteCascade = append(srv.deleteCascade, cascade) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *lbTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with load-balancer endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +func lbKubeName(cluster, suffix string) string { + return fmt.Sprintf("kube_service_%s_%s", cluster, suffix) +} + +// ── US2.1: Identify Floating IPs of a cluster ──────────────────────────────── + +// Scenario: FIP belonging to the cluster → included in deletion +// Scenario: FIP from another cluster → excluded +// Scenario: FIP without Kubernetes description → excluded +func TestDeleteFloatingIPs_Filtering(t *testing.T) { + tests := []struct { + name string + description string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + description: fipDesc("mycluster"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + description: fipDesc("othercluster"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "non-kubernetes description", + description: "Some unrelated floating IP", + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "empty description", + description: "", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-001", Description: tt.description} + // List: FIP present before deletion; empty after (verification passes) + srv.fipLists = [][]fipRecord{{fip}, {}} + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedFIPs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but FIP was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: FIPs from multiple clusters → only those matching the target cluster are deleted +func TestDeleteFloatingIPs_MultipleIPsPartialMatch(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, // match + {ID: "fip-002", Description: fipDesc("othercluster")}, // no match + {ID: "fip-003", Description: "Some other description"}, // no match + {ID: "fip-004", Description: fipDesc("mycluster")}, // match + } + srv.fipLists = [][]fipRecord{fips, {}} + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := make(map[string]bool, len(srv.deletedFIPs)) + for _, id := range srv.deletedFIPs { + deleted[id] = true + } + srv.mu.Unlock() + + if !deleted["fip-001"] || !deleted["fip-004"] { + t.Errorf("expected fip-001 and fip-004 to be deleted, got: %v", srv.deletedFIPs) + } + if deleted["fip-002"] || deleted["fip-003"] { + t.Errorf("expected fip-002 and fip-003 to NOT be deleted, got: %v", srv.deletedFIPs) + } +} + +// ── US2.2: Delete Floating IPs ─────────────────────────────────────────────── + +// Scenario: Successful deletion +// Given a FIP belonging to cluster "mycluster" +// When the FIP purge is triggered +// Then the FIP is deleted via the Neutron API +func TestDeleteFloatingIPs_SuccessfulDeletion(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, + {ID: "fip-002", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips, {}} // first: FIPs present; verification: empty + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedFIPs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 FIPs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, fip := range fips { + if !idSet[fip.ID] { + t.Errorf("expected FIP %s to be deleted", fip.ID) + } + } +} + +// Scenario: HTTP 400 error during deletion +// Then a warning is emitted +// And deletion continues for other FIPs +// And verification is triggered (check_fips = true) +func TestDeleteFloatingIPs_TransientError400_ContinuesAndVerifies(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-001", Description: fipDesc("mycluster")}, // returns HTTP 400 + {ID: "fip-002", Description: fipDesc("mycluster")}, // returns HTTP 204 + } + srv.fipLists = [][]fipRecord{fips, {}} // verification returns empty + srv.deleteStatus["fip-001"] = http.StatusBadRequest + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + // Transient error must NOT be propagated + if err != nil { + t.Fatalf("expected no error for transient HTTP 400, got: %v", err) + } + + // Both FIPs must have been attempted + srv.mu.Lock() + attempted := len(srv.deletedFIPs) + getCount := srv.fipGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both FIPs to be attempted, got %d DELETE calls", attempted) + } + // Verification GET must have been triggered (deleted=true even on transient error) + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: HTTP 409 error (Conflict) → same behaviour as 400 +func TestDeleteFloatingIPs_TransientError409_Continues(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-conflict", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips, {}} + srv.deleteStatus["fip-conflict"] = http.StatusConflict + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } +} + +// Scenario: HTTP 500 error → exception propagated +func TestDeleteFloatingIPs_HTTP500_PropagatesError(t *testing.T) { + srv := newNetworkTestServer(t) + fips := []fipRecord{ + {ID: "fip-server-error", Description: fipDesc("mycluster")}, + } + srv.fipLists = [][]fipRecord{fips} + srv.deleteStatus["fip-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: FIPs still present after deletion → error returned +// (the controller will retry via the retry annotation) +func TestDeleteFloatingIPs_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-persistent", Description: fipDesc("mycluster")} + // Verification also returns the FIP — OpenStack has not deleted it yet + srv.fipLists = [][]fipRecord{{fip}, {fip}} + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when FIP persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: context is canceled while polling → polling stops immediately, +// without an extra wasted list round-trip +func TestDeleteFloatingIPs_ContextCanceledDuringPolling_StopsImmediately(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-persistent", Description: fipDesc("mycluster")} + srv.fipLists = [][]fipRecord{{fip}, {fip}, {fip}, {fip}} // always present + + session := srv.authenticate(t) + ctx, cancel := context.WithCancel(context.Background()) + session.SleepFunc = func(d time.Duration) { cancel() } + + err := session.DeleteFloatingIPs(ctx, logr.Discard(), "mycluster") + + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } + srv.mu.Lock() + getCount := srv.fipGetCount + srv.mu.Unlock() + // 1 pre-delete list + 1 poll (attempt 0, before any sleep) = 2 GET calls. + // The sleep before attempt 1 cancels the context, so attempt 1's list call + // must never happen. + if getCount != 2 { + t.Errorf("expected exactly 2 GET calls (no wasted round-trip after cancellation), got %d", getCount) + } +} + +// Scenario: FIP disappears during polling → polling succeeds without error +func TestDeleteFloatingIPs_SlowDeletion_PollingSucceeds(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-slow", Description: fipDesc("mycluster")} + // idx 0: pre-delete list (FIP present) + // idx 1: first poll (FIP still present — OpenStack PENDING_DELETE) + // idx 2: second poll (FIP gone — deletion complete) + srv.fipLists = [][]fipRecord{{fip}, {fip}, {}} + + session := srv.authenticate(t) + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error when FIP disappears during polling, got: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedFIPs + getCount := srv.fipGetCount + srv.mu.Unlock() + + if len(deletedIDs) != 1 { + t.Errorf("expected 1 FIP deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + // 1 pre-delete list + 2 polls = 3 GET calls + if getCount != 3 { + t.Errorf("expected 3 GET calls (1 list + 2 polls), got %d", getCount) + } +} + +// Scenario: FIP already gone on the first verification check → no wait incurred +func TestDeleteFloatingIPs_ImmediateVerification_NoSleep(t *testing.T) { + srv := newNetworkTestServer(t) + fip := fipRecord{ID: "fip-fast", Description: fipDesc("mycluster")} + srv.fipLists = [][]fipRecord{{fip}, {}} // gone by the first verification check + + session := srv.authenticate(t) + var sleepCalls int + session.SleepFunc = func(d time.Duration) { sleepCalls++ } + err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sleepCalls != 0 { + t.Errorf("expected no sleep when the resource is already gone, got %d sleep calls", sleepCalls) + } +} + +// Scenario: No matching FIP → no deletion, no verification +func TestDeleteFloatingIPs_NothingToDelete_NoVerification(t *testing.T) { + srv := newNetworkTestServer(t) + srv.fipLists = [][]fipRecord{{}} // empty list + + session := srv.authenticate(t) + if err := session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no FIPs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedFIPs) + getCount := srv.fipGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedFIPs) + } + // No verification GET should be triggered when nothing was found + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// Scenario: Pas d'endpoint "network" dans le catalogue → CatalogError +func TestDeleteFloatingIPs_NoNetworkEndpoint_ReturnsCatalogError(t *testing.T) { + // Use a Keystone server that only advertises a "compute" endpoint (no "network") + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("compute", + endpoint("public", "RegionOne", "http://compute.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteFloatingIPs(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected CatalogError when network endpoint is absent") + } + var target *openstack.CatalogError + if !errorAs(err, &target) { + t.Errorf("expected *CatalogError, got %T: %v", err, err) + } +} + +// ── Epic 3: Octavia Load Balancer Cleanup ───────────────────────────────────── + +// ── US3.1: Identify Kubernetes Load Balancers ───────────────────────────────── + +// Scenario: LB belonging to the cluster → included in deletion +// Scenario: LB from another cluster → excluded +// Scenario: LB without kube_service prefix → excluded +func TestDeleteLoadBalancers_Filtering(t *testing.T) { + tests := []struct { + name string + lbName string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + lbName: lbKubeName("mycluster", "api"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + lbName: lbKubeName("othercluster", "api"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "wrong prefix", + lbName: "fake_service_mycluster_api", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-001", Name: tt.lbName} + srv.lbLists = [][]lbRecord{{lb}, {}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedLBs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but LB was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// ── US3.2: HTTP error during listing (PR #261) ─────────────────────────────── + +// Scenario: HTTP error during LB listing → ERROR log, no exception +func TestDeleteLoadBalancers_ListError_LogsAndSkips(t *testing.T) { + srv := newLBTestServer(t) + srv.listStatusOverride = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected nil when list returns HTTP 500, got: %v", err) + } + srv.mu.Lock() + deleted := len(srv.deletedLBs) + srv.mu.Unlock() + if deleted != 0 { + t.Errorf("expected no DELETE calls when list fails, got %d", deleted) + } +} + +// Scenario: HTTP error while verifying LB deletion after polling → ERROR log, no exception +// (unlike other resource types, LB verification failures are non-fatal: Octavia +// listing is known to be slower/less reliable, see PR #261.) +func TestDeleteLoadBalancers_VerificationListError_LogsAndSucceeds(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-001", Name: lbKubeName("mycluster", "svc")} + srv.lbLists = [][]lbRecord{{lb}} + srv.listStatusOverride = http.StatusInternalServerError + srv.listFailFrom = 2 // pre-delete list (call 1) succeeds; verification (call 2+) fails + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected nil when verification list fails after deletion, got: %v", err) + } + srv.mu.Lock() + deleted := len(srv.deletedLBs) + srv.mu.Unlock() + if deleted != 1 { + t.Errorf("expected the LB delete to have been attempted, got %d DELETE calls", deleted) + } +} + +// ── US3.3: Delete Load Balancers with cascade ───────────────────────────────── + +// Scenario: Successful deletion with cascade=true +func TestDeleteLoadBalancers_SuccessfulDeletion(t *testing.T) { + srv := newLBTestServer(t) + lbs := []lbRecord{ + {ID: "lb-001", Name: lbKubeName("mycluster", "svc1")}, + {ID: "lb-002", Name: lbKubeName("mycluster", "svc2")}, + } + srv.lbLists = [][]lbRecord{lbs, {}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedLBs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 LBs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, lb := range lbs { + if !idSet[lb.ID] { + t.Errorf("expected LB %s to be deleted", lb.ID) + } + } +} + +// Scenario: DELETE issued with cascade=true +func TestDeleteLoadBalancers_CascadeDelete(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{ + {{ID: "lb-cascade", Name: lbKubeName("mycluster", "svc")}}, + {}, + } + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + cascades := srv.deleteCascade + srv.mu.Unlock() + + if len(cascades) == 0 { + t.Fatal("expected a DELETE call, got none") + } + for i, c := range cascades { + if c != "true" { + t.Errorf("DELETE call %d: expected cascade=true, got %q", i, c) + } + } +} + +// Scenario: HTTP 400 error during deletion → warning, continues, verification triggered +func TestDeleteLoadBalancers_TransientError400_ContinuesAndVerifies(t *testing.T) { + srv := newLBTestServer(t) + lbs := []lbRecord{ + {ID: "lb-001", Name: lbKubeName("mycluster", "svc1")}, // returns HTTP 400 + {ID: "lb-002", Name: lbKubeName("mycluster", "svc2")}, // returns HTTP 204 + } + srv.lbLists = [][]lbRecord{lbs, {}} + srv.deleteStatus["lb-001"] = http.StatusBadRequest + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 400, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedLBs) + getCount := srv.lbGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both LBs to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: HTTP 500 error → exception propagated +func TestDeleteLoadBalancers_HTTP500_PropagatesError(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{ + {{ID: "lb-server-error", Name: lbKubeName("mycluster", "svc")}}, + } + srv.deleteStatus["lb-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: LBs still present after deletion → error returned +func TestDeleteLoadBalancers_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newLBTestServer(t) + lb := lbRecord{ID: "lb-persistent", Name: lbKubeName("mycluster", "svc")} + srv.lbLists = [][]lbRecord{{lb}, {lb}} + + session := srv.authenticate(t) + err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when LB persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: No matching LB → no deletion, no verification +func TestDeleteLoadBalancers_NothingToDelete_NoVerification(t *testing.T) { + srv := newLBTestServer(t) + srv.lbLists = [][]lbRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no LBs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedLBs) + getCount := srv.lbGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedLBs) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// ── SG mock server ──────────────────────────────────────────────────────────── + +type sgRecord struct { + ID string `json:"id"` + Description string `json:"description"` +} + +// sgTestServer is a mock OpenStack server that handles Keystone auth + +// a Neutron-like security group API. The catalog advertises the server's own +// URL as the "network" endpoint. +type sgTestServer struct { + *httptest.Server + mu sync.Mutex + sgLists [][]sgRecord + sgGetCount int + listStatusOverride int // if non-zero, all GETs return this status + deleteStatus map[string]int + deletedSGs []string +} + +func newSGTestServer(t *testing.T) *sgTestServer { + t.Helper() + srv := &sgTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-sg-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "network" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "network", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/v2.0/security-groups", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + override := srv.listStatusOverride + srv.sgGetCount++ + idx := srv.sgGetCount - 1 + var list []sgRecord + if len(srv.sgLists) > 0 { + if idx < len(srv.sgLists) { + list = srv.sgLists[idx] + } else { + list = srv.sgLists[len(srv.sgLists)-1] + } + } + srv.mu.Unlock() + if override != 0 { + w.WriteHeader(override) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"security_groups": list}) + }) + + mux.HandleFunc("/v2.0/security-groups/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/v2.0/security-groups/") + srv.mu.Lock() + srv.deletedSGs = append(srv.deletedSGs, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *sgTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with network endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +func sgDesc(cluster string) string { + return fmt.Sprintf("Security Group for Service LoadBalancer in cluster %s", cluster) +} + +// Scenario: No "load-balancer" endpoint in catalog → return nil (LBs skipped) +func TestDeleteLoadBalancers_NoLoadBalancerEndpoint_Skips(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteLoadBalancers(context.Background(), logr.Discard(), "mycluster") + if err != nil { + t.Fatalf("expected nil when load-balancer endpoint is absent, got: %v", err) + } +} + +// ── Epic 4: Security Group Cleanup ─────────────────────────────────────────── + +// ── US4.1: Identify Security Groups of a cluster ───────────────────────────── + +// Scenario: SG belonging to the cluster → included in deletion +// Scenario: SG from another cluster → excluded +// Scenario: Non-matching description → excluded +func TestDeleteSecurityGroups_Filtering(t *testing.T) { + tests := []struct { + name string + description string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + description: sgDesc("mycluster"), + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + description: sgDesc("othercluster"), + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "wrong prefix", + description: "Group for Service LoadBalancer in cluster mycluster", + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "unrelated description", + description: "Some other security group", + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newSGTestServer(t) + sg := sgRecord{ID: "sg-001", Description: tt.description} + srv.sgLists = [][]sgRecord{{sg}, {}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedSGs) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but SG was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// ── US4.2: Delete Security Groups ──────────────────────────────────────────── + +// Scenario: Successful deletion +func TestDeleteSecurityGroups_SuccessfulDeletion(t *testing.T) { + srv := newSGTestServer(t) + sgs := []sgRecord{ + {ID: "sg-001", Description: sgDesc("mycluster")}, + {ID: "sg-002", Description: sgDesc("mycluster")}, + } + srv.sgLists = [][]sgRecord{sgs, {}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedSGs + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 SGs deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, sg := range sgs { + if !idSet[sg.ID] { + t.Errorf("expected SG %s to be deleted", sg.ID) + } + } +} + +// Scenario: SG still in use (HTTP 409) → warning, continues, verification triggered +func TestDeleteSecurityGroups_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newSGTestServer(t) + sgs := []sgRecord{ + {ID: "sg-001", Description: sgDesc("mycluster")}, // returns HTTP 409 + {ID: "sg-002", Description: sgDesc("mycluster")}, // returns HTTP 204 + } + srv.sgLists = [][]sgRecord{sgs, {}} + srv.deleteStatus["sg-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedSGs) + getCount := srv.sgGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both SGs to be attempted, got %d DELETE calls", attempted) + } + // Verification GET must have been triggered (deleted=true even on transient error) + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: HTTP 500 error → exception propagated +func TestDeleteSecurityGroups_HTTP500_PropagatesError(t *testing.T) { + srv := newSGTestServer(t) + srv.sgLists = [][]sgRecord{ + {{ID: "sg-server-error", Description: sgDesc("mycluster")}}, + } + srv.deleteStatus["sg-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: SGs still present after deletion → error returned +func TestDeleteSecurityGroups_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newSGTestServer(t) + sg := sgRecord{ID: "sg-persistent", Description: sgDesc("mycluster")} + srv.sgLists = [][]sgRecord{{sg}, {sg}} + + session := srv.authenticate(t) + err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when SG persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: No matching SG → no deletion, no verification +func TestDeleteSecurityGroups_NothingToDelete_NoVerification(t *testing.T) { + srv := newSGTestServer(t) + srv.sgLists = [][]sgRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteSecurityGroups(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no SGs: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedSGs) + getCount := srv.sgGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedSGs) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// ── Cinder Volume mock server ───────────────────────────────────────────────── + +type cinderVolumeRecord struct { + ID string `json:"id"` + Metadata map[string]string `json:"metadata"` +} + +// cinderTestServer is a mock OpenStack server that handles Keystone auth + +// a Cinder-like volumes API. The catalog advertises the server's own URL as +// the "volumev3" endpoint (first type checked by cinderEndpoint). +type cinderTestServer struct { + *httptest.Server + mu sync.Mutex + volumeLists [][]cinderVolumeRecord // sequence of list responses; last entry is reused + volumeGetCount int + deleteStatus map[string]int + deletedVolumes []string +} + +func newCinderTestServer(t *testing.T) *cinderTestServer { + t.Helper() + srv := &cinderTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-cinder-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "volumev3" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "volumev3", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Cinder: list volumes (exact path — takes priority over /volumes/ subtree) + mux.HandleFunc("/volumes/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.volumeGetCount++ + idx := srv.volumeGetCount - 1 + var list []cinderVolumeRecord + if len(srv.volumeLists) > 0 { + if idx < len(srv.volumeLists) { + list = srv.volumeLists[idx] + } else { + list = srv.volumeLists[len(srv.volumeLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"volumes": list}) + }) + + // Cinder: delete volume by ID (subtree pattern handles /volumes/{id}) + mux.HandleFunc("/volumes/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/volumes/") + srv.mu.Lock() + srv.deletedVolumes = append(srv.deletedVolumes, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *cinderTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with volumev3 endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +// ── Epic 5: Cinder Volume Management ───────────────────────────────────────── + +// ── US5.1: Identify volumes of a cluster ───────────────────────────────────── + +// Scenario: Volume from the correct cluster without keep → deleted +// Scenario: Volume with keep=true → kept +// Scenario: Volume from another cluster → excluded +// Scenario: Volume without CSI metadata → excluded +func TestDeleteVolumes_Filtering(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster, no keep", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "matching cluster, keep=true", + metadata: map[string]string{ + "cinder.csi.openstack.org/cluster": "mycluster", + "janitor.capi.azimuth-cloud.com/keep": "true", + }, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "different cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "othercluster"}, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "no CSI metadata", + metadata: map[string]string{}, + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newCinderTestServer(t) + vol := cinderVolumeRecord{ID: "vol-001", Metadata: tt.metadata} + srv.volumeLists = [][]cinderVolumeRecord{{vol}, {}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedVolumes) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but volume was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: Successful deletion of multiple volumes +func TestDeleteVolumes_SuccessfulDeletion(t *testing.T) { + srv := newCinderTestServer(t) + vols := []cinderVolumeRecord{ + {ID: "vol-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "vol-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.volumeLists = [][]cinderVolumeRecord{vols, {}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedVolumes + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 volumes deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, vol := range vols { + if !idSet[vol.ID] { + t.Errorf("expected volume %s to be deleted", vol.ID) + } + } +} + +// Scenario: Transient HTTP 409 error → warning, continues, verification triggered +func TestDeleteVolumes_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newCinderTestServer(t) + vols := []cinderVolumeRecord{ + {ID: "vol-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "vol-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.volumeLists = [][]cinderVolumeRecord{vols, {}} + srv.deleteStatus["vol-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedVolumes) + getCount := srv.volumeGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both volumes to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: HTTP 500 error → exception propagated +func TestDeleteVolumes_HTTP500_PropagatesError(t *testing.T) { + srv := newCinderTestServer(t) + srv.volumeLists = [][]cinderVolumeRecord{ + {{ID: "vol-server-error", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}}, + } + srv.deleteStatus["vol-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Volume still present after deletion → error returned +func TestDeleteVolumes_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newCinderTestServer(t) + vol := cinderVolumeRecord{ + ID: "vol-persistent", + Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + } + srv.volumeLists = [][]cinderVolumeRecord{{vol}, {vol}} + + session := srv.authenticate(t) + err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when volume persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// Scenario: No matching volume → no deletion, no verification +func TestDeleteVolumes_NothingToDelete_NoVerification(t *testing.T) { + srv := newCinderTestServer(t) + srv.volumeLists = [][]cinderVolumeRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteVolumes(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no volumes: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedVolumes) + getCount := srv.volumeGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedVolumes) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} + +// ── Cinder Snapshot mock server ─────────────────────────────────────────────── + +// snapshotTestServer is a mock OpenStack server handling Keystone auth + +// a Cinder-like snapshots API. Catalog advertises the server's own URL as +// the "volumev3" endpoint (same as volumes — same Cinder service). +type snapshotTestServer struct { + *httptest.Server + mu sync.Mutex + snapshotLists [][]cinderVolumeRecord // reuse volumeItem shape (id + metadata) + snapshotGetCount int + deleteStatus map[string]int + deletedSnapshots []string +} + +func newSnapshotTestServer(t *testing.T) *snapshotTestServer { + t.Helper() + srv := &snapshotTestServer{ + deleteStatus: make(map[string]int), + } + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-snapshot-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "volumev3", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Cinder: list snapshots (exact path — takes priority over /snapshots/ subtree) + mux.HandleFunc("/snapshots/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.snapshotGetCount++ + idx := srv.snapshotGetCount - 1 + var list []cinderVolumeRecord + if len(srv.snapshotLists) > 0 { + if idx < len(srv.snapshotLists) { + list = srv.snapshotLists[idx] + } else { + list = srv.snapshotLists[len(srv.snapshotLists)-1] + } + } + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"snapshots": list}) + }) + + // Cinder: delete snapshot by ID (subtree pattern handles /snapshots/{id}) + mux.HandleFunc("/snapshots/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + id := strings.TrimPrefix(r.URL.Path, "/snapshots/") + srv.mu.Lock() + srv.deletedSnapshots = append(srv.deletedSnapshots, id) + status, ok := srv.deleteStatus[id] + if !ok { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *snapshotTestServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with volumev3 endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +// ── Epic 6: Cinder Snapshot Management ─────────────────────────────────────── + +// ── US6.1: Identify and delete snapshots of a cluster ──────────────────────── + +// Scenario: Snapshot from the correct cluster → deleted +// Scenario: Snapshot from another cluster → excluded +func TestDeleteSnapshots_Filtering(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + cluster string + shouldDelete bool + }{ + { + name: "matching cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + cluster: "mycluster", + shouldDelete: true, + }, + { + name: "different cluster", + metadata: map[string]string{"cinder.csi.openstack.org/cluster": "othercluster"}, + cluster: "mycluster", + shouldDelete: false, + }, + { + name: "no CSI metadata", + metadata: map[string]string{}, + cluster: "mycluster", + shouldDelete: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newSnapshotTestServer(t) + snap := cinderVolumeRecord{ID: "snap-001", Metadata: tt.metadata} + srv.snapshotLists = [][]cinderVolumeRecord{{snap}, {}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), tt.cluster); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := len(srv.deletedSnapshots) > 0 + srv.mu.Unlock() + + if deleted != tt.shouldDelete { + t.Errorf("shouldDelete=%v but snapshot was deleted=%v", tt.shouldDelete, deleted) + } + }) + } +} + +// Scenario: Successful deletion of multiple snapshots +func TestDeleteSnapshots_SuccessfulDeletion(t *testing.T) { + srv := newSnapshotTestServer(t) + snaps := []cinderVolumeRecord{ + {ID: "snap-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "snap-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.snapshotLists = [][]cinderVolumeRecord{snaps, {}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deletedIDs := srv.deletedSnapshots + srv.mu.Unlock() + + if len(deletedIDs) != 2 { + t.Errorf("expected 2 snapshots deleted, got %d: %v", len(deletedIDs), deletedIDs) + } + idSet := make(map[string]bool) + for _, id := range deletedIDs { + idSet[id] = true + } + for _, snap := range snaps { + if !idSet[snap.ID] { + t.Errorf("expected snapshot %s to be deleted", snap.ID) + } + } +} + +// Scenario: Transient HTTP 409 error → warning, continues, verification triggered +func TestDeleteSnapshots_TransientError409_ContinuesAndVerifies(t *testing.T) { + srv := newSnapshotTestServer(t) + snaps := []cinderVolumeRecord{ + {ID: "snap-001", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + {ID: "snap-002", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}, + } + srv.snapshotLists = [][]cinderVolumeRecord{snaps, {}} + srv.deleteStatus["snap-001"] = http.StatusConflict + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err != nil { + t.Fatalf("expected no error for transient HTTP 409, got: %v", err) + } + + srv.mu.Lock() + attempted := len(srv.deletedSnapshots) + getCount := srv.snapshotGetCount + srv.mu.Unlock() + + if attempted != 2 { + t.Errorf("expected both snapshots to be attempted, got %d DELETE calls", attempted) + } + if getCount < 2 { + t.Errorf("expected at least 2 GET calls (list + verification), got %d", getCount) + } +} + +// Scenario: HTTP 500 error → exception propagated +func TestDeleteSnapshots_HTTP500_PropagatesError(t *testing.T) { + srv := newSnapshotTestServer(t) + srv.snapshotLists = [][]cinderVolumeRecord{ + {{ID: "snap-server-error", Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}}}, + } + srv.deleteStatus["snap-server-error"] = http.StatusInternalServerError + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Snapshot still present after deletion → error returned +func TestDeleteSnapshots_StillPresentAfterDeletion_ReturnsError(t *testing.T) { + srv := newSnapshotTestServer(t) + snap := cinderVolumeRecord{ + ID: "snap-persistent", + Metadata: map[string]string{"cinder.csi.openstack.org/cluster": "mycluster"}, + } + srv.snapshotLists = [][]cinderVolumeRecord{{snap}, {snap}} + + session := srv.authenticate(t) + err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster") + + if err == nil { + t.Fatal("expected error when snapshot persists after deletion") + } + if !strings.Contains(err.Error(), "mycluster") { + t.Errorf("expected error to mention cluster name, got: %v", err) + } +} + +// ── Identity (AppCredential) mock server ────────────────────────────────────── + +// identityTestServer is a mock OpenStack server handling Keystone auth + +// the Identity API for application credentials. The catalog advertises the +// server's own URL as the "identity" endpoint. +type identityTestServer struct { + *httptest.Server + mu sync.Mutex + deleteStatus int // HTTP status for the DELETE request; 0 means 204 + deletedAppcredID string +} + +func newIdentityTestServer(t *testing.T) *identityTestServer { + t.Helper() + srv := &identityTestServer{} + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-identity-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + // Self-referential catalog: "identity" endpoint = this server. + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": "identity", + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + // Identity: DELETE /v3/users/{userID}/application_credentials/{appcredID} + mux.HandleFunc("/v3/users/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + parts := strings.Split(r.URL.Path, "/application_credentials/") + srv.mu.Lock() + if len(parts) == 2 { + srv.deletedAppcredID = parts[1] + } + status := srv.deleteStatus + if status == 0 { + status = http.StatusNoContent + } + srv.mu.Unlock() + w.WriteHeader(status) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +// authenticate returns both the session and the cloudsYAML used to build it, +// so the same YAML can be passed to DeleteAppCredential. +func (srv *identityTestServer) authenticate(t *testing.T) (*openstack.Session, string) { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: test-appcred-id + application_credential_secret: test-secret + interface: public + region_name: RegionOne +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session with identity endpoint") + } + session.SleepFunc = func(d time.Duration) {} + return session, cloudsYAML +} + +// ── Epic 7: Application Credential Management ───────────────────────────────── + +// ── US7.1: Delete the OpenStack Application Credential ─────────────────────── + +// Scenario: Deletion authorised → HTTP 204, return nil +func TestDeleteAppCredential_SuccessfulDeletion(t *testing.T) { + srv := newIdentityTestServer(t) + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + srv.mu.Lock() + deleted := srv.deletedAppcredID + srv.mu.Unlock() + + if deleted != "test-appcred-id" { + t.Errorf("expected appcred-id %q to be deleted, got %q", "test-appcred-id", deleted) + } +} + +// Scenario: Application Credential already deleted → HTTP 404, return nil +func TestDeleteAppCredential_AlreadyDeleted_404(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusNotFound + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("expected nil for HTTP 404 (already deleted), got: %v", err) + } +} + +// Scenario: Application Credential cannot be deleted (403) → warning, return nil +func TestDeleteAppCredential_Forbidden_403(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusForbidden + session, cloudsYAML := srv.authenticate(t) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack"); err != nil { + t.Fatalf("expected nil for HTTP 403 (restricted), got: %v", err) + } +} + +// Scenario: HTTP 500 error → error propagated +func TestDeleteAppCredential_HTTP500_PropagatesError(t *testing.T) { + srv := newIdentityTestServer(t) + srv.deleteStatus = http.StatusInternalServerError + session, cloudsYAML := srv.authenticate(t) + + err := session.DeleteAppCredential(context.Background(), logr.Discard(), cloudsYAML, "openstack") + if err == nil { + t.Fatal("expected error for HTTP 500, got nil") + } +} + +// Scenario: Password auth has no application credential → deletion is a no-op +func TestDeleteAppCredential_PasswordAuth_NoOp(t *testing.T) { + srv := newIdentityTestServer(t) + session, _ := srv.authenticate(t) + + passwordYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3password + auth: + auth_url: %s + username: alice + password: s3cret + project_id: proj-123 + user_domain_name: Default + interface: public + region_name: RegionOne +`, srv.URL) + + if err := session.DeleteAppCredential(context.Background(), logr.Discard(), passwordYAML, "openstack"); err != nil { + t.Fatalf("expected nil for password auth (no appcred), got: %v", err) + } + + srv.mu.Lock() + deleted := srv.deletedAppcredID + srv.mu.Unlock() + if deleted != "" { + t.Errorf("expected no DELETE call, but appcred %q was deleted", deleted) + } +} + +// Scenario: No "identity" endpoint in catalog → CatalogError +func TestDeleteAppCredential_NoIdentityEndpoint_ReturnsCatalogError(t *testing.T) { + ks := newKeystoneServer(t) + ks.catalog = catalogWith( + serviceEntry("network", + endpoint("public", "RegionOne", "http://network.example.com"), + ), + ) + + clouds := buildCloudsYAML(ks.URL, "v3applicationcredential") + session, err := openstack.Authenticate(context.Background(), clouds, "openstack", "") + if err != nil { + t.Fatalf("unexpected authenticate error: %v", err) + } + + err = session.DeleteAppCredential(context.Background(), logr.Discard(), clouds, "openstack") + + if err == nil { + t.Fatal("expected CatalogError when identity endpoint is absent") + } + var target *openstack.CatalogError + if !errorAs(err, &target) { + t.Errorf("expected *CatalogError, got %T: %v", err, err) + } +} + +// Scenario: No matching snapshot → no deletion, no verification +func TestDeleteSnapshots_NothingToDelete_NoVerification(t *testing.T) { + srv := newSnapshotTestServer(t) + srv.snapshotLists = [][]cinderVolumeRecord{{}} + + session := srv.authenticate(t) + if err := session.DeleteSnapshots(context.Background(), logr.Discard(), "mycluster"); err != nil { + t.Fatalf("unexpected error with no snapshots: %v", err) + } + + srv.mu.Lock() + deletedCount := len(srv.deletedSnapshots) + getCount := srv.snapshotGetCount + srv.mu.Unlock() + + if deletedCount != 0 { + t.Errorf("expected no DELETE calls, got %d: %v", deletedCount, srv.deletedSnapshots) + } + if getCount != 1 { + t.Errorf("expected exactly 1 GET call (no verification), got %d", getCount) + } +} diff --git a/internal/openstack/robustness_test.go b/internal/openstack/robustness_test.go new file mode 100644 index 0000000..1f227c6 --- /dev/null +++ b/internal/openstack/robustness_test.go @@ -0,0 +1,156 @@ +package openstack_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/internal/openstack" +) + +// ── US12.1 : Timeout HTTP ───────────────────────────────────────────────────── + +// Scenario: context already cancelled → Authenticate returns an error immediately +func TestAuthenticate_ReturnsError_WhenContextAlreadyCancelled(t *testing.T) { + // Use a real (if short-lived) server so Authenticate reaches the HTTP call. + ks := newKeystoneServer(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before the call + + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: cred-id + application_credential_secret: cred-secret +`, ks.URL) + + _, err := openstack.Authenticate(ctx, cloudsYAML, "openstack", "") + if err == nil { + t.Fatal("expected error for cancelled context, got nil") + } +} + +// ── US12.2 : Alias Cinder ──────────────────────────────────────────────────── + +// cinderAliasServer is a minimal Cinder mock that advertises a configurable +// service type in the catalog (allows testing "volume", "block-storage", etc.). +type cinderAliasServer struct { + *httptest.Server + mu sync.Mutex + volumeGetCount int +} + +func newCinderAliasServer(t *testing.T, serviceType string) *cinderAliasServer { + t.Helper() + srv := &cinderAliasServer{} + mux := http.NewServeMux() + + mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("X-Subject-Token", "test-alias-token") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "token": map[string]any{"user": map[string]any{"id": "test-user"}}, + }) + }) + + mux.HandleFunc("/v3/auth/catalog", func(w http.ResponseWriter, r *http.Request) { + selfURL := "http://" + r.Host + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "catalog": []any{ + map[string]any{ + "type": serviceType, // "volume", "block-storage", … + "endpoints": []any{ + map[string]any{ + "interface": "public", + "region_id": "RegionOne", + "url": selfURL, + }, + }, + }, + }, + }) + }) + + mux.HandleFunc("/volumes/detail", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + srv.mu.Lock() + srv.volumeGetCount++ + srv.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"volumes": []any{}}) + }) + + srv.Server = httptest.NewServer(mux) + t.Cleanup(srv.Server.Close) + return srv +} + +func (srv *cinderAliasServer) authenticate(t *testing.T) *openstack.Session { + t.Helper() + cloudsYAML := fmt.Sprintf(` +clouds: + openstack: + auth_type: v3applicationcredential + auth: + auth_url: %s + application_credential_id: cred-id + application_credential_secret: cred-secret +`, srv.URL) + session, err := openstack.Authenticate(context.Background(), cloudsYAML, "openstack", "") + if err != nil { + t.Fatalf("authenticate: %v", err) + } + if !session.IsAuthenticated() { + t.Fatal("expected authenticated session") + } + session.SleepFunc = func(d time.Duration) {} + return session +} + +// Scenario: catalog with "block-storage" → cinderEndpoint resolves the correct endpoint +func TestCinderEndpoint_FallsBackToBlockStorage(t *testing.T) { + srv := newCinderAliasServer(t, "block-storage") + session := srv.authenticate(t) + + err := session.DeleteVolumes(context.Background(), logr.Discard(), "test-cluster") + if err != nil { + t.Fatalf("DeleteVolumes with block-storage catalog: %v", err) + } + if srv.volumeGetCount == 0 { + t.Error("expected at least one GET /volumes/detail call") + } +} + +// Scenario: catalog with "volume" only → cinderEndpoint uses the legacy alias +func TestCinderEndpoint_FallsBackToVolumeAlias(t *testing.T) { + srv := newCinderAliasServer(t, "volume") + session := srv.authenticate(t) + + err := session.DeleteVolumes(context.Background(), logr.Discard(), "test-cluster") + if err != nil { + t.Fatalf("DeleteVolumes with volume alias catalog: %v", err) + } + if srv.volumeGetCount == 0 { + t.Error("expected at least one GET /volumes/detail call") + } +} diff --git a/internal/openstack/tls.go b/internal/openstack/tls.go new file mode 100644 index 0000000..0ba61ea --- /dev/null +++ b/internal/openstack/tls.go @@ -0,0 +1,19 @@ +package openstack + +import ( + "crypto/tls" + "crypto/x509" + "fmt" +) + +func loadCACert(cfg *tls.Config, cacert string) error { + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM([]byte(cacert)) { + return fmt.Errorf("failed to append CA certificate") + } + cfg.RootCAs = pool + return nil +} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..77b9bd2 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,80 @@ +{ pkgs ? import ./nixpkgs.nix }: + +let + # Build the manager binary for a given package set (native or cross). + buildManager = p: p.buildGoModule { + pname = "cluster-api-janitor-openstack"; + version = "0.0.0-dev"; + src = ../.; + subPackages = [ "cmd" ]; + env.CGO_ENABLED = "0"; + ldflags = [ "-s" "-w" ]; + # Run `nix-build nix -A manager` once; it will fail and print the real hash. + vendorHash = "sha256-5p5z+fzRkBk6rIb3DWwA3jsF4MdMVAwKHz7xza09fCc="; + postInstall = '' + mv $out/bin/cmd $out/bin/manager + ''; + meta.mainProgram = "manager"; + }; + + # Build a layered OCI image for a given package set. + buildImage = p: m: p.dockerTools.buildLayeredImage { + name = "ghcr.io/azimuth-cloud/cluster-api-janitor-openstack"; + tag = "latest"; + contents = [ pkgs.cacert m ]; + config = { + Entrypoint = [ "/bin/manager" ]; + ExposedPorts."8081/tcp" = {}; + User = "65532:65532"; + Labels = { + "org.opencontainers.image.source" = + "https://github.com/azimuth-cloud/cluster-api-janitor-openstack"; + "org.opencontainers.image.licenses" = "Apache-2.0"; + }; + }; + }; + + manager = buildManager pkgs; + image = buildImage pkgs manager; + + # arm64 cross-compiled on an amd64 host. + crossPkgs = pkgs.pkgsCross.aarch64-multiplatform; + manager-arm64 = buildManager crossPkgs; + image-arm64 = buildImage crossPkgs manager-arm64; + + # SBOM — reads Go build-info embedded in the static binary (survives -s -w). + sbom = pkgs.runCommand "sbom.cdx.json" { + nativeBuildInputs = [ pkgs.syft ]; + } '' + syft scan ${manager}/bin/manager \ + --output cyclonedx-json \ + --file $out + ''; + + # CI check: go fmt + go vet + unit tests (native only — arm64 cross tests cannot + # run on an amd64 host, so doCheck is NOT set in buildManager itself). + tests = (buildManager pkgs).overrideAttrs (_: { + pname = "cluster-api-janitor-openstack-tests"; + subPackages = []; # build all packages, not just cmd/ + doCheck = true; + checkPhase = '' + runHook preCheck + + bad=$(gofmt -l $(find . -name '*.go' \ + -not -path './vendor/*' -not -path './.git/*')) + if [ -n "$bad" ]; then + echo "Files not formatted with go fmt:" + printf '%s\n' $bad + exit 1 + fi + + go vet ./... + + go test -v $(go list ./... | grep -v '/test/e2e') + + runHook postCheck + ''; + installPhase = "touch $out"; + }); + +in { inherit manager image image-arm64 sbom tests; } diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix new file mode 100644 index 0000000..45a564c --- /dev/null +++ b/nix/nixpkgs.nix @@ -0,0 +1,5 @@ +import (builtins.fetchTarball { + # nixos-26.05 — require Go 1.26+ + # To update: nix-prefetch-url --unpack https://github.com/NixOS/nixpkgs/archive/.tar.gz + url = "https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-26.05.tar.gz"; +}) {} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index b68657a..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,44 +0,0 @@ -[tool.ruff] -line-length = 88 -target-version = "py312" - -[tool.ruff.format] -line-ending = "lf" - -[tool.ruff.lint] -select = [ - # pycodestyle - "E", - # Pyflakes - "F", - # flake8-builtins - "A", - # Ruff rules - "RUF", - # flake8-async - "ASYNC", - # pyupgrade - "UP", - # tidy imports - "TID", - # sorted imports - "I", - # check complexity - "C90", - # pep8 naming - "N", -] - -ignore = [ - "UP017", # remove and fix once 3.10 not tested in CI -] - -[tool.mypy] -follow_imports = "silent" -warn_redundant_casts = true -warn_unused_ignores = true -check_untyped_defs = true - -[tool.ruff.lint.mccabe] -# Flag errors (`C901`) whenever the complexity level exceeds 5. -max-complexity = 14 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 868cbe1..0000000 --- a/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -aiohappyeyeballs==2.7.1 -aiohttp==3.14.1 -aiosignal==1.4.0 -anyio==4.14.1 -async-timeout==5.0.1 -attrs==26.1.0 -certifi==2026.6.17 -click==8.4.2 -easykube==0.6.0 -exceptiongroup==1.3.1 -frozenlist==1.8.0 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.18 -iso8601==2.1.0 -kopf==1.44.6 -multidict==6.7.1 -propcache==0.5.2 -python-json-logger==4.1.0 -PyYAML==6.0.3 -sniffio==1.3.1 -typing_extensions==4.16.0 -yarl==1.24.2 diff --git a/setup.cfg b/setup.cfg deleted file mode 100755 index 17d5b3f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[metadata] -name = cluster-api-janitor-openstack -version = 0.1.0 -description = Kubernetes operator that cleans up external resources for Cluster API OpenStack clusters. -author = Matt Pryor -author_email = matt@stackhpc.com -url = https://github.com/azimuth-cloud/cluster-api-janitor-openstack -python-requires = ">=3.10" - -[options] -zip_safe = False -include_package_data = True -packages = find_namespace: -install_requires = - easykube - httpx - kopf - pyyaml diff --git a/setup.py b/setup.py deleted file mode 100755 index bac24a4..0000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import setuptools - -if __name__ == "__main__": - setuptools.setup() diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 0000000..bf935ec --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,122 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/test/utils" +) + +var ( + // managerImage is the manager image to be built and loaded for testing. + // Must match the name/tag hardcoded in nix/default.nix's `image` derivation. + managerImage = "ghcr.io/azimuth-cloud/cluster-api-janitor-openstack:latest" + // shouldCleanupCertManager tracks whether CertManager was installed by this suite. + shouldCleanupCertManager = false +) + +// TestE2E runs the e2e test suite to validate the solution in an isolated environment. +// The default setup requires Kind and CertManager. +// +// To enable kubectl kuberc (use custom kubectl configurations), set: KUBECTL_KUBERC=true +// By default, kuberc is disabled to ensure consistent test behavior across different environments. +// To skip CertManager installation, set: CERT_MANAGER_INSTALL_SKIP=true +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting cluster-api-janitor-openstack e2e test suite\n") + RunSpecs(t, "e2e suite") +} + +var _ = BeforeSuite(func() { + By("building the manager image (Nix)") + imageArchive := filepath.Join(os.TempDir(), "cluster-api-janitor-openstack-e2e-image.tar.gz") + cmd := exec.Command("nix-build", "nix", "-A", "image", "-o", imageArchive) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") + + // TODO(user): If you want to change the e2e test vendor from Kind, + // ensure the image is built and available, then remove the following block. + By("loading the manager image on Kind") + err = utils.LoadImageArchiveToKindClusterWithName(imageArchive) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + + configureKubectlKubeRC() + setupCertManager() +}) + +var _ = AfterSuite(func() { + teardownCertManager() +}) + +// Disable kubectl kuberc by default for test isolation. +// This prevents local kubectl configurations from affecting test behavior. +// To enable kuberc, set: KUBECTL_KUBERC=true +func configureKubectlKubeRC() { + if os.Getenv("KUBECTL_KUBERC") != "true" { + By("disabling kubectl kuberc for test isolation") + err := os.Setenv("KUBECTL_KUBERC", "false") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to disable kubectl kuberc") + _, _ = fmt.Fprintf(GinkgoWriter, + "kubectl kuberc disabled for consistent test behavior (override with KUBECTL_KUBERC=true)\n") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "kubectl kuberc enabled (KUBECTL_KUBERC=true)\n") + } +} + +// setupCertManager installs CertManager if needed for webhook tests. +// Skips installation if CERT_MANAGER_INSTALL_SKIP=true or if already present. +func setupCertManager() { + if os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager installation (CERT_MANAGER_INSTALL_SKIP=true)\n") + return + } + + By("checking if CertManager is already installed") + if utils.IsCertManagerCRDsInstalled() { + _, _ = fmt.Fprintf(GinkgoWriter, "CertManager is already installed. Skipping installation.\n") + return + } + + // Mark for cleanup before installation to handle interruptions and partial installs. + shouldCleanupCertManager = true + + By("installing CertManager") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") +} + +// teardownCertManager uninstalls CertManager if it was installed by setupCertManager. +// This ensures we only remove what we installed. +func teardownCertManager() { + if !shouldCleanupCertManager { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager cleanup (not installed by this suite)\n") + return + } + + By("uninstalling CertManager") + utils.UninstallCertManager() +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..1aa9d47 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,339 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/azimuth-cloud/cluster-api-janitor-openstack/test/utils" +) + +// namespace where the project is deployed in +const namespace = "cluster-api-janitor-openstack-system" + +// serviceAccountName created for the project +const serviceAccountName = "cluster-api-janitor-openstack-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "cluster-api-janitor-openstack-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "cluster-api-janitor-openstack-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + By("getting the name of the controller-manager pod") + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + By("validating the pod's status") + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=cluster-api-janitor-openstack-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("ensuring the controller pod is ready") + verifyControllerPodReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("True"), "Controller pod not ready") + } + Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Serving metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed()) + + // +kubebuilder:scaffold:e2e-metrics-webhooks-readiness + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": [ + "for i in $(seq 1 30); do curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics && exit 0 || sleep 2; done; exit 1" + ], + "securityContext": { + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccountName": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + verifyMetricsAvailable := func(g Gomega) { + metricsOutput, err := getMetricsOutput() + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + g.Expect(metricsOutput).NotTo(BeEmpty()) + g.Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + } + Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput, err := getMetricsOutput() + // Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) + }) +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + By("creating temporary file to store the token request") + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + By("executing kubectl command to create the token") + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + By("parsing the JSON output to extract the token") + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() (string, error) { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + return utils.Run(cmd) +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 0000000..0367c9f --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,228 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck +) + +const ( + certmanagerVersion = "v1.20.2" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" + + defaultKindBinary = "kind" + defaultKindCluster = "kind" +) + +func warnError(err error) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + } + + return string(output), nil +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } + + // Delete leftover leases in kube-system (not cleaned by default) + kubeSystemLeases := []string{ + "cert-manager-cainjector-leader-election", + "cert-manager-controller", + } + for _, lease := range kubeSystemLeases { + cmd = exec.Command("kubectl", "delete", "lease", lease, + "-n", "kube-system", "--ignore-not-found", "--force", "--grace-period=0") + if _, err := Run(cmd); err != nil { + warnError(err) + } + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageArchiveToKindClusterWithName loads an OCI image archive, as +// produced by `nix-build nix -A image`, into the kind cluster. Unlike +// `kind load docker-image`, this does not require a local Docker daemon. +func LoadImageArchiveToKindClusterWithName(archivePath string) error { + cluster := defaultKindCluster + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "image-archive", archivePath, "--name", cluster} + kindBinary := defaultKindBinary + if v, ok := os.LookupEnv("KIND"); ok { + kindBinary = v + } + cmd := exec.Command(kindBinary, kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.SplitSeq(output, "\n") + for element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, fmt.Errorf("failed to get current working directory: %w", err) + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncommented", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +} diff --git a/tox.ini b/tox.ini deleted file mode 100644 index eaaa863..0000000 --- a/tox.ini +++ /dev/null @@ -1,55 +0,0 @@ -[tox] -minversion = 4.0.0 -# We run autofix last, to ensure CI fails, -# even though we do our best to autofix locally -envlist = py3,ruff,codespell,mypy,autofix -skipsdist = True - -[testenv] -basepython = python3 -usedevelop = True -setenv = - PYTHONWARNINGS=default::DeprecationWarning - OS_STDOUT_CAPTURE=1 - OS_STDERR_CAPTURE=1 - OS_TEST_TIMEOUT=60 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = stestr run {posargs} - -[testenv:autofix] -commands = - ruff format {tox_root} - codespell {tox_root} -w --skip venv - ruff check {tox_root} --fix - -[testenv:black] -# TODO: understand why ruff doesn't fix -# line lengths as well as black does -commands = black {tox_root} --check - -[testenv:codespell] -commands = codespell --skip ./venv {posargs} - -[testenv:ruff] -description = Run Ruff checks -commands = - ruff check {tox_root} - ruff format {tox_root} --check - -[testenv:venv] -commands = {posargs} - -[testenv:cover] -setenv = - VIRTUAL_ENV={envdir} - PYTHON=coverage run --source capi_janitor --parallel-mode -commands = - stestr run {posargs} - coverage combine - coverage html -d cover - coverage xml -o cover/coverage.xml - coverage report - -[testenv:mypy] -commands = mypy {tox_root} {posargs}