diff --git a/api/v1/kustomization_types.go b/api/v1/kustomization_types.go index 945626983..a4620f66a 100644 --- a/api/v1/kustomization_types.go +++ b/api/v1/kustomization_types.go @@ -199,6 +199,13 @@ type KustomizationSpec struct { // The expressions are evaluated only when Wait or HealthChecks are specified. // +optional HealthCheckExprs []kustomize.CustomHealthCheck `json:"healthCheckExprs,omitempty"` + + // Ignore is a list of rules for specifying which changes to ignore + // during drift detection. These rules are applied to the resources managed + // by the Kustomization and are used to exclude specific JSON pointer paths + // from the drift detection and apply process. + // +optional + Ignore []IgnoreRule `json:"ignore,omitempty"` } // BuildMetadataOption defines the supported buildMetadata options. @@ -225,6 +232,22 @@ type CommonMetadata struct { Labels map[string]string `json:"labels,omitempty"` } +// IgnoreRule defines a rule to selectively disregard specific changes during +// the drift detection process. +type IgnoreRule struct { + // Paths is a list of JSON Pointer (RFC 6901) paths to be excluded from + // consideration in a Kubernetes object. + // +required + Paths []string `json:"paths"` + + // Target is a selector for specifying Kubernetes objects to which this + // rule applies. + // If Target is not set, the Paths will be ignored for all Kubernetes + // objects within the manifest of the Kustomization. + // +optional + Target *kustomize.Selector `json:"target,omitempty"` +} + // Decryption defines how decryption is handled for Kubernetes manifests. type Decryption struct { // Provider is the name of the decryption engine. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 19dfacc25..b48fccea1 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -91,6 +91,31 @@ func (in *Decryption) DeepCopy() *Decryption { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IgnoreRule) DeepCopyInto(out *IgnoreRule) { + *out = *in + if in.Paths != nil { + in, out := &in.Paths, &out.Paths + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Target != nil { + in, out := &in.Target, &out.Target + *out = new(kustomize.Selector) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IgnoreRule. +func (in *IgnoreRule) DeepCopy() *IgnoreRule { + if in == nil { + return nil + } + out := new(IgnoreRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Kustomization) DeepCopyInto(out *Kustomization) { *out = *in @@ -222,6 +247,13 @@ func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) { *out = make([]kustomize.CustomHealthCheck, len(*in)) copy(*out, *in) } + if in.Ignore != nil { + in, out := &in.Ignore, &out.Ignore + *out = make([]IgnoreRule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KustomizationSpec. diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index d079db4d0..1f77cb8d5 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -231,6 +231,73 @@ spec: - name type: object type: array + ignore: + description: |- + Ignore is a list of rules for specifying which changes to ignore + during drift detection. These rules are applied to the resources managed + by the Kustomization and are used to exclude specific JSON pointer paths + from the drift detection and apply process. + items: + description: |- + IgnoreRule defines a rule to selectively disregard specific changes during + the drift detection process. + properties: + paths: + description: |- + Paths is a list of JSON Pointer (RFC 6901) paths to be excluded from + consideration in a Kubernetes object. + items: + type: string + type: array + target: + description: |- + Target is a selector for specifying Kubernetes objects to which this + rule applies. + If Target is not set, the Paths will be ignored for all Kubernetes + objects within the manifest of the Kustomization. + properties: + annotationSelector: + description: |- + AnnotationSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource annotations. + type: string + group: + description: |- + Group is the API group to select resources from. + Together with Version and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + kind: + description: |- + Kind of the API Group to select resources from. + Together with Group and Version it is capable of unambiguously + identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + labelSelector: + description: |- + LabelSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource labels. + type: string + name: + description: Name to match resources with. + type: string + namespace: + description: Namespace to select resources from. + type: string + version: + description: |- + Version of the API Group to select resources from. + Together with Group and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + type: object + required: + - paths + type: object + type: array ignoreMissingComponents: description: |- IgnoreMissingComponents instructs the controller to ignore Components paths diff --git a/docs/api/v1/kustomize.md b/docs/api/v1/kustomize.md index baccead81..f29924825 100644 --- a/docs/api/v1/kustomize.md +++ b/docs/api/v1/kustomize.md @@ -442,6 +442,23 @@ health of custom resources using Common Expression Language (CEL). The expressions are evaluated only when Wait or HealthChecks are specified.

+ + +ignore
+ + +[]IgnoreRule + + + + +(Optional) +

Ignore is a list of rules for specifying which changes to ignore +during drift detection. These rules are applied to the resources managed +by the Kustomization and are used to exclude specific JSON pointer paths +from the drift detection and apply process.

+ + @@ -647,6 +664,57 @@ field.

+

IgnoreRule +

+

+(Appears on: +KustomizationSpec) +

+

IgnoreRule defines a rule to selectively disregard specific changes during +the drift detection process.

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+paths
+ +[]string + +		 +

Paths is a list of JSON Pointer (RFC 6901) paths to be excluded from +consideration in a Kubernetes object.

+
+target
+ + +github.com/fluxcd/pkg/apis/kustomize.Selector + + +		 +(Optional) +

Target is a selector for specifying Kubernetes objects to which this +rule applies. +If Target is not set, the Paths will be ignored for all Kubernetes +objects within the manifest of the Kustomization.

+
+
+

KustomizationSpec

@@ -1038,6 +1106,23 @@ health of custom resources using Common Expression Language (CEL). The expressions are evaluated only when Wait or HealthChecks are specified.

+ + +ignore
+ + +[]IgnoreRule + + + + +(Optional) +

Ignore is a list of rules for specifying which changes to ignore +during drift detection. These rules are applied to the resources managed +by the Kustomization and are used to exclude specific JSON pointer paths +from the drift detection and apply process.

+ + diff --git a/docs/spec/v1/kustomizations.md b/docs/spec/v1/kustomizations.md index bd6e6f693..481d24628 100644 --- a/docs/spec/v1/kustomizations.md +++ b/docs/spec/v1/kustomizations.md @@ -893,6 +893,113 @@ kustomize.toolkit.fluxcd.io/force: enabled This way, only the targeted resources are force-replaced when immutable field changes are made. The annotation should be removed after the change is applied. +### Ignore Rules + +`.spec.ignore` is an optional list used to selectively ignore changes +to specific fields during drift detection and correction. This allows external +controllers or tools to manage certain fields on Kubernetes resources without +having those changes reverted by the kustomize-controller during reconciliation. + +Each item in the list must have the following fields: + +- `paths` (required): A list of [JSON Pointer (RFC 6901)](https://datatracker.ietf.org/doc/html/rfc6901) + paths to exclude from drift detection. These paths refer to specific fields + within the Kubernetes object manifest. +- `target` (optional): A selector to scope the rule to specific Kubernetes + resources. If not set, the paths are ignored for all resources in the + Kustomization. + +**Warning:** Omitting the `target` selector causes the rule to match **all** +objects managed by the Kustomization. Always scope rules to specific resources +using `target` unless you intentionally want to ignore the specified paths +across every resource. + +The `target` selector supports the following fields: + +| Field | Description | +|----------------------|--------------------------------------| +| `group` | API group (regex) | +| `version` | API version (regex) | +| `kind` | Resource kind (regex) | +| `name` | Resource name (regex) | +| `namespace` | Resource namespace (regex) | +| `labelSelector` | Kubernetes label selector expression | +| `annotationSelector` | Kubernetes annotation selector expression | + +**Note:** The `group`, `version`, `kind`, `name`, and `namespace` fields +support regex patterns. The `labelSelector` and `annotationSelector` fields +use the standard Kubernetes +[label selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) +syntax. + +**Note:** For JSON Pointer paths that contain `/` in the key name (e.g. +annotation keys), the `/` must be escaped as `~1` per +[RFC 6901](https://datatracker.ietf.org/doc/html/rfc6901#section-3). +For example, the annotation `external-dns.alpha.kubernetes.io/hostname` +would be referenced as `/metadata/annotations/external-dns.alpha.kubernetes.io~1hostname`. + +To ignore fields only on resources that match a target selector: + +```yaml +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: app + namespace: flux-system +spec: + # ...omitted for brevity + ignore: + - paths: + - "/spec/replicas" + target: + kind: Deployment + - paths: + - "/metadata/annotations/external-dns.alpha.kubernetes.io~1hostname" + target: + kind: Service + name: my-service + - paths: + - "/spec/template/spec/containers/0/resources" + target: + kind: Deployment + name: my-app +``` + +In the above example: + +- The `/spec/replicas` field is ignored on all Deployments, allowing + an HPA or other autoscaler to manage the replica count without + interference from the kustomize-controller. +- The `external-dns.alpha.kubernetes.io/hostname` annotation is ignored + on a specific Service named `my-service`, allowing external-dns to + manage this annotation. +- The entire `/resources` subtree under container spec is ignored on + a specific Deployment named `my-app`, allowing a VPA or other resource + management tool to adjust container resources. + +**Note:** Changes to ignored fields alone do not trigger a reconciliation. +The controller excludes ignored paths when comparing the desired state against +the live object, so modifications made by external controllers to those fields +will not cause unnecessary applies or resource version bumps. + +**Important:** Drift ignore rules work with the +[server-side apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/) +field ownership model. When a reconciliation is triggered (e.g. by a source +revision change or drift in non-ignored fields), +the controller resolves each ignored path using one of two strategies based on +field ownership: + +- **Strip** — If the ignored field is owned by another Apply-type field manager + (e.g. another controller using server-side apply), the field is removed from + the apply payload. This relinquishes the controller's ownership and allows the + other manager to retain full control of the field. +- **Adopt** — If the controller is the sole Apply-type field manager for the + field, the in-cluster value is copied into the apply payload. This preserves + the current value without reverting changes made by Update-type operations + (e.g. `kubectl patch`, `kubectl edit`, or client-go Update calls) while + keeping the controller's field ownership intact. + ### KubeConfig (Remote clusters) With the `.spec.kubeConfig` field a Kustomization diff --git a/internal/controller/kustomization_controller.go b/internal/controller/kustomization_controller.go index c6aa48de1..095d0008e 100644 --- a/internal/controller/kustomization_controller.go +++ b/internal/controller/kustomization_controller.go @@ -63,6 +63,7 @@ import ( "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/statusreaders" "github.com/fluxcd/pkg/ssa" + "github.com/fluxcd/pkg/ssa/jsondiff" "github.com/fluxcd/pkg/ssa/normalize" ssautil "github.com/fluxcd/pkg/ssa/utils" "github.com/fluxcd/pkg/tar" @@ -859,6 +860,27 @@ func (r *KustomizationReconciler) apply(ctx context.Context, applyOpts.CustomStageKinds = r.CustomStageKinds applyOpts.MigrateAPIVersion = r.MigrateAPIVersion + if len(obj.Spec.Ignore) > 0 { + ignoreRules := make([]jsondiff.IgnoreRule, len(obj.Spec.Ignore)) + for i, rule := range obj.Spec.Ignore { + ignoreRules[i] = jsondiff.IgnoreRule{ + Paths: rule.Paths, + } + if rule.Target != nil { + ignoreRules[i].Selector = &jsondiff.Selector{ + Group: rule.Target.Group, + Version: rule.Target.Version, + Kind: rule.Target.Kind, + Name: rule.Target.Name, + Namespace: rule.Target.Namespace, + AnnotationSelector: rule.Target.AnnotationSelector, + LabelSelector: rule.Target.LabelSelector, + } + } + } + applyOpts.DriftIgnoreRules = ignoreRules + } + fieldManagers := []ssa.FieldManager{ { // to undo changes made with 'kubectl apply --server-side --force-conflicts' diff --git a/internal/controller/kustomization_drift_ignore_test.go b/internal/controller/kustomization_drift_ignore_test.go new file mode 100644 index 000000000..6e404109d --- /dev/null +++ b/internal/controller/kustomization_drift_ignore_test.go @@ -0,0 +1,196 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/testserver" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" +) + +func TestKustomizationReconciler_DriftIgnoreRules(t *testing.T) { + g := NewWithT(t) + id := "drift-ignore-" + randStringRunes(5) + revision := "v1.0.0" + + err := createNamespace(id) + g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + + err = createKubeConfigSecret(id) + g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret") + + manifests := func(name, data string) []testserver.File { + return []testserver.File{ + { + Name: "configmap.yaml", + Body: fmt.Sprintf(`--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: %[1]s +data: + managed-key: "%[2]s" + ignored-key: "original" +`, name, data), + }, + } + } + + artifact, err := testServer.ArtifactFromFiles(manifests(id, "v1")) + g.Expect(err).NotTo(HaveOccurred(), "failed to create artifact from files") + + repositoryName := types.NamespacedName{ + Name: fmt.Sprintf("drift-ignore-%s", randStringRunes(5)), + Namespace: id, + } + + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + kustomizationKey := types.NamespacedName{ + Name: fmt.Sprintf("drift-ignore-%s", randStringRunes(5)), + Namespace: id, + } + kustomization := &kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: kustomizationKey.Name, + Namespace: kustomizationKey.Namespace, + }, + Spec: kustomizev1.KustomizationSpec{ + Interval: metav1.Duration{Duration: reconciliationInterval}, + Path: "./", + KubeConfig: &meta.KubeConfigReference{ + SecretRef: &meta.SecretKeyReference{ + Name: "kubeconfig", + }, + }, + SourceRef: kustomizev1.CrossNamespaceSourceReference{ + Name: repositoryName.Name, + Namespace: repositoryName.Namespace, + Kind: sourcev1.GitRepositoryKind, + }, + TargetNamespace: id, + Force: false, + Ignore: []kustomizev1.IgnoreRule{ + { + Paths: []string{"/data/ignored-key"}, + }, + }, + }, + } + + g.Expect(k8sClient.Create(context.Background(), kustomization)).To(Succeed()) + + resultK := &kustomizev1.Kustomization{} + resultCM := &corev1.ConfigMap{} + + t.Run("creates configmap with initial data", func(t *testing.T) { + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + return resultK.Status.LastAppliedRevision == revision + }, timeout, time.Second).Should(BeTrue()) + logStatus(t, resultK) + + kstatusCheck.CheckErr(ctx, resultK) + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, resultCM)).Should(Succeed()) + g.Expect(resultCM.Data["managed-key"]).To(Equal("v1")) + g.Expect(resultCM.Data["ignored-key"]).To(Equal("original")) + }) + + t.Run("out-of-band change to ignored field is preserved on re-reconcile", func(t *testing.T) { + // Modify the ignored field out-of-band. + patch := client.RawPatch(types.MergePatchType, []byte(`{"data":{"ignored-key":"modified-externally"}}`)) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: id, + Namespace: id, + }, + } + err = k8sClient.Patch(context.Background(), configMap, patch) + g.Expect(err).NotTo(HaveOccurred()) + + // Verify the out-of-band change was applied. + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, resultCM)).Should(Succeed()) + g.Expect(resultCM.Data["ignored-key"]).To(Equal("modified-externally")) + + // Trigger reconciliation by updating the source revision. + artifact, err = testServer.ArtifactFromFiles(manifests(id, "v2")) + g.Expect(err).NotTo(HaveOccurred()) + revision = "v2.0.0" + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + // Wait for the new revision to be applied. + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + return resultK.Status.LastAppliedRevision == revision + }, timeout, time.Second).Should(BeTrue()) + logStatus(t, resultK) + + kstatusCheck.CheckErr(ctx, resultK) + + // Verify that the managed field was updated but the ignored field was preserved. + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, resultCM)).Should(Succeed()) + g.Expect(resultCM.Data["managed-key"]).To(Equal("v2")) + g.Expect(resultCM.Data["ignored-key"]).To(Equal("modified-externally")) + }) + + t.Run("non-ignored field change is reverted on re-reconcile", func(t *testing.T) { + // Modify the managed field out-of-band. + patch := client.RawPatch(types.MergePatchType, []byte(`{"data":{"managed-key":"tampered"}}`)) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: id, + Namespace: id, + }, + } + err = k8sClient.Patch(context.Background(), configMap, patch) + g.Expect(err).NotTo(HaveOccurred()) + + // Trigger reconciliation by updating the source. + artifact, err = testServer.ArtifactFromFiles(manifests(id, "v3")) + g.Expect(err).NotTo(HaveOccurred()) + revision = "v3.0.0" + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + return resultK.Status.LastAppliedRevision == revision + }, timeout, time.Second).Should(BeTrue()) + logStatus(t, resultK) + + kstatusCheck.CheckErr(ctx, resultK) + + // The managed field should be restored, the ignored field should still be preserved. + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, resultCM)).Should(Succeed()) + g.Expect(resultCM.Data["managed-key"]).To(Equal("v3")) + g.Expect(resultCM.Data["ignored-key"]).To(Equal("modified-externally")) + }) +}