diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 95d30ffbb9..9879102a61 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -71,8 +71,8 @@ jobs: - name: Start the operator run: | if [[ "${{ inputs.test }}" == "test_flink_operator_ha.sh" ]]; then - sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml - sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml + sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/config.yaml + sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/config.yaml sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml fi if [[ "${{ inputs.test }}" == "test_dynamic_config.sh" ]]; then diff --git a/docs/content.zh/docs/development/guide.md b/docs/content.zh/docs/development/guide.md index 5edf352904..9983219724 100644 --- a/docs/content.zh/docs/development/guide.md +++ b/docs/content.zh/docs/development/guide.md @@ -121,7 +121,7 @@ basic-session-example-rest LoadBalancer 10.96.36.250 127.0.0.1 8081:30 ``` The operator picks up the default log and flink configurations from `/opt/flink/conf`. You can put the rest configuration parameters here: ```bash -cat /opt/flink/conf/flink-conf.yaml +cat /opt/flink/conf/config.yaml rest.port: 8081 rest.address: localhost kubernetes.rest-service.exposed.type: LoadBalancer diff --git a/docs/content.zh/docs/operations/configuration.md b/docs/content.zh/docs/operations/configuration.md index 8aff224aca..ba62e9a318 100644 --- a/docs/content.zh/docs/operations/configuration.md +++ b/docs/content.zh/docs/operations/configuration.md @@ -48,6 +48,23 @@ defaultConfiguration: kubernetes.operator.observer.progress-check.interval: 5 s ``` +The configuration can also be supplied in Flink's modern YAML 1.2 `config.yaml` format (introduced in Flink 1.19 and the only format Flink 2.0 accepts) by defining the `config.yaml` key under `defaultConfiguration`: + +``` +defaultConfiguration: + create: true + append: true + config.yaml: |+ + kubernetes.operator: + metrics.reporter.slf4j: + factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory + interval: 5 MINUTE + reconcile.interval: 15 s + observer.progress-check.interval: 5 s +``` + +When both `config.yaml` and `flink-conf.yaml` are set, `config.yaml` takes precedence. The operator always mounts the resolved configuration as `config.yaml`, so prefer the nested form shown above: a flat, dotted-key override of a key already present in the chart's default `conf/config.yaml` would otherwise be a duplicate key under Flink's strict YAML parser. + To learn more about metrics and logging configuration please refer to the dedicated [docs page]({{< ref "docs/operations/metrics-logging" >}}). ### Flink Version and Namespace specific defaults diff --git a/docs/content.zh/docs/operations/helm.md b/docs/content.zh/docs/operations/helm.md index 74eb2125d2..f49341a831 100644 --- a/docs/content.zh/docs/operations/helm.md +++ b/docs/content.zh/docs/operations/helm.md @@ -71,9 +71,9 @@ The configurable parameters of the Helm chart and which default values as detail | Parameters | Description | Default Value | |------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | defaultConfiguration.append | Whether to append configuration files with configs. | true | -| defaultConfiguration.config.yaml | The newer configuration file format for flink that will be enforced in Flink 2.0. Note this was introduced in flink 1.19. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | +| defaultConfiguration.config.yaml | Modern YAML 1.2 config format (introduced in Flink 1.19, required by Flink 2.0). Takes precedence over flink-conf.yaml when set; the resolved config is always mounted as config.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | | defaultConfiguration.create | Whether to enable default configuration to create for flink-kubernetes-operator. | true | -| defaultConfiguration.flink-conf.yaml | The default configuration of flink-conf.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | +| defaultConfiguration.flink-conf.yaml | Legacy flat config format. Used when config.yaml is not set; the resolved config is always mounted as config.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | | defaultConfiguration.log4j-console.properties | The default configuration of log4j-console.properties. | | | defaultConfiguration.log4j-operator.properties | The default configuration of log4j-operator.properties. | | | defaultConfiguration.logback-operator.xml | The default configuration of logback-operator.xml. Used when `logging.framework` is set to `logback`. | | diff --git a/docs/content/docs/development/guide.md b/docs/content/docs/development/guide.md index 5edf352904..9983219724 100644 --- a/docs/content/docs/development/guide.md +++ b/docs/content/docs/development/guide.md @@ -121,7 +121,7 @@ basic-session-example-rest LoadBalancer 10.96.36.250 127.0.0.1 8081:30 ``` The operator picks up the default log and flink configurations from `/opt/flink/conf`. You can put the rest configuration parameters here: ```bash -cat /opt/flink/conf/flink-conf.yaml +cat /opt/flink/conf/config.yaml rest.port: 8081 rest.address: localhost kubernetes.rest-service.exposed.type: LoadBalancer diff --git a/docs/content/docs/operations/configuration.md b/docs/content/docs/operations/configuration.md index 1839562999..1dddc9ffbb 100644 --- a/docs/content/docs/operations/configuration.md +++ b/docs/content/docs/operations/configuration.md @@ -48,6 +48,23 @@ defaultConfiguration: kubernetes.operator.observer.progress-check.interval: 5 s ``` +The configuration can also be supplied in Flink's modern YAML 1.2 `config.yaml` format (introduced in Flink 1.19 and the only format Flink 2.0 accepts) by defining the `config.yaml` key under `defaultConfiguration`: + +``` +defaultConfiguration: + create: true + append: true + config.yaml: |+ + kubernetes.operator: + metrics.reporter.slf4j: + factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory + interval: 5 MINUTE + reconcile.interval: 15 s + observer.progress-check.interval: 5 s +``` + +When both `config.yaml` and `flink-conf.yaml` are set, `config.yaml` takes precedence. The operator always mounts the resolved configuration as `config.yaml`, so prefer the nested form shown above: a flat, dotted-key override of a key already present in the chart's default `conf/config.yaml` would otherwise be a duplicate key under Flink's strict YAML parser. + To learn more about metrics and logging configuration please refer to the dedicated [docs page]({{< ref "docs/operations/metrics-logging" >}}). ### Flink Version and Namespace specific defaults diff --git a/docs/content/docs/operations/helm.md b/docs/content/docs/operations/helm.md index 07678aff1a..f49341a831 100644 --- a/docs/content/docs/operations/helm.md +++ b/docs/content/docs/operations/helm.md @@ -71,9 +71,9 @@ The configurable parameters of the Helm chart and which default values as detail | Parameters | Description | Default Value | |------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | defaultConfiguration.append | Whether to append configuration files with configs. | true | -| defaultConfiguration.config.yaml | The newer configuration file format for flink that will enforced in Flink 2.0. Note this was introduced in flink 1.19. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | +| defaultConfiguration.config.yaml | Modern YAML 1.2 config format (introduced in Flink 1.19, required by Flink 2.0). Takes precedence over flink-conf.yaml when set; the resolved config is always mounted as config.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | | defaultConfiguration.create | Whether to enable default configuration to create for flink-kubernetes-operator. | true | -| defaultConfiguration.flink-conf.yaml | The default configuration of flink-conf.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | +| defaultConfiguration.flink-conf.yaml | Legacy flat config format. Used when config.yaml is not set; the resolved config is always mounted as config.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE
kubernetes.operator.reconcile.interval: 15 s
kubernetes.operator.observer.progress-check.interval: 5 s | | defaultConfiguration.log4j-console.properties | The default configuration of log4j-console.properties. | | | defaultConfiguration.log4j-operator.properties | The default configuration of log4j-operator.properties. | | | defaultConfiguration.logback-operator.xml | The default configuration of logback-operator.xml. Used when `logging.framework` is set to `logback`. | | diff --git a/helm/flink-kubernetes-operator/conf/flink-conf.yaml b/helm/flink-kubernetes-operator/conf/config.yaml similarity index 98% rename from helm/flink-kubernetes-operator/conf/flink-conf.yaml rename to helm/flink-kubernetes-operator/conf/config.yaml index 0f6cd74604..a04a8237a2 100644 --- a/helm/flink-kubernetes-operator/conf/flink-conf.yaml +++ b/helm/flink-kubernetes-operator/conf/config.yaml @@ -16,10 +16,6 @@ # limitations under the License. ################################################################################ -# Flink job/cluster related configs -taskmanager.numberOfTaskSlots: 1 -parallelism.default: 1 - # These parameters are required for Java 17 support. # Flink 1.18 uses env.java.opts.all, if a user supplies their own version of these opts in their FlinkDeployment the options below will be overridden. # env.java.default-opts.all is used for 1.19 onwards so users can supply their own opts.all in their Job deployments and have these appended. diff --git a/helm/flink-kubernetes-operator/templates/controller/configmap.yaml b/helm/flink-kubernetes-operator/templates/controller/configmap.yaml index 5ab29cd876..18f61e9b8c 100644 --- a/helm/flink-kubernetes-operator/templates/controller/configmap.yaml +++ b/helm/flink-kubernetes-operator/templates/controller/configmap.yaml @@ -25,24 +25,14 @@ metadata: data: config.yaml: |+ {{- if .Values.defaultConfiguration.append }} - {{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}} + {{- $.Files.Get "conf/config.yaml" | nindent 4 -}} {{- end }} -{{- if hasKey (.Values.defaultConfiguration) "config.yaml" }} - {{- index (.Values.defaultConfiguration) "config.yaml" | nindent 4 -}} -{{- end }} -{{- if .Values.watchNamespaces }} - kubernetes.operator.watched.namespaces: {{ join "," .Values.watchNamespaces }} -{{- end }} -{{- if index .Values "operatorHealth" }} - kubernetes.operator.health.probe.enabled: true - kubernetes.operator.health.probe.port: {{ .Values.operatorHealth.port }} -{{- end }} - flink-conf.yaml: |+ -{{- if .Values.defaultConfiguration.append }} - {{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}} -{{- end }} -{{- if hasKey (.Values.defaultConfiguration) "flink-conf.yaml" }} - {{- index (.Values.defaultConfiguration) "flink-conf.yaml" | nindent 4 -}} +{{- with (index .Values.defaultConfiguration "config.yaml") }} + {{- . | nindent 4 -}} +{{- else }} + {{- with (index .Values.defaultConfiguration "flink-conf.yaml") }} + {{- . | nindent 4 -}} + {{- end }} {{- end }} {{- if .Values.watchNamespaces }} kubernetes.operator.watched.namespaces: {{ join "," .Values.watchNamespaces }} diff --git a/helm/flink-kubernetes-operator/templates/controller/deployment.yaml b/helm/flink-kubernetes-operator/templates/controller/deployment.yaml index a31175c9bb..036e271613 100644 --- a/helm/flink-kubernetes-operator/templates/controller/deployment.yaml +++ b/helm/flink-kubernetes-operator/templates/controller/deployment.yaml @@ -244,13 +244,8 @@ spec: configMap: name: flink-operator-config items: - {{- if hasKey .Values.defaultConfiguration "flink-conf.yaml" }} - - key: flink-conf.yaml - path: flink-conf.yaml - {{- else }} - key: config.yaml path: config.yaml - {{- end }} {{- if eq .Values.logging.framework "logback" }} - key: logback-operator.xml path: logback-operator.xml diff --git a/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml b/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml index 90476695aa..74ec611173 100644 --- a/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml +++ b/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml @@ -68,3 +68,64 @@ tests: asserts: - isNotNull: path: data["log4j-console.properties"] + + - it: Emits a single config.yaml key with today's effective defaults and no redundant seed keys + asserts: + - isNotNull: + path: data["config.yaml"] + - notExists: + path: data["flink-conf.yaml"] + - matchRegex: + path: data["config.yaml"] + pattern: "(?m)^\\s*kubernetes.operator.metrics.reporter.slf4j.factory.class:" + - matchRegex: + path: data["config.yaml"] + pattern: "(?m)^\\s*kubernetes.operator.reconcile.interval: 15 s" + - matchRegex: + path: data["config.yaml"] + pattern: "(?m)^\\s*kubernetes.operator.health.probe.enabled: true" + - notMatchRegex: + path: data["config.yaml"] + pattern: "(?m)^\\s*taskmanager.numberOfTaskSlots:" + - notMatchRegex: + path: data["config.yaml"] + pattern: "(?m)^\\s*parallelism.default:" + + - it: A user flink-conf.yaml override flows into config.yaml when config.yaml is unset + set: + defaultConfiguration: + flink-conf.yaml: |+ + kubernetes.operator.reconcile.interval: 99 s + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "reconcile.interval: 99 s" + + - it: When both are set, config.yaml wins and flink-conf.yaml is ignored + set: + defaultConfiguration: + config.yaml: |+ + kubernetes.operator: + reconcile.interval: 30 s + flink-conf.yaml: |+ + kubernetes.operator.reconcile.interval: 99 s + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "reconcile.interval: 30 s" + - notMatchRegex: + path: data["config.yaml"] + pattern: "reconcile.interval: 99 s" + + - it: A flat config.yaml override of a former seed key is not duplicated (relies on dropping it from the seed) + set: + defaultConfiguration: + config.yaml: |+ + taskmanager.numberOfTaskSlots: 2 + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "taskmanager.numberOfTaskSlots: 2" + - notMatchRegex: + path: data["config.yaml"] + pattern: "taskmanager.numberOfTaskSlots: 1" diff --git a/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml b/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml index e6333d76ea..08dda10d6e 100644 --- a/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml +++ b/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml @@ -383,3 +383,16 @@ tests: key: log4j-operator.properties path: log4j-operator.properties +- it: Should mount the operator config as config.yaml by default + asserts: + - contains: + path: spec.template.spec.volumes[0].configMap.items + content: + key: config.yaml + path: config.yaml + - notContains: + path: spec.template.spec.volumes[0].configMap.items + content: + key: flink-conf.yaml + path: flink-conf.yaml + diff --git a/helm/flink-kubernetes-operator/values.yaml b/helm/flink-kubernetes-operator/values.yaml index 8eaa936ec8..aecf1d01f7 100644 --- a/helm/flink-kubernetes-operator/values.yaml +++ b/helm/flink-kubernetes-operator/values.yaml @@ -174,7 +174,16 @@ defaultConfiguration: # This option has not effect, if create is equal to false. append: true -# Uncomment to use the new config.yaml format, but also comment out the flink-config.yaml key. +# Operator configuration overrides go under ONE of the two keys below. If both are set, +# config.yaml wins; either way the operator mounts the result as config.yaml. +# +# config.yaml - modern YAML 1.2 format (Flink 1.19+, required by Flink 2.0). Recommended. +# flink-conf.yaml - legacy flat "key: value" format (shipped as the default below). +# +# Prefer the nested config.yaml form: a flat, dotted-key override of a key the chart already +# ships (e.g. the Java 17 opts in conf/config.yaml) would be a duplicate key under Flink's +# strict YAML parser and fail at operator startup. Example, equivalent to the default below: +# # config.yaml: |+ # kubernetes.operator: # metrics.reporter.slf4j: