diff --git a/charts/sn-platform-slim/templates/_helpers.tpl b/charts/sn-platform-slim/templates/_helpers.tpl index a26c81bc6..4d4c45a99 100644 --- a/charts/sn-platform-slim/templates/_helpers.tpl +++ b/charts/sn-platform-slim/templates/_helpers.tpl @@ -189,6 +189,33 @@ Pulsar Cluster Name. {{- end }} {{- end }} +{{/* +Package management values validation. +*/}} +{{- define "pulsar.functions.packageManagement.validate" -}} +{{- $pm := .Values.functions.packageManagement | default dict -}} +{{- if $pm.enabled -}} +{{- $functionmesh := .Values.broker.functionmesh | default dict -}} +{{- $mesh := $functionmesh.mesh | default dict -}} +{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}} +{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}} +{{- end -}} +{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} +{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} +{{- end -}} +{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}} +{{- end -}} +{{- if eq $provider "cloudStorage" -}} +{{- $cloud := $pm.cloudStorage | default dict -}} +{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Metadata provider selection. */}} diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml index 0e09467b9..b8f5fda88 100644 --- a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml @@ -3,6 +3,7 @@ # # deploy PulsarBroker only when `components.broker and operator.enabled` is true {{- if .Values.components.broker }} +{{- include "pulsar.functions.packageManagement.validate" . }} apiVersion: pulsar.streamnative.io/v1alpha1 kind: PulsarBroker metadata: @@ -336,10 +337,13 @@ spec: {{- end }} custom: PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet" + {{- $pm := .Values.functions.packageManagement | default dict }} {{- if include "pulsar.metadata.isOxia" . }} PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" + {{- if not $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "false" + {{- end }} PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true" PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true" PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true" @@ -351,6 +355,21 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} + {{- if $pm.enabled }} + PULSAR_PREFIX_enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }} + PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }} + PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} {{- with .Values.broker.configData }} {{ toYaml . | indent 6 }} {{- end }} diff --git a/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml b/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml index c83527543..c36f8d624 100644 --- a/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml +++ b/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml @@ -3,6 +3,7 @@ # {{- if .Values.components.functions }} +{{- include "pulsar.functions.packageManagement.validate" . }} ## function config map apiVersion: v1 kind: ConfigMap @@ -27,7 +28,26 @@ data: tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt" {{- end }} {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if and $pm.enabled (eq $pm.provider "bookkeeper") }} + numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }} + {{- else }} numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }} + {{- end }} + {{- if $pm.enabled }} + enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + packagesCloudStorageType: {{ $cloud.type | quote }} + packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} pulsarFunctionsCluster: {{ template "pulsar.fullname" . }} functionRuntimeFactoryConfigs: jobNamespace: {{ template "pulsar.functions.namespace" . }} diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index 2066f4e77..36c758ac7 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -1562,6 +1562,29 @@ functions: clusterRole: true autoRollDeployment: true annotations: {} + packageManagement: + # Whether to render explicit package management service config. + # Defaults to false to preserve existing behavior, especially for Oxia deployments. + enabled: false + # Storage provider selector. Valid values: bookkeeper, cloudStorage. + # When Oxia metadata is enabled, cloudStorage is required if package management is enabled. + provider: bookkeeper + bookkeeper: + storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider" + # Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty. + replicas: "" + ledgerRootPath: "/ledgers" + cloudStorage: + storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider" + # Valid values: S3, GCS, AZUREBLOB. + type: "" + # Examples: s3a://bucket, gs://bucket, abfs://container@account. + bucket: "" + # Examples: /packages, /sub/path. + bucketPath: "" + # Optional documentation-only compatibility field matching cloud-api-server. + # Helm does not parse this; set type, bucket, and bucketPath explicitly. + storagePath: "" ports: http: 8080 https: 8443 diff --git a/charts/sn-platform/templates/_helpers.tpl b/charts/sn-platform/templates/_helpers.tpl index 3681d51c4..467199cd4 100644 --- a/charts/sn-platform/templates/_helpers.tpl +++ b/charts/sn-platform/templates/_helpers.tpl @@ -187,6 +187,33 @@ Pulsar Cluster Name. {{- end }} {{- end }} +{{/* +Package management values validation. +*/}} +{{- define "pulsar.functions.packageManagement.validate" -}} +{{- $pm := .Values.functions.packageManagement | default dict -}} +{{- if $pm.enabled -}} +{{- $functionmesh := .Values.broker.functionmesh | default dict -}} +{{- $mesh := $functionmesh.mesh | default dict -}} +{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}} +{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}} +{{- end -}} +{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} +{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} +{{- end -}} +{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}} +{{- end -}} +{{- if eq $provider "cloudStorage" -}} +{{- $cloud := $pm.cloudStorage | default dict -}} +{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Metadata provider selection. */}} diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml index 8a0df8834..35b56acde 100644 --- a/charts/sn-platform/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform/templates/broker/broker-cluster.yaml @@ -3,6 +3,7 @@ # # deploy PulsarBroker only when `components.broker and operator.enabled` is true {{- if .Values.components.broker }} +{{- include "pulsar.functions.packageManagement.validate" . }} apiVersion: pulsar.streamnative.io/v1alpha1 kind: PulsarBroker metadata: @@ -337,10 +338,13 @@ spec: {{- end }} custom: PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet" + {{- $pm := .Values.functions.packageManagement | default dict }} {{- if include "pulsar.metadata.isOxia" . }} PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" + {{- if not $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "false" + {{- end }} PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true" PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true" PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true" @@ -352,6 +356,21 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} + {{- if $pm.enabled }} + PULSAR_PREFIX_enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }} + PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }} + PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} {{- with .Values.broker.configData }} {{ toYaml . | indent 6 }} {{- end }} diff --git a/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml b/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml index c83527543..c36f8d624 100644 --- a/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml +++ b/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml @@ -3,6 +3,7 @@ # {{- if .Values.components.functions }} +{{- include "pulsar.functions.packageManagement.validate" . }} ## function config map apiVersion: v1 kind: ConfigMap @@ -27,7 +28,26 @@ data: tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt" {{- end }} {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if and $pm.enabled (eq $pm.provider "bookkeeper") }} + numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }} + {{- else }} numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }} + {{- end }} + {{- if $pm.enabled }} + enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + packagesCloudStorageType: {{ $cloud.type | quote }} + packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} pulsarFunctionsCluster: {{ template "pulsar.fullname" . }} functionRuntimeFactoryConfigs: jobNamespace: {{ template "pulsar.functions.namespace" . }} diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 9e7b72595..4c49e5172 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -1638,6 +1638,29 @@ functions: clusterRole: true autoRollDeployment: true annotations: {} + packageManagement: + # Whether to render explicit package management service config. + # Defaults to false to preserve existing behavior, especially for Oxia deployments. + enabled: false + # Storage provider selector. Valid values: bookkeeper, cloudStorage. + # When Oxia metadata is enabled, cloudStorage is required if package management is enabled. + provider: bookkeeper + bookkeeper: + storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider" + # Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty. + replicas: "" + ledgerRootPath: "/ledgers" + cloudStorage: + storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider" + # Valid values: S3, GCS, AZUREBLOB. + type: "" + # Examples: s3a://bucket, gs://bucket, abfs://container@account. + bucket: "" + # Examples: /packages, /sub/path. + bucketPath: "" + # Optional documentation-only compatibility field matching cloud-api-server. + # Helm does not parse this; set type, bucket, and bucketPath explicitly. + storagePath: "" ports: http: 8080 https: 8443