From d519de6c70f0f7c1f7548b722a43e8563ea630f5 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 19 Jun 2026 11:00:23 +0800 Subject: [PATCH 1/3] feat(config): add package management support --- charts/sn-platform/templates/_helpers.tpl | 22 ++++++++++++++++++ .../templates/broker/broker-cluster.yaml | 17 ++++++++++++++ .../function-worker-configfile-configmap.yaml | 20 ++++++++++++++++ charts/sn-platform/values.yaml | 23 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/charts/sn-platform/templates/_helpers.tpl b/charts/sn-platform/templates/_helpers.tpl index 3681d51c4..46a0e3855 100644 --- a/charts/sn-platform/templates/_helpers.tpl +++ b/charts/sn-platform/templates/_helpers.tpl @@ -187,6 +187,28 @@ Pulsar Cluster Name. {{- end }} {{- end }} +{{/* +Package management values validation. +*/}} +{{- define "pulsar.functions.packageManagement.validate" -}} +{{- $pm := .Values.functions.packageManagement | default dict -}} +{{- if $pm.enabled -}} +{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} +{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} +{{- end -}} +{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}} +{{- end -}} +{{- if eq $provider "cloudStorage" -}} +{{- $cloud := $pm.cloudStorage | default dict -}} +{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Metadata provider selection. */}} diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml index 8a0df8834..b432fb962 100644 --- a/charts/sn-platform/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform/templates/broker/broker-cluster.yaml @@ -3,6 +3,7 @@ # # deploy PulsarBroker only when `components.broker and operator.enabled` is true {{- if .Values.components.broker }} +{{- include "pulsar.functions.packageManagement.validate" . }} apiVersion: pulsar.streamnative.io/v1alpha1 kind: PulsarBroker metadata: @@ -352,6 +353,22 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if $pm.enabled }} + PULSAR_PREFIX_enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }} + PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }} + PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} {{- with .Values.broker.configData }} {{ toYaml . | indent 6 }} {{- end }} diff --git a/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml b/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml index c83527543..c36f8d624 100644 --- a/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml +++ b/charts/sn-platform/templates/broker/function-worker-configfile-configmap.yaml @@ -3,6 +3,7 @@ # {{- if .Values.components.functions }} +{{- include "pulsar.functions.packageManagement.validate" . }} ## function config map apiVersion: v1 kind: ConfigMap @@ -27,7 +28,26 @@ data: tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt" {{- end }} {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if and $pm.enabled (eq $pm.provider "bookkeeper") }} + numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }} + {{- else }} numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }} + {{- end }} + {{- if $pm.enabled }} + enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + packagesCloudStorageType: {{ $cloud.type | quote }} + packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} pulsarFunctionsCluster: {{ template "pulsar.fullname" . }} functionRuntimeFactoryConfigs: jobNamespace: {{ template "pulsar.functions.namespace" . }} diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 9e7b72595..4c49e5172 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -1638,6 +1638,29 @@ functions: clusterRole: true autoRollDeployment: true annotations: {} + packageManagement: + # Whether to render explicit package management service config. + # Defaults to false to preserve existing behavior, especially for Oxia deployments. + enabled: false + # Storage provider selector. Valid values: bookkeeper, cloudStorage. + # When Oxia metadata is enabled, cloudStorage is required if package management is enabled. + provider: bookkeeper + bookkeeper: + storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider" + # Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty. + replicas: "" + ledgerRootPath: "/ledgers" + cloudStorage: + storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider" + # Valid values: S3, GCS, AZUREBLOB. + type: "" + # Examples: s3a://bucket, gs://bucket, abfs://container@account. + bucket: "" + # Examples: /packages, /sub/path. + bucketPath: "" + # Optional documentation-only compatibility field matching cloud-api-server. + # Helm does not parse this; set type, bucket, and bucketPath explicitly. + storagePath: "" ports: http: 8080 https: 8443 From 8ede9c70377e2d1565c81ab6eae06f428d46bc73 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Sun, 21 Jun 2026 10:07:17 +0800 Subject: [PATCH 2/3] feat(functions): add package management support --- .../sn-platform-slim/templates/_helpers.tpl | 22 ++++++++++++++++++ .../templates/broker/broker-cluster.yaml | 17 ++++++++++++++ .../function-worker-configfile-configmap.yaml | 20 ++++++++++++++++ charts/sn-platform-slim/values.yaml | 23 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/charts/sn-platform-slim/templates/_helpers.tpl b/charts/sn-platform-slim/templates/_helpers.tpl index a26c81bc6..e5888fc05 100644 --- a/charts/sn-platform-slim/templates/_helpers.tpl +++ b/charts/sn-platform-slim/templates/_helpers.tpl @@ -189,6 +189,28 @@ Pulsar Cluster Name. {{- end }} {{- end }} +{{/* +Package management values validation. +*/}} +{{- define "pulsar.functions.packageManagement.validate" -}} +{{- $pm := .Values.functions.packageManagement | default dict -}} +{{- if $pm.enabled -}} +{{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} +{{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} +{{- end -}} +{{- if and (include "pulsar.metadata.isOxia" .) (ne $provider "cloudStorage") -}} +{{- fail "functions.packageManagement.provider must be cloudStorage when Oxia metadata is enabled" -}} +{{- end -}} +{{- if eq $provider "cloudStorage" -}} +{{- $cloud := $pm.cloudStorage | default dict -}} +{{- $_ := required "functions.packageManagement.cloudStorage.type is required when provider=cloudStorage" $cloud.type -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucket is required when provider=cloudStorage" $cloud.bucket -}} +{{- $_ := required "functions.packageManagement.cloudStorage.bucketPath is required when provider=cloudStorage" $cloud.bucketPath -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Metadata provider selection. */}} diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml index 0e09467b9..6111c312b 100644 --- a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml @@ -3,6 +3,7 @@ # # deploy PulsarBroker only when `components.broker and operator.enabled` is true {{- if .Values.components.broker }} +{{- include "pulsar.functions.packageManagement.validate" . }} apiVersion: pulsar.streamnative.io/v1alpha1 kind: PulsarBroker metadata: @@ -351,6 +352,22 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if $pm.enabled }} + PULSAR_PREFIX_enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + PULSAR_PREFIX_packagesCloudStorageType: {{ $cloud.type | quote }} + PULSAR_PREFIX_packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + PULSAR_PREFIX_packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + PULSAR_PREFIX_packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + PULSAR_PREFIX_packagesReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize $bookkeeper.replicas | quote }} + PULSAR_PREFIX_packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} {{- with .Values.broker.configData }} {{ toYaml . | indent 6 }} {{- end }} diff --git a/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml b/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml index c83527543..c36f8d624 100644 --- a/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml +++ b/charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml @@ -3,6 +3,7 @@ # {{- if .Values.components.functions }} +{{- include "pulsar.functions.packageManagement.validate" . }} ## function config map apiVersion: v1 kind: ConfigMap @@ -27,7 +28,26 @@ data: tlsTrustCertsFilePath: "/pulsar/certs/ca/ca.crt" {{- end }} {{- end }} + {{- $pm := .Values.functions.packageManagement | default dict }} + {{- if and $pm.enabled (eq $pm.provider "bookkeeper") }} + numFunctionPackageReplicas: {{ default .Values.broker.configData.managedLedgerDefaultEnsembleSize .Values.functions.packageManagement.bookkeeper.replicas }} + {{- else }} numFunctionPackageReplicas: {{ .Values.broker.configData.managedLedgerDefaultEnsembleSize }} + {{- end }} + {{- if $pm.enabled }} + enablePackagesManagement: "true" + {{- if eq $pm.provider "cloudStorage" }} + {{- $cloud := $pm.cloudStorage | default dict }} + packagesManagementStorageProvider: {{ $cloud.storageProvider | quote }} + packagesCloudStorageType: {{ $cloud.type | quote }} + packagesCloudStorageBucket: {{ $cloud.bucket | quote }} + packagesCloudStorageBucketPath: {{ $cloud.bucketPath | quote }} + {{- else }} + {{- $bookkeeper := $pm.bookkeeper | default dict }} + packagesManagementStorageProvider: {{ $bookkeeper.storageProvider | quote }} + packagesManagementLedgerRootPath: {{ $bookkeeper.ledgerRootPath | quote }} + {{- end }} + {{- end }} pulsarFunctionsCluster: {{ template "pulsar.fullname" . }} functionRuntimeFactoryConfigs: jobNamespace: {{ template "pulsar.functions.namespace" . }} diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index 2066f4e77..36c758ac7 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -1562,6 +1562,29 @@ functions: clusterRole: true autoRollDeployment: true annotations: {} + packageManagement: + # Whether to render explicit package management service config. + # Defaults to false to preserve existing behavior, especially for Oxia deployments. + enabled: false + # Storage provider selector. Valid values: bookkeeper, cloudStorage. + # When Oxia metadata is enabled, cloudStorage is required if package management is enabled. + provider: bookkeeper + bookkeeper: + storageProvider: "org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider" + # Defaults to broker.configData.managedLedgerDefaultEnsembleSize when empty. + replicas: "" + ledgerRootPath: "/ledgers" + cloudStorage: + storageProvider: "io.streamnative.packages.storage.cloud.CloudStorageProvider" + # Valid values: S3, GCS, AZUREBLOB. + type: "" + # Examples: s3a://bucket, gs://bucket, abfs://container@account. + bucket: "" + # Examples: /packages, /sub/path. + bucketPath: "" + # Optional documentation-only compatibility field matching cloud-api-server. + # Helm does not parse this; set type, bucket, and bucketPath explicitly. + storagePath: "" ports: http: 8080 https: 8443 From 1c912fd193b804d13d1a18f6ed9865c6aa5e9e33 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 23 Jun 2026 08:46:10 +0800 Subject: [PATCH 3/3] refactor(broker): consolidate package management validation --- charts/sn-platform-slim/templates/_helpers.tpl | 5 +++++ charts/sn-platform-slim/templates/broker/broker-cluster.yaml | 4 +++- charts/sn-platform/templates/_helpers.tpl | 5 +++++ charts/sn-platform/templates/broker/broker-cluster.yaml | 4 +++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/charts/sn-platform-slim/templates/_helpers.tpl b/charts/sn-platform-slim/templates/_helpers.tpl index e5888fc05..4d4c45a99 100644 --- a/charts/sn-platform-slim/templates/_helpers.tpl +++ b/charts/sn-platform-slim/templates/_helpers.tpl @@ -195,6 +195,11 @@ Package management values validation. {{- define "pulsar.functions.packageManagement.validate" -}} {{- $pm := .Values.functions.packageManagement | default dict -}} {{- if $pm.enabled -}} +{{- $functionmesh := .Values.broker.functionmesh | default dict -}} +{{- $mesh := $functionmesh.mesh | default dict -}} +{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}} +{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}} +{{- end -}} {{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} {{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} {{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml index 6111c312b..b8f5fda88 100644 --- a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml @@ -337,10 +337,13 @@ spec: {{- end }} custom: PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet" + {{- $pm := .Values.functions.packageManagement | default dict }} {{- if include "pulsar.metadata.isOxia" . }} PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" + {{- if not $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "false" + {{- end }} PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true" PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true" PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true" @@ -352,7 +355,6 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} - {{- $pm := .Values.functions.packageManagement | default dict }} {{- if $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "true" {{- if eq $pm.provider "cloudStorage" }} diff --git a/charts/sn-platform/templates/_helpers.tpl b/charts/sn-platform/templates/_helpers.tpl index 46a0e3855..467199cd4 100644 --- a/charts/sn-platform/templates/_helpers.tpl +++ b/charts/sn-platform/templates/_helpers.tpl @@ -193,6 +193,11 @@ Package management values validation. {{- define "pulsar.functions.packageManagement.validate" -}} {{- $pm := .Values.functions.packageManagement | default dict -}} {{- if $pm.enabled -}} +{{- $functionmesh := .Values.broker.functionmesh | default dict -}} +{{- $mesh := $functionmesh.mesh | default dict -}} +{{- if and $functionmesh.enabled (hasKey $mesh "uploadEnabled") (not $mesh.uploadEnabled) -}} +{{- fail "functions.packageManagement.enabled=true conflicts with broker.functionmesh.mesh.uploadEnabled=false" -}} +{{- end -}} {{- $provider := required "functions.packageManagement.provider is required when functions.packageManagement.enabled=true" $pm.provider -}} {{- if and (ne $provider "bookkeeper") (ne $provider "cloudStorage") -}} {{- fail "functions.packageManagement.provider must be one of: bookkeeper, cloudStorage" -}} diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml index b432fb962..35b56acde 100644 --- a/charts/sn-platform/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform/templates/broker/broker-cluster.yaml @@ -338,10 +338,13 @@ spec: {{- end }} custom: PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet" + {{- $pm := .Values.functions.packageManagement | default dict }} {{- if include "pulsar.metadata.isOxia" . }} PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}" + {{- if not $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "false" + {{- end }} PULSAR_PREFIX_kafkaGroupOffsetsStoreInMetadata: "true" PULSAR_PREFIX_kafkaProducerStateStoreInMetadata: "true" PULSAR_PREFIX_kafkaSchemaRegistryStoreInOxia: "true" @@ -353,7 +356,6 @@ spec: PULSAR_PREFIX_schemaRegistryStorageClassName: {{ default "io.streamnative.pulsar.schema.OxiaSchemaStorageFactory" $storageCatalog.schemaRegistryStorageClassName | quote }} PULSAR_PREFIX_topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService" {{- end }} - {{- $pm := .Values.functions.packageManagement | default dict }} {{- if $pm.enabled }} PULSAR_PREFIX_enablePackagesManagement: "true" {{- if eq $pm.provider "cloudStorage" }}